def __call__(self, **argv):
#Setting default destination if -o is not provided
if not self.dest:
self.dest = os.path.join(self.requestarea, 'results')
#Creating the destination directory if necessary
if not os.path.exists(self.dest):
self.logger.debug("Creating directory %s " % self.dest)
os.makedirs(self.dest)
elif not os.path.isdir(self.dest):
raise ConfigurationException('Destination directory is a file')
self.logger.info("Setting the destination directory to %s " %
self.dest)
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' %
self.cachedinfo['RequestName'])
inputlist = [('workflow', self.cachedinfo['RequestName'])]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' %
self.options.quantity)
inputlist.append(('limit', self.options.quantity))
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids)
inputlist.extend(self.options.jobids)
server = HTTPRequests(self.serverurl, self.proxyfilename)
dictresult, status, reason = server.get(self.uri, data=inputlist)
self.logger.debug('Server result: %s' % dictresult)
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (
str(inputlist), str(dictresult), str(reason))
raise ConfigurationException(msg)
totalfiles = len(dictresult['result'])
cpresults = []
# for workflow in dictresult['result']: TODO re-enable this when we will have resubmissions
workflow = dictresult[
'result'] #TODO assigning workflow to dictresult. for the moment we have only one wf
arglist = ['-d', self.dest, '-i', workflow]
if self.options.skipProxy:
arglist.append('-p')
if len(workflow) > 0:
self.logger.info("Retrieving %s files" % totalfiles)
copyoutput = remote_copy(self.logger, arglist)
copyoutput()
if totalfiles == 0:
self.logger.info("No files to retrieve")
def __call__(self, **argv):
#Setting default destination if -o is not provided
if not self.dest:
self.dest = os.path.join(self.requestarea, 'results')
#Creating the destination directory if necessary
if not os.path.exists( self.dest ):
self.logger.debug("Creating directory %s " % self.dest)
os.makedirs( self.dest )
elif not os.path.isdir( self.dest ):
raise ConfigurationException('Destination directory is a file')
self.logger.info("Setting the destination directory to %s " % self.dest )
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' % self.cachedinfo['RequestName'] )
inputlist = [ ('workflow', self.cachedinfo['RequestName']) ]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' % self.options.quantity )
inputlist.append( ('limit',self.options.quantity) )
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids )
inputlist.extend( self.options.jobids )
server = HTTPRequests(self.serverurl, self.proxyfilename)
dictresult, status, reason = server.get(self.uri, data = inputlist)
self.logger.debug('Server result: %s' % dictresult )
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise ConfigurationException(msg)
totalfiles = len( dictresult['result'] )
cpresults = []
# for workflow in dictresult['result']: TODO re-enable this when we will have resubmissions
workflow = dictresult['result'] #TODO assigning workflow to dictresult. for the moment we have only one wf
arglist = ['-d', self.dest, '-i', workflow]
if self.options.skipProxy:
arglist.append('-p')
if len(workflow) > 0:
self.logger.info("Retrieving %s files" % totalfiles )
copyoutput = remote_copy( self.logger, arglist )
copyoutput()
if totalfiles == 0:
self.logger.info("No files to retrieve")
def __call__(self, **argv):
## Retrieve the transferLogs parameter from the task database.
taskdbparam, configparam = '', ''
if argv.get('subresource') == 'logs':
taskdbparam = 'tm_save_logs'
configparam = "General.transferLogs"
elif argv.get('subresource') == 'data':
taskdbparam = 'tm_transfer_outputs'
configparam = "General.transferOutputs"
transferFlag = 'unknown'
inputlist = {'subresource': 'search', 'workflow': self.cachedinfo['RequestName']}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
uri = self.getUrl(self.instance, resource = 'task')
dictresult, status, reason = server.get(uri, data = inputlist)
self.logger.debug('Server result: %s' % dictresult)
if status == 200:
if 'desc' in dictresult and 'columns' in dictresult['desc']:
position = dictresult['desc']['columns'].index(taskdbparam)
transferFlag = dictresult['result'][position] #= 'T' or 'F'
else:
self.logger.debug("Unable to locate %s in server result." % (taskdbparam))
## If transferFlag = False, there is nothing to retrieve.
if transferFlag == 'F':
msg = "No files to retrieve. Files not transferred to storage since task configuration parameter %s is False." % (configparam)
self.logger.info(msg)
return {'success': {}, 'failed': {}}
## Retrieve tm_edm_outfiles, tm_tfile_outfiles and tm_outfiles from the task database and check if they are empty.
if argv.get('subresource') == 'data' and status == 200:
if 'desc' in dictresult and 'columns' in dictresult['desc']:
position = dictresult['desc']['columns'].index('tm_edm_outfiles')
tm_edm_outfiles = dictresult['result'][position]
position = dictresult['desc']['columns'].index('tm_tfile_outfiles')
tm_tfile_outfiles = dictresult['result'][position]
position = dictresult['desc']['columns'].index('tm_outfiles')
tm_outfiles = dictresult['result'][position]
if tm_edm_outfiles == '[]' and tm_tfile_outfiles == '[]' and tm_outfiles == '[]':
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " There are no output files to retrieve, because CRAB could not detect any in the CMSSW configuration"
msg += " nor was any explicitly specified in the CRAB configuration."
self.logger.warning(msg)
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' % self.cachedinfo['RequestName'])
inputlist = [('workflow', self.cachedinfo['RequestName'])]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' % self.options.quantity)
inputlist.append(('limit', self.options.quantity))
else:
self.logger.debug('Retrieving all file locations')
inputlist.append(('limit', -1))
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids)
inputlist.extend(self.options.jobids)
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, reason = server.get(self.uri, data = urllib.urlencode(inputlist))
self.logger.debug('Server result: %s' % dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
totalfiles = len(dictresult['result'])
workflow = dictresult['result']
if len(workflow) > 0:
if self.options.dump or self.options.xroot:
self.logger.debug("Getting url info")
else:
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
if self.options.xroot:
self.logger.debug("XRootD urls are requested")
xrootlfn = ["root://cms-xrd-global.cern.ch/%s" % link['lfn'] for link in workflow]
self.logger.info("\n".join(xrootlfn))
returndict = {'xrootd': xrootlfn}
elif self.options.dump:
jobid_pfn_lfn_list = sorted(map(lambda x: (x['jobid'], x['pfn'], x['lfn']), workflow))
lastjobid = -1
filecounter = 1
msg = ""
for jobid, pfn, lfn in jobid_pfn_lfn_list:
if jobid != lastjobid:
msg += "%s=== Files from job %s:" % ('\n' if lastjobid != -1 else '', jobid)
lastjobid = jobid
filecounter = 1
msg += "\n%d) PFN: %s" % (filecounter, pfn)
msg += "\n%s LFN: %s" % (' '*(len(str(filecounter))), lfn)
filecounter += 1
self.logger.info(msg)
returndict = {'pfn': [pfn for _, pfn, _ in jobid_pfn_lfn_list], 'lfn': [lfn for _, _, lfn in jobid_pfn_lfn_list]}
else:
self.logger.info("Retrieving %s files" % (totalfiles))
arglist = ['--destination', self.dest, '--input', workflow, '--dir', self.options.projdir, \
'--proxy', self.proxyfilename, '--parallel', self.options.nparallel, '--wait', self.options.waittime, \
'--checksum', self.checksum, '--command', self.command]
copyoutput = remote_copy(self.logger, arglist)
successdict, faileddict = copyoutput()
#need to use deepcopy because successdict and faileddict are dict that is under the a manage dict, accessed multithreadly
returndict = {'success': copy.deepcopy(successdict) , 'failed': copy.deepcopy(faileddict)}
if totalfiles == 0:
self.logger.info("No files to retrieve.")
returndict = {'success': {} , 'failed': {}}
if transferFlag == 'unknown':
if ('success' in returndict and not returndict['success']) and \
('failed' in returndict and not returndict['failed']):
msg = "This is normal behavior if %s = False in the task configuration." % (configparam)
self.logger.info(msg)
return returndict
def __call__(self, **argv): # pylint: disable=arguments-differ
## Retrieve the transferLogs parameter from the task database.
taskdbparam, configparam = '', ''
if argv.get('subresource') in ['logs', 'logs2']:
taskdbparam = 'tm_save_logs'
configparam = "General.transferLogs"
elif argv.get('subresource') in ['data', 'data2']:
taskdbparam = 'tm_transfer_outputs'
configparam = "General.transferOutputs"
transferFlag = 'unknown'
inputlist = {'subresource': 'search', 'workflow': self.cachedinfo['RequestName']}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
uri = getUrl(self.instance, resource = 'task')
dictresult, status, _ = server.get(uri, data = inputlist)
self.logger.debug('Server result: %s' % dictresult)
splitting = None
if status == 200:
if 'desc' in dictresult and 'columns' in dictresult['desc']:
position = dictresult['desc']['columns'].index(taskdbparam)
transferFlag = dictresult['result'][position] #= 'T' or 'F'
position = dictresult['desc']['columns'].index('tm_split_algo')
splitting = dictresult['result'][position]
else:
self.logger.debug("Unable to locate %s in server result." % (taskdbparam))
## If transferFlag = False, there is nothing to retrieve.
if transferFlag == 'F':
msg = "No files to retrieve. Files not transferred to storage since task configuration parameter %s is False." % (configparam)
self.logger.info(msg)
return {'success': {}, 'failed': {}}
## Retrieve tm_edm_outfiles, tm_tfile_outfiles and tm_outfiles from the task database and check if they are empty.
if argv.get('subresource') in ['data', 'data2'] and status == 200:
if 'desc' in dictresult and 'columns' in dictresult['desc']:
position = dictresult['desc']['columns'].index('tm_edm_outfiles')
tm_edm_outfiles = dictresult['result'][position]
position = dictresult['desc']['columns'].index('tm_tfile_outfiles')
tm_tfile_outfiles = dictresult['result'][position]
position = dictresult['desc']['columns'].index('tm_outfiles')
tm_outfiles = dictresult['result'][position]
if tm_edm_outfiles == '[]' and tm_tfile_outfiles == '[]' and tm_outfiles == '[]':
msg = "%sWarning%s:" % (colors.RED, colors.NORMAL)
msg += " There are no output files to retrieve, because CRAB could not detect any in the CMSSW configuration"
msg += " nor was any explicitly specified in the CRAB configuration."
self.logger.warning(msg)
#check the format of jobids
if getattr(self.options, 'jobids', None):
self.options.jobids = validateJobids(self.options.jobids, splitting != 'Automatic')
self.processAndStoreJobIds()
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' % self.cachedinfo['RequestName'])
inputlist = [('workflow', self.cachedinfo['RequestName'])]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' % self.options.quantity)
inputlist.append(('limit', self.options.quantity))
else:
self.logger.debug('Retrieving all file locations')
inputlist.append(('limit', -1))
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids)
inputlist.extend(self.options.jobids)
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, reason = server.get(self.uri, data = urllib.urlencode(inputlist))
self.logger.debug('Server result: %s' % dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
totalfiles = len(dictresult['result'])
fileInfoList = dictresult['result']
self.insertXrootPfns(fileInfoList)
if len(fileInfoList) > 0:
if self.options.dump or self.options.xroot:
self.logger.debug("Getting url info")
else:
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
if self.options.xroot:
self.logger.debug("XRootD urls are requested")
xrootlfn = ["root://cms-xrd-global.cern.ch/%s" % link['lfn'] for link in fileInfoList]
self.logger.info("\n".join(xrootlfn))
returndict = {'xrootd': xrootlfn}
elif self.options.dump:
jobid_pfn_lfn_list = sorted(map(lambda x: (x['jobid'], x['pfn'], x['lfn']), fileInfoList)) # pylint: disable=deprecated-lambda
lastjobid = -1
filecounter = 1
msg = ""
for jobid, pfn, lfn in jobid_pfn_lfn_list:
if jobid != lastjobid:
msg += "%s=== Files from job %s:" % ('\n' if lastjobid != -1 else '', jobid)
lastjobid = jobid
filecounter = 1
msg += "\n%d) PFN: %s" % (filecounter, pfn)
msg += "\n%s LFN: %s" % (' '*(len(str(filecounter))), lfn)
filecounter += 1
self.logger.info(msg)
returndict = {'pfn': [pfn for _, pfn, _ in jobid_pfn_lfn_list], 'lfn': [lfn for _, _, lfn in jobid_pfn_lfn_list]}
else:
self.logger.info("Retrieving %s files" % (totalfiles))
arglist = ['--destination', self.dest, '--input', fileInfoList, '--dir', self.options.projdir, \
'--proxy', self.proxyfilename, '--parallel', self.options.nparallel, '--wait', self.options.waittime, \
'--checksum', self.checksum, '--command', self.command]
copyoutput = remote_copy(self.logger, arglist)
successdict, faileddict = copyoutput()
#need to use deepcopy because successdict and faileddict are dict that is under the a manage dict, accessed multithreadly
returndict = {'success': copy.deepcopy(successdict) , 'failed': copy.deepcopy(faileddict)}
if totalfiles == 0:
self.logger.info("No files to retrieve.")
returndict = {'success': {} , 'failed': {}}
if transferFlag == 'unknown':
if ('success' in returndict and not returndict['success']) and \
('failed' in returndict and not returndict['failed']):
msg = "This is normal behavior if %s = False in the task configuration." % (configparam)
self.logger.info(msg)
return returndict
def __call__(self, **argv):
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' %
self.cachedinfo['RequestName'])
inputlist = [('workflow', self.cachedinfo['RequestName'])]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' %
self.options.quantity)
inputlist.append(('limit', self.options.quantity))
else:
self.logger.debug('Retrieving all file locations')
inputlist.append(('limit', -1))
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids)
inputlist.extend(self.options.jobids)
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl,
self.proxyfilename,
self.proxyfilename,
version=__version__)
dictresult, status, reason = server.get(self.uri, data=inputlist)
self.logger.debug('Server result: %s' % dictresult)
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (
str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
totalfiles = len(dictresult['result'])
cpresults = []
# for workflow in dictresult['result']: TODO re-enable this when we will have resubmissions
workflow = dictresult[
'result'] #TODO assigning workflow to dictresult. for the moment we have only one wf
if len(workflow) > 0:
if self.options.dump or self.options.xroot:
self.logger.debug("Getting url info")
else:
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
if self.options.xroot:
self.logger.debug("XRootD urls are requested")
xrootlfn = [
"root://cms-xrd-global.cern.ch/%s" % link['lfn']
for link in workflow
]
self.logger.info("\n".join(xrootlfn))
returndict = {'xrootd': xrootlfn}
elif self.dump:
jobid_pfn_lfn_list = map(
lambda x: (x['jobid'], x['pfn'], x['lfn']), workflow)
jobid_pfn_lfn_list.sort()
lastjobid = -1
filecounter = 1
msg = ""
for jobid, pfn, lfn in jobid_pfn_lfn_list:
if jobid != lastjobid:
msg += "%s=== Files from job %s:" % (
'\n' if lastjobid != -1 else '', jobid)
lastjobid = jobid
filecounter = 1
msg += "\n%d) PFN: %s" % (filecounter, pfn)
msg += "\n%s LFN: %s" % (' ' *
(len(str(filecounter))), lfn)
filecounter += 1
self.logger.info(msg)
returndict = {
'pfn': [pfn for _, pfn, _ in jobid_pfn_lfn_list],
'lfn': [lfn for _, _, lfn in jobid_pfn_lfn_list]
}
else:
self.logger.info("Retrieving %s files" % (totalfiles))
arglist = ['--destination', self.dest, '--input', workflow, '--dir', self.options.task, \
'--proxy', self.proxyfilename, '--parallel', self.options.nparallel, '--wait', self.options.waittime]
copyoutput = remote_copy(self.logger, arglist)
successdict, faileddict = copyoutput()
#need to use deepcopy because successdict and faileddict are dict that is under the a manage dict, accessed multithreadly
returndict = {
'success': copy.deepcopy(successdict),
'failed': copy.deepcopy(faileddict)
}
if totalfiles == 0:
## TODO: we should use an API to retrieve from the TaskDB what are the transfer flag values for the task.
## If the corresponding transfer flag is False, the user should not expect to be able to retrieve the files.
self.logger.info("No files to retrieve.")
returndict = {'success': {}, 'failed': {}}
return returndict
def __call__(self, **argv):
#Setting default destination if -o is not provided
if not self.dest:
self.dest = os.path.join(self.requestarea, 'results')
# Destination is a URL.
if re.match("^[a-z]+://", self.dest):
if not self.dest.endswith("/"):
self.dest += "/"
#Creating the destination directory if necessary
elif not os.path.exists( self.dest ):
self.logger.debug("Creating directory %s " % self.dest)
os.makedirs( self.dest)
elif not os.path.isdir( self.dest ):
raise ConfigurationException('Destination directory is a file')
if self.options.dump or self.options.xroot:
self.logger.debug("Getting url info")
else:
self.logger.info("Setting the destination to %s " % self.dest )
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' % self.cachedinfo['RequestName'] )
inputlist = [ ('workflow', self.cachedinfo['RequestName']) ]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' % self.options.quantity )
inputlist.append( ('limit',self.options.quantity) )
else:
self.logger.debug('Retrieving all file locations')
inputlist.append( ('limit', -1) )
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids )
inputlist.extend( self.options.jobids )
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, reason = server.get(self.uri, data = inputlist)
self.logger.debug('Server result: %s' % dictresult )
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
totalfiles = len(dictresult['result'])
cpresults = []
# for workflow in dictresult['result']: TODO re-enable this when we will have resubmissions
workflow = dictresult['result'] #TODO assigning workflow to dictresult. for the moment we have only one wf
arglist = ['--destination', self.dest, '--input', workflow, '--dir', self.options.task, '--proxy', self.proxyfilename, '--parallel', self.options.nparallel, '--wait', self.options.waittime]
if len(workflow) > 0:
if self.options.xroot:
self.logger.debug("XRootD urls are requested")
xrootlfn = ["root://cms-xrd-global.cern.ch/%s" % link['lfn'] for link in workflow]
self.logger.info("\n".join(xrootlfn))
returndict = {'xrootd': xrootlfn}
elif self.dump:
jobid_pfn_lfn_list = map(lambda x: (x['jobid'], x['pfn'], x['lfn']), workflow)
jobid_pfn_lfn_list.sort()
lastjobid = -1
filecounter = 1
msg = ""
for jobid, pfn, lfn in jobid_pfn_lfn_list:
if jobid != lastjobid:
msg += "%s=== Files from job %s:" % ('\n' if lastjobid != -1 else '', jobid)
lastjobid = jobid
filecounter = 1
msg += "\n%d) PFN: %s" % (filecounter, pfn)
msg += "\n%s LFN: %s" % (' '*(len(str(filecounter))), lfn)
filecounter += 1
self.logger.info(msg)
returndict = {'pfn': [pfn for _, pfn, _ in jobid_pfn_lfn_list], 'lfn': [lfn for _, _, lfn in jobid_pfn_lfn_list]}
else:
self.logger.info("Retrieving %s files" % totalfiles )
copyoutput = remote_copy( self.logger, arglist )
successdict, faileddict = copyoutput()
#need to use deepcopy because successdict and faileddict are dict that is under the a manage dict, accessed multithreadly
returndict = {'success' : copy.deepcopy(successdict) , 'failed' : copy.deepcopy(faileddict)}
if totalfiles == 0:
self.logger.info("No files to retrieve")
returndict = {'success' : {} , 'failed' : {}}
return returndict
def __call__(self, **argv):
#Setting default destination if -o is not provided
if not self.dest:
self.dest = os.path.join(self.requestarea, 'results')
# Destination is a URL.
if re.match("^[a-z]+://", self.dest):
if not self.dest.endswith("/"):
self.dest += "/"
#Creating the destination directory if necessary
elif not os.path.exists( self.dest ):
self.logger.debug("Creating directory %s " % self.dest)
os.makedirs( self.dest)
elif not os.path.isdir( self.dest ):
raise ConfigurationException('Destination directory is a file')
if self.options.dump or self.options.xroot:
self.logger.debug("Getting url info")
else:
self.logger.info("Setting the destination to %s " % self.dest )
#Retrieving output files location from the server
self.logger.debug('Retrieving locations for task %s' % self.cachedinfo['RequestName'] )
inputlist = [ ('workflow', self.cachedinfo['RequestName']) ]
inputlist.extend(list(argv.iteritems()))
if getattr(self.options, 'quantity', None):
self.logger.debug('Retrieving %s file locations' % self.options.quantity )
inputlist.append( ('limit',self.options.quantity) )
else:
self.logger.debug('Retrieving all file locations')
inputlist.append( ('limit', -1) )
if getattr(self.options, 'jobids', None):
self.logger.debug('Retrieving jobs %s' % self.options.jobids )
inputlist.extend( self.options.jobids )
server = HTTPRequests(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
dictresult, status, reason = server.get(self.uri, data = inputlist)
self.logger.debug('Server result: %s' % dictresult )
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise ConfigurationException(msg)
totalfiles = len( dictresult['result'] )
cpresults = []
# for workflow in dictresult['result']: TODO re-enable this when we will have resubmissions
workflow = dictresult['result'] #TODO assigning workflow to dictresult. for the moment we have only one wf
arglist = ['-d', self.dest, '-i', workflow, '-t', self.options.task, '-p', self.proxyfilename, '-l',self.options.nparallel, '-w',self.options.waittime]
if len(workflow) > 0:
if self.options.xroot:
self.logger.debug("XRootD url is requested")
for fileinfo in workflow:
self.logger.info("root://cms-xrd-global.cern.ch/%s" % fileinfo['lfn'])
elif self.dump:
for fileInfo in workflow:
self.logger.info(fileInfo['pfn'])
else:
self.logger.info("Retrieving %s files" % totalfiles )
copyoutput = remote_copy( self.logger, arglist )
copyoutput()
if totalfiles == 0:
self.logger.info("No files to retrieve")
