def __call__(self):
if self.options.short:
taskname = self.cachedinfo['RequestName']
inputlist = {'subresource': 'webdir', 'workflow': taskname}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
uri = self.getUrl(self.instance, resource = 'task')
webdir = getProxiedWebDir(taskname, self.serverurl, uri, self.proxyfilename, self.logger.debug)
if not webdir:
dictresult, status, reason = server.get(uri, data = inputlist)
webdir = dictresult['result'][0]
self.logger.info('Server result: %s' % webdir)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
failed, success = self.retrieveShortLogs(webdir, self.proxyfilename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED,colors.NORMAL,failed)
self.logger.info(msg)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN,colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
returndict = getcommand.__call__(self, subresource = 'logs')
if ('success' in returndict and not returndict['success']) or \
('failed' in returndict and returndict['failed']):
msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler."
self.logger.info(msg)
return returndict
def __call__(self): # pylint: disable=arguments-differ
if self.options.short:
taskname = self.cachedinfo['RequestName']
inputlist = {'subresource': 'search', 'workflow': taskname}
server = self.crabserver
webdir = getProxiedWebDir(crabserver=self.crabserver, task=taskname, logFunction=self.logger.debug)
dictresult, status, reason = server.get(api='task', data=inputlist)
if not webdir:
webdir = dictresult['result'][0]
self.logger.info('Server result: %s' % webdir)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
splitting = getColumn(dictresult, 'tm_split_algo')
if getattr(self.options, 'jobids', None):
self.options.jobids = validateJobids(self.options.jobids, splitting != 'Automatic')
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
failed, success = self.retrieveShortLogs(webdir, self.proxyfilename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED, colors.NORMAL, failed)
self.logger.info(msg)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN, colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
# Different from the old getlog code: set 'logs2' as subresource so that 'getcommand' uses the new logic.
returndict = getcommand.__call__(self, subresource='logs2')
if ('success' in returndict and not returndict['success']) or \
('failed' in returndict and returndict['failed']):
msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler."
self.logger.info(msg)
return returndict
def __call__(self):
returndict = getcommand.__call__(self, subresource='data')
if not returndict.get('success', {}) and not returndict.get(
'failed', {}):
msg = "This is normal behavior if General.transferOutputs=False is present in the task configuration."
self.logger.info(msg)
return returndict
def __call__(self):
if self.options.short:
inputlist = {'subresource': 'webdir', 'workflow': self.cachedinfo['RequestName']}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
uri = self.getUrl(self.instance, resource = 'task')
dictresult, status, reason = server.get(uri, data = inputlist)
self.logger.info('Server result: %s' % dictresult['result'][0])
dictresult = self.processServerResult(dictresult)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
self.logger.info("Retrieving...")
success = []
failed = []
for item in self.options.jobids:
jobid = str(item[1])
filename = 'job_out.'+jobid+'.0.txt'
url = dictresult['result'][0]+'/'+filename
try:
getFileFromURL(url, self.dest+'/'+filename)
self.logger.info ('Retrieved %s' % (filename))
success.append(filename)
retry = 1
#To retrieve retried joblog, if there is any.
while urllib.urlopen(dictresult['result'][0]+'/'+'job_out.'+jobid+'.'+str(retry)+'.txt').getcode() == 200:
filename = 'job_out.'+jobid+'.'+str(retry)+'.txt'
url = dictresult['result'][0]+'/'+filename
getFileFromURL(url, self.dest+'/'+filename)
self.logger.info ('Retrieved %s' % (filename))
success.append(filename)
retry = retry + 1
except ClientException as ex:
self.logger.debug(str(ex))
failed.append(filename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED,colors.NORMAL,failed)
self.logger.info(msg)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN,colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
returndict = getcommand.__call__(self, subresource = 'logs')
if ('success' in returndict and not returndict['success']) or \
('failed' in returndict and returndict['failed']):
msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler."
self.logger.info(msg)
return returndict
def __call__(self):
if self.options.short:
#Check if splitting is automatic
try:
splitting=self.cachedinfo['OriginalConfig'].Data.splitting
except AttributeError: #Default setting is 'Automatic'
splitting='Automatic'
except KeyError: #crab remade task does not have 'OriginalConfig' key, need to fetch from DB
splitting='Unknown'
taskname = self.cachedinfo['RequestName']
inputlist = {'subresource': 'webdir', 'workflow': taskname}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl, self.proxyfilename, self.proxyfilename, version=__version__)
uri = self.getUrl(self.instance, resource = 'task')
webdir=None
if splitting!='Unknown':
webdir = getProxiedWebDir(taskname, self.serverurl, uri, self.proxyfilename, self.logger.debug)
if not webdir:
dictresult, status, reason = server.get(uri, data = inputlist)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
if splitting=='Unknown':
splitting=getColumn(dictresult,'tm_split_algo')
webdir = dictresult['result'][0]
self.logger.info('Server result: %s' % webdir)
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
#check the format of jobids
self.options.jobids = validateJobids(self.options.jobids,splitting!='Automatic')
failed, success = self.retrieveShortLogs(webdir, self.proxyfilename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED,colors.NORMAL,failed)
self.logger.info(msg)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN,colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
# Different from the old getlog code: set 'logs2' as subresource so that 'getcommand' uses the new logic.
returndict = getcommand.__call__(self, subresource = 'logs2')
if ('success' in returndict and not returndict['success']) or \
('failed' in returndict and returndict['failed']):
msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler."
self.logger.info(msg)
return returndict
def __call__(self):
if self.options.short:
taskname = self.cachedinfo['RequestName']
inputlist = {'subresource': 'webdir', 'workflow': taskname}
serverFactory = CRABClient.Emulator.getEmulator('rest')
server = serverFactory(self.serverurl,
self.proxyfilename,
self.proxyfilename,
version=__version__)
uri = self.getUrl(self.instance, resource='task')
webdir = getProxiedWebDir(taskname, self.serverurl, uri,
self.proxyfilename, self.logger.debug)
if not webdir:
dictresult, status, reason = server.get(uri, data=inputlist)
webdir = dictresult['result'][0]
self.logger.info('Server result: %s' % webdir)
if status != 200:
msg = "Problem retrieving information from the server:\ninput:%s\noutput:%s\nreason:%s" % (
str(inputlist), str(dictresult), str(reason))
raise RESTCommunicationException(msg)
self.setDestination()
self.logger.info("Setting the destination to %s " % self.dest)
failed, success = self.retrieveShortLogs(webdir,
self.proxyfilename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (
colors.RED, colors.NORMAL, failed)
self.logger.info(msg)
else:
self.logger.info(
"%sSuccess%s: All files successfully retrieved." %
(colors.GREEN, colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
# Different from the old getlog code: set 'logs2' as subresource so that 'getcommand' uses the new logic.
returndict = getcommand.__call__(self, subresource='logs2')
if ('success' in returndict and not returndict['success']) or \
('failed' in returndict and returndict['failed']):
msg = "You can use the --short option to retrieve a short version of the log files from the Grid scheduler."
self.logger.info(msg)
return returndict
def __call__(self):
getcommand.__call__(self, subresource = 'logs')
def __call__(self):
returndict = getcommand.__call__(self, subresource = 'data')
if not returndict.get('success', {}) and not returndict.get('failed', {}):
msg = "This is normal behavior if General.transferOutputs=False is present in the task configuration."
self.logger.info(msg)
return returndict
def __call__(self):
returndict = getcommand.__call__(self, subresource = 'data2')
return returndict
def __call__(self):
getcommand.__call__(self, subresource = 'data')
def __call__(self):
return getcommand.__call__(self, subresource='logs')
url = dictresult['result'][0]+'/'+filename
getFileFromURL(url, self.dest+'/'+filename)
self.logger.info ('Retrieved %s' % (filename))
success.append(filename)
retry = retry + 1
except ClientException, ex:
self.logger.debug(str(ex))
failed.append(filename)
if failed:
msg = "%sError%s: Failed to retrieve the following files: %s" % (colors.RED,colors.NORMAL,failed)
self.logger.info(msg)
else:
self.logger.info("%sSuccess%s: All files successfully retrieved." % (colors.GREEN,colors.NORMAL))
returndict = {'success': success, 'failed': failed}
else:
returndict = getcommand.__call__(self, subresource = 'logs')
if not returndict.get('success', {}) and not returndict.get('failed', {}):
msg = "This is normal behavior unless General.transferLogs=True is present in the task configuration."
self.logger.info(msg)
return returndict
def setOptions(self):
"""
__setOptions__
This allows to set specific command options
"""
self.parser.add_option( '--quantity',
dest = 'quantity',
help = 'The number of logs you want to retrieve (or "all"). Ignored if --jobids is used.' )
self.parser.add_option( '--parallel',
def __call__(self):
returndict = getcommand.__call__(self, subresource = 'data')
return returndict
def __call__(self):
getcommand.__call__(self, subresource='data')
