def executeAction(self, nextinput, work):
""" Execute an action and deal with the error handling and upload of the tasklogfile to the crabcache
"""
try:
output = work.execute(nextinput, task=self._task, tempDir=self.tempDir)
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
raise WorkerHandlerException(str(twe), retry = twe.retry) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
finally:
#TODO: we need to do that also in Worker.py otherwise some messages might only be in the TW file but not in the crabcache.
logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname'])
if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {'endpoint':self._task['tm_cache_url'], 'cert':self._task['user_proxy'], 'key':self._task['user_proxy']}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task['tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = "Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" % (self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result)
self.logger.error(msg)
except Exception: #pylint: disable=broad-except
msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname']
self.logger.exception(msg) #upload logfile of the task to the crabcache
return output
def executeAction(self, nextinput, work):
""" Execute an action and deal with the error handling and upload of the tasklogfile to the crabcache
"""
try:
output = work.execute(nextinput, task=self._task, tempDir=self.tempDir)
except TapeDatasetException as tde:
raise TapeDatasetException(str(tde))
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
raise WorkerHandlerException(str(twe), retry = twe.retry) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
finally:
#TODO: we need to do that also in Worker.py otherwise some messages might only be in the TW file but not in the crabcache.
logpath = self.config.TaskWorker.logsDir+'/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname'])
if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {'endpoint':self._task['tm_cache_url'], 'cert':self._task['user_proxy'], 'key':self._task['user_proxy']}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task['tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = "Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" % (self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result)
self.logger.error(msg)
except Exception: #pylint: disable=broad-except
msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname']
self.logger.exception(msg) #upload logfile of the task to the crabcache
return output
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
#set the logger to save the tasklog
formatter = logging.Formatter("%(asctime)s:%(levelname)s:%(module)s:%(message)s")
taskdirname = "logs/tasks/%s/" % self._task['tm_username']
if not os.path.isdir(taskdirname):
os.mkdir(taskdirname)
taskhandler = FileHandler(taskdirname + self._task['tm_taskname'] + '.log')
taskhandler.setLevel(logging.DEBUG)
self.logger.addHandler(taskhandler)
for work in self.getWorks():
self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler as sh:
msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh))
self.logger.error(msg)
nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
finally:
#upload logfile of the task to the crabcache
logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname'])
if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {'endpoint': self._task['tm_cache_url'], 'cert' : self._task['user_proxy'], 'key' : self._task['user_proxy']}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task['tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = ("Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" %
(self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result))
self.logger.error(msg)
except Exception as e:
msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname']
self.logger.exception(msg)
t1 = time.time()
self.logger.info("Finished %s on %s in %d seconds" % (str(work), self._task['tm_taskname'], t1-t0))
try:
nextinput = output.result
except AttributeError:
nextinput = output
self.removeTaskLogHandler(taskhandler)
return nextinput
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
taskhandler = self.addTaskLogHandler()
# I know it looks like a duplicated printout from the process logs (proc.N.log) perspective.
# Infact we have a smilar printout in the processWorker function of the Worker module, but
# it does not go to the task logfile and it is useful imho.
self.logger.debug("Process %s is starting %s on task %s" % (self.procnum, self.workFunction, self._task['tm_taskname']))
for work in self.getWorks():
#Loop that iterates over the actions to be performed
self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler as sh:
msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh))
self.logger.error(msg)
nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
finally:
#upload logfile of the task to the crabcache
logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname'])
if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {'endpoint': self._task['tm_cache_url'], 'cert' : self._task['user_proxy'], 'key' : self._task['user_proxy']}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task['tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = ("Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" %
(self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result))
self.logger.error(msg)
except Exception:
msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname']
self.logger.exception(msg)
t1 = time.time()
self.logger.info("Finished %s on %s in %d seconds" % (str(work), self._task['tm_taskname'], t1 - t0))
try:
nextinput = output.result
except AttributeError:
nextinput = output
self.removeTaskLogHandler(taskhandler)
return nextinput
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz'
) #file to upload
upfileLog = path.join(
currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt'
) #file to upload
ufc = UserFileCache({
'endpoint': os.environ['UFCURL'],
'pycurl': True
})
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
#hashkey deletion
ufc.removeFile(res['hashkey'])
#log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog + '.downloaded')
self.assertTrue(filecmp.cmp(upfileLog, upfileLog + '.downloaded'))
def executeInternal(self, *args, **kw):
tempDir = args[0][0]
inputFiles = args[0][3]
splitterResult = args[0][4]
cwd = os.getcwd()
try:
os.chdir(tempDir)
splittingSummary = SplittingSummary(kw['task']['tm_split_algo'])
for jobgroup in splitterResult:
jobs = jobgroup.getJobs()
splittingSummary.addJobs(jobs)
splittingSummary.dump('splitting-summary.json')
inputFiles.append('splitting-summary.json')
self.packSandbox(inputFiles)
self.logger.info('Uploading dry run tarball to the user file cache')
ufc = UserFileCache(dict={'cert': kw['task']['user_proxy'], 'key': kw['task']['user_proxy'], 'endpoint': kw['task']['tm_cache_url']})
result = ufc.uploadLog('dry-run-sandbox.tar.gz')
os.remove('dry-run-sandbox.tar.gz')
if 'hashkey' not in result:
raise TaskWorkerException('Failed to upload dry-run-sandbox.tar.gz to the user file cache: ' + str(result))
else:
self.logger.info('Uploaded dry run tarball to the user file cache: ' + str(result))
update = {'workflow': kw['task']['tm_taskname'], 'subresource': 'state', 'status': 'UPLOADED'}
self.logger.debug('Updating task status: %s' % str(update))
self.server.post(self.resturi, data=urllib.urlencode(update))
finally:
os.chdir(cwd)
return Result(task=kw['task'], result=args[0])
def executeInternal(self, *args, **kw):
inputFiles = args[0][2]
splitterResult = args[0][3][0]
cwd = os.getcwd()
try:
os.chdir(kw['tempDir'])
splittingSummary = SplittingSummary(kw['task']['tm_split_algo'])
for jobgroup in splitterResult:
jobs = jobgroup.getJobs()
splittingSummary.addJobs(jobs)
splittingSummary.dump('splitting-summary.json')
inputFiles.append('splitting-summary.json')
self.packSandbox(inputFiles)
self.logger.info('Uploading dry run tarball to the user file cache')
ufc = UserFileCache(mydict={'cert': kw['task']['user_proxy'], 'key': kw['task']['user_proxy'], 'endpoint': kw['task']['tm_cache_url']})
result = ufc.uploadLog('dry-run-sandbox.tar.gz')
os.remove('dry-run-sandbox.tar.gz')
if 'hashkey' not in result:
raise TaskWorkerException('Failed to upload dry-run-sandbox.tar.gz to the user file cache: ' + str(result))
else:
self.logger.info('Uploaded dry run tarball to the user file cache: ' + str(result))
update = {'workflow': kw['task']['tm_taskname'], 'subresource': 'state', 'status': 'UPLOADED'}
self.logger.debug('Updating task status: %s' % str(update))
self.server.post(self.resturi, data=urllib.urlencode(update))
finally:
os.chdir(cwd)
return Result(task=kw['task'], result=args[0])
def executeInternal(self, *args, **kw):
inputFiles = args[0][2]
splitterResult = args[0][3][0]
cwd = os.getcwd()
try:
os.chdir(kw['tempDir'])
splittingSummary = SplittingSummary(kw['task']['tm_split_algo'])
for jobgroup in splitterResult:
jobs = jobgroup.getJobs()
splittingSummary.addJobs(jobs)
splittingSummary.dump('splitting-summary.json')
inputFiles.append('splitting-summary.json')
self.packSandbox(inputFiles)
self.logger.info(
'Uploading dry run tarball to the user file cache')
if 'S3' in kw['task']['tm_cache_url'].upper():
uploadToS3(crabserver=self.crabserver,
filepath='dry-run-sandbox.tar.gz',
objecttype='runtimefiles',
taskname=kw['task']['tm_taskname'],
logger=self.logger)
result = {
'hashkey': 'ok'
} # a dummy one to keep same semantics as when using UserFileCache
os.remove('dry-run-sandbox.tar.gz')
else:
ufc = UserFileCache(
mydict={
'cert': kw['task']['user_proxy'],
'key': kw['task']['user_proxy'],
'endpoint': kw['task']['tm_cache_url']
})
result = ufc.uploadLog('dry-run-sandbox.tar.gz')
os.remove('dry-run-sandbox.tar.gz')
if 'hashkey' not in result:
raise TaskWorkerException(
'Failed to upload dry-run-sandbox.tar.gz to the user file cache: '
+ str(result))
self.logger.info(
'Uploaded dry run tarball to the user file cache: %s',
str(result))
update = {
'workflow': kw['task']['tm_taskname'],
'subresource': 'state',
'status': 'UPLOADED'
}
self.logger.debug('Updating task status: %s', str(update))
self.crabserver.post(api='workflowdb',
data=urllib.urlencode(update))
finally:
os.chdir(cwd)
return Result(task=kw['task'], result=args[0])
def testUploadDownload(self):
if 'UFCURL' in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/test_file.tgz') #file to upload
upfileLog = path.join(currdir, 'WMCore_t/Services_t/UserFileCache_t/uplog.txt') #file to upload
ufc = UserFileCache({'endpoint':os.environ['UFCURL']})
#hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res['hashkey'], output='pippo_publish_down.tgz')
#log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog+'.downloaded')
self.assertTrue(filecmp.cmp(upfileLog, upfileLog+'.downloaded'))
def testUploadDownload(self):
if "UFCURL" in os.environ:
currdir = getTestBase()
upfile = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/test_file.tgz") # file to upload
upfileLog = path.join(currdir, "WMCore_t/Services_t/UserFileCache_t/uplog.txt") # file to upload
ufc = UserFileCache({"endpoint": os.environ["UFCURL"], "pycurl": True})
# hashkey upload/download
res = ufc.upload(upfile)
ufc.download(res["hashkey"], output="pippo_publish_down.tgz")
# hashkey deletion
ufc.removeFile(res["hashkey"])
# log upload/download
res = ufc.uploadLog(upfileLog)
ufc.downloadLog(upfileLog, upfileLog + ".downloaded")
self.assertTrue(filecmp.cmp(upfileLog, upfileLog + ".downloaded"))
doupload = False
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
baseurl = getUrl(instance = instance , resource = 'info')
if doupload:
cacheurl = server_info('backendurls', serverurl, proxyfilename, baseurl)
cacheurl = cacheurl['cacheSSL']
cacheurldict = {'endpoint': cacheurl}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" % (cacheurl, logfilename))
logger.info("Uploading log file...")
ufc.uploadLog(logpath, logfilename)
logger.info("%sSuccess%s: Log file uploaded successfully." % (colors.GREEN, colors.NORMAL))
logfileurl = cacheurl + '/logfile?name='+str(logfilename)
if not username:
username = getUsernameFromSiteDB_wrapped(logger, quiet = True)
if username:
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
def getPlugins(namespace, plugins, skip):
def uploadlogfile(logger,
proxyfilename,
logfilename=None,
logpath=None,
instance='prod',
serverurl=None,
username=None):
## WMCore dependencies. Moved here to minimize dependencies in the bootstrap script
from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
doupload = True
if logfilename == None:
logfilename = str(time.strftime("%Y-%m-%d_%H%M%S")) + '_crab.log'
logger.info('Fetching user enviroment to log file')
try:
cmd = 'env'
logger.debug('Running env command')
pipe = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True)
stdout, dummyStderr = pipe.communicate()
logger.debug('\n\n\nUSER ENVIROMENT\n%s' % stdout)
except Exception as se:
logger.debug('Failed to get the user env\nException message: %s' %
(se))
if logpath != None:
if not os.path.exists(logpath):
doupload = False
logger.debug('%sError%s: %s does not exist' %
(colors.RED, colors.NORMAL, logpath))
else:
if os.path.exists(str(os.getcwd()) + '/crab.log'):
logpath = str(os.getcwd()) + '/crab.log'
else:
logger.debug(
'%sError%s: Failed to find crab.log in current directory %s' %
(colors.RED, colors.NORMAL, str(os.getcwd())))
if serverurl == None and instance in SERVICE_INSTANCES.keys():
serverurl = SERVICE_INSTANCES[instance]
elif not instance in SERVICE_INSTANCES.keys() and serverurl != None:
instance = 'private'
elif not instance in SERVICE_INSTANCES.keys() and serverurl == None:
logger.debug('%sError%s: serverurl is None' %
(colors.RED, colors.NORMAL))
doupload = False
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
baseurl = getUrl(instance=instance, resource='info')
if doupload:
cacheurl = server_info('backendurls', serverurl, proxyfilename,
baseurl)
# Encode in ascii because old pycurl present in old CMSSW versions
# doesn't support unicode.
cacheurl = cacheurl['cacheSSL'].encode('ascii')
cacheurldict = {'endpoint': cacheurl, "pycurl": True}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" %
(cacheurl, logfilename))
logger.info("Uploading log file...")
ufc.uploadLog(logpath, logfilename)
logger.info("%sSuccess%s: Log file uploaded successfully." %
(colors.GREEN, colors.NORMAL))
logfileurl = cacheurl + '/logfile?name=' + str(logfilename)
if not username:
username = getUserDNandUsername(logger).get('username')
if username:
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
def uploadlogfile(logger, proxyfilename, logfilename = None, logpath = None, instance = 'prod', serverurl = None, username = None):
## WMCore dependencies. Moved here to minimize dependencies in the bootstrap script
from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
doupload = True
if logfilename == None:
logfilename = str(time.strftime("%Y-%m-%d_%H%M%S"))+'_crab.log'
logger.info('Fetching user enviroment to log file')
try:
cmd = 'env'
logger.debug('Running env command')
pipe = subprocess.Popen(cmd, stdout = subprocess.PIPE, stderr = subprocess.PIPE, shell = True)
stdout, stderr = pipe.communicate()
logger.debug('\n\n\nUSER ENVIROMENT\n%s' % stdout)
except Exception as se:
logger.debug('Failed to get the user env\nException message: %s' % (se))
if logpath != None:
if not os.path.exists(logpath):
doupload = False
logger.debug('%sError%s: %s does not exist' %(colors.RED, colors.NORMAL, logpath))
else:
if os.path.exists(str(os.getcwd()) + '/crab.log'):
logpath = str(os.getcwd())+'/crab.log'
else:
logger.debug('%sError%s: Failed to find crab.log in current directory %s' % (colors.RED, colors.NORMAL, str(os.getcwd())))
if serverurl == None and instance in SERVICE_INSTANCES.keys():
serverurl = SERVICE_INSTANCES[instance]
elif not instance in SERVICE_INSTANCES.keys() and serverurl != None:
instance = 'private'
elif not instance in SERVICE_INSTANCES.keys() and serverurl == None:
logger.debug('%sError%s: serverurl is None' % (colors.RED, colors.NORMAL))
doupload = False
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
baseurl = getUrl(instance = instance , resource = 'info')
if doupload:
cacheurl = server_info('backendurls', serverurl, proxyfilename, baseurl)
cacheurl = cacheurl['cacheSSL']
cacheurldict = {'endpoint': cacheurl}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" % (cacheurl, logfilename))
logger.info("Uploading log file...")
ufc.uploadLog(logpath, logfilename)
logger.info("%sSuccess%s: Log file uploaded successfully." % (colors.GREEN, colors.NORMAL))
logfileurl = cacheurl + '/logfile?name='+str(logfilename)
if not username:
username = getUsernameFromSiteDB_wrapped(logger, quiet = True)
if username:
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
def executeInternal(self, *args, **kw):
inputFiles = args[0][2]
splitterResult = args[0][3][0]
cwd = os.getcwd()
try:
os.chdir(kw['tempDir'])
splittingSummary = SplittingSummary(kw['task']['tm_split_algo'])
for jobgroup in splitterResult:
jobs = jobgroup.getJobs()
splittingSummary.addJobs(jobs)
splittingSummary.dump('splitting-summary.json')
inputFiles.append('splitting-summary.json')
self.packSandbox(inputFiles)
self.logger.info(
'Uploading dry run tarball to the user file cache')
t0 = time.time()
if 'S3' in kw['task']['tm_cache_url'].upper():
uploadToS3(crabserver=self.crabserver,
filepath='dry-run-sandbox.tar.gz',
objecttype='runtimefiles',
taskname=kw['task']['tm_taskname'],
logger=self.logger)
result = {
'hashkey': 'ok'
} # a dummy one to keep same semantics as when using UserFileCache
os.remove('dry-run-sandbox.tar.gz')
else:
ufc = UserFileCache(
mydict={
'cert': kw['task']['user_proxy'],
'key': kw['task']['user_proxy'],
'endpoint': kw['task']['tm_cache_url']
})
result = ufc.uploadLog('dry-run-sandbox.tar.gz')
os.remove('dry-run-sandbox.tar.gz')
if 'hashkey' not in result:
raise TaskWorkerException(
'Failed to upload dry-run-sandbox.tar.gz to the user file cache: '
+ str(result))
self.logger.info(
'Uploaded dry run tarball to the user file cache: %s',
str(result))
# wait until tarball is available, S3 may take a few seconds for this (ref. issue #6706 )
t1 = time.time()
lt1 = time.strftime("%H:%M:%S", time.localtime(t1))
uploadTime = t1 - t0
self.logger.debug(
'runtimefiles upload took %s secs and completed at %s',
uploadTime, lt1)
self.logger.debug('check if tarball is available')
tarballOK = False
while not tarballOK:
try:
self.logger.debug('download tarball to /dev/null')
downloadFromS3(crabserver=self.crabserver,
filepath='/dev/null',
objecttype='runtimefiles',
taskname=kw['task']['tm_taskname'],
logger=self.logger)
self.logger.debug('OK, it worked')
tarballOK = True
except Exception as e:
self.logger.debug('runtimefiles tarball not ready yet')
self.logger.debug('Exception was raised: %s', e)
self.logger.debug('Sleep 5 sec')
time.sleep(5)
update = {
'workflow': kw['task']['tm_taskname'],
'subresource': 'state',
'status': 'UPLOADED'
}
self.logger.debug('Updating task status: %s', str(update))
self.crabserver.post(api='workflowdb',
data=urllib.urlencode(update))
finally:
os.chdir(cwd)
return Result(task=kw['task'], result=args[0])
def uploadlogfile(logger,
proxyfilename,
taskname=None,
logfilename=None,
logpath=None,
instance=None,
serverurl=None,
username=None):
## WMCore dependencies. Moved here to minimize dependencies in the bootstrap script
from WMCore.Services.UserFileCache.UserFileCache import UserFileCache
doupload = True
if logfilename == None:
logfilename = str(time.strftime("%Y-%m-%d_%H%M%S")) + '_crab.log'
logger.info('Fetching user enviroment to log file')
try:
logger.debug('Running env command')
stdout, _, _ = execute_command(command='env')
logger.debug('\n\n\nUSER ENVIROMENT\n%s' % stdout)
except Exception as se:
logger.debug('Failed to get the user env\nException message: %s' %
(se))
if logpath != None:
if not os.path.exists(logpath):
doupload = False
logger.debug('%sError%s: %s does not exist' %
(colors.RED, colors.NORMAL, logpath))
else:
if os.path.exists(str(os.getcwd()) + '/crab.log'):
logpath = str(os.getcwd()) + '/crab.log'
else:
logger.debug(
'%sError%s: Failed to find crab.log in current directory %s' %
(colors.RED, colors.NORMAL, str(os.getcwd())))
if proxyfilename == None:
logger.debug('No proxy was given')
doupload = False
if doupload:
# uploadLog is executed directly from crab main script, does not inherit from SubCommand
# so it needs its own REST server instantiation
restClass = CRABClient.Emulator.getEmulator('rest')
crabserver = restClass(hostname=serverurl,
localcert=proxyfilename,
localkey=proxyfilename,
retry=2,
logger=logger,
verbose=False,
version=__version__,
userAgent='CRABClient')
crabserver.setDbInstance(instance)
cacheurl = server_info(crabserver=crabserver,
subresource='backendurls')['cacheSSL']
logger.info("Uploading log file...")
if 'S3' in cacheurl.upper():
objecttype = 'clientlog'
uploadToS3(crabserver=crabserver,
filepath=logpath,
objecttype=objecttype,
taskname=taskname,
logger=logger)
logfileurl = getDownloadUrlFromS3(crabserver=crabserver,
objecttype=objecttype,
taskname=taskname,
logger=logger)
else:
cacheurldict = {'endpoint': cacheurl, "pycurl": True}
ufc = UserFileCache(cacheurldict)
logger.debug("cacheURL: %s\nLog file name: %s" %
(cacheurl, logfilename))
ufc.uploadLog(logpath, logfilename)
logfileurl = cacheurl + '/logfile?name=' + str(logfilename)
if not username:
from CRABClient.UserUtilities import getUsername
username = getUsername(proxyFile=proxyfilename, logger=logger)
logfileurl += '&username='******'Failed to upload the log file')
logfileurl = False
return logfileurl
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
taskhandler = self.addTaskLogHandler()
# I know it looks like a duplicated printout from the process logs (proc.N.log) perspective.
# Infact we have a smilar printout in the processWorker function of the Worker module, but
# it does not go to the task logfile and it is useful imho.
self.logger.debug(
"Process %s is starting %s on task %s" %
(self.procnum, self.workFunction, self._task['tm_taskname']))
for work in self.getWorks():
#Loop that iterates over the actions to be performed
self.logger.debug("Starting %s on %s" %
(str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler as sh:
msg = "Controlled stop of handler for %s on %s " % (self._task,
str(sh))
self.logger.error(msg)
nextinput = Result(
task=self._task,
result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())
) #print the stacktrace only in debug mode
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(
str(twe)
) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (
self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(
msg) #Errors not foreseen. Print everything!
finally:
#upload logfile of the task to the crabcache
logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'],
self._task['tm_taskname'])
if os.path.isfile(
logpath
) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {
'endpoint': self._task['tm_cache_url'],
'cert': self._task['user_proxy'],
'key': self._task['user_proxy']
}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task[
'tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = (
"Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s"
% (self._task['tm_cache_url'],
self._task['tm_taskname'], hte.headers,
hte.result))
self.logger.error(msg)
except Exception:
msg = "Unknown error while uploading the logfile for task %s" % self._task[
'tm_taskname']
self.logger.exception(msg)
t1 = time.time()
self.logger.info("Finished %s on %s in %d seconds" %
(str(work), self._task['tm_taskname'], t1 - t0))
try:
nextinput = output.result
except AttributeError:
nextinput = output
self.removeTaskLogHandler(taskhandler)
return nextinput
