def executeAction(self, nextinput, work):
""" Execute an action and deal with the error handling and upload of the tasklogfile to the crabcache
"""
try:
output = work.execute(nextinput, task=self._task, tempDir=self.tempDir)
except TapeDatasetException as tde:
raise TapeDatasetException(str(tde))
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
raise WorkerHandlerException(str(twe), retry = twe.retry) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
finally:
#TODO: we need to do that also in Worker.py otherwise some messages might only be in the TW file but not in the crabcache.
logpath = self.config.TaskWorker.logsDir+'/tasks/%s/%s.log' % (self._task['tm_username'], self._task['tm_taskname'])
if os.path.isfile(logpath) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {'endpoint':self._task['tm_cache_url'], 'cert':self._task['user_proxy'], 'key':self._task['user_proxy']}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task['tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = "Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s" % (self._task['tm_cache_url'], self._task['tm_taskname'], hte.headers, hte.result)
self.logger.error(msg)
except Exception: #pylint: disable=broad-except
msg = "Unknown error while uploading the logfile for task %s" % self._task['tm_taskname']
self.logger.exception(msg) #upload logfile of the task to the crabcache
return output
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
for work in self.getWorks():
self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler, sh:
msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh))
self.logger.error(msg)
nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException, twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST
class TaskHandler(object):
"""Handling the set of operations to be performed."""
def __init__(self, task):
"""Initializer
:arg TaskWorker.DataObjects.Task task: the task to work on."""
self.logger = logging.getLogger()
self._work = []
self._task = task
def addWork(self, work):
"""Appending a new action to be performed on the task
:arg callable work: a new callable to be called :)"""
if work not in self._work:
self._work.append( work )
def getWorks(self):
"""Retrieving the queued actions
:return: generator of actions to be performed."""
for w in self._work:
yield w
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
for work in self.getWorks():
self.logger.debug("Starting %s on %s" % (str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler, sh:
msg = "Controlled stop of handler for %s on %s " % (self._task, str(sh))
self.logger.error(msg)
nextinput = Result(task=self._task, result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException, twe:
self.logger.debug(str(traceback.format_exc())) #print the stacktrace only in debug mode
raise WorkerHandlerException(str(twe)) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception, exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
raise WorkerHandlerException(msg) #Errors not foreseen. Print everything!
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
for work in self.getWorks():
self.logger.debug("Starting %s on %s" %
(str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler, sh:
msg = "Controlled stop of handler for %s on %s " % (self._task,
str(sh))
self.logger.error(msg)
nextinput = Result(
task=self._task,
result='StopHandler exception received, controlled stop')
break
except Exception, exc:
msg = "Problem handling %s because of %s failure, tracebak follows\n" % (
self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
raise WorkerHandlerException(msg)
def actionWork(self, *args, **kwargs):
"""Performing the set of actions"""
nextinput = args
taskhandler = self.addTaskLogHandler()
# I know it looks like a duplicated printout from the process logs (proc.N.log) perspective.
# Infact we have a smilar printout in the processWorker function of the Worker module, but
# it does not go to the task logfile and it is useful imho.
self.logger.debug(
"Process %s is starting %s on task %s" %
(self.procnum, self.workFunction, self._task['tm_taskname']))
for work in self.getWorks():
#Loop that iterates over the actions to be performed
self.logger.debug("Starting %s on %s" %
(str(work), self._task['tm_taskname']))
t0 = time.time()
try:
output = work.execute(nextinput, task=self._task)
except StopHandler as sh:
msg = "Controlled stop of handler for %s on %s " % (self._task,
str(sh))
self.logger.error(msg)
nextinput = Result(
task=self._task,
result='StopHandler exception received, controlled stop')
break #exit normally. Worker will not notice there was an error
except TaskWorkerException as twe:
self.logger.debug(str(traceback.format_exc())
) #print the stacktrace only in debug mode
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(
str(twe)
) #TaskWorker error, do not add traceback to the error propagated to the REST
except Exception as exc:
msg = "Problem handling %s because of %s failure, traceback follows\n" % (
self._task['tm_taskname'], str(exc))
msg += str(traceback.format_exc())
self.logger.error(msg)
self.removeTaskLogHandler(taskhandler)
raise WorkerHandlerException(
msg) #Errors not foreseen. Print everything!
finally:
#upload logfile of the task to the crabcache
logpath = 'logs/tasks/%s/%s.log' % (self._task['tm_username'],
self._task['tm_taskname'])
if os.path.isfile(
logpath
) and 'user_proxy' in self._task: #the user proxy might not be there if myproxy retrieval failed
cacheurldict = {
'endpoint': self._task['tm_cache_url'],
'cert': self._task['user_proxy'],
'key': self._task['user_proxy']
}
try:
ufc = UserFileCache(cacheurldict)
logfilename = self._task[
'tm_taskname'] + '_TaskWorker.log'
ufc.uploadLog(logpath, logfilename)
except HTTPException as hte:
msg = (
"Failed to upload the logfile to %s for task %s. More details in the http headers and body:\n%s\n%s"
% (self._task['tm_cache_url'],
self._task['tm_taskname'], hte.headers,
hte.result))
self.logger.error(msg)
except Exception:
msg = "Unknown error while uploading the logfile for task %s" % self._task[
'tm_taskname']
self.logger.exception(msg)
t1 = time.time()
self.logger.info("Finished %s on %s in %d seconds" %
(str(work), self._task['tm_taskname'], t1 - t0))
try:
nextinput = output.result
except AttributeError:
nextinput = output
self.removeTaskLogHandler(taskhandler)
return nextinput