def _run(self):
"""Wraps Run() to update and validate the task's status.
Uncaught exceptions are fatal.
Returns:
Task completion status.
"""
try:
self._start_time = datetime.datetime.now()
try:
self._state = self.run()
finally:
self._end_time = datetime.datetime.now()
if not self.completed:
logging.error(
'%r returned invalid task completion code: %r',
type(self).run, self._state)
# A task status that is neither SUCCESS nor FAILURE is a programming
# error: program should not continue in such a scenario.
base.shutdown()
return self._state
except:
logging.error('Unhandled exception from Task.run().')
traceback.print_exc()
# It is arguable whether an uncaught exception should cause the program
# to stop. We current assume an uncaught exception is a programming error.
# This could be made configurable via a flag.
base.shutdown()
def _run(self):
"""Worker loop."""
while True:
task = self._task_queue.pick()
if task is None:
logging.debug('Shutting down worker %s.', self)
return
logging.info('Worker %s running task %s', self._worker_id, task.task_id)
# Task exceptions should not kill the worker:
try:
task._run()
task.workflow._report_task_complete(task)
except:
logging.error('Unhandled exception from Task.run(): task not completed')
# TODO: retry task?
traceback.print_exc()
base.shutdown()