def execute(self, task):
if not isinstance(task, QueueTask):
raise TypeError('Unknown object: %s' % task)
try:
result = task.execute()
except Exception as exc:
if self.store_errors:
metadata = task.get_metadata()
metadata['error'] = repr(exc)
metadata['traceback'] = traceback.format_exc()
self.put(task.task_id, Error(metadata))
self.put_error(metadata)
raise
if self.result_store and not isinstance(task, PeriodicQueueTask):
if result is not None or self.store_none:
self.put(task.task_id, result)
if task.on_complete:
next_task = task.on_complete
next_task.extend_data(result)
self.enqueue(next_task)
return result
def _execute(self, task, timestamp):
if self._pre_execute:
try:
self._run_pre_execute(task)
except CancelExecution:
self._emit(S.SIGNAL_CANCELED, task)
return
start = time.time()
exception = None
task_value = None
try:
try:
task_value = task.execute()
finally:
duration = time.time() - start
except TaskLockedException as exc:
logger.warning('Task %s not run, unable to acquire lock.', task.id)
exception = exc
self._emit(S.SIGNAL_LOCKED, task)
except RetryTask as exc:
logger.info('Task %s raised RetryTask, retrying.', task.id)
task.retries += 1
exception = exc
except KeyboardInterrupt:
logger.warning('Received exit signal, %s did not finish.', task.id)
return
except Exception as exc:
logger.exception('Unhandled exception in task %s.', task.id)
exception = exc
self._emit(S.SIGNAL_ERROR, task, exc)
else:
logger.info('%s executed in %0.3fs', task, duration)
if self.results and not isinstance(task, PeriodicTask):
if exception is not None:
try:
tb = traceback.format_exc()
except AttributeError: # Seems to only happen on 3.4.
tb = '- unable to resolve traceback on Python 3.4 -'
self.put_result(task.id, Error({
'error': repr(exception),
'retries': task.retries,
'traceback': tb,
'task_id': task.id,
}))
elif task_value is not None or self.store_none:
self.put_result(task.id, task_value)
if self._post_execute:
self._run_post_execute(task, task_value, exception)
if exception is None:
# Task executed successfully, send the COMPLETE signal.
self._emit(S.SIGNAL_COMPLETE, task)
if task.on_complete and exception is None:
next_task = task.on_complete
next_task.extend_data(task_value)
self.enqueue(next_task)
elif task.on_error and exception is not None:
next_task = task.on_error
next_task.extend_data(exception)
self.enqueue(next_task)
if exception is not None and task.retries:
self._emit(S.SIGNAL_RETRYING, task)
self._requeue_task(task, self._get_timestamp())
return task_value
def _execute(self, task, timestamp):
if self._pre_execute:
try:
self._run_pre_execute(task)
except CancelExecution:
self._emit(S.SIGNAL_CANCELED, task)
return
start = time_clock()
exception = None
task_value = None
try:
self._tasks_in_flight.add(task)
try:
task_value = task.execute()
finally:
self._tasks_in_flight.remove(task)
duration = time_clock() - start
except TaskLockedException as exc:
logger.warning('Task %s not run, unable to acquire lock.', task.id)
exception = exc
self._emit(S.SIGNAL_LOCKED, task)
except RetryTask as exc:
logger.info('Task %s raised RetryTask, retrying.', task.id)
task.retries += 1
exception = exc
except CancelExecution as exc:
if exc.retry or (exc.retry is None and task.retries):
task.retries = max(task.retries, 1)
msg = '(task will be retried)'
else:
task.retries = 0
msg = '(aborted, will not be retried)'
logger.warning('Task %s raised CancelExecution %s.', task.id, msg)
self._emit(S.SIGNAL_CANCELED, task)
exception = exc
except KeyboardInterrupt:
logger.warning('Received exit signal, %s did not finish.', task.id)
self._emit(S.SIGNAL_INTERRUPTED, task)
return
except Exception as exc:
logger.exception('Unhandled exception in task %s.', task.id)
exception = exc
self._emit(S.SIGNAL_ERROR, task, exc)
else:
logger.info('%s executed in %0.3fs', task, duration)
if self.results and not isinstance(task, PeriodicTask):
if exception is not None:
error_data = self.build_error_result(task, exception)
self.put_result(task.id, Error(error_data))
elif task_value is not None or self.store_none:
self.put_result(task.id, task_value)
if self._post_execute:
self._run_post_execute(task, task_value, exception)
if exception is None:
# Task executed successfully, send the COMPLETE signal.
self._emit(S.SIGNAL_COMPLETE, task)
if task.on_complete and exception is None:
next_task = task.on_complete
next_task.extend_data(task_value)
self.enqueue(next_task)
elif task.on_error and exception is not None:
next_task = task.on_error
next_task.extend_data(exception)
self.enqueue(next_task)
if exception is not None and task.retries:
self._emit(S.SIGNAL_RETRYING, task)
self._requeue_task(task, self._get_timestamp())
return task_value