def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
This overrides the error handler run by the worker when the task fails.
It updates state, finish_time and traceback of the relevant task status
for asynchronous tasks. Skip updating status for synchronous tasks.
:param exc: The exception raised by the task.
:param task_id: Unique id of the failed task.
:param args: Original arguments for the executed task.
:param kwargs: Original keyword arguments for the executed task.
:param einfo: celery's ExceptionInfo instance, containing serialized traceback.
"""
if isinstance(exc, PulpCodedException):
_logger.info(_('Task failed : [%(task_id)s] : %(msg)s') %
{'task_id': task_id, 'msg': str(exc)})
_logger.debug(traceback.format_exc())
else:
_logger.info(_('Task failed : [%s]') % task_id)
# celery will log the traceback
if kwargs.get('scheduled_call_id') is not None:
utils.increment_failure_count(kwargs['scheduled_call_id'])
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
finish_time = dateutils.format_iso8601_datetime(now)
task_status = TaskStatus.objects.get(task_id=task_id)
task_status['state'] = constants.CALL_ERROR_STATE
task_status['finish_time'] = finish_time
task_status['traceback'] = einfo.traceback
if not isinstance(exc, PulpException):
exc = PulpException(str(exc))
task_status['error'] = exc.to_dict()
task_status.save()
self._handle_cProfile(task_id)
common_utils.delete_working_directory()
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
This overrides the error handler run by the worker when the task fails.
It updates state, finish_time and traceback of the relevant task status
for asynchronous tasks. Skip updating status for synchronous tasks.
:param exc: The exception raised by the task.
:param task_id: Unique id of the failed task.
:param args: Original arguments for the executed task.
:param kwargs: Original keyword arguments for the executed task.
:param einfo: celery's ExceptionInfo instance, containing serialized traceback.
"""
_logger.debug(traceback.format_exc())
if isinstance(exc, PulpCodedException):
_logger.info(
_('Task failed : [%(task_id)s] : %(msg)s') % {
'task_id': task_id,
'msg': str(exc)
})
else:
_logger.info(_('Task failed : [%s]') % task_id)
if kwargs.get('scheduled_call_id') is not None:
utils.increment_failure_count(kwargs['scheduled_call_id'])
try:
called_directly = self.request.called_directly
except Exception: # a workaround for when celery's internal state is bad
self._handle_on_failure_cleanup(task_id, exc, einfo)
raise
if not called_directly:
self._handle_on_failure_cleanup(task_id, exc, einfo)
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""
This overrides the error handler run by the worker when the task fails.
It updates state, finish_time and traceback of the relevant task status
for asynchronous tasks. Skip updating status for synchronous tasks.
:param exc: The exception raised by the task.
:param task_id: Unique id of the failed task.
:param args: Original arguments for the executed task.
:param kwargs: Original keyword arguments for the executed task.
:param einfo: celery's ExceptionInfo instance, containing serialized traceback.
"""
if isinstance(exc, PulpCodedException):
_logger.info(_('Task failed : [%(task_id)s] : %(msg)s') %
{'task_id': task_id, 'msg': str(exc)})
_logger.debug(traceback.format_exc())
else:
_logger.info(_('Task failed : [%s]') % task_id)
# celery will log the traceback
if kwargs.get('scheduled_call_id') is not None:
utils.increment_failure_count(kwargs['scheduled_call_id'])
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
finish_time = dateutils.format_iso8601_datetime(now)
task_status = TaskStatus.objects.get(task_id=task_id)
task_status['state'] = constants.CALL_ERROR_STATE
task_status['finish_time'] = finish_time
task_status['traceback'] = einfo.traceback
if not isinstance(exc, PulpException):
exc = PulpException(str(exc))
task_status['error'] = exc.to_dict()
task_status.save()
self._handle_cProfile(task_id)
common_utils.delete_working_directory()
def test_not_found(self, mock_from_db, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_find.return_value = None
utils.increment_failure_count(self.schedule_id)
# from_db() gets called if find_and_modify returns anything.
self.assertEqual(mock_from_db.call_count, 0)
def test_not_found(self, mock_from_db, mock_find):
mock_find.return_value = None
mock_find.__name__ = 'find_and_modify'
utils.increment_failure_count(self.schedule_id)
# from_db() gets called if find_and_modify returns anything.
self.assertEqual(mock_from_db.call_count, 0)
def test_not_found(self, mock_from_db, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_find.return_value = None
utils.increment_failure_count(self.schedule_id)
# from_db() gets called if find_and_modify returns anything.
self.assertEqual(mock_from_db.call_count, 0)
def test_threshold_none(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
mock_find.return_value = SCHEDULES[1]
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule
self.assertEqual(mock_update.call_count, 0)
def test_threshold_none(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
mock_find.return_value = SCHEDULES[1]
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule
self.assertEqual(mock_update.call_count, 0)
def test_threshold_none(self, mock_update, mock_find):
mock_find.return_value = SCHEDULES[1]
mock_find.__name__ = 'find_and_modify'
mock_update.__name__ = 'update'
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule
self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
schedule = SCHEDULES[0].copy()
schedule['enabled'] = False
mock_find.return_value = schedule
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule, since it's already disabled
self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
schedule = SCHEDULES[0].copy()
schedule['enabled'] = False
mock_find.return_value = schedule
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule, since it's already disabled
self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_update, mock_find):
schedule = SCHEDULES[0].copy()
schedule['enabled'] = False
mock_find.return_value = schedule
mock_find.__name__ = 'find_and_modify'
mock_update.__name__ = 'update'
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we didn't disable the schedule, since it's already disabled
self.assertEqual(mock_update.call_count, 0)
def test_update(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_find.return_value = SCHEDULES[0]
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
self.assertEqual(len(mock_find.call_args[0]), 0)
self.assertEqual(mock_find.call_args[1]['query'], {'_id': ObjectId(self.schedule_id)})
self.assertEqual(mock_find.call_args[1]['update']['$inc']['consecutive_failures'], 1)
last_updated = mock_find.call_args[1]['update']['$set']['last_updated']
# make sure the last_updated value is within the last tenth of a second
self.assertTrue(time.time() - last_updated < .1)
# make sure it asks for the new version of the schedule to be returned
self.assertTrue(mock_find.call_args[1]['new'] is True)
def test_update(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_find.return_value = SCHEDULES[0]
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
self.assertEqual(len(mock_find.call_args[0]), 0)
self.assertEqual(mock_find.call_args[1]['query'], {'_id': ObjectId(self.schedule_id)})
self.assertEqual(mock_find.call_args[1]['update']['$inc']['consecutive_failures'], 1)
last_updated = mock_find.call_args[1]['update']['$set']['last_updated']
# make sure the last_updated value is within the last tenth of a second
self.assertTrue(time.time() - last_updated < .1)
# make sure it asks for the new version of the schedule to be returned
self.assertTrue(mock_find.call_args[1]['new'] is True)
def test_update(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_find.return_value = SCHEDULES[0]
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
self.assertEqual(len(mock_find.call_args[0]), 0)
self.assertEqual(mock_find.call_args[1]["query"], {"_id": ObjectId(self.schedule_id)})
self.assertEqual(mock_find.call_args[1]["update"]["$inc"]["consecutive_failures"], 1)
last_updated = mock_find.call_args[1]["update"]["$set"]["last_updated"]
# make sure the last_updated value is within the last tenth of a second
self.assertTrue(time.time() - last_updated < 0.1)
# make sure it asks for the new version of the schedule to be returned
self.assertTrue(mock_find.call_args[1]["new"] is True)
def handle_failed_task(self, event):
"""
Celery event handler for failed tasks. This will check if we are
watching the task for failure, and if so, increments the corresponding
schedule's failure count. If it has met or exceeded its failure
threshold, the schedule will be disabled.
:param event: dictionary of poorly-documented data about a celery task.
At a minimum, this method depends on the key 'uuid'
being present and representing the task's ID.
:type event: dict
"""
schedule_id, has_failure = self.pop(event['uuid'])
if schedule_id:
_logger.info(_('incrementing consecutive failure count for schedule %(id)s') % {'id': schedule_id})
utils.increment_failure_count(schedule_id)
def handle_failed_task(self, event):
"""
Celery event handler for failed tasks. This will check if we are
watching the task for failure, and if so, increments the corresponding
schedule's failure count. If it has met or exceeded its failure
threshold, the schedule will be disabled.
:param event: dictionary of poorly-documented data about a celery task.
At a minimum, this method depends on the key 'uuid'
being present and representing the task's ID.
:type event: dict
"""
schedule_id, has_failure = self.pop(event['uuid'])
if schedule_id:
msg = _('incrementing consecutive failure count for schedule %s') % schedule_id
_logger.info(msg)
utils.increment_failure_count(schedule_id)
def test_disable_schedule(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
schedule = SCHEDULES[0].copy()
schedule['consecutive_failures'] = 2
mock_find.return_value = schedule
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we disable the schedule
self.assertEqual(mock_update.call_count, 1)
self.assertEqual(mock_update.call_args[0][0], {'_id': ObjectId(self.schedule_id)})
self.assertTrue(mock_update.call_args[0][1]['$set']['enabled'] is False)
last_updated = mock_update.call_args[0][1]['$set']['last_updated']
# make sure the last_updated value is within the last tenth of a second
self.assertTrue(time.time() - last_updated < .1)
def test_disable_schedule(self, mock_get_collection):
mock_find = mock_get_collection.return_value.find_and_modify
mock_update = mock_get_collection.return_value.update
schedule = SCHEDULES[0].copy()
schedule['consecutive_failures'] = 2
mock_find.return_value = schedule
utils.increment_failure_count(self.schedule_id)
self.assertEqual(mock_find.call_count, 1)
# make sure we disable the schedule
self.assertEqual(mock_update.call_count, 1)
self.assertEqual(mock_update.call_args[0][0], {'_id': ObjectId(self.schedule_id)})
self.assertTrue(mock_update.call_args[0][1]['$set']['enabled'] is False)
last_updated = mock_update.call_args[0][1]['$set']['last_updated']
# make sure the last_updated value is within the last tenth of a second
self.assertTrue(time.time() - last_updated < .1)