def on_failure(self, exc, task_id, args, kwargs, einfo): """ This overrides the error handler run by the worker when the task fails. It updates state, finish_time and traceback of the relevant task status for asynchronous tasks. Skip updating status for synchronous tasks. :param exc: The exception raised by the task. :param task_id: Unique id of the failed task. :param args: Original arguments for the executed task. :param kwargs: Original keyword arguments for the executed task. :param einfo: celery's ExceptionInfo instance, containing serialized traceback. """ if isinstance(exc, PulpCodedException): _logger.info(_('Task failed : [%(task_id)s] : %(msg)s') % {'task_id': task_id, 'msg': str(exc)}) _logger.debug(traceback.format_exc()) else: _logger.info(_('Task failed : [%s]') % task_id) # celery will log the traceback if kwargs.get('scheduled_call_id') is not None: utils.increment_failure_count(kwargs['scheduled_call_id']) if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) finish_time = dateutils.format_iso8601_datetime(now) task_status = TaskStatus.objects.get(task_id=task_id) task_status['state'] = constants.CALL_ERROR_STATE task_status['finish_time'] = finish_time task_status['traceback'] = einfo.traceback if not isinstance(exc, PulpException): exc = PulpException(str(exc)) task_status['error'] = exc.to_dict() task_status.save() self._handle_cProfile(task_id) common_utils.delete_working_directory()
def on_failure(self, exc, task_id, args, kwargs, einfo): """ This overrides the error handler run by the worker when the task fails. It updates state, finish_time and traceback of the relevant task status for asynchronous tasks. Skip updating status for synchronous tasks. :param exc: The exception raised by the task. :param task_id: Unique id of the failed task. :param args: Original arguments for the executed task. :param kwargs: Original keyword arguments for the executed task. :param einfo: celery's ExceptionInfo instance, containing serialized traceback. """ _logger.debug(traceback.format_exc()) if isinstance(exc, PulpCodedException): _logger.info( _('Task failed : [%(task_id)s] : %(msg)s') % { 'task_id': task_id, 'msg': str(exc) }) else: _logger.info(_('Task failed : [%s]') % task_id) if kwargs.get('scheduled_call_id') is not None: utils.increment_failure_count(kwargs['scheduled_call_id']) try: called_directly = self.request.called_directly except Exception: # a workaround for when celery's internal state is bad self._handle_on_failure_cleanup(task_id, exc, einfo) raise if not called_directly: self._handle_on_failure_cleanup(task_id, exc, einfo)
def on_failure(self, exc, task_id, args, kwargs, einfo): """ This overrides the error handler run by the worker when the task fails. It updates state, finish_time and traceback of the relevant task status for asynchronous tasks. Skip updating status for synchronous tasks. :param exc: The exception raised by the task. :param task_id: Unique id of the failed task. :param args: Original arguments for the executed task. :param kwargs: Original keyword arguments for the executed task. :param einfo: celery's ExceptionInfo instance, containing serialized traceback. """ if isinstance(exc, PulpCodedException): _logger.info(_('Task failed : [%(task_id)s] : %(msg)s') % {'task_id': task_id, 'msg': str(exc)}) _logger.debug(traceback.format_exc()) else: _logger.info(_('Task failed : [%s]') % task_id) # celery will log the traceback if kwargs.get('scheduled_call_id') is not None: utils.increment_failure_count(kwargs['scheduled_call_id']) if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) finish_time = dateutils.format_iso8601_datetime(now) task_status = TaskStatus.objects.get(task_id=task_id) task_status['state'] = constants.CALL_ERROR_STATE task_status['finish_time'] = finish_time task_status['traceback'] = einfo.traceback if not isinstance(exc, PulpException): exc = PulpException(str(exc)) task_status['error'] = exc.to_dict() task_status.save() self._handle_cProfile(task_id) common_utils.delete_working_directory()
def test_not_found(self, mock_from_db, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_find.return_value = None utils.increment_failure_count(self.schedule_id) # from_db() gets called if find_and_modify returns anything. self.assertEqual(mock_from_db.call_count, 0)
def test_not_found(self, mock_from_db, mock_find): mock_find.return_value = None mock_find.__name__ = 'find_and_modify' utils.increment_failure_count(self.schedule_id) # from_db() gets called if find_and_modify returns anything. self.assertEqual(mock_from_db.call_count, 0)
def test_not_found(self, mock_from_db, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_find.return_value = None utils.increment_failure_count(self.schedule_id) # from_db() gets called if find_and_modify returns anything. self.assertEqual(mock_from_db.call_count, 0)
def test_threshold_none(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update mock_find.return_value = SCHEDULES[1] utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule self.assertEqual(mock_update.call_count, 0)
def test_threshold_none(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update mock_find.return_value = SCHEDULES[1] utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule self.assertEqual(mock_update.call_count, 0)
def test_threshold_none(self, mock_update, mock_find): mock_find.return_value = SCHEDULES[1] mock_find.__name__ = 'find_and_modify' mock_update.__name__ = 'update' utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update schedule = SCHEDULES[0].copy() schedule['enabled'] = False mock_find.return_value = schedule utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule, since it's already disabled self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update schedule = SCHEDULES[0].copy() schedule['enabled'] = False mock_find.return_value = schedule utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule, since it's already disabled self.assertEqual(mock_update.call_count, 0)
def test_schedule_already_disabled(self, mock_update, mock_find): schedule = SCHEDULES[0].copy() schedule['enabled'] = False mock_find.return_value = schedule mock_find.__name__ = 'find_and_modify' mock_update.__name__ = 'update' utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we didn't disable the schedule, since it's already disabled self.assertEqual(mock_update.call_count, 0)
def test_update(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_find.return_value = SCHEDULES[0] utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) self.assertEqual(len(mock_find.call_args[0]), 0) self.assertEqual(mock_find.call_args[1]['query'], {'_id': ObjectId(self.schedule_id)}) self.assertEqual(mock_find.call_args[1]['update']['$inc']['consecutive_failures'], 1) last_updated = mock_find.call_args[1]['update']['$set']['last_updated'] # make sure the last_updated value is within the last tenth of a second self.assertTrue(time.time() - last_updated < .1) # make sure it asks for the new version of the schedule to be returned self.assertTrue(mock_find.call_args[1]['new'] is True)
def test_update(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_find.return_value = SCHEDULES[0] utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) self.assertEqual(len(mock_find.call_args[0]), 0) self.assertEqual(mock_find.call_args[1]['query'], {'_id': ObjectId(self.schedule_id)}) self.assertEqual(mock_find.call_args[1]['update']['$inc']['consecutive_failures'], 1) last_updated = mock_find.call_args[1]['update']['$set']['last_updated'] # make sure the last_updated value is within the last tenth of a second self.assertTrue(time.time() - last_updated < .1) # make sure it asks for the new version of the schedule to be returned self.assertTrue(mock_find.call_args[1]['new'] is True)
def test_update(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_find.return_value = SCHEDULES[0] utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) self.assertEqual(len(mock_find.call_args[0]), 0) self.assertEqual(mock_find.call_args[1]["query"], {"_id": ObjectId(self.schedule_id)}) self.assertEqual(mock_find.call_args[1]["update"]["$inc"]["consecutive_failures"], 1) last_updated = mock_find.call_args[1]["update"]["$set"]["last_updated"] # make sure the last_updated value is within the last tenth of a second self.assertTrue(time.time() - last_updated < 0.1) # make sure it asks for the new version of the schedule to be returned self.assertTrue(mock_find.call_args[1]["new"] is True)
def handle_failed_task(self, event): """ Celery event handler for failed tasks. This will check if we are watching the task for failure, and if so, increments the corresponding schedule's failure count. If it has met or exceeded its failure threshold, the schedule will be disabled. :param event: dictionary of poorly-documented data about a celery task. At a minimum, this method depends on the key 'uuid' being present and representing the task's ID. :type event: dict """ schedule_id, has_failure = self.pop(event['uuid']) if schedule_id: _logger.info(_('incrementing consecutive failure count for schedule %(id)s') % {'id': schedule_id}) utils.increment_failure_count(schedule_id)
def handle_failed_task(self, event): """ Celery event handler for failed tasks. This will check if we are watching the task for failure, and if so, increments the corresponding schedule's failure count. If it has met or exceeded its failure threshold, the schedule will be disabled. :param event: dictionary of poorly-documented data about a celery task. At a minimum, this method depends on the key 'uuid' being present and representing the task's ID. :type event: dict """ schedule_id, has_failure = self.pop(event['uuid']) if schedule_id: msg = _('incrementing consecutive failure count for schedule %s') % schedule_id _logger.info(msg) utils.increment_failure_count(schedule_id)
def test_disable_schedule(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update schedule = SCHEDULES[0].copy() schedule['consecutive_failures'] = 2 mock_find.return_value = schedule utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we disable the schedule self.assertEqual(mock_update.call_count, 1) self.assertEqual(mock_update.call_args[0][0], {'_id': ObjectId(self.schedule_id)}) self.assertTrue(mock_update.call_args[0][1]['$set']['enabled'] is False) last_updated = mock_update.call_args[0][1]['$set']['last_updated'] # make sure the last_updated value is within the last tenth of a second self.assertTrue(time.time() - last_updated < .1)
def test_disable_schedule(self, mock_get_collection): mock_find = mock_get_collection.return_value.find_and_modify mock_update = mock_get_collection.return_value.update schedule = SCHEDULES[0].copy() schedule['consecutive_failures'] = 2 mock_find.return_value = schedule utils.increment_failure_count(self.schedule_id) self.assertEqual(mock_find.call_count, 1) # make sure we disable the schedule self.assertEqual(mock_update.call_count, 1) self.assertEqual(mock_update.call_args[0][0], {'_id': ObjectId(self.schedule_id)}) self.assertTrue(mock_update.call_args[0][1]['$set']['enabled'] is False) last_updated = mock_update.call_args[0][1]['$set']['last_updated'] # make sure the last_updated value is within the last tenth of a second self.assertTrue(time.time() - last_updated < .1)