def cancel(task_id): """ Cancel the task that is represented by the given task_id. This method cancels only the task with given task_id, not the spawned tasks. This also updates task's state to 'canceled'. :param task_id: The ID of the task you wish to cancel :type task_id: basestring :raises MissingResource: if a task with given task_id does not exist :raises PulpCodedException: if given task is already in a complete state """ try: task_status = TaskStatus.objects.get(task_id=task_id) except DoesNotExist: raise MissingResource(task_id) if task_status['state'] in constants.CALL_COMPLETE_STATES: # If the task is already done, just stop msg = _('Task [%(task_id)s] already in a completed state: %(state)s') _logger.info(msg % {'task_id': task_id, 'state': task_status['state']}) return controller.revoke(task_id, terminate=True) TaskStatus.objects(task_id=task_id, state__nin=constants.CALL_COMPLETE_STATES).\ update_one(set__state=constants.CALL_CANCELED_STATE) msg = _('Task canceled: %(task_id)s.') msg = msg % {'task_id': task_id} _logger.info(msg)
def test_task_status_update(self): """ Tests the successful operation of task status update. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' TaskStatus(task_id, worker_name, tags, state).save() now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = {'start_time': start_time, 'state': 'running', 'progress_report': {'report-id': 'my-progress'}} TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['start_time'], delta['start_time']) # Make sure that parse_iso8601_datetime is able to parse the start_time without errors dateutils.parse_iso8601_datetime(task_status['start_time']) self.assertEqual(task_status['state'], delta['state']) self.assertEqual(task_status['progress_report'], delta['progress_report']) self.assertEqual(task_status['worker_name'], worker_name)
def __call__(self, *args, **kwargs): """ This overrides CeleryTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status[ 'state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) return super(Task, self).__call__(*args, **kwargs)
def failed(self, reply): """ Notification (reply) indicating an RMI failed. This information used to update the task status. :param reply: A failure reply object. :type reply: gofer.rmi.async.Failed """ _logger.info(_('Task RMI (failed): %(r)s'), {'r': reply}) call_context = dict(reply.data) action = call_context.get('action') task_id = call_context['task_id'] traceback = reply.xstate['trace'] finished = reply.timestamp if not finished: now = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(now) TaskStatus.objects(task_id=task_id).update_one( set__finish_time=finished, set__state=constants.CALL_ERROR_STATE, set__traceback=traceback) if action == 'bind': ReplyHandler._bind_failed(task_id, call_context) return if action == 'unbind': ReplyHandler._unbind_failed(task_id, call_context) return
def test_task_status_update(self): """ Tests the successful operation of task status update. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' TaskStatus(task_id, worker_name, tags, state).save() now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = { 'start_time': start_time, 'state': 'running', 'progress_report': { 'report-id': 'my-progress' } } TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['start_time'], delta['start_time']) # Make sure that parse_iso8601_datetime is able to parse the start_time without errors dateutils.parse_iso8601_datetime(task_status['start_time']) self.assertEqual(task_status['state'], delta['state']) self.assertEqual(task_status['progress_report'], delta['progress_report']) self.assertEqual(task_status['worker_name'], worker_name)
def succeeded(self, reply): """ Notification (reply) indicating an RMI succeeded. This information is relayed to the task coordinator. :param reply: A successful reply object. :type reply: gofer.rmi.async.Succeeded """ _logger.info(_('Task RMI (succeeded): %(r)s'), {'r': reply}) call_context = dict(reply.data) action = call_context.get('action') task_id = call_context['task_id'] result = dict(reply.retval) finished = reply.timestamp if not finished: now = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(now) TaskStatus.objects(task_id=task_id).update_one( set__finish_time=finished, set__state=constants.CALL_FINISHED_STATE, set__result=result) if action == 'bind': if result['succeeded']: ReplyHandler._bind_succeeded(task_id, call_context) else: ReplyHandler._bind_failed(task_id, call_context) return if action == 'unbind': if result['succeeded']: ReplyHandler._unbind_succeeded(call_context) else: ReplyHandler._unbind_failed(task_id, call_context) return
def test_task_status_update_fires_notification(self, mock_send): """ Test that update_one() also fires a notification. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' ts = TaskStatus(task_id, worker_name, tags, state) ts.save() # ensure event was fired for save() mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id) now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = {'start_time': start_time, 'state': 'running', 'progress_report': {'report-id': 'my-progress'}} self.assertEquals(len(mock_send.call_args_list), 1) TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) # ensure event was fired for update_one() self.assertEquals(len(mock_send.call_args_list), 2) mock_send.assert_called_with(ts, routing_key="tasks.%s" % task_id)
def __call__(self, *args, **kwargs): """ This overrides CeleryTask's __call__() method. We use this method for task state tracking of Pulp tasks. """ # Check task status and skip running the task if task state is 'canceled'. try: task_status = TaskStatus.objects.get(task_id=self.request.id) except DoesNotExist: task_status = None if task_status and task_status['state'] == constants.CALL_CANCELED_STATE: _logger.debug("Task cancel received for task-id : [%s]" % self.request.id) return # Update start_time and set the task state to 'running' for asynchronous tasks. # Skip updating status for eagerly executed tasks, since we don't want to track # synchronous tasks in our database. if not self.request.called_directly: now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) # Using 'upsert' to avoid a possible race condition described in the apply_async method # above. TaskStatus.objects(task_id=self.request.id).update_one( set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True) # Run the actual task _logger.debug("Running task : [%s]" % self.request.id) return super(Task, self).__call__(*args, **kwargs)
def failed(self, reply): """ Notification (reply) indicating an RMI failed. This information used to update the task status. :param reply: A failure reply object. :type reply: gofer.rmi.async.Failed """ _logger.info(_('Task RMI (failed): %(r)s'), {'r': reply}) call_context = dict(reply.data) action = call_context.get('action') task_id = call_context['task_id'] traceback = reply.xstate['trace'] finished = reply.timestamp if not finished: now = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(now) TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished, set__state=constants.CALL_ERROR_STATE, set__traceback=traceback) if action == 'bind': ReplyHandler._bind_failed(task_id, call_context) return if action == 'unbind': ReplyHandler._unbind_failed(task_id, call_context) return
def succeeded(self, reply): """ Notification (reply) indicating an RMI succeeded. This information is relayed to the task coordinator. :param reply: A successful reply object. :type reply: gofer.rmi.async.Succeeded """ _logger.info(_('Task RMI (succeeded): %(r)s'), {'r': reply}) call_context = dict(reply.data) action = call_context.get('action') task_id = call_context['task_id'] result = dict(reply.retval) finished = reply.timestamp if not finished: now = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(now) TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished, set__state=constants.CALL_FINISHED_STATE, set__result=result) if action == 'bind': if result['succeeded']: ReplyHandler._bind_succeeded(task_id, call_context) else: ReplyHandler._bind_failed(task_id, call_context) return if action == 'unbind': if result['succeeded']: ReplyHandler._unbind_succeeded(call_context) else: ReplyHandler._unbind_failed(task_id, call_context) return
def set_progress(self, status): """ Informs the server of the current state of the publish operation. The contents of the status is dependent on how the distributor implementation chooses to divide up the publish process. @param status: contains arbitrary data to describe the state of the publish; the contents may contain whatever information is relevant to the distributor implementation so long as it is serializable """ if self.task_id is None: # not running within a task return try: self.progress_report[self.report_id] = status TaskStatus.objects(task_id=self.task_id).update_one(set__progress_report=self.progress_report) except Exception, e: logger.exception('Exception from server setting progress for report [%s]' % self.report_id) try: logger.error('Progress value: %s' % str(status)) except Exception: # Best effort to print this, but if its that grossly unserializable # the log will tank and we don't want that exception to bubble up pass raise self.exception_class(e), None, sys.exc_info()[2]
def test_set_accepted(self): task_id = self.get_random_uuid() TaskStatus(task_id, state=constants.CALL_WAITING_STATE).save() TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\ update_one(set__state=constants.CALL_ACCEPTED_STATE) task_status = TaskStatus.objects.get(task_id=task_id) self.assertTrue(task_status['state'], constants.CALL_ACCEPTED_STATE)
def progress(self, reply): """ Notification (reply) indicating an RMI has reported status. This information is relayed to the task coordinator. :param reply: A progress reply object. :type reply: gofer.rmi.async.Progress """ call_context = dict(reply.data) task_id = call_context['task_id'] TaskStatus.objects(task_id=task_id).update_one(set__progress_report=reply.details)
def GET(self): valid_filters = ['tag'] filters = self.filters(valid_filters) tags = filters.get('tag', []) if tags: raw_tasks = TaskStatus.objects(tags__all=tags) else: raw_tasks = TaskStatus.objects() serialized_task_statuses = [task_serializer(task) for task in raw_tasks] return self.ok(serialized_task_statuses)
def progress(self, reply): """ Notification (reply) indicating an RMI has reported status. This information is relayed to the task coordinator. :param reply: A progress reply object. :type reply: gofer.rmi.async.Progress """ call_context = dict(reply.data) task_id = call_context['task_id'] TaskStatus.objects(task_id=task_id).update_one( set__progress_report=reply.details)
def accepted(self, reply): """ Notification that an RMI has started executing in the agent. The task status is updated in the pulp DB. :param reply: A status reply object. :type reply: gofer.rmi.async.Accepted """ _logger.debug(_('Task RMI (accepted): %(r)s'), {'r': reply}) call_context = dict(reply.data) task_id = call_context['task_id'] TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\ update_one(set__state=constants.CALL_ACCEPTED_STATE)
def test_set_succeeded_with_timestamp(self): task_id = self.get_random_uuid() TaskStatus(task_id).save() result = 'done' now = '2014-11-21 05:21:38.829678' TaskStatus.objects(task_id=task_id).update_one(set__finish_time=now, set__state=constants.CALL_FINISHED_STATE, set__result=result) task_status = TaskStatus.objects(task_id=task_id).first() self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE) self.assertTrue(task_status['finish_time'], now) self.assertTrue(task_status['result'], result)
def test_set_failed_with_timestamp(self): task_id = self.get_random_uuid() TaskStatus(task_id).save() traceback = 'abcdef' finished = '2014-11-21 05:21:38.829678' TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished, set__state=constants.CALL_ERROR_STATE, set__traceback=traceback) task_status = TaskStatus.objects.get(task_id=task_id) self.assertTrue(task_status['state'], constants.CALL_ERROR_STATE) self.assertTrue(task_status['finish_time'], finished) self.assertTrue(task_status['traceback'], traceback)
def test_set_failed_with_timestamp(self): task_id = self.get_random_uuid() TaskStatus(task_id).save() traceback = 'abcdef' finished = '2014-11-21 05:21:38.829678' TaskStatus.objects(task_id=task_id).update_one( set__finish_time=finished, set__state=constants.CALL_ERROR_STATE, set__traceback=traceback) task_status = TaskStatus.objects.get(task_id=task_id) self.assertTrue(task_status['state'], constants.CALL_ERROR_STATE) self.assertTrue(task_status['finish_time'], finished) self.assertTrue(task_status['traceback'], traceback)
def test_set_succeeded_with_timestamp(self): task_id = self.get_random_uuid() TaskStatus(task_id).save() result = 'done' now = '2014-11-21 05:21:38.829678' TaskStatus.objects(task_id=task_id).update_one( set__finish_time=now, set__state=constants.CALL_FINISHED_STATE, set__result=result) task_status = TaskStatus.objects(task_id=task_id).first() self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE) self.assertTrue(task_status['finish_time'], now) self.assertTrue(task_status['result'], result)
def test_creates_task_status(self, apply_async): args = [1, 'b', 'iii'] kwargs = { 'a': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_1 } apply_async.return_value = celery.result.AsyncResult('test_task_id') task = tasks.Task() task.apply_async(*args, **kwargs) task_statuses = TaskStatus.objects() self.assertEqual(len(task_statuses), 1) new_task_status = task_statuses[0] self.assertEqual(new_task_status['task_id'], 'test_task_id') self.assertEqual(new_task_status['worker_name'], WORKER_1) self.assertEqual(new_task_status['tags'], kwargs['tags']) self.assertEqual(new_task_status['state'], 'waiting') self.assertEqual(new_task_status['error'], None) self.assertEqual(new_task_status['spawned_tasks'], []) self.assertEqual(new_task_status['progress_report'], {}) self.assertEqual(new_task_status['task_type'], 'pulp.server.async.tasks.Task') self.assertEqual(new_task_status['start_time'], None) self.assertEqual(new_task_status['finish_time'], None) self.assertEqual(new_task_status['result'], None)
def test_updates_task_status_correctly(self, mock_request): exc = Exception() task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = {'1': 'for the money', 'tags': ['test_tags']} class EInfo(object): """ on_failure handler expects an instance of celery's ExceptionInfo class as one of the attributes. It stores string representation of traceback in it's traceback instance variable. This is a stub to imitate that behavior. """ def __init__(self): self.traceback = "string_repr_of_traceback" einfo = EInfo() mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) self.assertEqual(task_status['traceback'], None) task = tasks.Task() task.on_failure(exc, task_id, args, kwargs, einfo) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'error') self.assertFalse(new_task_status['finish_time'] is None) # Make sure that parse_iso8601_datetime is able to parse the finish_time without errors dateutils.parse_iso8601_datetime(new_task_status['finish_time']) self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_spawned_task_status(self, mock_request): async_result = AsyncResult('foo-id') retval = tasks.TaskResult(error=PulpException('error-foo'), result='bar') retval.spawned_tasks = [async_result] task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2} mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) task = tasks.Task() task.on_success(retval, task_id, args, kwargs) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'finished') self.assertEqual(new_task_status['result'], 'bar') self.assertEqual(new_task_status['error']['description'], 'error-foo') self.assertFalse(new_task_status['finish_time'] is None) # Make sure that parse_iso8601_datetime is able to parse the finish_time without errors dateutils.parse_iso8601_datetime(new_task_status['finish_time']) self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_async_result(self, mock_request): retval = AsyncResult('foo-id') task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = { '1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2 } mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) task = tasks.Task() task.on_success(retval, task_id, args, kwargs) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'finished') self.assertEqual(new_task_status['result'], None) self.assertFalse(new_task_status['finish_time'] is None) # Make sure that parse_iso8601_datetime is able to parse the finish_time without errors dateutils.parse_iso8601_datetime(new_task_status['finish_time']) self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def _delete_worker(name, normal_shutdown=False): """ Delete the Worker with _id name from the database, cancel any associated tasks and reservations If the worker shutdown normally, no message is logged, otherwise an error level message is logged. Default is to assume the worker did not shut down normally. Any resource reservations associated with this worker are cleaned up by this function. Any tasks associated with this worker are explicitly canceled. :param name: The name of the worker you wish to delete. :type name: basestring :param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to False. :type normal_shutdown: bool """ if normal_shutdown is False: msg = _('The worker named %(name)s is missing. Canceling the tasks in its queue.') msg = msg % {'name': name} _logger.error(msg) # Delete the worker document Worker.objects(name=name).delete() # Delete all reserved_resource documents for the worker ReservedResource.get_collection().remove({'worker_name': name}) # Cancel all of the tasks that were assigned to this worker's queue for task_status in TaskStatus.objects(worker_name=name, state__in=constants.CALL_INCOMPLETE_STATES): cancel(task_status['task_id']) # Delete working directory common_utils.delete_worker_working_directory(name)
def test_spawned_task_dict(self, mock_request): retval = tasks.TaskResult(spawned_tasks=[{ 'task_id': 'foo-id' }], result='bar') task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = { '1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2 } mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) task = tasks.Task() task.on_success(retval, task_id, args, kwargs) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'finished') self.assertEqual(new_task_status['result'], 'bar') self.assertFalse(new_task_status['finish_time'] is None) self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_save_update_defaults(self): """ Test the save method with default arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = { 'what do we want?': 'progress!', 'when do we want it?': 'now!' } task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus(task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Let's go ahead and insert the object ts.save() # Now let's alter it a bit, and make sure the alteration makes it to the DB correctly. new_state = constants.CALL_RUNNING_STATE ts.state = new_state # This should update ts in the database ts.save() ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) # The state should have been updated self.assertEqual(ts['state'], new_state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_set_succeeded(self, mock_date): task_id = self.get_random_uuid() TaskStatus(task_id).save() result = 'done' now = '2014-11-21 05:21:38.829678' mock_date.return_value = now t = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(t) TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished, set__state=constants.CALL_FINISHED_STATE, set__result=result) task_status = TaskStatus.objects(task_id=task_id).first() self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE) self.assertTrue(task_status['finish_time'], now) self.assertTrue(task_status['result'], result)
def test_save_update_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' old_start_time = start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus( task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Put the object in the database, and then change some of it settings. ts.save() new_worker_name = 'a different_worker' new_state = constants.CALL_SUSPENDED_STATE new_start_time = old_start_time + timedelta(minutes=10) new_start_time = dateutils.format_iso8601_datetime(new_start_time) ts.worker_name = new_worker_name ts.state = new_state ts.start_time = new_start_time # This should update the worker_name on ts in the database, but should not update the state # or start_time ts.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) # Queue should have been updated self.assertEqual(ts['worker_name'], new_worker_name) self.assertEqual(ts['tags'], tags) # state should not have been updated self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) # start_time should not have been updated self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_set_succeeded(self, mock_date): task_id = self.get_random_uuid() TaskStatus(task_id).save() result = 'done' now = '2014-11-21 05:21:38.829678' mock_date.return_value = now t = datetime.now(dateutils.utc_tz()) finished = dateutils.format_iso8601_datetime(t) TaskStatus.objects(task_id=task_id).update_one( set__finish_time=finished, set__state=constants.CALL_FINISHED_STATE, set__result=result) task_status = TaskStatus.objects(task_id=task_id).first() self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE) self.assertTrue(task_status['finish_time'], now) self.assertTrue(task_status['result'], result)
def started(self, reply): """ Notification that an RMI has started executing in the agent. The task status is updated in the pulp DB. :param reply: A status reply object. :type reply: gofer.rmi.async.Started """ _logger.debug(_('Task RMI (started): %(r)s'), {'r': reply}) call_context = dict(reply.data) task_id = call_context['task_id'] started = reply.timestamp if not started: now = datetime.now(dateutils.utc_tz()) started = dateutils.format_iso8601_datetime(now) TaskStatus.objects(task_id=task_id).update_one(set__start_time=started) TaskStatus.objects(task_id=task_id, state__in=[constants.CALL_WAITING_STATE, constants.CALL_ACCEPTED_STATE]).\ update_one(set__state=constants.CALL_RUNNING_STATE)
def test_DELETE_doesnt_cancel_spawned_celery_task(self, revoke): """ Test the DELETE() which should cause a revoke call to Celery's Controller. This also tests that the spawned tasks are canceled as well. """ task_id = '1234abcd' spawned_task_id = 'spawned_task' spawned_by_spawned_task_id = 'spawned_by_spawned_task' TaskStatus(task_id).save() TaskStatus(spawned_task_id).save() TaskStatus(spawned_by_spawned_task_id).save() TaskStatus.objects(task_id=task_id).update_one(set__spawned_tasks=[spawned_task_id]) TaskStatus.objects(task_id=spawned_task_id).update_one( set__spawned_tasks=[spawned_by_spawned_task_id]) self.task_resource.DELETE(task_id) self.assertEqual(revoke.call_count, 1) revoke.assert_called_once_with(task_id, terminate=True)
def test_DELETE_completed_celery_task(self): """ Test the DELETE() method does not change the state of a task that is already complete """ task_id = '1234abcd' TaskStatus(task_id, state=constants.CALL_FINISHED_STATE).save() self.task_resource.DELETE(task_id) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], constants.CALL_FINISHED_STATE)
def test_save_insert_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is not already in the database. """ task_id = str(uuid4()) worker_name = 'some_worker' tags = ['tag_1', 'tag_2'] state = constants.CALL_RUNNING_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = { 'what do we want?': 'progress!', 'when do we want it?': 'now!' } task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus(task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # This should cause ts to be in the database ts.save_with_set_on_insert( fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_cancel_after_task_canceled(self, *unused_mocks): """ Test that canceling a task that was already canceled results in no change to the task state. """ task_id = '1234abcd' TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save() tasks.cancel(task_id) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_after_task_finished(self, _logger, revoke): """ Test that canceling a task that is already finished results in no change to the task state. """ task_id = '1234abcd' TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save() tasks.cancel(task_id) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_successful(self, _logger, revoke): task_id = '1234abcd' TaskStatus(task_id).save() tasks.cancel(task_id) revoke.assert_called_once_with(task_id, terminate=True) self.assertEqual(_logger.info.call_count, 1) log_msg = _logger.info.mock_calls[0][1][0] self.assertTrue(task_id in log_msg) self.assertTrue('Task canceled' in log_msg) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_illegal_multi_arg(self): """ Test that we receive an exception if we try to use the 'multi' kwarg """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' ts = TaskStatus(task_id, worker_name, tags, state) ts.save() self.assertRaises(NotImplementedError, TaskStatus.objects(task_id=task_id).update_one, multi=True)
def test_task_status_not_modified_when_task_status_exists(self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags']} task_id = 'test_task_id' TaskStatus(task_id, 'test-worker', state=CALL_CANCELED_STATE).save() apply_async.return_value = celery.result.AsyncResult(task_id) task = tasks.Task() task.apply_async(*args, **kwargs) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], 'canceled') self.assertEqual(task_status['start_time'], None)
def test_task_status_not_modified_when_task_status_exists( self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags']} task_id = 'test_task_id' TaskStatus(task_id, 'test-worker', state=CALL_CANCELED_STATE).save() apply_async.return_value = celery.result.AsyncResult(task_id) task = tasks.Task() task.apply_async(*args, **kwargs) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], 'canceled') self.assertEqual(task_status['start_time'], None)
def test_save_update_defaults(self): """ Test the save method with default arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus( task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Let's go ahead and insert the object ts.save() # Now let's alter it a bit, and make sure the alteration makes it to the DB correctly. new_state = constants.CALL_RUNNING_STATE ts.state = new_state # This should update ts in the database ts.save() ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) # The state should have been updated self.assertEqual(ts['state'], new_state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_create_task_status_defaults(self): """ Tests TaskStatus creation with minimal information, to ensure that defaults are handled properly. """ task_id = self.get_random_uuid() TaskStatus(task_id).save() task_statuses = TaskStatus.objects() self.assertEqual(1, len(task_statuses)) self.assertEqual(task_id, task_statuses[0]['task_id']) self.assertEqual(None, task_statuses[0]['worker_name']) self.assertEqual([], task_statuses[0]['tags']) self.assertEqual('waiting', task_statuses[0]['state'])
def test_saves_passed_in_routing_key_as_worker_name(self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags'], 'routing_key': 'othername'} apply_async.return_value = celery.result.AsyncResult('test_task_id') task = tasks.Task() task.apply_async(*args, **kwargs) task_statuses = TaskStatus.objects() self.assertEqual(len(task_statuses), 1) new_task_status = task_statuses[0] self.assertEqual(new_task_status['task_id'], 'test_task_id') self.assertEqual(new_task_status['worker_name'], 'othername') self.assertEqual(new_task_status['tags'], kwargs['tags']) self.assertEqual(new_task_status['state'], 'waiting')
def test_saves_default_routing_key_as_worker_name(self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags']} apply_async.return_value = celery.result.AsyncResult('test_task_id') task = tasks.Task() task.apply_async(*args, **kwargs) task_statuses = TaskStatus.objects() self.assertEqual(len(task_statuses), 1) new_task_status = task_statuses[0] self.assertEqual(new_task_status['task_id'], 'test_task_id') self.assertEqual(new_task_status['worker_name'], defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default) self.assertEqual(new_task_status['tags'], kwargs['tags']) self.assertEqual(new_task_status['state'], 'waiting')
def test_saves_default_routing_key_as_worker_name(self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags']} apply_async.return_value = celery.result.AsyncResult('test_task_id') task = tasks.Task() task.apply_async(*args, **kwargs) task_statuses = TaskStatus.objects() self.assertEqual(len(task_statuses), 1) new_task_status = task_statuses[0] self.assertEqual(new_task_status['task_id'], 'test_task_id') self.assertEqual( new_task_status['worker_name'], defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default) self.assertEqual(new_task_status['tags'], kwargs['tags']) self.assertEqual(new_task_status['state'], 'waiting')
def test_saves_passed_in_routing_key_as_worker_name(self, apply_async): args = [1, 'b', 'iii'] kwargs = { 'a': 'for the money', 'tags': ['test_tags'], 'routing_key': 'othername' } apply_async.return_value = celery.result.AsyncResult('test_task_id') task = tasks.Task() task.apply_async(*args, **kwargs) task_statuses = TaskStatus.objects() self.assertEqual(len(task_statuses), 1) new_task_status = task_statuses[0] self.assertEqual(new_task_status['task_id'], 'test_task_id') self.assertEqual(new_task_status['worker_name'], 'othername') self.assertEqual(new_task_status['tags'], kwargs['tags']) self.assertEqual(new_task_status['state'], 'waiting')
def _delete_worker(name, normal_shutdown=False): """ Delete the Worker with _id name from the database, cancel any associated tasks and reservations If the worker shutdown normally, no message is logged, otherwise an error level message is logged. Default is to assume the worker did not shut down normally. Any resource reservations associated with this worker are cleaned up by this function. Any tasks associated with this worker are explicitly canceled. :param name: The name of the worker you wish to delete. In the database, the _id field is the name. :type name: basestring :param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to False. :type normal_shutdown: bool """ if normal_shutdown is False: msg = _( 'The worker named %(name)s is missing. Canceling the tasks in its queue.' ) msg = msg % {'name': name} _logger.error(msg) # Delete the worker document worker_list = list( resources.filter_workers(Criteria(filters={'_id': name}))) if len(worker_list) > 0: worker_document = worker_list[0] worker_document.delete() # Delete all reserved_resource documents for the worker ReservedResource.get_collection().remove({'worker_name': name}) # Cancel all of the tasks that were assigned to this worker's queue worker = Worker.from_bson({'_id': name}) for task_status in TaskStatus.objects( worker_name=worker.name, state__in=constants.CALL_INCOMPLETE_STATES): cancel(task_status['task_id'])