def test_task_id_validation(self): # Valid task_id valid_task_id = str(uuid4()) TaskStatus(valid_task_id).save() # Invalid task_id invalid_task_ids = [4, {}, None, uuid4(), ('a', 'b'), object(), []] for invalid_task_id in invalid_task_ids: self.assertRaises(ValidationError, TaskStatus(invalid_task_id).save)
def test_worker_name_validation(self): # Valid worker_name task_id = str(uuid4()) valid_worker_name = 'worker_name' TaskStatus(task_id=task_id, worker_name=valid_worker_name).save() # Invalid worker_name invalid_worker_names = [4, {}, uuid4(), ('a', 'b'), object(), []] for invalid_worker_name in invalid_worker_names: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, worker_name=invalid_worker_name).save)
def test_tags_validation(self): # Valid tags task_id = str(uuid4()) valid_tags = ['tag1', 'tag2'] TaskStatus(task_id=task_id, tags=valid_tags).save() # Invalid tags invalid_tags = [4, {}, uuid4(), object(), 'tags', [1, 2]] for invalid_tag in invalid_tags: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, tags=invalid_tag).save)
def test_error_validation(self): # Valid error task_id = str(uuid4()) valid_error = {'error': 'some error'} TaskStatus(task_id=task_id, error=valid_error).save() # Invalid error invalid_errors = [4, uuid4(), object(), 'tags', [1, 2]] for invalid_error in invalid_errors: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, error=invalid_error).save)
def test_spawned_tasks_validation(self): # Valid spawned_tasks task_id = str(uuid4()) valid_spawned_tasks = ['spawned1', 'spawned2'] TaskStatus(task_id=task_id, spawned_tasks=valid_spawned_tasks).save() # Invalid spawned_tasks invalid_spawned_tasks = [4, uuid4(), object(), 'tags', [1, 2], {}] for invalid_spawned_task in invalid_spawned_tasks: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, spawned_tasks=invalid_spawned_task).save)
def test_create_task_status_duplicate_task_id(self): """ Tests TaskStatus creation with a duplicate task id. """ task_id = self.get_random_uuid() TaskStatus(task_id).save() try: TaskStatus(task_id).save() except Exception, e: self.assertTrue(task_id in str(e))
def test_task_type_validation(self): # Valid task_type task_id = str(uuid4()) valid_task_type = 'task_type' TaskStatus(task_id=task_id, task_type=valid_task_type).save() # Invalid task_type invalid_task_types = [4, {}, uuid4(), ('a', 'b'), object(), []] for invalid_task_type in invalid_task_types: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, task_type=invalid_task_type).save)
def test_finish_time_validation(self): # Valid finish_time task_id = str(uuid4()) valid_finish_time = dateutils.format_iso8601_datetime(datetime.now()) TaskStatus(task_id=task_id, finish_time=valid_finish_time).save() # Invalid finish_time invalid_finish_times = [4, {}, uuid4(), ('a', 'b'), object(), [], datetime.now()] for invalid_finish_time in invalid_finish_times: self.assertRaises(ValidationError, TaskStatus(task_id=task_id, task_type=invalid_finish_time).save)
def test_state_validation(self): # Valid state valid_states = constants.CALL_STATES for valid_state in valid_states: TaskStatus(task_id=str(uuid4()), state=valid_state).save() # Invalid state invalid_states = [4, {}, uuid4(), object(), 'invalid_state', []] for invalid_state in invalid_states: self.assertRaises(ValidationError, TaskStatus(task_id=str(uuid4()), state=invalid_state).save)
def test_progress_report_validation(self): # Valid progress_report task_id = str(uuid4()) valid_progress_report = {'progress': 'going good'} TaskStatus(task_id=task_id, progress_report=valid_progress_report).save() # Invalid progress_report invalid_progress_reports = [4, uuid4(), object(), 'tags', [1, 2], ()] for invalid_progress_report in invalid_progress_reports: self.assertRaises( ValidationError, TaskStatus(task_id=task_id, progress_report=invalid_progress_report).save)
def test_updates_task_status_correctly(self, mock_request): exc = Exception() task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = {'1': 'for the money', 'tags': ['test_tags']} class EInfo(object): """ on_failure handler expects an instance of celery's ExceptionInfo class as one of the attributes. It stores string representation of traceback in it's traceback instance variable. This is a stub to imitate that behavior. """ def __init__(self): self.traceback = "string_repr_of_traceback" einfo = EInfo() mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) self.assertEqual(task_status['traceback'], None) task = tasks.Task() task.on_failure(exc, task_id, args, kwargs, einfo) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'error') self.assertFalse(new_task_status['finish_time'] is None) # Make sure that parse_iso8601_datetime is able to parse the finish_time without errors dateutils.parse_iso8601_datetime(new_task_status['finish_time']) self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_spawned_task_dict(self, mock_request): retval = tasks.TaskResult(spawned_tasks=[{ 'task_id': 'foo-id' }], result='bar') task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = { '1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2 } mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) task = tasks.Task() task.on_success(retval, task_id, args, kwargs) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'finished') self.assertEqual(new_task_status['result'], 'bar') self.assertFalse(new_task_status['finish_time'] is None) self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_async_result(self, mock_request): retval = AsyncResult('foo-id') task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = { '1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2 } mock_request.called_directly = False task_status = TaskStatus(task_id).save() self.assertEqual(task_status['state'], 'waiting') self.assertEqual(task_status['finish_time'], None) task = tasks.Task() task.on_success(retval, task_id, args, kwargs) new_task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(new_task_status['state'], 'finished') self.assertEqual(new_task_status['result'], None) self.assertFalse(new_task_status['finish_time'] is None) # Make sure that parse_iso8601_datetime is able to parse the finish_time without errors dateutils.parse_iso8601_datetime(new_task_status['finish_time']) self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_task_status_not_modified_when_task_status_exists( self, apply_async): args = [1, 'b', 'iii'] kwargs = {'a': 'for the money', 'tags': ['test_tags']} task_id = 'test_task_id' # This simulates the case where a task had already completed # prior to apply_sync attempting to create a TaskStatus. # https://pulp.plan.io/issues/2959 TaskStatus(task_id, 'test-worker', state=CALL_FINISHED_STATE, result='any_result', start_time='2017-09-20T10:00:00Z', finish_time='2017-09-20T11:00:00Z').save() apply_async.return_value = celery.result.AsyncResult(task_id) task = tasks.Task() task.apply_async(*args, **kwargs) task_status = TaskStatus.objects(task_id=task_id).first() # Fields which were missing on the object have been added self.assertEqual(task_status['task_type'], 'pulp.server.async.tasks.Task') self.assertEqual(task_status['tags'], ['test_tags']) # Fields which already existed on the object are retained self.assertEqual(task_status['state'], 'finished') self.assertEqual(task_status['start_time'], '2017-09-20T10:00:00Z') self.assertEqual(task_status['finish_time'], '2017-09-20T11:00:00Z') self.assertEqual(task_status['result'], 'any_result')
def apply_async(self, *args, **kwargs): """ A wrapper around the PulpTask apply_async method. It allows us to accept a few more parameters than Celery does for our own purposes, listed below. It also allows us to create and update task status which can be used to track status of this task during it's lifetime. :param queue: The queue that the task has been placed into (optional, defaults to the general Celery queue.) :type queue: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :param group_id: The id that identifies which group of tasks a task belongs to :type group_id: uuid.UUID :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ routing_key = kwargs.get('routing_key', defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default) tag_list = kwargs.pop('tags', []) group_id = kwargs.pop('group_id', None) async_result = super(Task, self).apply_async(*args, **kwargs) async_result.tags = tag_list # Create a new task status with the task id and tags. task_status = TaskStatus( task_id=async_result.id, task_type=self.name, state=constants.CALL_WAITING_STATE, worker_name=routing_key, tags=tag_list, group_id=group_id) # To avoid the race condition where __call__ method below is called before # this change is propagated to all db nodes, using an 'upsert' here and setting # the task state to 'waiting' only on an insert. task_status.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time']) return async_result
def test_task_status_update(self): """ Tests the successful operation of task status update. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' TaskStatus(task_id, worker_name, tags, state).save() now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = {'start_time': start_time, 'state': 'running', 'progress_report': {'report-id': 'my-progress'}} TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['start_time'], delta['start_time']) # Make sure that parse_iso8601_datetime is able to parse the start_time without errors dateutils.parse_iso8601_datetime(task_status['start_time']) self.assertEqual(task_status['state'], delta['state']) self.assertEqual(task_status['progress_report'], delta['progress_report']) self.assertEqual(task_status['worker_name'], worker_name)
def test_task_status_update_fires_notification(self, mock_send): """ Test that update_one() also fires a notification. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' ts = TaskStatus(task_id, worker_name, tags, state) ts.save() # ensure event was fired for save() mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id) now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = {'start_time': start_time, 'state': 'running', 'progress_report': {'report-id': 'my-progress'}} self.assertEquals(len(mock_send.call_args_list), 1) TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) # ensure event was fired for update_one() self.assertEquals(len(mock_send.call_args_list), 2) mock_send.assert_called_with(ts, routing_key="tasks.%s" % task_id)
def test_with_scheduled_call_none(self, mock_increment_failure, mock_request): exc = Exception() task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = { '1': 'for the money', 'tags': ['test_tags'], 'scheduled_call_id': None } class EInfo(object): """ on_failure handler expects an instance of celery's ExceptionInfo class as one of the attributes. It stores string representation of traceback in it's traceback instance variable. This is a stub to imitate that behavior. """ def __init__(self): self.traceback = "string_repr_of_traceback" einfo = EInfo() mock_request.called_directly = False TaskStatus(task_id).save() task = tasks.Task() task.on_failure(exc, task_id, args, kwargs, einfo) self.assertFalse(mock_increment_failure.called)
def test_create_task_status_invalid_task_id(self): """ Test that TaskStatus creation with an invalid task id raises the correct error. """ try: TaskStatus(None).save() except ValidationError, e: self.assertTrue('task_id' in e.message)
def test_set_accepted(self): task_id = self.get_random_uuid() TaskStatus(task_id, state=constants.CALL_WAITING_STATE).save() TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\ update_one(set__state=constants.CALL_ACCEPTED_STATE) task_status = TaskStatus.objects.get(task_id=task_id) self.assertTrue(task_status['state'], constants.CALL_ACCEPTED_STATE)
def test_save_update_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' old_start_time = start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus( task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Put the object in the database, and then change some of it settings. ts.save() new_worker_name = 'a different_worker' new_state = constants.CALL_SUSPENDED_STATE new_start_time = old_start_time + timedelta(minutes=10) new_start_time = dateutils.format_iso8601_datetime(new_start_time) ts.worker_name = new_worker_name ts.state = new_state ts.start_time = new_start_time # This should update the worker_name on ts in the database, but should not update the state # or start_time ts.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) # Queue should have been updated self.assertEqual(ts['worker_name'], new_worker_name) self.assertEqual(ts['tags'], tags) # state should not have been updated self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) # start_time should not have been updated self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def apply_async_with_reservation(self, resource_type, resource_id, *args, **kwargs): """ This method allows the caller to schedule the ReservedTask to run asynchronously just like Celery's apply_async(), while also making the named resource. No two tasks that claim the same resource reservation can execute concurrently. It accepts type and id of a resource and combines them to form a resource id. This does not dispatch the task directly, but instead promises to dispatch it later by encapsulating the desired task through a call to a _queue_reserved_task task. See the docblock on _queue_reserved_task for more information on this. This method creates a TaskStatus as a placeholder for later updates. Pulp expects to poll on a task just after calling this method, so a TaskStatus entry needs to exist for it before it returns. For a list of parameters accepted by the *args and **kwargs parameters, please see the docblock for the apply_async() method. :param resource_type: A string that identifies type of a resource :type resource_type: basestring :param resource_id: A string that identifies some named resource, guaranteeing that only one task reserving this same string can happen at a time. :type resource_id: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :param group_id: The id to identify which group of tasks a task belongs to :type group_id: uuid.UUID :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ # Form a resource_id for reservation by combining given resource type and id. This way, # two different resources having the same id will not block each other. resource_id = ":".join((resource_type, resource_id)) inner_task_id = str(uuid.uuid4()) task_name = self.name tag_list = kwargs.get('tags', []) group_id = kwargs.get('group_id', None) # Create a new task status with the task id and tags. task_status = TaskStatus(task_id=inner_task_id, task_type=task_name, state=constants.CALL_WAITING_STATE, tags=tag_list, group_id=group_id) # To avoid the race condition where __call__ method below is called before # this change is propagated to all db nodes, using an 'upsert' here and setting # the task state to 'waiting' only on an insert. task_status.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time']) try: _queue_reserved_task.apply_async( args=[task_name, inner_task_id, resource_id, args, kwargs], queue=RESOURCE_MANAGER_QUEUE ) except Exception: TaskStatus.objects(task_id=task_status.task_id).update(state=constants.CALL_ERROR_STATE) raise return AsyncResult(inner_task_id)
def test_save_task_status_fires_notification(self, mock_send): """ Test that saving a TaskStatus fires an event notification. """ task_id = self.get_random_uuid() ts = TaskStatus(task_id) ts.save() mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id)
def test_find_by_criteria_with_result(self): tags = ['test', 'tags'] TaskStatus(task_id='1').save() TaskStatus(task_id='2', tags=tags).save() result = 'done' TaskStatus(task_id='3', tags=tags, state=constants.CALL_FINISHED_STATE, result=result).save() filters = {'tags': tags, 'task_id': {'$in': ['1', '3']}} fields = ['task_id', 'tags', 'result'] limit = 1 sort = (('task_id', DESCENDING), ) criteria = Criteria(filters=filters, fields=fields, limit=limit, sort=sort) query_set = TaskStatus.objects.find_by_criteria(criteria) self.assertEqual(len(query_set), 1) self.assertEqual(query_set[0].task_id, '3') self.assertEqual(query_set[0].result, result) task_state_default = constants.CALL_WAITING_STATE self.assertEqual(query_set[0].state, task_state_default)
def apply_async(self, *args, **kwargs): """ A wrapper around the PulpTask apply_async method. It allows us to accept a few more parameters than Celery does for our own purposes, listed below. It also allows us to create and update task status which can be used to track status of this task during it's lifetime. :param queue: The queue that the task has been placed into (optional, defaults to the general Celery queue.) :type queue: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :param group_id: The id that identifies which group of tasks a task belongs to :type group_id: uuid.UUID :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ if celery_version.startswith('4'): routing_key = kwargs.get( 'routing_key', defaults.NAMESPACES['task']['default_routing_key'].default) else: routing_key = kwargs.get( 'routing_key', defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default) tag_list = kwargs.pop('tags', []) group_id = kwargs.pop('group_id', None) try: async_result = super(Task, self).apply_async(*args, **kwargs) except Exception: if 'task_id' in kwargs: TaskStatus.objects(task_id=kwargs['task_id']).update( state=constants.CALL_ERROR_STATE) raise async_result.tags = tag_list # Create a new task status with the task id and tags. task_status = TaskStatus(task_id=async_result.id, task_type=self.name, state=constants.CALL_WAITING_STATE, worker_name=routing_key, tags=tag_list, group_id=group_id) # We're now racing with __call__, on_failure and on_success, any of which may # have completed by now. To avoid overwriting TaskStatus updates from those callbacks, # we'll do an upsert and only touch the fields listed below if we've inserted the object. task_status.save_with_set_on_insert(fields_to_set_on_insert=[ 'state', 'start_time', 'finish_time', 'result', 'error', 'spawned_tasks', 'traceback' ]) return async_result
def test_with_scheduled_call(self, mock_reset_failure, mock_request): retval = 'random_return_value' task_id = str(uuid.uuid4()) args = [1, 'b', 'iii'] kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2, 'scheduled_call_id': '12345'} mock_request.called_directly = False TaskStatus(task_id).save() task = tasks.Task() task.on_success(retval, task_id, args, kwargs) mock_reset_failure.assert_called_once_with('12345')
def test_cancel_after_task_canceled(self, *unused_mocks): """ Test that canceling a task that was already canceled results in no change to the task state. """ task_id = '1234abcd' TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save() tasks.cancel(task_id) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_after_task_finished(self, _logger, revoke): """ Test that canceling a task that is already finished results in no change to the task state. """ task_id = '1234abcd' TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save() tasks.cancel(task_id) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_successful(self, _logger, revoke): task_id = '1234abcd' TaskStatus(task_id).save() tasks.cancel(task_id) revoke.assert_called_once_with(task_id, terminate=True) self.assertEqual(_logger.info.call_count, 1) log_msg = _logger.info.mock_calls[0][1][0] self.assertTrue(task_id in log_msg) self.assertTrue('Task canceled' in log_msg) task_status = TaskStatus.objects(task_id=task_id).first() self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_event_instantiation(self, mock_current_task): mock_current_task.request.id = 'fake_id' fake_task_status = TaskStatus('fake_id') fake_task_status.save() event_type = 'test_type' payload = 'test_payload' try: event = event_data.Event(event_type, payload) except Exception, e: self.fail(e.message)