def apply_async(self, *args, **kwargs): """ A wrapper around the Celery apply_async method. It allows us to accept a few more parameters than Celery does for our own purposes, listed below. It also allows us to create and update task status which can be used to track status of this task during it's lifetime. :param queue: The queue that the task has been placed into (optional, defaults to the general Celery queue.) :type queue: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ queue = kwargs.get('queue', defaults.NAMESPACES['CELERY']['DEFAULT_QUEUE'].default) tags = kwargs.pop('tags', []) async_result = super(Task, self).apply_async(*args, **kwargs) async_result.tags = tags # Create a new task status with the task id and tags. task_status = TaskStatus( task_id=async_result.id, task_type=self.name, state=constants.CALL_WAITING_STATE, queue=queue, tags=tags) # To avoid the race condition where __call__ method below is called before # this change is propagated to all db nodes, using an 'upsert' here and setting # the task state to 'waiting' only on an insert. task_status.save(fields_to_set_on_insert=['state', 'start_time']) return async_result
def apply_async(self, *args, **kwargs): """ A wrapper around the Celery apply_async method. It allows us to accept a few more parameters than Celery does for our own purposes, listed below. It also allows us to create and update task status which can be used to track status of this task during it's lifetime. :param queue: The queue that the task has been placed into (optional, defaults to the general Celery queue.) :type queue: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ routing_key = kwargs.get( 'routing_key', defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default) tags = kwargs.pop('tags', []) async_result = super(Task, self).apply_async(*args, **kwargs) async_result.tags = tags # Create a new task status with the task id and tags. task_status = TaskStatus(task_id=async_result.id, task_type=self.name, state=constants.CALL_WAITING_STATE, worker_name=routing_key, tags=tags) # To avoid the race condition where __call__ method below is called before # this change is propagated to all db nodes, using an 'upsert' here and setting # the task state to 'waiting' only on an insert. task_status.save(fields_to_set_on_insert=['state', 'start_time']) return async_result
def test_task_status_update_fires_notification(self, mock_send): """ Test that update_one() also fires a notification. """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' ts = TaskStatus(task_id, worker_name, tags, state) ts.save() # ensure event was fired for save() mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id) now = datetime.now(dateutils.utc_tz()) start_time = dateutils.format_iso8601_datetime(now) delta = {'start_time': start_time, 'state': 'running', 'progress_report': {'report-id': 'my-progress'}} self.assertEquals(len(mock_send.call_args_list), 1) TaskStatus.objects(task_id=task_id).update_one( set__start_time=delta['start_time'], set__state=delta['state'], set__progress_report=delta['progress_report']) # ensure event was fired for update_one() self.assertEquals(len(mock_send.call_args_list), 2) mock_send.assert_called_with(ts, routing_key="tasks.%s" % task_id)
def test_save_update_defaults(self): """ Test the save method with default arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = { 'what do we want?': 'progress!', 'when do we want it?': 'now!' } task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus(task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Let's go ahead and insert the object ts.save() # Now let's alter it a bit, and make sure the alteration makes it to the DB correctly. new_state = constants.CALL_RUNNING_STATE ts.state = new_state # This should update ts in the database ts.save() ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) # The state should have been updated self.assertEqual(ts['state'], new_state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_save_update_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' old_start_time = start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus( task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Put the object in the database, and then change some of it settings. ts.save() new_worker_name = 'a different_worker' new_state = constants.CALL_SUSPENDED_STATE new_start_time = old_start_time + timedelta(minutes=10) new_start_time = dateutils.format_iso8601_datetime(new_start_time) ts.worker_name = new_worker_name ts.state = new_state ts.start_time = new_start_time # This should update the worker_name on ts in the database, but should not update the state # or start_time ts.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) # Queue should have been updated self.assertEqual(ts['worker_name'], new_worker_name) self.assertEqual(ts['tags'], tags) # state should not have been updated self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) # start_time should not have been updated self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_save_task_status_fires_notification(self, mock_send): """ Test that saving a TaskStatus fires an event notification. """ task_id = self.get_random_uuid() ts = TaskStatus(task_id) ts.save() mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id)
def save_update_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is already in the database. """ task_id = 'a_task_id' queue = 'some_queue' tags = ['tag_1', 'tag_2'] state = 'a state' spawned_tasks = ['foo'] error = 'some_error' progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) result = None ts = TaskStatus( task_id, queue, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Put the object in the database, and then change some of it settings. ts.save() new_queue = 'a_different_queue' new_state = 'some_other_state' new_start_time = start_time + timedelta(minutes=10) ts.queue = new_queue ts.state = new_state ts.start_time = new_start_time # This should update the queue on ts in the database, but should not update the state or # start_time ts.save(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.get_collection().find() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) # Queue should have been updated self.assertEqual(ts['queue'], new_queue) self.assertEqual(ts['tags'], tags) # state should not have been updated self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) # start_time should not have been updated self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_data_call(self, mock_current_task): mock_current_task.request.id = 'fake_id' fake_task_status = TaskStatus('fake_id') fake_task_status.save() data = {'event_type': 'test_type', 'payload': 'test_payload', 'call_report': fake_task_status} event = event_data.Event(data['event_type'], data['payload']) self.assertEqual(data, event.data())
def save_insert_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is not already in the database. """ task_id = 'a_task_id' worker_name = 'some_worker' tags = ['tag_1', 'tag_2'] state = 'a state' spawned_tasks = ['foo'] error = 'some_error' progress_report = { 'what do we want?': 'progress!', 'when do we want it?': 'now!' } task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) result = None ts = TaskStatus(task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # This should cause ts to be in the database ts.save(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.get_collection().find() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_event_instantiation(self, mock_current_task): mock_current_task.request.id = 'fake_id' fake_task_status = TaskStatus('fake_id') fake_task_status.save() event_type = 'test_type' payload = 'test_payload' try: event = event_data.Event(event_type, payload) except Exception, e: self.fail(e.message)
def test_illegal_multi_arg(self): """ Test that we receive an exception if we try to use the 'multi' kwarg """ task_id = self.get_random_uuid() worker_name = 'special_worker_name' tags = ['test-tag1', 'test-tag2'] state = 'waiting' ts = TaskStatus(task_id, worker_name, tags, state) ts.save() self.assertRaises(NotImplementedError, TaskStatus.objects(task_id=task_id).update_one, multi=True)
def test_event_instantiation(self, mock_current_task): mock_current_task.request.id = 'fake_id' fake_task_status = TaskStatus('fake_id') fake_task_status.save() event_type = 'test_type' payload = 'test_payload' try: event = event_data.Event(event_type, payload) except Exception, e: self.fail(e.message)
def test_data_call(self, mock_current_task): mock_current_task.request.id = 'fake_id' fake_task_status = TaskStatus('fake_id') fake_task_status.save() data = { 'event_type': 'test_type', 'payload': 'test_payload', 'call_report': fake_task_status } event = event_data.Event(data['event_type'], data['payload']) self.assertEqual(data, event.data())
def apply_async_with_reservation(self, resource_type, resource_id, *args, **kwargs): """ This method allows the caller to schedule the ReservedTask to run asynchronously just like Celery's apply_async(), while also making the named resource. No two tasks that claim the same resource reservation can execute concurrently. It accepts type and id of a resource and combines them to form a resource id. This does not dispatch the task directly, but instead promises to dispatch it later by encapsulating the desired task through a call to a _queue_reserved_task task. See the docblock on _queue_reserved_task for more information on this. This method creates a TaskStatus as a placeholder for later updates. Pulp expects to poll on a task just after calling this method, so a TaskStatus entry needs to exist for it before it returns. For a list of parameters accepted by the *args and **kwargs parameters, please see the docblock for the apply_async() method. :param resource_type: A string that identifies type of a resource :type resource_type: basestring :param resource_id: A string that identifies some named resource, guaranteeing that only one task reserving this same string can happen at a time. :type resource_id: basestring :param tags: A list of tags (strings) to place onto the task, used for searching for tasks by tag :type tags: list :return: An AsyncResult instance as returned by Celery's apply_async :rtype: celery.result.AsyncResult """ # Form a resource_id for reservation by combining given resource type and id. This way, # two different resources having the same id will not block each other. resource_id = ":".join((resource_type, resource_id)) inner_task_id = str(uuid.uuid4()) task_name = self.name tags = kwargs.get('tags', []) # Create a new task status with the task id and tags. task_status = TaskStatus(task_id=inner_task_id, task_type=task_name, state=constants.CALL_WAITING_STATE, tags=tags) # To avoid the race condition where __call__ method below is called before # this change is propagated to all db nodes, using an 'upsert' here and setting # the task state to 'waiting' only on an insert. task_status.save(fields_to_set_on_insert=['state', 'start_time']) _queue_reserved_task.apply_async( args=[task_name, inner_task_id, resource_id, args, kwargs], queue=RESOURCE_MANAGER_QUEUE) return AsyncResult(inner_task_id)
def test_save_update_defaults(self): """ Test the save method with default arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus( task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Let's go ahead and insert the object ts.save() # Now let's alter it a bit, and make sure the alteration makes it to the DB correctly. new_state = constants.CALL_RUNNING_STATE ts.state = new_state # This should update ts in the database ts.save() ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['worker_name'], worker_name) self.assertEqual(ts['tags'], tags) # The state should have been updated self.assertEqual(ts['state'], new_state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def save_update_defaults(self): """ Test the save method with default arguments when the object is already in the database. """ task_id = 'a_task_id' queue = 'some_queue' tags = ['tag_1', 'tag_2'] state = 'a state' spawned_tasks = ['foo'] error = 'some_error' progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) result = None ts = TaskStatus( task_id, queue, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Let's go ahead and insert the object ts.save() # Now let's alter it a bit, and make sure the alteration makes it to the DB correctly. new_state = 'some_altered_state_of_consciousness' ts.state = new_state # This should update ts in the database ts.save() ts = TaskStatus.get_collection().find() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['queue'], queue) self.assertEqual(ts['tags'], tags) # The state should have been updated self.assertEqual(ts['state'], new_state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def save_insert_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is not already in the database. """ task_id = 'a_task_id' queue = 'some_queue' tags = ['tag_1', 'tag_2'] state = 'a state' spawned_tasks = ['foo'] error = 'some_error' progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'} task_type = 'some.task' start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) result = None ts = TaskStatus( task_id, queue, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # This should cause ts to be in the database ts.save(fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.get_collection().find() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) self.assertEqual(ts['queue'], queue) self.assertEqual(ts['tags'], tags) self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)
def test_save_update_with_set_on_insert(self): """ Test the save method with set on insert arguments when the object is already in the database. """ task_id = str(uuid4()) worker_name = 'worker_name' tags = ['tag_1', 'tag_2'] state = constants.CALL_ACCEPTED_STATE spawned_tasks = ['foo'] error = {'error': 'some_error'} progress_report = { 'what do we want?': 'progress!', 'when do we want it?': 'now!' } task_type = 'some.task' old_start_time = start_time = datetime.now() finish_time = start_time + timedelta(minutes=5) start_time = dateutils.format_iso8601_datetime(start_time) finish_time = dateutils.format_iso8601_datetime(finish_time) result = None ts = TaskStatus(task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error, progress_report=progress_report, task_type=task_type, start_time=start_time, finish_time=finish_time, result=result) # Put the object in the database, and then change some of it settings. ts.save() new_worker_name = 'a different_worker' new_state = constants.CALL_SUSPENDED_STATE new_start_time = old_start_time + timedelta(minutes=10) new_start_time = dateutils.format_iso8601_datetime(new_start_time) ts.worker_name = new_worker_name ts.state = new_state ts.start_time = new_start_time # This should update the worker_name on ts in the database, but should not update the state # or start_time ts.save_with_set_on_insert( fields_to_set_on_insert=['state', 'start_time']) ts = TaskStatus.objects() # There should only be one TaskStatus in the db self.assertEqual(len(ts), 1) ts = ts[0] # Make sure all the attributes are correct self.assertEqual(ts['task_id'], task_id) # Queue should have been updated self.assertEqual(ts['worker_name'], new_worker_name) self.assertEqual(ts['tags'], tags) # state should not have been updated self.assertEqual(ts['state'], state) self.assertEqual(ts['error'], error) self.assertEqual(ts['spawned_tasks'], spawned_tasks) self.assertEqual(ts['progress_report'], progress_report) self.assertEqual(ts['task_type'], task_type) # start_time should not have been updated self.assertEqual(ts['start_time'], start_time) self.assertEqual(ts['finish_time'], finish_time) self.assertEqual(ts['result'], result) # These are always None self.assertEqual(ts['traceback'], None) self.assertEqual(ts['exception'], None)