def test_task_status_update(self):
"""
Tests the successful operation of task status update.
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatus(task_id, worker_name, tags, state).save()
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {'start_time': start_time,
'state': 'running',
'progress_report': {'report-id': 'my-progress'}}
TaskStatus.objects(task_id=task_id).update_one(
set__start_time=delta['start_time'], set__state=delta['state'],
set__progress_report=delta['progress_report'])
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['start_time'], delta['start_time'])
# Make sure that parse_iso8601_datetime is able to parse the start_time without errors
dateutils.parse_iso8601_datetime(task_status['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'], delta['progress_report'])
self.assertEqual(task_status['worker_name'], worker_name)
def test_task_status_update_fires_notification(self, mock_send):
"""
Test that update_one() also fires a notification.
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
ts = TaskStatus(task_id, worker_name, tags, state)
ts.save()
# ensure event was fired for save()
mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id)
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {'start_time': start_time,
'state': 'running',
'progress_report': {'report-id': 'my-progress'}}
self.assertEquals(len(mock_send.call_args_list), 1)
TaskStatus.objects(task_id=task_id).update_one(
set__start_time=delta['start_time'], set__state=delta['state'],
set__progress_report=delta['progress_report'])
# ensure event was fired for update_one()
self.assertEquals(len(mock_send.call_args_list), 2)
mock_send.assert_called_with(ts, routing_key="tasks.%s" % task_id)
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id. This method cancels only the task
with given task_id, not the spawned tasks. This also updates task's state to 'canceled'.
:param task_id: The ID of the task you wish to cancel
:type task_id: basestring
:raises MissingResource: if a task with given task_id does not exist
:raises PulpCodedException: if given task is already in a complete state
"""
try:
task_status = TaskStatus.objects.get(task_id=task_id)
except DoesNotExist:
raise MissingResource(task_id)
if task_status['state'] in constants.CALL_COMPLETE_STATES:
# If the task is already done, just stop
msg = _('Task [%(task_id)s] already in a completed state: %(state)s')
_logger.info(msg % {'task_id': task_id, 'state': task_status['state']})
return
controller.revoke(task_id, terminate=True)
TaskStatus.objects(task_id=task_id, state__nin=constants.CALL_COMPLETE_STATES).\
update_one(set__state=constants.CALL_CANCELED_STATE)
msg = _('Task canceled: %(task_id)s.')
msg = msg % {'task_id': task_id}
_logger.info(msg)
def clean(self):
super(RepoManagerTests, self).clean()
model.Repository.drop_collection()
RepoImporter.get_collection().remove()
RepoDistributor.get_collection().remove()
TaskStatus.objects().delete()
def __call__(self, *args, **kwargs):
"""
This overrides CeleryTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" %
self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
return super(Task, self).__call__(*args, **kwargs)
def clean(self):
super(RepoManagerTests, self).clean()
model.Repository.drop_collection()
RepoImporter.get_collection().remove()
RepoDistributor.get_collection().remove()
TaskStatus.objects().delete()
def clean(self):
super(RepoManagerTests, self).clean()
model.Repository.objects.delete()
model.Importer.objects.delete()
model.Distributor.objects.delete()
TaskStatus.objects().delete()
def clean(self):
super(RepoManagerTests, self).clean()
model.Repository.objects.delete()
model.Importer.objects.delete()
model.Distributor.objects.delete()
TaskStatus.objects().delete()
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id. This method cancels only the task
with given task_id, not the spawned tasks. This also updates task's state to 'canceled'.
:param task_id: The ID of the task you wish to cancel
:type task_id: basestring
:raises MissingResource: if a task with given task_id does not exist
:raises PulpCodedException: if given task is already in a complete state
"""
try:
task_status = TaskStatus.objects.get(task_id=task_id)
except DoesNotExist:
raise MissingResource(task_id)
if task_status['state'] in constants.CALL_COMPLETE_STATES:
# If the task is already done, just stop
msg = _('Task [%(task_id)s] already in a completed state: %(state)s')
_logger.info(msg % {'task_id': task_id, 'state': task_status['state']})
return
controller.revoke(task_id, terminate=True)
TaskStatus.objects(task_id=task_id, state__nin=constants.CALL_COMPLETE_STATES).\
update_one(set__state=constants.CALL_CANCELED_STATE)
msg = _('Task canceled: %(task_id)s.')
msg = msg % {'task_id': task_id}
_logger.info(msg)
def __call__(self, *args, **kwargs):
"""
This overrides CeleryTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status['state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" % self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
return super(Task, self).__call__(*args, **kwargs)
def succeeded(self, reply):
"""
Notification (reply) indicating an RMI succeeded.
This information is relayed to the task coordinator.
:param reply: A successful reply object.
:type reply: gofer.rmi.async.Succeeded
"""
_logger.info(_('Task RMI (succeeded): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
result = dict(reply.retval)
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
if action == 'bind':
if result['succeeded']:
ReplyHandler._bind_succeeded(task_id, call_context)
else:
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
if result['succeeded']:
ReplyHandler._unbind_succeeded(call_context)
else:
ReplyHandler._unbind_failed(task_id, call_context)
return
def failed(self, reply):
"""
Notification (reply) indicating an RMI failed.
This information used to update the task status.
:param reply: A failure reply object.
:type reply: gofer.rmi.async.Failed
"""
_logger.info(_('Task RMI (failed): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
traceback = reply.xstate['trace']
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
if action == 'bind':
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
ReplyHandler._unbind_failed(task_id, call_context)
return
def set_progress(self, status):
"""
Informs the server of the current state of the publish operation. The
contents of the status is dependent on how the distributor
implementation chooses to divide up the publish process.
@param status: contains arbitrary data to describe the state of the
publish; the contents may contain whatever information is relevant
to the distributor implementation so long as it is serializable
"""
if self.task_id is None:
# not running within a task
return
try:
self.progress_report[self.report_id] = status
TaskStatus.objects(task_id=self.task_id).update_one(
set__progress_report=self.progress_report)
except Exception, e:
_logger.exception(
'Exception from server setting progress for report [%s]' % self.report_id)
try:
_logger.error('Progress value: %s' % str(status))
except Exception:
# Best effort to print this, but if its that grossly unserializable
# the log will tank and we don't want that exception to bubble up
pass
raise self.exception_class(e), None, sys.exc_info()[2]
def test_set_accepted(self):
task_id = self.get_random_uuid()
TaskStatus(task_id, state=constants.CALL_WAITING_STATE).save()
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ACCEPTED_STATE)
def apply_async_with_reservation(self, resource_type, resource_id, *args, **kwargs):
"""
This method allows the caller to schedule the ReservedTask to run asynchronously just like
Celery's apply_async(), while also making the named resource. No two tasks that claim the
same resource reservation can execute concurrently. It accepts type and id of a resource
and combines them to form a resource id.
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a _queue_reserved_task task. See the
docblock on _queue_reserved_task for more information on this.
This method creates a TaskStatus as a placeholder for later updates. Pulp expects to poll
on a task just after calling this method, so a TaskStatus entry needs to exist for it
before it returns.
For a list of parameters accepted by the *args and **kwargs parameters, please see the
docblock for the apply_async() method.
:param resource_type: A string that identifies type of a resource
:type resource_type: basestring
:param resource_id: A string that identifies some named resource, guaranteeing that only
one task reserving this same string can happen at a time.
:type resource_id: basestring
:param tags: A list of tags (strings) to place onto the task, used for searching
for tasks by tag
:type tags: list
:param group_id: The id to identify which group of tasks a task belongs to
:type group_id: uuid.UUID
:return: An AsyncResult instance as returned by Celery's apply_async
:rtype: celery.result.AsyncResult
"""
# Form a resource_id for reservation by combining given resource type and id. This way,
# two different resources having the same id will not block each other.
resource_id = ":".join((resource_type, resource_id))
inner_task_id = str(uuid.uuid4())
task_name = self.name
tag_list = kwargs.get('tags', [])
group_id = kwargs.get('group_id', None)
# Create a new task status with the task id and tags.
task_status = TaskStatus(task_id=inner_task_id, task_type=task_name,
state=constants.CALL_WAITING_STATE, tags=tag_list,
group_id=group_id)
# To avoid the race condition where __call__ method below is called before
# this change is propagated to all db nodes, using an 'upsert' here and setting
# the task state to 'waiting' only on an insert.
task_status.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time'])
try:
_queue_reserved_task.apply_async(
args=[task_name, inner_task_id, resource_id, args, kwargs],
queue=RESOURCE_MANAGER_QUEUE
)
except Exception:
TaskStatus.objects(task_id=task_status.task_id).update(state=constants.CALL_ERROR_STATE)
raise
return AsyncResult(inner_task_id)
def apply_async_with_reservation(self, resource_type, resource_id, *args, **kwargs):
"""
This method allows the caller to schedule the ReservedTask to run asynchronously just like
Celery's apply_async(), while also making the named resource. No two tasks that claim the
same resource reservation can execute concurrently. It accepts type and id of a resource
and combines them to form a resource id.
This does not dispatch the task directly, but instead promises to dispatch it later by
encapsulating the desired task through a call to a _queue_reserved_task task. See the
docblock on _queue_reserved_task for more information on this.
This method creates a TaskStatus as a placeholder for later updates. Pulp expects to poll
on a task just after calling this method, so a TaskStatus entry needs to exist for it
before it returns.
For a list of parameters accepted by the *args and **kwargs parameters, please see the
docblock for the apply_async() method.
:param resource_type: A string that identifies type of a resource
:type resource_type: basestring
:param resource_id: A string that identifies some named resource, guaranteeing that only
one task reserving this same string can happen at a time.
:type resource_id: basestring
:param tags: A list of tags (strings) to place onto the task, used for searching
for tasks by tag
:type tags: list
:param group_id: The id to identify which group of tasks a task belongs to
:type group_id: uuid.UUID
:return: An AsyncResult instance as returned by Celery's apply_async
:rtype: celery.result.AsyncResult
"""
# Form a resource_id for reservation by combining given resource type and id. This way,
# two different resources having the same id will not block each other.
resource_id = ":".join((resource_type, resource_id))
inner_task_id = str(uuid.uuid4())
task_name = self.name
tag_list = kwargs.get('tags', [])
group_id = kwargs.get('group_id', None)
# Create a new task status with the task id and tags.
task_status = TaskStatus(task_id=inner_task_id, task_type=task_name,
state=constants.CALL_WAITING_STATE, tags=tag_list,
group_id=group_id)
# To avoid the race condition where __call__ method below is called before
# this change is propagated to all db nodes, using an 'upsert' here and setting
# the task state to 'waiting' only on an insert.
task_status.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time'])
try:
_queue_reserved_task.apply_async(
args=[task_name, inner_task_id, resource_id, args, kwargs],
queue=RESOURCE_MANAGER_QUEUE
)
except Exception:
TaskStatus.objects(task_id=task_status.task_id).update(state=constants.CALL_ERROR_STATE)
raise
return AsyncResult(inner_task_id)
def progress(self, reply):
"""
Notification (reply) indicating an RMI has reported status.
This information is relayed to the task coordinator.
:param reply: A progress reply object.
:type reply: gofer.rmi.async.Progress
"""
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id).update_one(set__progress_report=reply.details)
def apply_async(self, *args, **kwargs):
"""
A wrapper around the PulpTask apply_async method. It allows us to accept a few more
parameters than Celery does for our own purposes, listed below. It also allows us
to create and update task status which can be used to track status of this task
during it's lifetime.
:param queue: The queue that the task has been placed into (optional, defaults to
the general Celery queue.)
:type queue: basestring
:param tags: A list of tags (strings) to place onto the task, used for searching for
tasks by tag
:type tags: list
:param group_id: The id that identifies which group of tasks a task belongs to
:type group_id: uuid.UUID
:return: An AsyncResult instance as returned by Celery's apply_async
:rtype: celery.result.AsyncResult
"""
if celery_version.startswith('4'):
routing_key = kwargs.get(
'routing_key',
defaults.NAMESPACES['task']['default_routing_key'].default)
else:
routing_key = kwargs.get(
'routing_key',
defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default)
tag_list = kwargs.pop('tags', [])
group_id = kwargs.pop('group_id', None)
try:
async_result = super(Task, self).apply_async(*args, **kwargs)
except Exception:
if 'task_id' in kwargs:
TaskStatus.objects(task_id=kwargs['task_id']).update(
state=constants.CALL_ERROR_STATE)
raise
async_result.tags = tag_list
# Create a new task status with the task id and tags.
task_status = TaskStatus(task_id=async_result.id,
task_type=self.name,
state=constants.CALL_WAITING_STATE,
worker_name=routing_key,
tags=tag_list,
group_id=group_id)
# We're now racing with __call__, on_failure and on_success, any of which may
# have completed by now. To avoid overwriting TaskStatus updates from those callbacks,
# we'll do an upsert and only touch the fields listed below if we've inserted the object.
task_status.save_with_set_on_insert(fields_to_set_on_insert=[
'state', 'start_time', 'finish_time', 'result', 'error',
'spawned_tasks', 'traceback'
])
return async_result
def accepted(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Accepted
"""
_logger.debug(_('Task RMI (accepted): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
def apply_async(self, *args, **kwargs):
"""
A wrapper around the PulpTask apply_async method. It allows us to accept a few more
parameters than Celery does for our own purposes, listed below. It also allows us
to create and update task status which can be used to track status of this task
during it's lifetime.
:param queue: The queue that the task has been placed into (optional, defaults to
the general Celery queue.)
:type queue: basestring
:param tags: A list of tags (strings) to place onto the task, used for searching for
tasks by tag
:type tags: list
:param group_id: The id that identifies which group of tasks a task belongs to
:type group_id: uuid.UUID
:return: An AsyncResult instance as returned by Celery's apply_async
:rtype: celery.result.AsyncResult
"""
if celery_version.startswith('4'):
routing_key = kwargs.get(
'routing_key',
defaults.NAMESPACES['task']['default_routing_key'].default)
else:
routing_key = kwargs.get(
'routing_key',
defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default)
tag_list = kwargs.pop('tags', [])
group_id = kwargs.pop('group_id', None)
try:
async_result = super(Task, self).apply_async(*args, **kwargs)
except:
if 'task_id' in kwargs:
TaskStatus.objects(task_id=kwargs['task_id']).update(
state=constants.CALL_ERROR_STATE)
raise
async_result.tags = tag_list
# Create a new task status with the task id and tags.
task_status = TaskStatus(task_id=async_result.id,
task_type=self.name,
state=constants.CALL_WAITING_STATE,
worker_name=routing_key,
tags=tag_list,
group_id=group_id)
# To avoid the race condition where __call__ method below is called before
# this change is propagated to all db nodes, using an 'upsert' here and setting
# the task state to 'waiting' only on an insert.
task_status.save_with_set_on_insert(
fields_to_set_on_insert=['state', 'start_time'])
return async_result
def test_set_succeeded_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=now,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def test_set_failed_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
traceback = 'abcdef'
finished = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ERROR_STATE)
self.assertTrue(task_status['finish_time'], finished)
self.assertTrue(task_status['traceback'], traceback)
def apply_async(self, *args, **kwargs):
"""
A wrapper around the PulpTask apply_async method. It allows us to accept a few more
parameters than Celery does for our own purposes, listed below. It also allows us
to create and update task status which can be used to track status of this task
during it's lifetime.
:param queue: The queue that the task has been placed into (optional, defaults to
the general Celery queue.)
:type queue: basestring
:param tags: A list of tags (strings) to place onto the task, used for searching for
tasks by tag
:type tags: list
:param group_id: The id that identifies which group of tasks a task belongs to
:type group_id: uuid.UUID
:return: An AsyncResult instance as returned by Celery's apply_async
:rtype: celery.result.AsyncResult
"""
if celery_version.startswith('4'):
routing_key = kwargs.get('routing_key',
defaults.NAMESPACES['task']['default_routing_key'].default)
else:
routing_key = kwargs.get('routing_key',
defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default)
tag_list = kwargs.pop('tags', [])
group_id = kwargs.pop('group_id', None)
try:
async_result = super(Task, self).apply_async(*args, **kwargs)
except Exception:
if 'task_id' in kwargs:
TaskStatus.objects(task_id=kwargs['task_id']).update(
state=constants.CALL_ERROR_STATE
)
raise
async_result.tags = tag_list
# Create a new task status with the task id and tags.
task_status = TaskStatus(
task_id=async_result.id, task_type=self.name,
state=constants.CALL_WAITING_STATE, worker_name=routing_key, tags=tag_list,
group_id=group_id)
# We're now racing with __call__, on_failure and on_success, any of which may
# have completed by now. To avoid overwriting TaskStatus updates from those callbacks,
# we'll do an upsert and only touch the fields listed below if we've inserted the object.
task_status.save_with_set_on_insert(fields_to_set_on_insert=[
'state', 'start_time', 'finish_time', 'result', 'error',
'spawned_tasks', 'traceback'])
return async_result
def test_task_status_not_modified_when_task_status_exists(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
# This simulates the case where a task had already completed
# prior to apply_sync attempting to create a TaskStatus.
# https://pulp.plan.io/issues/2959
TaskStatus(task_id, 'test-worker',
state=CALL_FINISHED_STATE,
result='any_result',
start_time='2017-09-20T10:00:00Z',
finish_time='2017-09-20T11:00:00Z').save()
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatus.objects(task_id=task_id).first()
# Fields which were missing on the object have been added
# self.assertEqual(task_status['task_type'], 'pulp.server.async.tasks.Task')
self.assertEqual(task_status['tags'], ['test_tags'])
# Fields which already existed on the object are retained
self.assertEqual(task_status['state'], 'finished')
self.assertEqual(task_status['start_time'], '2017-09-20T10:00:00Z')
self.assertEqual(task_status['finish_time'], '2017-09-20T11:00:00Z')
self.assertEqual(task_status['result'], 'any_result')
def test_creates_task_status_with_group_id(self, apply_async):
args = [1, 'b', 'iii']
group_id = uuid.uuid4()
kwargs = {'a': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_1,
'group_id': group_id}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(task_statuses.count(), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['group_id'], group_id)
self.assertEqual(new_task_status['worker_name'], WORKER_1)
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
self.assertEqual(new_task_status['error'], None)
self.assertEqual(new_task_status['spawned_tasks'], [])
self.assertEqual(new_task_status['progress_report'], {})
self.assertEqual(new_task_status['task_type'], 'pulp.server.async.tasks.Task')
self.assertEqual(new_task_status['start_time'], None)
self.assertEqual(new_task_status['finish_time'], None)
self.assertEqual(new_task_status['result'], None)
def test_spawned_task_dict(self, mock_request):
retval = tasks.TaskResult(spawned_tasks=[{
'task_id': 'foo-id'
}],
result='bar')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertFalse(new_task_status['finish_time'] is None)
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_spawned_task_status(self, mock_request):
async_result = AsyncResult('foo-id')
retval = tasks.TaskResult(error=PulpException('error-foo'),
result='bar')
retval.spawned_tasks = [async_result]
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertEqual(new_task_status['error']['description'], 'error-foo')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_updates_task_status_correctly(self, mock_request):
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_creates_task_status_with_group_id(self, apply_async):
args = [1, 'b', 'iii']
group_id = uuid.uuid4()
kwargs = {
'a': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_1,
'group_id': group_id
}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(task_statuses.count(), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['group_id'], group_id)
self.assertEqual(new_task_status['worker_name'], WORKER_1)
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
self.assertEqual(new_task_status['error'], None)
self.assertEqual(new_task_status['spawned_tasks'], [])
self.assertEqual(new_task_status['progress_report'], {})
self.assertEqual(new_task_status['task_type'],
'pulp.server.async.tasks.Task')
self.assertEqual(new_task_status['start_time'], None)
self.assertEqual(new_task_status['finish_time'], None)
self.assertEqual(new_task_status['result'], None)
def test_task_status_not_modified_when_task_status_exists(
self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
# This simulates the case where a task had already completed
# prior to apply_sync attempting to create a TaskStatus.
# https://pulp.plan.io/issues/2959
TaskStatus(task_id,
'test-worker',
state=CALL_FINISHED_STATE,
result='any_result',
start_time='2017-09-20T10:00:00Z',
finish_time='2017-09-20T11:00:00Z').save()
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatus.objects(task_id=task_id).first()
# Fields which were missing on the object have been added
self.assertEqual(task_status['task_type'],
'pulp.server.async.tasks.Task')
self.assertEqual(task_status['tags'], ['test_tags'])
# Fields which already existed on the object are retained
self.assertEqual(task_status['state'], 'finished')
self.assertEqual(task_status['start_time'], '2017-09-20T10:00:00Z')
self.assertEqual(task_status['finish_time'], '2017-09-20T11:00:00Z')
self.assertEqual(task_status['result'], 'any_result')
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _('The worker named %(name)s is missing. Canceling the tasks in its queue.')
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(worker_name=name,
state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
# Delete working directory
common_utils.delete_worker_working_directory(name)
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.objects(worker_name=name).delete()
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(
worker_name=name, state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
def test_updates_task_status_correctly(self, mock_request):
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_async_result(self, mock_request):
retval = AsyncResult('foo-id')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], None)
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def __call__(self, *args, **kwargs):
"""
This overrides PulpTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is in one of complete states.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] in constants.CALL_COMPLETE_STATES:
_logger.debug("Task is in the %s state, task-id : [%s]" %
(task_status['state'], self.request.id))
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Also update the worker_name to cover cases where apply_async was called without
# providing the worker name up-front. Skip updating status for eagerly executed tasks,
# since we don't want to track synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
worker_name = self.request.hostname
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
try:
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
set__worker_name=worker_name,
upsert=True)
except NotUniqueError:
# manually retry the upsert. see https://jira.mongodb.org/browse/SERVER-14322
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
set__worker_name=worker_name,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
if config.getboolean('profiling', 'enabled') is True:
self.pr = cProfile.Profile()
self.pr.enable()
return super(Task, self).__call__(*args, **kwargs)
def test_set_succeeded(self, mock_date):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
mock_date.return_value = now
t = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(t)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def test_save_update_with_set_on_insert(self):
"""
Test the save method with set on insert arguments when the object is already in the
database.
"""
task_id = str(uuid4())
worker_name = 'worker_name'
tags = ['tag_1', 'tag_2']
state = constants.CALL_ACCEPTED_STATE
spawned_tasks = ['foo']
error = {'error': 'some_error'}
progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'}
task_type = 'some.task'
old_start_time = start_time = datetime.now()
finish_time = start_time + timedelta(minutes=5)
start_time = dateutils.format_iso8601_datetime(start_time)
finish_time = dateutils.format_iso8601_datetime(finish_time)
result = None
ts = TaskStatus(
task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error,
progress_report=progress_report, task_type=task_type, start_time=start_time,
finish_time=finish_time, result=result)
# Put the object in the database, and then change some of it settings.
ts.save()
new_worker_name = 'a different_worker'
new_state = constants.CALL_SUSPENDED_STATE
new_start_time = old_start_time + timedelta(minutes=10)
new_start_time = dateutils.format_iso8601_datetime(new_start_time)
ts.worker_name = new_worker_name
ts.state = new_state
ts.start_time = new_start_time
# This should update the worker_name on ts in the database, but should not update the state
# or start_time
ts.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time'])
ts = TaskStatus.objects()
# There should only be one TaskStatus in the db
self.assertEqual(len(ts), 1)
ts = ts[0]
# Make sure all the attributes are correct
self.assertEqual(ts['task_id'], task_id)
# Queue should have been updated
self.assertEqual(ts['worker_name'], new_worker_name)
self.assertEqual(ts['tags'], tags)
# state should not have been updated
self.assertEqual(ts['state'], state)
self.assertEqual(ts['error'], error)
self.assertEqual(ts['spawned_tasks'], spawned_tasks)
self.assertEqual(ts['progress_report'], progress_report)
self.assertEqual(ts['task_type'], task_type)
# start_time should not have been updated
self.assertEqual(ts['start_time'], start_time)
self.assertEqual(ts['finish_time'], finish_time)
self.assertEqual(ts['result'], result)
# These are always None
self.assertEqual(ts['traceback'], None)
self.assertEqual(ts['exception'], None)
def get(self, request):
"""
Return a response containing a list of all tasks or a response containg a list of all tasks
that can be filtered by the optional GET parameter 'tags'.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:return: Response containing a serialized list of dicts, one for each task
:rtype: django.http.HttpResponse
"""
tags = request.GET.getlist('tag')
if tags:
raw_tasks = TaskStatus.objects(tags__all=tags)
else:
raw_tasks = TaskStatus.objects()
serialized_task_statuses = [task_serializer(task) for task in raw_tasks]
return generate_json_response_with_pulp_encoder(serialized_task_statuses)
def started(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Started
"""
_logger.debug(_('Task RMI (started): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
started = reply.timestamp
if not started:
now = datetime.now(dateutils.utc_tz())
started = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__start_time=started)
TaskStatus.objects(task_id=task_id, state__in=[constants.CALL_WAITING_STATE,
constants.CALL_ACCEPTED_STATE]).\
update_one(set__state=constants.CALL_RUNNING_STATE)
def get(self, request):
"""
Return a response containing a list of all tasks or a response containing
a list of tasks filtered by the optional GET parameter 'tags'.
:param request: WSGI request object
:type request: django.core.handlers.wsgi.WSGIRequest
:return: Response containing a serialized list of dicts, one for each task
:rtype: django.http.HttpResponse
"""
tags = request.GET.getlist('tag')
if tags:
raw_tasks = TaskStatus.objects(tags__all=tags)
else:
raw_tasks = TaskStatus.objects()
serialized_task_statuses = [task_serializer(task) for task in raw_tasks]
return generate_json_response_with_pulp_encoder(serialized_task_statuses)
def test_cancel_after_task_finished(self, _logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_after_task_finished(self, _logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_successful(self, _logger, revoke):
task_id = '1234abcd'
TaskStatus(task_id).save()
tasks.cancel(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
self.assertEqual(_logger.info.call_count, 1)
log_msg = _logger.info.mock_calls[0][1][0]
self.assertTrue(task_id in log_msg)
self.assertTrue('Task canceled' in log_msg)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_illegal_multi_arg(self):
"""
Test that we receive an exception if we try to use the 'multi' kwarg
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
ts = TaskStatus(task_id, worker_name, tags, state)
ts.save()
self.assertRaises(NotImplementedError, TaskStatus.objects(task_id=task_id).update_one,
multi=True)
def test_task_status_not_modified_when_task_status_exists(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatus(task_id, 'test-worker', state=CALL_CANCELED_STATE).save()
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
