def cancel(task_id):
"""
Cancel the task that is represented by the given task_id. This method cancels only the task
with given task_id, not the spawned tasks. This also updates task's state to 'canceled'.
:param task_id: The ID of the task you wish to cancel
:type task_id: basestring
:raises MissingResource: if a task with given task_id does not exist
:raises PulpCodedException: if given task is already in a complete state
"""
try:
task_status = TaskStatus.objects.get(task_id=task_id)
except DoesNotExist:
raise MissingResource(task_id)
if task_status['state'] in constants.CALL_COMPLETE_STATES:
# If the task is already done, just stop
msg = _('Task [%(task_id)s] already in a completed state: %(state)s')
_logger.info(msg % {'task_id': task_id, 'state': task_status['state']})
return
controller.revoke(task_id, terminate=True)
TaskStatus.objects(task_id=task_id, state__nin=constants.CALL_COMPLETE_STATES).\
update_one(set__state=constants.CALL_CANCELED_STATE)
msg = _('Task canceled: %(task_id)s.')
msg = msg % {'task_id': task_id}
_logger.info(msg)
def test_task_status_update(self):
"""
Tests the successful operation of task status update.
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatus(task_id, worker_name, tags, state).save()
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {'start_time': start_time,
'state': 'running',
'progress_report': {'report-id': 'my-progress'}}
TaskStatus.objects(task_id=task_id).update_one(
set__start_time=delta['start_time'], set__state=delta['state'],
set__progress_report=delta['progress_report'])
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['start_time'], delta['start_time'])
# Make sure that parse_iso8601_datetime is able to parse the start_time without errors
dateutils.parse_iso8601_datetime(task_status['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'], delta['progress_report'])
self.assertEqual(task_status['worker_name'], worker_name)
def __call__(self, *args, **kwargs):
"""
This overrides CeleryTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status[
'state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" %
self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE,
set__start_time=start_time,
upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
return super(Task, self).__call__(*args, **kwargs)
def failed(self, reply):
"""
Notification (reply) indicating an RMI failed.
This information used to update the task status.
:param reply: A failure reply object.
:type reply: gofer.rmi.async.Failed
"""
_logger.info(_('Task RMI (failed): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
traceback = reply.xstate['trace']
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(
set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
if action == 'bind':
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
ReplyHandler._unbind_failed(task_id, call_context)
return
def test_task_status_update(self):
"""
Tests the successful operation of task status update.
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatus(task_id, worker_name, tags, state).save()
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {
'start_time': start_time,
'state': 'running',
'progress_report': {
'report-id': 'my-progress'
}
}
TaskStatus.objects(task_id=task_id).update_one(
set__start_time=delta['start_time'],
set__state=delta['state'],
set__progress_report=delta['progress_report'])
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['start_time'], delta['start_time'])
# Make sure that parse_iso8601_datetime is able to parse the start_time without errors
dateutils.parse_iso8601_datetime(task_status['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'],
delta['progress_report'])
self.assertEqual(task_status['worker_name'], worker_name)
def cancel(task_id):
"""
Cancel the task that is represented by the given task_id. This method cancels only the task
with given task_id, not the spawned tasks. This also updates task's state to 'canceled'.
:param task_id: The ID of the task you wish to cancel
:type task_id: basestring
:raises MissingResource: if a task with given task_id does not exist
:raises PulpCodedException: if given task is already in a complete state
"""
try:
task_status = TaskStatus.objects.get(task_id=task_id)
except DoesNotExist:
raise MissingResource(task_id)
if task_status['state'] in constants.CALL_COMPLETE_STATES:
# If the task is already done, just stop
msg = _('Task [%(task_id)s] already in a completed state: %(state)s')
_logger.info(msg % {'task_id': task_id, 'state': task_status['state']})
return
controller.revoke(task_id, terminate=True)
TaskStatus.objects(task_id=task_id, state__nin=constants.CALL_COMPLETE_STATES).\
update_one(set__state=constants.CALL_CANCELED_STATE)
msg = _('Task canceled: %(task_id)s.')
msg = msg % {'task_id': task_id}
_logger.info(msg)
def succeeded(self, reply):
"""
Notification (reply) indicating an RMI succeeded.
This information is relayed to the task coordinator.
:param reply: A successful reply object.
:type reply: gofer.rmi.async.Succeeded
"""
_logger.info(_('Task RMI (succeeded): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
result = dict(reply.retval)
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(
set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
if action == 'bind':
if result['succeeded']:
ReplyHandler._bind_succeeded(task_id, call_context)
else:
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
if result['succeeded']:
ReplyHandler._unbind_succeeded(call_context)
else:
ReplyHandler._unbind_failed(task_id, call_context)
return
def test_task_status_update_fires_notification(self, mock_send):
"""
Test that update_one() also fires a notification.
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
ts = TaskStatus(task_id, worker_name, tags, state)
ts.save()
# ensure event was fired for save()
mock_send.assert_called_once_with(ts, routing_key="tasks.%s" % task_id)
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {'start_time': start_time,
'state': 'running',
'progress_report': {'report-id': 'my-progress'}}
self.assertEquals(len(mock_send.call_args_list), 1)
TaskStatus.objects(task_id=task_id).update_one(
set__start_time=delta['start_time'], set__state=delta['state'],
set__progress_report=delta['progress_report'])
# ensure event was fired for update_one()
self.assertEquals(len(mock_send.call_args_list), 2)
mock_send.assert_called_with(ts, routing_key="tasks.%s" % task_id)
def __call__(self, *args, **kwargs):
"""
This overrides CeleryTask's __call__() method. We use this method
for task state tracking of Pulp tasks.
"""
# Check task status and skip running the task if task state is 'canceled'.
try:
task_status = TaskStatus.objects.get(task_id=self.request.id)
except DoesNotExist:
task_status = None
if task_status and task_status['state'] == constants.CALL_CANCELED_STATE:
_logger.debug("Task cancel received for task-id : [%s]" % self.request.id)
return
# Update start_time and set the task state to 'running' for asynchronous tasks.
# Skip updating status for eagerly executed tasks, since we don't want to track
# synchronous tasks in our database.
if not self.request.called_directly:
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
# Using 'upsert' to avoid a possible race condition described in the apply_async method
# above.
TaskStatus.objects(task_id=self.request.id).update_one(
set__state=constants.CALL_RUNNING_STATE, set__start_time=start_time, upsert=True)
# Run the actual task
_logger.debug("Running task : [%s]" % self.request.id)
return super(Task, self).__call__(*args, **kwargs)
def failed(self, reply):
"""
Notification (reply) indicating an RMI failed.
This information used to update the task status.
:param reply: A failure reply object.
:type reply: gofer.rmi.async.Failed
"""
_logger.info(_('Task RMI (failed): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
traceback = reply.xstate['trace']
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
if action == 'bind':
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
ReplyHandler._unbind_failed(task_id, call_context)
return
def succeeded(self, reply):
"""
Notification (reply) indicating an RMI succeeded.
This information is relayed to the task coordinator.
:param reply: A successful reply object.
:type reply: gofer.rmi.async.Succeeded
"""
_logger.info(_('Task RMI (succeeded): %(r)s'), {'r': reply})
call_context = dict(reply.data)
action = call_context.get('action')
task_id = call_context['task_id']
result = dict(reply.retval)
finished = reply.timestamp
if not finished:
now = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
if action == 'bind':
if result['succeeded']:
ReplyHandler._bind_succeeded(task_id, call_context)
else:
ReplyHandler._bind_failed(task_id, call_context)
return
if action == 'unbind':
if result['succeeded']:
ReplyHandler._unbind_succeeded(call_context)
else:
ReplyHandler._unbind_failed(task_id, call_context)
return
def set_progress(self, status):
"""
Informs the server of the current state of the publish operation. The
contents of the status is dependent on how the distributor
implementation chooses to divide up the publish process.
@param status: contains arbitrary data to describe the state of the
publish; the contents may contain whatever information is relevant
to the distributor implementation so long as it is serializable
"""
if self.task_id is None:
# not running within a task
return
try:
self.progress_report[self.report_id] = status
TaskStatus.objects(task_id=self.task_id).update_one(set__progress_report=self.progress_report)
except Exception, e:
logger.exception('Exception from server setting progress for report [%s]' % self.report_id)
try:
logger.error('Progress value: %s' % str(status))
except Exception:
# Best effort to print this, but if its that grossly unserializable
# the log will tank and we don't want that exception to bubble up
pass
raise self.exception_class(e), None, sys.exc_info()[2]
def set_progress(self, status):
"""
Informs the server of the current state of the publish operation. The
contents of the status is dependent on how the distributor
implementation chooses to divide up the publish process.
@param status: contains arbitrary data to describe the state of the
publish; the contents may contain whatever information is relevant
to the distributor implementation so long as it is serializable
"""
if self.task_id is None:
# not running within a task
return
try:
self.progress_report[self.report_id] = status
TaskStatus.objects(task_id=self.task_id).update_one(set__progress_report=self.progress_report)
except Exception, e:
logger.exception('Exception from server setting progress for report [%s]' % self.report_id)
try:
logger.error('Progress value: %s' % str(status))
except Exception:
# Best effort to print this, but if its that grossly unserializable
# the log will tank and we don't want that exception to bubble up
pass
raise self.exception_class(e), None, sys.exc_info()[2]
def test_set_accepted(self):
task_id = self.get_random_uuid()
TaskStatus(task_id, state=constants.CALL_WAITING_STATE).save()
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ACCEPTED_STATE)
def test_set_accepted(self):
task_id = self.get_random_uuid()
TaskStatus(task_id, state=constants.CALL_WAITING_STATE).save()
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ACCEPTED_STATE)
def progress(self, reply):
"""
Notification (reply) indicating an RMI has reported status.
This information is relayed to the task coordinator.
:param reply: A progress reply object.
:type reply: gofer.rmi.async.Progress
"""
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id).update_one(set__progress_report=reply.details)
def GET(self):
valid_filters = ['tag']
filters = self.filters(valid_filters)
tags = filters.get('tag', [])
if tags:
raw_tasks = TaskStatus.objects(tags__all=tags)
else:
raw_tasks = TaskStatus.objects()
serialized_task_statuses = [task_serializer(task) for task in raw_tasks]
return self.ok(serialized_task_statuses)
def progress(self, reply):
"""
Notification (reply) indicating an RMI has reported status.
This information is relayed to the task coordinator.
:param reply: A progress reply object.
:type reply: gofer.rmi.async.Progress
"""
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id).update_one(
set__progress_report=reply.details)
def accepted(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Accepted
"""
_logger.debug(_('Task RMI (accepted): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
def accepted(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Accepted
"""
_logger.debug(_('Task RMI (accepted): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
TaskStatus.objects(task_id=task_id, state=constants.CALL_WAITING_STATE).\
update_one(set__state=constants.CALL_ACCEPTED_STATE)
def test_set_succeeded_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=now,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def test_set_failed_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
traceback = 'abcdef'
finished = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ERROR_STATE)
self.assertTrue(task_status['finish_time'], finished)
self.assertTrue(task_status['traceback'], traceback)
def test_set_failed_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
traceback = 'abcdef'
finished = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(
set__finish_time=finished,
set__state=constants.CALL_ERROR_STATE,
set__traceback=traceback)
task_status = TaskStatus.objects.get(task_id=task_id)
self.assertTrue(task_status['state'], constants.CALL_ERROR_STATE)
self.assertTrue(task_status['finish_time'], finished)
self.assertTrue(task_status['traceback'], traceback)
def test_set_succeeded_with_timestamp(self):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
TaskStatus.objects(task_id=task_id).update_one(
set__finish_time=now,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def test_creates_task_status(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {
'a': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_1
}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(len(task_statuses), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['worker_name'], WORKER_1)
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
self.assertEqual(new_task_status['error'], None)
self.assertEqual(new_task_status['spawned_tasks'], [])
self.assertEqual(new_task_status['progress_report'], {})
self.assertEqual(new_task_status['task_type'],
'pulp.server.async.tasks.Task')
self.assertEqual(new_task_status['start_time'], None)
self.assertEqual(new_task_status['finish_time'], None)
self.assertEqual(new_task_status['result'], None)
def test_updates_task_status_correctly(self, mock_request):
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_spawned_task_status(self, mock_request):
async_result = AsyncResult('foo-id')
retval = tasks.TaskResult(error=PulpException('error-foo'),
result='bar')
retval.spawned_tasks = [async_result]
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertEqual(new_task_status['error']['description'], 'error-foo')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_updates_task_status_correctly(self, mock_request):
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_async_result(self, mock_request):
retval = AsyncResult('foo-id')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], None)
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _('The worker named %(name)s is missing. Canceling the tasks in its queue.')
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
Worker.objects(name=name).delete()
# Delete all reserved_resource documents for the worker
ReservedResource.get_collection().remove({'worker_name': name})
# Cancel all of the tasks that were assigned to this worker's queue
for task_status in TaskStatus.objects(worker_name=name,
state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
# Delete working directory
common_utils.delete_worker_working_directory(name)
def test_spawned_task_dict(self, mock_request):
retval = tasks.TaskResult(spawned_tasks=[{
'task_id': 'foo-id'
}],
result='bar')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatus(task_id).save()
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertFalse(new_task_status['finish_time'] is None)
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_save_update_defaults(self):
"""
Test the save method with default arguments when the object is already in the database.
"""
task_id = str(uuid4())
worker_name = 'worker_name'
tags = ['tag_1', 'tag_2']
state = constants.CALL_ACCEPTED_STATE
spawned_tasks = ['foo']
error = {'error': 'some_error'}
progress_report = {
'what do we want?': 'progress!',
'when do we want it?': 'now!'
}
task_type = 'some.task'
start_time = datetime.now()
finish_time = start_time + timedelta(minutes=5)
start_time = dateutils.format_iso8601_datetime(start_time)
finish_time = dateutils.format_iso8601_datetime(finish_time)
result = None
ts = TaskStatus(task_id,
worker_name,
tags,
state,
spawned_tasks=spawned_tasks,
error=error,
progress_report=progress_report,
task_type=task_type,
start_time=start_time,
finish_time=finish_time,
result=result)
# Let's go ahead and insert the object
ts.save()
# Now let's alter it a bit, and make sure the alteration makes it to the DB correctly.
new_state = constants.CALL_RUNNING_STATE
ts.state = new_state
# This should update ts in the database
ts.save()
ts = TaskStatus.objects()
# There should only be one TaskStatus in the db
self.assertEqual(len(ts), 1)
ts = ts[0]
# Make sure all the attributes are correct
self.assertEqual(ts['task_id'], task_id)
self.assertEqual(ts['worker_name'], worker_name)
self.assertEqual(ts['tags'], tags)
# The state should have been updated
self.assertEqual(ts['state'], new_state)
self.assertEqual(ts['error'], error)
self.assertEqual(ts['spawned_tasks'], spawned_tasks)
self.assertEqual(ts['progress_report'], progress_report)
self.assertEqual(ts['task_type'], task_type)
self.assertEqual(ts['start_time'], start_time)
self.assertEqual(ts['finish_time'], finish_time)
self.assertEqual(ts['result'], result)
# These are always None
self.assertEqual(ts['traceback'], None)
self.assertEqual(ts['exception'], None)
def test_set_succeeded(self, mock_date):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
mock_date.return_value = now
t = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(t)
TaskStatus.objects(task_id=task_id).update_one(set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def test_save_update_with_set_on_insert(self):
"""
Test the save method with set on insert arguments when the object is already in the
database.
"""
task_id = str(uuid4())
worker_name = 'worker_name'
tags = ['tag_1', 'tag_2']
state = constants.CALL_ACCEPTED_STATE
spawned_tasks = ['foo']
error = {'error': 'some_error'}
progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'}
task_type = 'some.task'
old_start_time = start_time = datetime.now()
finish_time = start_time + timedelta(minutes=5)
start_time = dateutils.format_iso8601_datetime(start_time)
finish_time = dateutils.format_iso8601_datetime(finish_time)
result = None
ts = TaskStatus(
task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error,
progress_report=progress_report, task_type=task_type, start_time=start_time,
finish_time=finish_time, result=result)
# Put the object in the database, and then change some of it settings.
ts.save()
new_worker_name = 'a different_worker'
new_state = constants.CALL_SUSPENDED_STATE
new_start_time = old_start_time + timedelta(minutes=10)
new_start_time = dateutils.format_iso8601_datetime(new_start_time)
ts.worker_name = new_worker_name
ts.state = new_state
ts.start_time = new_start_time
# This should update the worker_name on ts in the database, but should not update the state
# or start_time
ts.save_with_set_on_insert(fields_to_set_on_insert=['state', 'start_time'])
ts = TaskStatus.objects()
# There should only be one TaskStatus in the db
self.assertEqual(len(ts), 1)
ts = ts[0]
# Make sure all the attributes are correct
self.assertEqual(ts['task_id'], task_id)
# Queue should have been updated
self.assertEqual(ts['worker_name'], new_worker_name)
self.assertEqual(ts['tags'], tags)
# state should not have been updated
self.assertEqual(ts['state'], state)
self.assertEqual(ts['error'], error)
self.assertEqual(ts['spawned_tasks'], spawned_tasks)
self.assertEqual(ts['progress_report'], progress_report)
self.assertEqual(ts['task_type'], task_type)
# start_time should not have been updated
self.assertEqual(ts['start_time'], start_time)
self.assertEqual(ts['finish_time'], finish_time)
self.assertEqual(ts['result'], result)
# These are always None
self.assertEqual(ts['traceback'], None)
self.assertEqual(ts['exception'], None)
def test_set_succeeded(self, mock_date):
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
result = 'done'
now = '2014-11-21 05:21:38.829678'
mock_date.return_value = now
t = datetime.now(dateutils.utc_tz())
finished = dateutils.format_iso8601_datetime(t)
TaskStatus.objects(task_id=task_id).update_one(
set__finish_time=finished,
set__state=constants.CALL_FINISHED_STATE,
set__result=result)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertTrue(task_status['state'], constants.CALL_FINISHED_STATE)
self.assertTrue(task_status['finish_time'], now)
self.assertTrue(task_status['result'], result)
def started(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Started
"""
_logger.debug(_('Task RMI (started): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
started = reply.timestamp
if not started:
now = datetime.now(dateutils.utc_tz())
started = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__start_time=started)
TaskStatus.objects(task_id=task_id, state__in=[constants.CALL_WAITING_STATE,
constants.CALL_ACCEPTED_STATE]).\
update_one(set__state=constants.CALL_RUNNING_STATE)
def started(self, reply):
"""
Notification that an RMI has started executing in the agent.
The task status is updated in the pulp DB.
:param reply: A status reply object.
:type reply: gofer.rmi.async.Started
"""
_logger.debug(_('Task RMI (started): %(r)s'), {'r': reply})
call_context = dict(reply.data)
task_id = call_context['task_id']
started = reply.timestamp
if not started:
now = datetime.now(dateutils.utc_tz())
started = dateutils.format_iso8601_datetime(now)
TaskStatus.objects(task_id=task_id).update_one(set__start_time=started)
TaskStatus.objects(task_id=task_id, state__in=[constants.CALL_WAITING_STATE,
constants.CALL_ACCEPTED_STATE]).\
update_one(set__state=constants.CALL_RUNNING_STATE)
def test_DELETE_doesnt_cancel_spawned_celery_task(self, revoke):
"""
Test the DELETE() which should cause a revoke call to Celery's Controller.
This also tests that the spawned tasks are canceled as well.
"""
task_id = '1234abcd'
spawned_task_id = 'spawned_task'
spawned_by_spawned_task_id = 'spawned_by_spawned_task'
TaskStatus(task_id).save()
TaskStatus(spawned_task_id).save()
TaskStatus(spawned_by_spawned_task_id).save()
TaskStatus.objects(task_id=task_id).update_one(set__spawned_tasks=[spawned_task_id])
TaskStatus.objects(task_id=spawned_task_id).update_one(
set__spawned_tasks=[spawned_by_spawned_task_id])
self.task_resource.DELETE(task_id)
self.assertEqual(revoke.call_count, 1)
revoke.assert_called_once_with(task_id, terminate=True)
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method does not change the state of a task that is already complete
"""
task_id = '1234abcd'
TaskStatus(task_id, state=constants.CALL_FINISHED_STATE).save()
self.task_resource.DELETE(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], constants.CALL_FINISHED_STATE)
def test_save_insert_with_set_on_insert(self):
"""
Test the save method with set on insert arguments when the object is not already in the
database.
"""
task_id = str(uuid4())
worker_name = 'some_worker'
tags = ['tag_1', 'tag_2']
state = constants.CALL_RUNNING_STATE
spawned_tasks = ['foo']
error = {'error': 'some_error'}
progress_report = {
'what do we want?': 'progress!',
'when do we want it?': 'now!'
}
task_type = 'some.task'
start_time = datetime.now()
finish_time = start_time + timedelta(minutes=5)
start_time = dateutils.format_iso8601_datetime(start_time)
finish_time = dateutils.format_iso8601_datetime(finish_time)
result = None
ts = TaskStatus(task_id,
worker_name,
tags,
state,
spawned_tasks=spawned_tasks,
error=error,
progress_report=progress_report,
task_type=task_type,
start_time=start_time,
finish_time=finish_time,
result=result)
# This should cause ts to be in the database
ts.save_with_set_on_insert(
fields_to_set_on_insert=['state', 'start_time'])
ts = TaskStatus.objects()
# There should only be one TaskStatus in the db
self.assertEqual(len(ts), 1)
ts = ts[0]
# Make sure all the attributes are correct
self.assertEqual(ts['task_id'], task_id)
self.assertEqual(ts['worker_name'], worker_name)
self.assertEqual(ts['tags'], tags)
self.assertEqual(ts['state'], state)
self.assertEqual(ts['error'], error)
self.assertEqual(ts['spawned_tasks'], spawned_tasks)
self.assertEqual(ts['progress_report'], progress_report)
self.assertEqual(ts['task_type'], task_type)
self.assertEqual(ts['start_time'], start_time)
self.assertEqual(ts['finish_time'], finish_time)
self.assertEqual(ts['result'], result)
# These are always None
self.assertEqual(ts['traceback'], None)
self.assertEqual(ts['exception'], None)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_after_task_finished(self, _logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_CANCELED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_after_task_finished(self, _logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatus(task_id, 'test_worker', state=CALL_FINISHED_STATE).save()
tasks.cancel(task_id)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_successful(self, _logger, revoke):
task_id = '1234abcd'
TaskStatus(task_id).save()
tasks.cancel(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
self.assertEqual(_logger.info.call_count, 1)
log_msg = _logger.info.mock_calls[0][1][0]
self.assertTrue(task_id in log_msg)
self.assertTrue('Task canceled' in log_msg)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_successful(self, _logger, revoke):
task_id = '1234abcd'
TaskStatus(task_id).save()
tasks.cancel(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
self.assertEqual(_logger.info.call_count, 1)
log_msg = _logger.info.mock_calls[0][1][0]
self.assertTrue(task_id in log_msg)
self.assertTrue('Task canceled' in log_msg)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_illegal_multi_arg(self):
"""
Test that we receive an exception if we try to use the 'multi' kwarg
"""
task_id = self.get_random_uuid()
worker_name = 'special_worker_name'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
ts = TaskStatus(task_id, worker_name, tags, state)
ts.save()
self.assertRaises(NotImplementedError, TaskStatus.objects(task_id=task_id).update_one,
multi=True)
def test_task_status_not_modified_when_task_status_exists(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatus(task_id, 'test-worker', state=CALL_CANCELED_STATE).save()
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_task_status_not_modified_when_task_status_exists(
self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatus(task_id, 'test-worker', state=CALL_CANCELED_STATE).save()
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatus.objects(task_id=task_id).first()
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_save_update_defaults(self):
"""
Test the save method with default arguments when the object is already in the database.
"""
task_id = str(uuid4())
worker_name = 'worker_name'
tags = ['tag_1', 'tag_2']
state = constants.CALL_ACCEPTED_STATE
spawned_tasks = ['foo']
error = {'error': 'some_error'}
progress_report = {'what do we want?': 'progress!', 'when do we want it?': 'now!'}
task_type = 'some.task'
start_time = datetime.now()
finish_time = start_time + timedelta(minutes=5)
start_time = dateutils.format_iso8601_datetime(start_time)
finish_time = dateutils.format_iso8601_datetime(finish_time)
result = None
ts = TaskStatus(
task_id, worker_name, tags, state, spawned_tasks=spawned_tasks, error=error,
progress_report=progress_report, task_type=task_type, start_time=start_time,
finish_time=finish_time, result=result)
# Let's go ahead and insert the object
ts.save()
# Now let's alter it a bit, and make sure the alteration makes it to the DB correctly.
new_state = constants.CALL_RUNNING_STATE
ts.state = new_state
# This should update ts in the database
ts.save()
ts = TaskStatus.objects()
# There should only be one TaskStatus in the db
self.assertEqual(len(ts), 1)
ts = ts[0]
# Make sure all the attributes are correct
self.assertEqual(ts['task_id'], task_id)
self.assertEqual(ts['worker_name'], worker_name)
self.assertEqual(ts['tags'], tags)
# The state should have been updated
self.assertEqual(ts['state'], new_state)
self.assertEqual(ts['error'], error)
self.assertEqual(ts['spawned_tasks'], spawned_tasks)
self.assertEqual(ts['progress_report'], progress_report)
self.assertEqual(ts['task_type'], task_type)
self.assertEqual(ts['start_time'], start_time)
self.assertEqual(ts['finish_time'], finish_time)
self.assertEqual(ts['result'], result)
# These are always None
self.assertEqual(ts['traceback'], None)
self.assertEqual(ts['exception'], None)
def test_create_task_status_defaults(self):
"""
Tests TaskStatus creation with minimal information, to ensure that defaults are handled
properly.
"""
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
task_statuses = TaskStatus.objects()
self.assertEqual(1, len(task_statuses))
self.assertEqual(task_id, task_statuses[0]['task_id'])
self.assertEqual(None, task_statuses[0]['worker_name'])
self.assertEqual([], task_statuses[0]['tags'])
self.assertEqual('waiting', task_statuses[0]['state'])
def test_saves_passed_in_routing_key_as_worker_name(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags'], 'routing_key': 'othername'}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(len(task_statuses), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['worker_name'], 'othername')
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
def test_create_task_status_defaults(self):
"""
Tests TaskStatus creation with minimal information, to ensure that defaults are handled
properly.
"""
task_id = self.get_random_uuid()
TaskStatus(task_id).save()
task_statuses = TaskStatus.objects()
self.assertEqual(1, len(task_statuses))
self.assertEqual(task_id, task_statuses[0]['task_id'])
self.assertEqual(None, task_statuses[0]['worker_name'])
self.assertEqual([], task_statuses[0]['tags'])
self.assertEqual('waiting', task_statuses[0]['state'])
def test_saves_default_routing_key_as_worker_name(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(len(task_statuses), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['worker_name'],
defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default)
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
def test_saves_default_routing_key_as_worker_name(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(len(task_statuses), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(
new_task_status['worker_name'],
defaults.NAMESPACES['CELERY']['DEFAULT_ROUTING_KEY'].default)
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
def test_saves_passed_in_routing_key_as_worker_name(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {
'a': 'for the money',
'tags': ['test_tags'],
'routing_key': 'othername'
}
apply_async.return_value = celery.result.AsyncResult('test_task_id')
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_statuses = TaskStatus.objects()
self.assertEqual(len(task_statuses), 1)
new_task_status = task_statuses[0]
self.assertEqual(new_task_status['task_id'], 'test_task_id')
self.assertEqual(new_task_status['worker_name'], 'othername')
self.assertEqual(new_task_status['tags'], kwargs['tags'])
self.assertEqual(new_task_status['state'], 'waiting')
def _delete_worker(name, normal_shutdown=False):
"""
Delete the Worker with _id name from the database, cancel any associated tasks and reservations
If the worker shutdown normally, no message is logged, otherwise an error level message is
logged. Default is to assume the worker did not shut down normally.
Any resource reservations associated with this worker are cleaned up by this function.
Any tasks associated with this worker are explicitly canceled.
:param name: The name of the worker you wish to delete. In the database, the _id
field is the name.
:type name: basestring
:param normal_shutdown: True if the worker shutdown normally, False otherwise. Defaults to
False.
:type normal_shutdown: bool
"""
if normal_shutdown is False:
msg = _(
'The worker named %(name)s is missing. Canceling the tasks in its queue.'
)
msg = msg % {'name': name}
_logger.error(msg)
# Delete the worker document
worker_list = list(
resources.filter_workers(Criteria(filters={'_id': name})))
if len(worker_list) > 0:
worker_document = worker_list[0]
worker_document.delete()
# Delete all reserved_resource documents for the worker
ReservedResource.get_collection().remove({'worker_name': name})
# Cancel all of the tasks that were assigned to this worker's queue
worker = Worker.from_bson({'_id': name})
for task_status in TaskStatus.objects(
worker_name=worker.name,
state__in=constants.CALL_INCOMPLETE_STATES):
cancel(task_status['task_id'])
