def test_GET_celery_tasks(self):
"""
Test the GET() method to get all current tasks.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
queue_1 = 'queue_1'
state1 = 'waiting'
task_id2 = str(uuid.uuid4())
queue_2 = 'queue_2'
state2 = 'running'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, queue_1, tags, state1)
TaskStatusManager.create_task_status(task_id2, queue_2, tags, state2)
status, body = self.get('/v2/tasks/')
# Validate
self.assertEqual(200, status)
self.assertTrue(len(body) == 2)
for task in body:
if task['task_id'] == task_id1:
self.assertEqual(
task['_href'],
serialization.dispatch.task_result_href(task)['_href'])
self.assertEquals(task['state'], state1)
self.assertEqual(task['queue'], queue_1)
else:
self.assertEqual(
task['_href'],
serialization.dispatch.task_result_href(task)['_href'])
self.assertEquals(task['state'], state2)
self.assertEqual(task['queue'], queue_2)
self.assertEquals(task['tags'], tags)
def test_update_task_status(self):
"""
Tests the successful operation of update_task_status().
"""
task_id = self.get_random_uuid()
queue = 'special_queue'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatusManager.create_task_status(task_id, queue, tags, state)
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {
'start_time': start_time,
'state': 'running',
'disregard': 'ignored',
'progress_report': {
'report-id': 'my-progress'
}
}
updated = TaskStatusManager.update_task_status(task_id, delta)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['start_time'], delta['start_time'])
# Make sure that parse_iso8601_datetime is able to parse the start_time without errors
dateutils.parse_iso8601_datetime(task_status['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'],
delta['progress_report'])
self.assertEqual(task_status['queue'], queue)
self.assertEqual(updated['start_time'], delta['start_time'])
self.assertEqual(updated['state'], delta['state'])
self.assertEqual(updated['progress_report'], delta['progress_report'])
self.assertTrue('disregard' not in updated)
self.assertTrue('disregard' not in task_status)
def test_GET_celery_tasks(self):
"""
Test the GET() method to get all current tasks.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
queue_1 = 'queue_1'
state1 = 'waiting'
task_id2 = str(uuid.uuid4())
queue_2 = 'queue_2'
state2 = 'running'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, queue_1, tags, state1)
TaskStatusManager.create_task_status(task_id2, queue_2, tags, state2)
status, body = self.get('/v2/tasks/')
# Validate
self.assertEqual(200, status)
self.assertTrue(len(body) == 2)
for task in body:
if task['task_id'] == task_id1:
self.assertEquals(task['state'], state1)
self.assertEqual(task['queue'], queue_1)
else:
self.assertEquals(task['state'], state2)
self.assertEqual(task['queue'], queue_2)
self.assertEquals(task['tags'], tags)
def test_update_task_status(self):
"""
Tests the successful operation of update_task_status().
"""
task_id = self.get_random_uuid()
queue = 'special_queue'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatusManager.create_task_status(task_id, queue, tags, state)
delta = {'start_time': dateutils.now_utc_timestamp(),
'state': 'running',
'disregard': 'ignored',
'progress_report': {'report-id': 'my-progress'}}
updated = TaskStatusManager.update_task_status(task_id, delta)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['start_time'], delta['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'], delta['progress_report'])
self.assertEqual(task_status['queue'], queue)
self.assertEqual(updated['start_time'], delta['start_time'])
self.assertEqual(updated['state'], delta['state'])
self.assertEqual(updated['progress_report'], delta['progress_report'])
self.assertTrue('disregard' not in updated)
self.assertTrue('disregard' not in task_status)
def test_GET_celery_tasks(self):
"""
Test the GET() method to get all current tasks.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
worker_1 = 'worker_1'
state1 = 'waiting'
task_id2 = str(uuid.uuid4())
worker_2 = 'worker_2'
state2 = 'running'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, worker_1, tags, state1)
TaskStatusManager.create_task_status(task_id2, worker_2, tags, state2)
status, body = self.get('/v2/tasks/')
# Validate
self.assertEqual(200, status)
self.assertTrue(len(body) == 2)
for task in body:
if task['task_id'] == task_id1:
self.assertEqual(task['_href'],
serialization.dispatch.task_result_href(task)['_href'])
self.assertEquals(task['state'], state1)
self.assertEqual(task['worker_name'], worker_1)
else:
self.assertEqual(task['_href'],
serialization.dispatch.task_result_href(task)['_href'])
self.assertEquals(task['state'], state2)
self.assertEqual(task['worker_name'], worker_2)
self.assertEquals(task['tags'], tags)
def test_update_task_status(self):
"""
Tests the successful operation of update_task_status().
"""
task_id = self.get_random_uuid()
queue = 'special_queue'
tags = ['test-tag1', 'test-tag2']
state = 'waiting'
TaskStatusManager.create_task_status(task_id, queue, tags, state)
now = datetime.now(dateutils.utc_tz())
start_time = dateutils.format_iso8601_datetime(now)
delta = {'start_time': start_time,
'state': 'running',
'disregard': 'ignored',
'progress_report': {'report-id': 'my-progress'}}
updated = TaskStatusManager.update_task_status(task_id, delta)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['start_time'], delta['start_time'])
# Make sure that parse_iso8601_datetime is able to parse the start_time without errors
dateutils.parse_iso8601_datetime(task_status['start_time'])
self.assertEqual(task_status['state'], delta['state'])
self.assertEqual(task_status['progress_report'], delta['progress_report'])
self.assertEqual(task_status['queue'], queue)
self.assertEqual(updated['start_time'], delta['start_time'])
self.assertEqual(updated['state'], delta['state'])
self.assertEqual(updated['progress_report'], delta['progress_report'])
self.assertTrue('disregard' not in updated)
self.assertTrue('disregard' not in task_status)
def test_create_task_status_invalid_task_id(self):
"""
Test that calling create_task_status() with an invalid task id raises the correct error.
"""
try:
TaskStatusManager.create_task_status(None)
except exceptions.InvalidValue, e:
self.assertTrue(e.property_names[0], 'task_id')
def test_create_task_status_invalid_task_id(self):
"""
Test that calling create_task_status() with an invalid task id raises the correct error.
"""
try:
TaskStatusManager.create_task_status(None)
except exceptions.InvalidValue, e:
self.assertTrue(e.property_names[0], 'task_id')
def test_cancel_after_task_finished(self, logger, revoke):
task_id = '1234abcd'
test_queue = AvailableQueue('test_queue')
TaskStatusManager.create_task_status(task_id, test_queue.name, state=CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, tasks.cancel, task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method raises a TaskComplete exception if the task is already complete.
"""
task_id = '1234abcd'
test_queue = AvailableQueue('test_queue')
TaskStatusManager.create_task_status(task_id, test_queue.name,
state=dispatch_constants.CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, self.task_resource.DELETE, task_id)
def test_cancel_after_task_finished(self, logger, revoke):
task_id = '1234abcd'
now = datetime.utcnow()
test_worker = Worker('test_worker', now)
TaskStatusManager.create_task_status(task_id, test_worker.name, state=CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, tasks.cancel, task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_GET_has_correct_worker_name_attribute(self):
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id, worker_name='worker1')
result = self.task_resource.GET(task_id)
result_json = json.loads(result)
self.assertTrue('worker_name' in result_json)
self.assertTrue(result_json['worker_name'] == 'worker1')
def test_GET_has_correct_worker_name_attribute(self):
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id, worker_name='worker1')
result = self.task_resource.GET(task_id)
result_json = json.loads(result)
self.assertTrue('worker_name' in result_json)
self.assertTrue(result_json['worker_name'] == 'worker1')
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method does not change the state of a task that is already complete
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id, state=constants.CALL_FINISHED_STATE)
self.task_resource.DELETE(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], constants.CALL_FINISHED_STATE)
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method raises a TaskComplete exception if the task is already complete.
"""
task_id = '1234abcd'
now = datetime.utcnow()
test_worker = Worker('test_worker', now)
TaskStatusManager.create_task_status(task_id, test_worker.name,
state=constants.CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, self.task_resource.DELETE, task_id)
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method does not change the state of a task that is already complete
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(
task_id, state=constants.CALL_FINISHED_STATE)
self.task_resource.DELETE(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], constants.CALL_FINISHED_STATE)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id, 'test_worker', state=CALL_CANCELED_STATE)
tasks.cancel(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_DELETE_completed_celery_task(self):
"""
Test the DELETE() method raises a TaskComplete exception if the task is already complete.
"""
task_id = '1234abcd'
now = datetime.utcnow()
test_worker = Worker('test_worker', now)
TaskStatusManager.create_task_status(
task_id, test_worker.name, state=constants.CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, self.task_resource.DELETE,
task_id)
def test_delete_task_status(self):
"""
Test delete_task_status() under normal circumstances.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id, 'a_queue')
TaskStatusManager.delete_task_status(task_id)
task_statuses = list(TaskStatusManager.find_all())
self.assertEqual(0, len(task_statuses))
def test_cancel_after_task_finished(self, logger, revoke):
task_id = '1234abcd'
now = datetime.utcnow()
test_worker = Worker('test_worker', now)
TaskStatusManager.create_task_status(task_id,
test_worker.name,
state=CALL_FINISHED_STATE)
self.assertRaises(PulpCodedException, tasks.cancel, task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_DELETE_celery_task(self, revoke):
"""
Test the DELETE() method with a UUID that does not correspond to a UUID that the
coordinator is aware of. This should cause a revoke call to Celery's Controller.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id)
self.task_resource.DELETE(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
def test_DELETE_celery_task(self, revoke):
"""
Test the DELETE() method with a UUID that does not correspond to a UUID that the
coordinator is aware of. This should cause a revoke call to Celery's Controller.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id)
self.task_resource.DELETE(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
def test_create_task_status_duplicate_task_id(self):
"""
Tests create_task_status() with a duplicate task id.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id)
try:
TaskStatusManager.create_task_status(task_id)
except exceptions.DuplicateResource, e:
self.assertTrue(task_id in e)
def test_delete_task_status(self):
"""
Test delete_task_status() under normal circumstances.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id, 'a_queue')
TaskStatusManager.delete_task_status(task_id)
task_statuses = list(TaskStatusManager.find_all())
self.assertEqual(0, len(task_statuses))
def test_create_task_status_duplicate_task_id(self):
"""
Tests create_task_status() with a duplicate task id.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id)
try:
TaskStatusManager.create_task_status(task_id)
except exceptions.DuplicateResource, e:
self.assertTrue(task_id in e)
def test_cancel_after_task_finished(self, logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id, 'test_worker', state=CALL_FINISHED_STATE)
tasks.cancel(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_successful(self, logger, revoke):
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id)
tasks.cancel(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
self.assertEqual(logger.info.call_count, 1)
log_msg = logger.info.mock_calls[0][1][0]
self.assertTrue(task_id in log_msg)
self.assertTrue('Task canceled' in log_msg)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_cancel_successful(self, logger, revoke):
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id)
tasks.cancel(task_id)
revoke.assert_called_once_with(task_id, terminate=True)
self.assertEqual(logger.info.call_count, 1)
log_msg = logger.info.mock_calls[0][1][0]
self.assertTrue(task_id in log_msg)
self.assertTrue('Task canceled' in log_msg)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_create_task_status_duplicate_task_id(self):
"""
Tests create_task_status() with a duplicate task id.
"""
task_id = self.get_random_uuid()
queue = 'a_queue'
TaskStatusManager.create_task_status(task_id, queue)
try:
TaskStatusManager.create_task_status(task_id, queue)
self.fail('Task status with a duplicate task id did not raise an exception')
except exceptions.DuplicateResource, e:
self.assertTrue(task_id in e)
def test_cancel_after_task_finished(self, logger, revoke):
"""
Test that canceling a task that is already finished results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id,
'test_worker',
state=CALL_FINISHED_STATE)
tasks.cancel(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_FINISHED_STATE)
def test_cancel_after_task_canceled(self, *unused_mocks):
"""
Test that canceling a task that was already canceled results in no change
to the task state.
"""
task_id = '1234abcd'
TaskStatusManager.create_task_status(task_id,
'test_worker',
state=CALL_CANCELED_STATE)
tasks.cancel(task_id)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], CALL_CANCELED_STATE)
def test_task_status_not_modified_when_task_status_exists(self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatusManager.create_task_status(task_id, 'test-worker', state=CALL_CANCELED_STATE)
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_create_task_status_duplicate_task_id(self):
"""
Tests create_task_status() with a duplicate task id.
"""
task_id = self.get_random_uuid()
queue = 'a_queue'
TaskStatusManager.create_task_status(task_id, queue)
try:
TaskStatusManager.create_task_status(task_id, queue)
self.fail(
'Task status with a duplicate task id did not raise an exception'
)
except exceptions.DuplicateResource, e:
self.assertTrue(task_id in e)
def test_create_task_status_invalid_attributes(self):
"""
Tests that calling create_task_status() with invalid attributes
results in an error
"""
task_id = self.get_random_uuid()
worker_name = ['not a string']
tags = 'not a list'
state = 1
try:
TaskStatusManager.create_task_status(task_id, worker_name, tags, state)
except exceptions.InvalidValue, e:
self.assertTrue('tags' in e.data_dict()['property_names'])
self.assertTrue('state' in e.data_dict()['property_names'])
self.assertTrue('worker_name' in e.data_dict()['property_names'])
def test_create_task_status_defaults(self):
"""
Tests create_task_status() with minimal information, to ensure that defaults are handled
properly.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id)
task_statuses = list(TaskStatus.get_collection().find())
self.assertEqual(1, len(task_statuses))
self.assertEqual(task_id, task_statuses[0]['task_id'])
self.assertEqual(None, task_statuses[0]['worker_name'])
self.assertEqual([], task_statuses[0]['tags'])
self.assertEqual('waiting', task_statuses[0]['state'])
def test_create_task_status_defaults(self):
"""
Tests create_task_status() with minimal information, to ensure that defaults are handled
properly.
"""
task_id = self.get_random_uuid()
TaskStatusManager.create_task_status(task_id)
task_statuses = list(TaskStatus.get_collection().find())
self.assertEqual(1, len(task_statuses))
self.assertEqual(task_id, task_statuses[0]['task_id'])
self.assertEqual(None, task_statuses[0]['worker_name'])
self.assertEqual([], task_statuses[0]['tags'])
self.assertEqual('waiting', task_statuses[0]['state'])
def test__delete_worker(self, logger, cancel, mock_add_consumer):
"""
Assert that the correct Tasks get canceled when their Worker is deleted, and that the Worker
is removed from the database.
"""
# cause two workers to be added to the database as having workers
worker_watcher.handle_worker_heartbeat({
'timestamp': time.time(),
'type': 'worker-heartbeat',
'hostname': WORKER_1,
})
worker_watcher.handle_worker_heartbeat({
'timestamp': time.time(),
'type': 'worker-heartbeat',
'hostname': WORKER_2,
})
# Let's simulate three tasks being assigned to WORKER_2, with two of them being
# in an incomplete state and one in a complete state. We will delete WORKER_2,
# which should cause the two to get canceled. Let's put task_1 in progress
TaskStatusManager.create_task_status('task_1',
WORKER_2_QUEUE,
state=CALL_RUNNING_STATE)
TaskStatusManager.create_task_status('task_2',
WORKER_2_QUEUE,
state=CALL_WAITING_STATE)
# This task shouldn't get canceled because it isn't in an incomplete state
TaskStatusManager.create_task_status('task_3',
WORKER_2_QUEUE,
state=CALL_FINISHED_STATE)
# Let's make a task in a worker that is still present just to make sure it isn't touched.
TaskStatusManager.create_task_status('task_4',
WORKER_1_QUEUE,
state=CALL_RUNNING_STATE)
# Let's just make sure the setup worked and that we have a Worker with RR2
worker_collection = Worker.get_collection()
self.assertEqual(worker_collection.find({'_id': WORKER_2}).count(), 1)
# Now let's delete the Worker named WORKER_2
tasks._delete_worker.apply_async(args=(WORKER_2, ),
queue=tasks.RESOURCE_MANAGER_QUEUE)
# cancel() should have been called twice with task_1 and task_2 as parameters
self.assertEqual(cancel.call_count, 2)
# Let's build a set out of the two times that cancel was called. We can't know for sure
# which order the Tasks got canceled in, but we can assert that the correct two tasks were
# canceled (task_3 should not appear in this set).
cancel_param_set = set([c[1] for c in cancel.mock_calls])
self.assertEqual(cancel_param_set, set([('task_1', ), ('task_2', )]))
# We should have logged that we are canceling the tasks
self.assertEqual(logger.call_count, 0)
self.assertTrue(WORKER_2 in logger.mock_calls[0][1][0])
self.assertTrue('Canceling the tasks' in logger.mock_calls[0][1][0])
# The Worker should have been deleted
self.assertEqual(worker_collection.find({'_id': WORKER_2}).count(), 0)
# the Worker for RW1 should remain
self.assertEqual(worker_collection.find({'_id': WORKER_1}).count(), 1)
def test_spawned_task_status(self, mock_request):
async_result = AsyncResult('foo-id')
retval = tasks.TaskResult(error=PulpException('error-foo'),
result='bar')
retval.spawned_tasks = [async_result]
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id)
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertEqual(new_task_status['error']['description'], 'error-foo')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_task_status_not_modified_when_task_status_exists(
self, apply_async):
args = [1, 'b', 'iii']
kwargs = {'a': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatusManager.create_task_status(task_id,
'test-worker',
state=CALL_CANCELED_STATE)
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_create_task_status_invalid_attributes(self):
"""
Tests that calling create_task_status() with invalid attributes
results in an error
"""
task_id = self.get_random_uuid()
worker_name = ['not a string']
tags = 'not a list'
state = 1
try:
TaskStatusManager.create_task_status(task_id, worker_name, tags,
state)
except exceptions.InvalidValue, e:
self.assertTrue('tags' in e.data_dict()['property_names'])
self.assertTrue('state' in e.data_dict()['property_names'])
self.assertTrue('worker_name' in e.data_dict()['property_names'])
def test_updates_task_status_correctly(self, mock_request):
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id)
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_spawned_task_dict(self, mock_request):
retval = tasks.TaskResult(spawned_tasks=[{
'task_id': 'foo-id'
}],
result='bar')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'routing_key': WORKER_2
}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id)
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertFalse(new_task_status['finish_time'] is None)
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_apply_async_task_canceled(self, apply_async):
"""
Assert that apply_async() honors 'canceled' task status.
"""
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
TaskStatusManager.create_task_status(task_id, AvailableQueue('test-queue'), state=CALL_CANCELED_STATE)
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_on_success_handler_async_result(self, mock_request):
"""
Make sure that overridden on_success handler updates task status correctly
"""
retval = AsyncResult('foo-id')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'queue': WORKER_2_QUEUE
}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(
task_id, 'some_queue')
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], None)
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_on_success_handler_spawned_task_dict(self, mock_request):
"""
Make sure that overridden on_success handler updates task status correctly
"""
retval = tasks.TaskResult(spawned_tasks=[{
'task_id': 'foo-id'
}],
result='bar')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'queue': WORKER_2_QUEUE
}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(
task_id, 'some_queue')
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertFalse(new_task_status['finish_time'] is None)
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_spawned_task_status(self, mock_request):
async_result = AsyncResult('foo-id')
retval = tasks.TaskResult(error=PulpException('error-foo'),
result='bar')
retval.spawned_tasks = [async_result]
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id)
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertEqual(new_task_status['error']['description'], 'error-foo')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_on_success_with_canceled_task(self, mock_request):
"""
Make sure on_success() does not move a canceled Task to 'finished' state.
"""
retval = 'random_return_value'
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {
'1': 'for the money',
'tags': ['test_tags'],
'queue': WORKER_2_QUEUE
}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(
task_id, 'some_queue', state=CALL_CANCELED_STATE)
task = tasks.Task()
# This should not update the task status to finished, since this task was canceled.
task.on_success(retval, task_id, args, kwargs)
updated_task_status = TaskStatusManager.find_by_task_id(task_id)
# Make sure the task is still canceled.
self.assertEqual(updated_task_status['state'], CALL_CANCELED_STATE)
self.assertEqual(updated_task_status['result'], retval)
self.assertFalse(updated_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(updated_task_status['finish_time'])
def test_on_failure_handler(self, mock_request):
"""
Make sure that overridden on_failure handler updates task status correctly
"""
exc = Exception()
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
class EInfo(object):
"""
on_failure handler expects an instance of celery's ExceptionInfo class
as one of the attributes. It stores string representation of traceback
in it's traceback instance variable. This is a stub to imitate that behavior.
"""
def __init__(self):
self.traceback = "string_repr_of_traceback"
einfo = EInfo()
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id, 'some_queue')
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
self.assertEqual(task_status['traceback'], None)
task = tasks.Task()
task.on_failure(exc, task_id, args, kwargs, einfo)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'error')
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['traceback'], einfo.traceback)
def test_on_success_handler_spawned_task_status(self, mock_request):
"""
Make sure that overridden on_success handler updates task status correctly
"""
async_result = AsyncResult('foo-id')
retval = tasks.TaskResult(error=PulpException('error-foo'),
result='bar')
retval.spawned_tasks = [async_result]
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'queue': RESERVED_WORKER_2}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id, 'some_queue')
self.assertEqual(task_status['state'], None)
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], 'bar')
self.assertEqual(new_task_status['error']['description'], 'error-foo')
self.assertFalse(new_task_status['finish_time'] == None)
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_on_success_handler_async_result(self, mock_request):
"""
Make sure that overridden on_success handler updates task status correctly
"""
retval = AsyncResult('foo-id')
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'queue': RESERVED_WORKER_2}
mock_request.called_directly = False
task_status = TaskStatusManager.create_task_status(task_id, 'some_queue')
self.assertEqual(task_status['state'], 'waiting')
self.assertEqual(task_status['finish_time'], None)
task = tasks.Task()
task.on_success(retval, task_id, args, kwargs)
new_task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(new_task_status['state'], 'finished')
self.assertEqual(new_task_status['result'], None)
self.assertFalse(new_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(new_task_status['finish_time'])
self.assertEqual(new_task_status['spawned_tasks'], ['foo-id'])
def test_create_task_status_invalid_attributes(self):
"""
Tests that calling create_task_status() with invalid attributes
results in an error
"""
task_id = self.get_random_uuid()
# queue is not allowed to be None
queue = None
tags = 'not a list'
state = 1
try:
TaskStatusManager.create_task_status(task_id, queue, tags, state)
self.fail('Invalid attributes did not cause create to raise an exception')
except exceptions.InvalidValue, e:
self.assertTrue('tags' in e.data_dict()['property_names'])
self.assertTrue('state' in e.data_dict()['property_names'])
self.assertTrue('queue' in e.data_dict()['property_names'])
def test__delete_queue(self, logger, cancel, active_queues, mock_add_consumer):
"""
Assert that the correct Tasks get canceled when their queue is deleted, and that the queue
is removed from the database.
"""
# cause two workers to be added to the database as having available queues
worker_watcher.handle_worker_heartbeat({
'timestamp': time.time(),
'type': 'worker-heartbeat',
'hostname': RESERVED_WORKER_1,
})
worker_watcher.handle_worker_heartbeat({
'timestamp': time.time(),
'type': 'worker-heartbeat',
'hostname': RESERVED_WORKER_2,
})
# Let's simulate three tasks being assigned to RESERVED_WORKER_2, with two of them being
# in an incomplete state and one in a complete state. We will delete RESERVED_WORKER_2's
# queue, which should cause the two to get canceled. Let's put task_1 in progress
TaskStatusManager.create_task_status('task_1', RESERVED_WORKER_2,
state=CALL_RUNNING_STATE)
TaskStatusManager.create_task_status('task_2', RESERVED_WORKER_2,
state=CALL_WAITING_STATE)
# This task shouldn't get canceled because it isn't in an incomplete state
TaskStatusManager.create_task_status('task_3', RESERVED_WORKER_2,
state=CALL_FINISHED_STATE)
# Let's make a task in a worker that is still present just to make sure it isn't touched.
TaskStatusManager.create_task_status('task_4', RESERVED_WORKER_1,
state=CALL_RUNNING_STATE)
# Let's just make sure the setup worked and that we have an AvailableQueue with RR2
aqc = AvailableQueue.get_collection()
self.assertEqual(aqc.find({'_id': RESERVED_WORKER_2}).count(), 1)
# Now let's delete the queue named RESERVED_WORKER_2
tasks._delete_queue.apply_async(args=(RESERVED_WORKER_2,),
queue=tasks.RESOURCE_MANAGER_QUEUE)
# cancel() should have been called twice with task_1 and task_2 as parameters
self.assertEqual(cancel.call_count, 2)
# Let's build a set out of the two times that cancel was called. We can't know for sure
# which order the Tasks got canceled in, but we can assert that the correct two tasks were
# canceled (task_3 should not appear in this set).
cancel_param_set = set([c[1] for c in cancel.mock_calls])
self.assertEqual(cancel_param_set, set([('task_1',), ('task_2',)]))
# We should have logged that we are canceling the tasks
self.assertEqual(logger.call_count, 0)
self.assertTrue(RESERVED_WORKER_2 in logger.mock_calls[0][1][0])
self.assertTrue('Canceling the tasks' in logger.mock_calls[0][1][0])
# The queue should have been deleted
self.assertEqual(aqc.find({'_id': RESERVED_WORKER_2}).count(), 0)
# the queue for RW1 should remain
self.assertEqual(aqc.find({'_id': RESERVED_WORKER_1}).count(), 1)
def test_GET_celery_task_by_missing_id(self):
"""
Test the GET() method to get a current task with given id.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
queue_1 = 'queue_1'
state1 = 'waiting'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, queue_1, tags, state1)
non_existing_task_id = str(uuid.uuid4())
status, body = self.get('/v2/tasks/%s/' % non_existing_task_id)
# Validate
self.assertEqual(404, status)
self.assertTrue(isinstance(body, dict))
self.assertTrue('Missing resource' in body['error_message'])
self.assertTrue(non_existing_task_id in body['error_message'])
def test_apply_async_task_canceled(self, apply_async):
"""
Assert that apply_async() honors 'canceled' task status.
"""
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags']}
task_id = 'test_task_id'
now = datetime.utcnow()
TaskStatusManager.create_task_status(task_id,
Worker('test-worker', now),
state=CALL_CANCELED_STATE)
apply_async.return_value = celery.result.AsyncResult(task_id)
task = tasks.Task()
task.apply_async(*args, **kwargs)
task_status = TaskStatusManager.find_by_task_id(task_id)
self.assertEqual(task_status['state'], 'canceled')
self.assertEqual(task_status['start_time'], None)
def test_GET_celery_task_by_missing_id(self):
"""
Test the GET() method to get a current task with given id.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
worker_1 = 'worker_1'
state1 = 'waiting'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, worker_1, tags, state1)
non_existing_task_id = str(uuid.uuid4())
status, body = self.get('/v2/tasks/%s/' % non_existing_task_id)
# Validate
self.assertEqual(404, status)
self.assertTrue(isinstance(body, dict))
self.assertTrue('Missing resource' in body['error_message'])
self.assertTrue(non_existing_task_id in body['error_message'])
def test_with_canceled_task(self, mock_request):
retval = 'random_return_value'
task_id = str(uuid.uuid4())
args = [1, 'b', 'iii']
kwargs = {'1': 'for the money', 'tags': ['test_tags'], 'routing_key': WORKER_2}
mock_request.called_directly = False
TaskStatusManager.create_task_status(task_id, state=CALL_CANCELED_STATE)
task = tasks.Task()
# This should not update the task status to finished, since this task was canceled.
task.on_success(retval, task_id, args, kwargs)
updated_task_status = TaskStatusManager.find_by_task_id(task_id)
# Make sure the task is still canceled.
self.assertEqual(updated_task_status['state'], CALL_CANCELED_STATE)
self.assertEqual(updated_task_status['result'], retval)
self.assertFalse(updated_task_status['finish_time'] is None)
# Make sure that parse_iso8601_datetime is able to parse the finish_time without errors
dateutils.parse_iso8601_datetime(updated_task_status['finish_time'])
def test_create_task_status_invalid_attributes(self):
"""
Tests that calling create_task_status() with invalid attributes
results in an error
"""
task_id = self.get_random_uuid()
# queue is not allowed to be None
queue = None
tags = 'not a list'
state = 1
try:
TaskStatusManager.create_task_status(task_id, queue, tags, state)
self.fail(
'Invalid attributes did not cause create to raise an exception'
)
except exceptions.InvalidValue, e:
self.assertTrue('tags' in e.data_dict()['property_names'])
self.assertTrue('state' in e.data_dict()['property_names'])
self.assertTrue('queue' in e.data_dict()['property_names'])
def bind(consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified bind. This method will be called
after the server-side representation of the binding has been created.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param repo_id: A repository ID.
:type repo_id: str
:param distributor_id: A distributor ID.
:type distributor_id: str
:param options: The options are handler specific.
:type options: dict
:return: The task created by the bind
:rtype: dict
"""
# track agent operations using a pseudo task
task_id = str(uuid4())
task_tags = [
tags.resource_tag(tags.RESOURCE_CONSUMER_TYPE, consumer_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_DISTRIBUTOR_TYPE, distributor_id),
tags.action_tag(tags.ACTION_AGENT_BIND)
]
task = TaskStatusManager.create_task_status(task_id, 'agent', tags=task_tags)
# agent request
consumer_manager = managers.consumer_manager()
binding_manager = managers.consumer_bind_manager()
consumer = consumer_manager.get_consumer(consumer_id)
binding = binding_manager.get_bind(consumer_id, repo_id, distributor_id)
agent_bindings = AgentManager._bindings([binding])
context = Context(
consumer,
task_id=task_id,
action='bind',
consumer_id=consumer_id,
repo_id=repo_id,
distributor_id=distributor_id)
agent = PulpAgent()
agent.consumer.bind(context, agent_bindings, options)
# bind action tracking
consumer_manager = managers.consumer_bind_manager()
consumer_manager.action_pending(
consumer_id,
repo_id,
distributor_id,
Bind.Action.BIND,
task_id)
return task
def unbind(consumer_id, repo_id, distributor_id, options):
"""
Request the agent to perform the specified unbind.
:param consumer_id: The consumer ID.
:type consumer_id: str
:param repo_id: A repository ID.
:type repo_id: str
:param distributor_id: A distributor ID.
:type distributor_id: str
:param options: The options are handler specific.
:type options: dict
:return: A task ID that may be used to track the agent request.
:rtype: str
"""
# track agent operations using a pseudo task
task_id = str(uuid4())
task_tags = [
tags.resource_tag(tags.RESOURCE_CONSUMER_TYPE, consumer_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_TYPE, repo_id),
tags.resource_tag(tags.RESOURCE_REPOSITORY_DISTRIBUTOR_TYPE, distributor_id),
tags.action_tag(tags.ACTION_AGENT_UNBIND)
]
task = TaskStatusManager.create_task_status(task_id, 'agent', tags=task_tags)
# agent request
manager = managers.consumer_manager()
consumer = manager.get_consumer(consumer_id)
binding = dict(repo_id=repo_id, distributor_id=distributor_id)
bindings = AgentManager._unbindings([binding])
context = Context(
consumer,
task_id=task_id,
action='unbind',
consumer_id=consumer_id,
repo_id=repo_id,
distributor_id=distributor_id)
agent = PulpAgent()
agent.consumer.unbind(context, bindings, options)
# unbind action tracking
manager = managers.consumer_bind_manager()
manager.action_pending(
consumer_id,
repo_id,
distributor_id,
Bind.Action.UNBIND,
task_id)
return task
def test_GET_celery_task_by_id(self):
"""
Test the GET() method to get a current task with given id.
"""
# Populate a couple of task statuses
task_id1 = str(uuid.uuid4())
queue_1 = 'queue_1'
state1 = 'waiting'
task_id2 = str(uuid.uuid4())
queue_2 = 'queue_2'
state2 = 'running'
tags = ['random', 'tags']
TaskStatusManager.create_task_status(task_id1, queue_1, tags, state1)
TaskStatusManager.create_task_status(task_id2, queue_2, tags, state2)
status, body = self.get('/v2/tasks/%s/' % task_id2)
# Validate
self.assertEqual(200, status)
self.assertTrue(isinstance(body, dict))
self.assertEquals(body['state'], state2)
self.assertEqual(body['queue'], queue_2)
self.assertEquals(body['tags'], tags)
