def test_no_workers_raised_when_there_are_no_workers(self):
self.mock_filter_workers.return_value = []
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}, {'worker_name': 'b'}]
try:
resources.get_unreserved_worker()
except NoWorkers:
pass
else:
self.fail("NoWorkers() Exception should have been raised.")
def test_no_workers_raised_when_there_are_no_workers(self):
self.mock_filter_workers.return_value = []
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}, {'worker_name': 'b'}]
try:
resources.get_unreserved_worker()
except NoWorkers:
pass
else:
self.fail("NoWorkers() Exception should have been raised.")
def test_reserved_resources_queried_correctly(self):
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}, {'worker_name': 'b'}]
try:
resources.get_unreserved_worker()
except NoWorkers:
pass
else:
self.fail("NoWorkers() Exception should have been raised.")
self.mock_resources.ReservedResource.get_collection.assert_called_once_with()
find.assert_called_once_with()
def test_reserved_resources_queried_correctly(self):
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}, {'worker_name': 'b'}]
try:
resources.get_unreserved_worker()
except NoWorkers:
pass
else:
self.fail("NoWorkers() Exception should have been raised.")
self.mock_resources.ReservedResource.get_collection.assert_called_once_with(
)
find.assert_called_once_with()
def _queue_reserved_task(name, task_id, resource_id, inner_args, inner_kwargs):
"""
A task that encapsulates another task to be dispatched later. This task being encapsulated is
called the "inner" task, and a task name, UUID, and accepts a list of positional args
and keyword args for the inner task. These arguments are named inner_args and inner_kwargs.
inner_args is a list, and inner_kwargs is a dictionary passed to the inner task as positional
and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
:param name: The name of the task to be called
:type name: basestring
:param inner_task_id: The UUID to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
:type inner_task_id: basestring
:param resource_id: The name of the resource you wish to reserve for your task. The system
will ensure that no other tasks that want that same reservation will run
concurrently with yours.
:type resource_id: basestring
:return: None
"""
while True:
try:
worker = resources.get_worker_for_reservation(resource_id)
except NoWorkers:
pass
else:
break
try:
worker = resources.get_unreserved_worker()
except NoWorkers:
pass
else:
break
# No worker is ready for this work, so we need to wait
time.sleep(0.25)
ReservedResource(task_id, worker['name'], resource_id).save()
inner_kwargs['routing_key'] = worker.name
inner_kwargs['exchange'] = DEDICATED_QUEUE_EXCHANGE
inner_kwargs['task_id'] = task_id
try:
celery.tasks[name].apply_async(*inner_args, **inner_kwargs)
finally:
_release_resource.apply_async((task_id, ),
routing_key=worker.name,
exchange=DEDICATED_QUEUE_EXCHANGE)
def _queue_reserved_task(name, task_id, resource_id, inner_args, inner_kwargs):
"""
A task that encapsulates another task to be dispatched later. This task being encapsulated is
called the "inner" task, and a task name, UUID, and accepts a list of positional args
and keyword args for the inner task. These arguments are named inner_args and inner_kwargs.
inner_args is a list, and inner_kwargs is a dictionary passed to the inner task as positional
and keyword arguments using the * and ** operators.
The inner task is dispatched into a dedicated queue for a worker that is decided at dispatch
time. The logic deciding which queue receives a task is controlled through the
find_worker function.
:param name: The name of the task to be called
:type name: basestring
:param inner_task_id: The UUID to be set on the task being called. By providing
the UUID, the caller can have an asynchronous reference to the inner task
that will be dispatched.
:type inner_task_id: basestring
:param resource_id: The name of the resource you wish to reserve for your task. The system
will ensure that no other tasks that want that same reservation will run
concurrently with yours.
:type resource_id: basestring
:return: None
"""
while True:
try:
worker = resources.get_worker_for_reservation(resource_id)
except NoWorkers:
pass
else:
break
try:
worker = resources.get_unreserved_worker()
except NoWorkers:
pass
else:
break
# No worker is ready for this work, so we need to wait
time.sleep(0.25)
ReservedResource(task_id, worker['name'], resource_id).save()
inner_kwargs['routing_key'] = worker.name
inner_kwargs['exchange'] = DEDICATED_QUEUE_EXCHANGE
inner_kwargs['task_id'] = task_id
try:
celery.tasks[name].apply_async(*inner_args, **inner_kwargs)
finally:
_release_resource.apply_async((task_id, ), routing_key=worker.name,
exchange=DEDICATED_QUEUE_EXCHANGE)
def test_worker_returned_when_one_worker_is_not_reserved(self):
self.mock_filter_workers.return_value = [{'name': 'a'}, {'name': 'b'}]
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}]
result = resources.get_unreserved_worker()
self.assertEqual(result, {'name': 'b'})
def test_workers_correctly_queried(self):
self.mock_filter_workers.return_value = [{'name': 'a'}, {'name': 'b'}]
resources.get_unreserved_worker()
self.mock_criteria.Criteria.assert_called_once_with()
self.mock_filter_workers.assert_called_once_with(self.mock_criteria.Criteria.return_value)
def test_worker_returned_when_one_worker_is_not_reserved(self):
self.mock_filter_workers.return_value = [{'name': 'a'}, {'name': 'b'}]
find = self.mock_resources.ReservedResource.get_collection.return_value.find
find.return_value = [{'worker_name': 'a'}]
result = resources.get_unreserved_worker()
self.assertEqual(result, {'name': 'b'})
def test_workers_correctly_queried(self):
self.mock_filter_workers.return_value = [{'name': 'a'}, {'name': 'b'}]
resources.get_unreserved_worker()
self.mock_criteria.Criteria.assert_called_once_with()
self.mock_filter_workers.assert_called_once_with(
self.mock_criteria.Criteria.return_value)