def handle_worker_heartbeat(event):
"""
Celery event handler for 'worker-heartbeat' events.
The event is first parsed and logged. Then the existing Worker objects are
searched for one to update. If an existing one is found, it is updated.
Otherwise a new Worker entry is created. Logging at the info and debug
level is also done.
:param event: A celery event to handle.
:type event: dict
"""
event_info = _parse_and_log_event(event)
find_worker_criteria = Criteria(filters={'_id': event_info['worker_name']},
fields=('_id', 'last_heartbeat'))
find_worker_list = list(resources.filter_workers(find_worker_criteria))
if find_worker_list:
Worker.get_collection().find_and_modify(
query={'_id': event_info['worker_name']},
update={'$set': {'last_heartbeat': event_info['timestamp']}}
)
else:
new_worker = Worker(event_info['worker_name'], event_info['timestamp'])
msg = _("New worker '%(worker_name)s' discovered") % event_info
_logger.info(msg)
new_worker.save()
def test__reserve_resource_with_existing_reservation(self):
"""
Test _reserve_resource() with a resource that has an existing reservation in the database.
It should return the queue listed in the database, and increment the reservation counter.
"""
# Set up a worker with a reservation count of 1
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
# Set up a resource reservation, using our worker from above
reserved_resource_1 = ReservedResource('resource_1',
worker_1.queue_name, 1)
reserved_resource_1.save()
# This should increase the reserved_resource_1 num_reservations to 2. worker_1's name should
# be returned
queue = tasks._reserve_resource('resource_1')
self.assertEqual(queue, WORKER_1_QUEUE)
# Make sure the ReservedResource is correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'], WORKER_1_QUEUE)
self.assertEqual(rr_1['num_reservations'], 2)
def test__release_resource_task_count_one(self):
"""
Test _release_resource() with a resource that has a task count of one. This should remove
the resource from the database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two reserved resources
reserved_resource_1 = ReservedResource('resource_1', worker_1.name, 7)
reserved_resource_1.save()
reserved_resource_2 = ReservedResource('resource_2', worker_2.name, 1)
reserved_resource_2.save()
# This should remove resource_2 from the _resource_map.
tasks._release_resource('resource_2')
# resource_2 should have been removed from the database
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'],
reserved_resource_1.assigned_queue)
self.assertEqual(rr_1['num_reservations'], 7)
def test__release_resource_task_count_two(self):
"""
Test _release_resource() with a resource that has a task count of two. This should simply
decrement the task_count for the resource, but should not remove it from the database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two resource reservations, using our workers from above
reserved_resource_1 = ReservedResource('resource_1', worker_1.name, 7)
reserved_resource_1.save()
reserved_resource_2 = ReservedResource('resource_2', worker_2.name, 2)
reserved_resource_2.save()
# This should reduce the reserved_resource_2 num_reservations to 1.
tasks._release_resource('resource_2')
# Make sure the ReservedResources are also correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 2)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'],
reserved_resource_1.assigned_queue)
self.assertEqual(rr_1['num_reservations'], 7)
rr_2 = rrc.find_one({'_id': reserved_resource_2.name})
self.assertEqual(rr_2['assigned_queue'],
reserved_resource_2.assigned_queue)
self.assertEqual(rr_2['num_reservations'], 1)
def handle_worker_heartbeat(event):
"""
Celery event handler for 'worker-heartbeat' events.
The event is first parsed and logged. If this event is from the resource manager, there is
no further processing to be done. Then the existing Worker objects are searched
for one to update. If an existing one is found, it is updated. Otherwise a new
Worker entry is created. Logging at the info and debug level is also done.
:param event: A celery event to handle.
:type event: dict
"""
event_info = _parse_and_log_event(event)
# if this is the resource_manager do nothing
if _is_resource_manager(event):
return
find_worker_criteria = Criteria(filters={'_id': event_info['worker_name']},
fields=('_id', 'last_heartbeat',
'num_reservations'))
find_worker_list = list(resources.filter_workers(find_worker_criteria))
if find_worker_list:
Worker.get_collection().find_and_modify(
query={'_id': event_info['worker_name']},
update={'$set': {
'last_heartbeat': event_info['timestamp']
}})
else:
new_worker = Worker(event_info['worker_name'], event_info['timestamp'])
msg = _("New worker '%(worker_name)s' discovered") % event_info
_logger.info(msg)
new_worker.save()
def test_resource_in_resource_map(self):
"""
Test _release_resource() with a valid resource. This should remove the resource from the
database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two reserved resources
reserved_resource_1 = ReservedResource(uuid.uuid4(), worker_1.name, 'resource_1')
reserved_resource_1.save()
reserved_resource_2 = ReservedResource(uuid.uuid4(), worker_2.name, 'resource_2')
reserved_resource_2.save()
# This should remove resource_2 from the _resource_map.
tasks._release_resource(reserved_resource_2.task_id)
# resource_2 should have been removed from the database
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.task_id})
self.assertEqual(rr_1['worker_name'], reserved_resource_1.worker_name)
self.assertEqual(rr_1['resource_id'], 'resource_1')
def test_resource_in_resource_map(self):
"""
Test _release_resource() with a valid resource. This should remove the resource from the
database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two reserved resources
reserved_resource_1 = ReservedResource(uuid.uuid4(), worker_1.name,
'resource_1')
reserved_resource_1.save()
reserved_resource_2 = ReservedResource(uuid.uuid4(), worker_2.name,
'resource_2')
reserved_resource_2.save()
# This should remove resource_2 from the _resource_map.
tasks._release_resource(reserved_resource_2.task_id)
# resource_2 should have been removed from the database
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.task_id})
self.assertEqual(rr_1['worker_name'], reserved_resource_1.worker_name)
self.assertEqual(rr_1['resource_id'], 'resource_1')
def test__release_resource_task_count_two(self):
"""
Test _release_resource() with a resource that has a task count of two. This should simply
decrement the task_count for the resource, but should not remove it from the database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two resource reservations, using our workers from above
reserved_resource_1 = ReservedResource('resource_1', worker_1.name, 7)
reserved_resource_1.save()
reserved_resource_2 = ReservedResource('resource_2', worker_2.name, 2)
reserved_resource_2.save()
# This should reduce the reserved_resource_2 num_reservations to 1.
tasks._release_resource('resource_2')
# Make sure the ReservedResources are also correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 2)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'], reserved_resource_1.assigned_queue)
self.assertEqual(rr_1['num_reservations'], 7)
rr_2 = rrc.find_one({'_id': reserved_resource_2.name})
self.assertEqual(rr_2['assigned_queue'], reserved_resource_2.assigned_queue)
self.assertEqual(rr_2['num_reservations'], 1)
def test__release_resource_task_count_one(self):
"""
Test _release_resource() with a resource that has a task count of one. This should remove
the resource from the database.
"""
# Set up two workers
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
worker_2 = Worker(WORKER_2, now)
worker_2.save()
# Set up two reserved resources
reserved_resource_1 = ReservedResource('resource_1', worker_1.name, 7)
reserved_resource_1.save()
reserved_resource_2 = ReservedResource('resource_2', worker_2.name, 1)
reserved_resource_2.save()
# This should remove resource_2 from the _resource_map.
tasks._release_resource('resource_2')
# resource_2 should have been removed from the database
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'], reserved_resource_1.assigned_queue)
self.assertEqual(rr_1['num_reservations'], 7)
def test__reserve_resource_without_existing_reservation(self):
"""
Test _reserve_resource() with a resource that does not have an existing reservation in the
database. It should find the least busy worker, add a reservation to the database with that
worker's queue, and then return the queue name.
"""
# Set up a worker
worker_1 = Worker(WORKER_1, datetime.utcnow())
worker_1.save()
queue = tasks._reserve_resource('resource_1')
self.assertEqual(queue, WORKER_1_QUEUE)
# Make sure the ReservedResource is correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': 'resource_1'})
self.assertEqual(rr_1['assigned_queue'], WORKER_1_QUEUE)
self.assertEqual(rr_1['num_reservations'], 1)
def test__reserve_resource_without_existing_reservation(self):
"""
Test _reserve_resource() with a resource that does not have an existing reservation in the
database. It should find the least busy worker, add a reservation to the database with that
worker's queue, and then return the queue name.
"""
# Set up a worker
worker_1 = Worker(WORKER_1, datetime.utcnow())
worker_1.save()
queue = tasks._reserve_resource('resource_1')
self.assertEqual(queue, WORKER_1_QUEUE)
# Make sure the ReservedResource is correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': 'resource_1'})
self.assertEqual(rr_1['assigned_queue'], WORKER_1_QUEUE)
self.assertEqual(rr_1['num_reservations'], 1)
def test_filter(self):
"""
Test a filter operation to make sure the results appear to be correct.
"""
# Make three workers. We'll filter for two of them.
now = datetime.utcnow()
kw_1 = Worker('worker_1', now)
kw_1.save()
kw_2 = Worker('worker_2', now)
kw_2.save()
kw_3 = Worker('worker_3', now)
kw_3.save()
criteria = Criteria(filters={'_id': {
'$gt': 'worker_1'
}},
sort=[('_id', pymongo.ASCENDING)])
workers = resources.filter_workers(criteria)
# Let's assert that workers is a generator, and then let's cast it to a list so it's easier
# to test that we got the correct instances back.
self.assertEqual(type(workers), types.GeneratorType)
workers = list(workers)
self.assertEqual(all([isinstance(w, Worker) for w in workers]), True)
self.assertEqual(workers[0].name, 'worker_2')
self.assertEqual(workers[1].name, 'worker_3')
def test__release_resource_not_in__resource_map(self):
"""
Test _release_resource() with a resource that is not in the database. This should be
gracefully handled, and result in no changes to the database.
"""
# Set up two workers
worker_1 = Worker(WORKER_1, datetime.utcnow())
worker_1.save()
worker_2 = Worker(WORKER_2, datetime.utcnow())
worker_2.save()
# Set up two resource reservations, using our workers from above
reserved_resource_1 = ReservedResource('resource_1', worker_1.name, 7)
reserved_resource_1.save()
reserved_resource_2 = ReservedResource('resource_2', worker_2.name, 3)
reserved_resource_2.save()
# This should not raise any Exception, but should also not alter either the Worker
# collection or the ReservedResource collection
tasks._release_resource('made_up_resource_id')
# Make sure that the workers collection has not been altered
worker_collection = Worker.get_collection()
self.assertEqual(worker_collection.count(), 2)
worker_1 = worker_collection.find_one({'_id': worker_1.name})
self.assertTrue(worker_1)
worker_2 = worker_collection.find_one({'_id': worker_2.name})
self.assertTrue(worker_2)
# Make sure that the reserved resources collection has not been altered
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 2)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'],
reserved_resource_1.assigned_queue)
self.assertEqual(rr_1['num_reservations'], 7)
rr_2 = rrc.find_one({'_id': reserved_resource_2.name})
self.assertEqual(rr_2['assigned_queue'],
reserved_resource_2.assigned_queue)
self.assertEqual(rr_2['num_reservations'], 3)
def test_resource_not_in_resource_map(self):
"""
Test _release_resource() with a resource that is not in the database. This should be
gracefully handled, and result in no changes to the database.
"""
# Set up two workers
worker_1 = Worker(WORKER_1, datetime.utcnow())
worker_1.save()
worker_2 = Worker(WORKER_2, datetime.utcnow())
worker_2.save()
# Set up two resource reservations, using our workers from above
reserved_resource_1 = ReservedResource(uuid.uuid4(), worker_1.name, 'resource_1')
reserved_resource_1.save()
reserved_resource_2 = ReservedResource(uuid.uuid4(), worker_2.name, 'resource_2')
reserved_resource_2.save()
# This should not raise any Exception, but should also not alter either the Worker
# collection or the ReservedResource collection
tasks._release_resource('made_up_resource_id')
# Make sure that the workers collection has not been altered
worker_collection = Worker.get_collection()
self.assertEqual(worker_collection.count(), 2)
worker_1 = worker_collection.find_one({'_id': worker_1.name})
self.assertTrue(worker_1)
worker_2 = worker_collection.find_one({'_id': worker_2.name})
self.assertTrue(worker_2)
# Make sure that the reserved resources collection has not been altered
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 2)
rr_1 = rrc.find_one({'_id': reserved_resource_1.task_id})
self.assertEqual(rr_1['worker_name'], reserved_resource_1.worker_name)
self.assertEqual(rr_1['resource_id'], 'resource_1')
rr_2 = rrc.find_one({'_id': reserved_resource_2.task_id})
self.assertEqual(rr_2['worker_name'], reserved_resource_2.worker_name)
self.assertEqual(rr_2['resource_id'], 'resource_2')
def test__reserve_resource_with_existing_reservation(self):
"""
Test _reserve_resource() with a resource that has an existing reservation in the database.
It should return the queue listed in the database, and increment the reservation counter.
"""
# Set up a worker with a reservation count of 1
now = datetime.utcnow()
worker_1 = Worker(WORKER_1, now)
worker_1.save()
# Set up a resource reservation, using our worker from above
reserved_resource_1 = ReservedResource('resource_1', worker_1.queue_name, 1)
reserved_resource_1.save()
# This should increase the reserved_resource_1 num_reservations to 2. worker_1's name should
# be returned
queue = tasks._reserve_resource('resource_1')
self.assertEqual(queue, WORKER_1_QUEUE)
# Make sure the ReservedResource is correct
rrc = ReservedResource.get_collection()
self.assertEqual(rrc.count(), 1)
rr_1 = rrc.find_one({'_id': reserved_resource_1.name})
self.assertEqual(rr_1['assigned_queue'], WORKER_1_QUEUE)
self.assertEqual(rr_1['num_reservations'], 2)
def test_filter(self):
"""
Test a filter operation to make sure the results appear to be correct.
"""
# Make three workers. We'll filter for two of them.
now = datetime.utcnow()
kw_1 = Worker('worker_1', now)
kw_1.save()
kw_2 = Worker('worker_2', now)
kw_2.save()
kw_3 = Worker('worker_3', now)
kw_3.save()
criteria = Criteria(filters={'_id': {'$gt': 'worker_1'}}, sort=[('_id', pymongo.ASCENDING)])
workers = resources.filter_workers(criteria)
# Let's assert that workers is a generator, and then let's cast it to a list so it's easier
# to test that we got the correct instances back.
self.assertEqual(type(workers), types.GeneratorType)
workers = list(workers)
self.assertEqual(all([isinstance(w, Worker) for w in workers]), True)
self.assertEqual(workers[0].name, 'worker_2')
self.assertEqual(workers[1].name, 'worker_3')
