def tearDown(self):
super(CoordinatorTests, self).tearDown()
self.coordinator = None
dispatch_factory._task_queue = self._task_queue_factory
self._task_queue_factory = None
self.collection.drop()
self.collection = None
QueuedCall.get_collection().drop()
ArchivedCall.get_collection().drop()
def tearDown(self):
super(CoordinatorTests, self).tearDown()
self.coordinator = None
dispatch_factory._task_queue = self._task_queue_factory
self._task_queue_factory = None
self.collection.drop()
self.collection = None
QueuedCall.get_collection().drop()
ArchivedCall.get_collection().drop()
def test_start_good_queued_call_collection(self):
request_1 = call.CallRequest(dummy_call)
request_2 = call.CallRequest(dummy_call)
request_1.group_id = request_2.group_id = 'my-group'
queued_request_1 = QueuedCall(request_1)
queued_request_2 = QueuedCall(request_2)
self.queued_call_collection.insert(queued_request_1)
self.queued_call_collection.insert(queued_request_2)
self.coordinator.start()
self.assertEqual(
self.coordinator.execute_call_asynchronously.call_count, 0)
self.assertEqual(self.coordinator.execute_multiple_calls.call_count, 1)
def start(self):
"""
Start the coordinator by clearing conflicting metadata and restarting any
interrupted tasks.
"""
# drop all previous knowledge of previous calls
self.call_resource_collection.remove(safe=True)
# re-start interrupted tasks
queued_call_collection = QueuedCall.get_collection()
queued_call_list = list(queued_call_collection.find().sort('timestamp'))
queued_call_collection.remove(safe=True)
queued_call_request_list = [CallRequest.deserialize(q['serialized_call_request']) for q in queued_call_list]
while queued_call_request_list:
call_request = queued_call_request_list[0]
# individually call un-grouped calls
if call_request.group_id is None:
queued_call_request_list.remove(call_request)
self.execute_call_asynchronously(call_request)
continue
# call grouped calls at all at once
call_request_group = [c for c in queued_call_request_list if c.group_id == call_request.group_id]
map(queued_call_request_list.remove, call_request_group)
try:
# NOTE (jconnor 2012-10-12) this will change the group_id, but I don't think I care
self.execute_multiple_calls(call_request_group)
except TopologicalSortError, e:
log_msg = _('Cannot execute call request group: %(g)s' % {'g': call_request.group_id})
_LOG.warn('\n'.join((log_msg, str(e))))
def DELETE(self, call_request_id):
coordinator = dispatch_factory.coordinator()
tasks = coordinator._find_tasks(call_request_id=call_request_id)
if not tasks:
raise QueuedCallNotFound(call_request_id)
collection = QueuedCall.get_collection()
collection.remove({'_id': tasks[0].queued_call_id}, safe=True)
link = serialization.link.current_link_obj()
return self.accepted(link)
def DELETE(self, task_id):
coordinator = dispatch_factory.coordinator()
tasks = coordinator.find_tasks(task_id=task_id)
if not tasks:
raise QueuedCallNotFound(task_id)
collection = QueuedCall.get_collection()
collection.remove({'_id': tasks[0].queued_call_id}, safe=True)
link = serialization.link.current_link_obj()
return self.accepted(link)
def test_start_good_queued_call(self):
request = call.CallRequest(dummy_call)
queued_request = QueuedCall(request)
self.queued_call_collection.insert(queued_request)
self.coordinator.start()
self.assertEqual(
self.coordinator.execute_call_asynchronously.call_count, 1)
self.assertEqual(self.coordinator.execute_multiple_calls.call_count, 0)
def test_queued_call_with_unicode_args(self):
args = [u'©üàº']
kwargs = {'display_name': u'Brasília'}
task = self.gen_task()
task.args = args
task.kwargs = kwargs
collection = QueuedCall.get_collection()
try:
self.queue.enqueue(task)
except:
self.fail(traceback.format_exc())
queued_call = collection.find({'_id': task.queued_call_id})
self.assertFalse(queued_call is None)
def test_queued_call_with_unicode_args(self):
args = [u'©üàº']
kwargs = {'display_name': u'Brasília'}
task = self.gen_task()
task.args = args
task.kwargs = kwargs
collection = QueuedCall.get_collection()
try:
self.queue.enqueue(task)
except:
self.fail(traceback.format_exc())
queued_call = collection.find({'_id': task.queued_call_id})
self.assertFalse(queued_call is None)
def start(self):
"""
Start the coordinator by clearing conflict metadata and restarting any
interrupted tasks.
"""
# drop all previous knowledge of running tasks
task_resource_collection = TaskResource.get_collection()
task_resource_collection.remove(safe=True)
# re-start interrupted tasks
queued_call_collection = QueuedCall.get_collection()
queued_call_list = list(queued_call_collection.find().sort('timestamp'))
queued_call_collection.remove(safe=True)
for queued_call in queued_call_list:
call_request = CallRequest.deserialize(queued_call['serialized_call_request'])
call_report = self.execute_call_asynchronously(call_request)
def test_queued_call_collection(self):
task = self.gen_task(call=call)
collection = QueuedCall.get_collection()
try:
self.queue.enqueue(task)
except:
self.fail(traceback.format_exc())
queued_call = collection.find_one({'_id': task.queued_call_id})
self.assertFalse(queued_call is None)
self.assertFalse(queued_call['serialized_call_request'] is None)
try:
call_request = CallRequest.deserialize(queued_call['serialized_call_request'])
except:
self.fail(traceback.format_exc())
self.assertFalse(call_request is None)
def test_queued_call_collection(self):
task = self.gen_task(call=call)
collection = QueuedCall.get_collection()
try:
self.queue.enqueue(task)
except:
self.fail(traceback.format_exc())
queued_call = collection.find_one({'_id': task.queued_call_id})
self.assertFalse(queued_call is None)
self.assertFalse(queued_call['serialized_call_request'] is None)
try:
call_request = CallRequest.deserialize(
queued_call['serialized_call_request'])
except:
self.fail(traceback.format_exc())
self.assertFalse(call_request is None)
def start(self):
"""
Start the coordinator by clearing conflicting metadata and restarting any
interrupted tasks.
"""
# drop all previous knowledge of previous calls
self.call_resource_collection.remove(safe=True)
# re-start interrupted tasks
queued_call_collection = QueuedCall.get_collection()
queued_call_list = list(
queued_call_collection.find().sort('timestamp'))
queued_call_collection.remove(safe=True)
queued_call_request_list = [
c for c in [
CallRequest.deserialize(q['serialized_call_request'])
for q in queued_call_list
] if c is not None
]
while queued_call_request_list:
call_request = queued_call_request_list[0]
# individually call un-grouped calls
if call_request.group_id is None:
queued_call_request_list.remove(call_request)
self.execute_call_asynchronously(call_request)
continue
# call grouped calls at all at once
call_request_group = [
c for c in queued_call_request_list
if c.group_id == call_request.group_id
]
map(queued_call_request_list.remove, call_request_group)
try:
# NOTE (jconnor 2012-10-12) this will change the group_id, but I don't think I care
self.execute_multiple_calls(call_request_group)
except TopologicalSortError, e:
log_msg = _('Cannot execute call request group: %(g)s' %
{'g': call_request.group_id})
_LOG.warn('\n'.join((log_msg, str(e))))
def enqueue(self, task):
"""
Enqueue (i.e. add) a task to the task queue.
@param task: task to be run
@type task: pulp.server.dispatch.task.Task
"""
self.__lock.acquire()
try:
queued_call = QueuedCall(task.call_request)
task.queued_call_id = queued_call['_id']
self.queued_call_collection.save(queued_call, safe=True)
task.complete_callback = self._complete
self._validate_call_request_dependencies(task)
self.__waiting_tasks.append(task)
task.call_life_cycle_callbacks(
dispatch_constants.CALL_ENQUEUE_LIFE_CYCLE_CALLBACK)
self.__condition.notify()
finally:
self.__lock.release()
def __init__(self,
concurrency_threshold,
dispatch_interval=0.5,
completed_task_cache_life=20.0):
self.concurrency_threshold = concurrency_threshold
self.dispatch_interval = dispatch_interval
self.completed_task_cache_life = timedelta(seconds=completed_task_cache_life)
self.queued_call_collection = QueuedCall.get_collection()
self.__waiting_tasks = []
self.__running_tasks = []
self.__canceled_tasks = []
self.__completed_tasks = []
self.__running_weight = 0
self.__exit = False
self.__lock = threading.RLock()
self.__condition = threading.Condition(self.__lock)
self.__dispatcher = None
def __init__(self,
concurrency_threshold,
dispatch_interval=0.5,
completed_task_cache_life=20.0):
self.concurrency_threshold = concurrency_threshold
self.dispatch_interval = dispatch_interval
self.completed_task_cache_life = timedelta(
seconds=completed_task_cache_life)
self.queued_call_collection = QueuedCall.get_collection()
self.__waiting_tasks = []
self.__running_tasks = []
self.__completed_tasks = []
self.__running_weight = 0
self.__exit = False
self.__lock = threading.RLock()
self.__condition = threading.Condition(self.__lock)
self.__dispatcher = None
def setUp(self):
QueuedCall.get_collection().remove()
dispatch_factory.initialize()
def setUp(self):
QueuedCall.get_collection().remove()
def setUp(self):
super(CoordinatorStartTests, self).setUp()
self.queued_call_collection = QueuedCall.get_collection()
self.coordinator.execute_call_asynchronously = mock.Mock()
self.coordinator.execute_multiple_calls = mock.Mock()
def setUp(self):
super(CoordinatorStartTests, self).setUp()
self.queued_call_collection = QueuedCall.get_collection()
self.coordinator.execute_call_asynchronously = mock.Mock()
self.coordinator.execute_multiple_calls = mock.Mock()
def tearDown(self):
super(TaskQueueTests, self).tearDown()
QueuedCall.get_collection().drop()
self.queue = None
def tearDown(self):
super(TaskQueueTests, self).tearDown()
QueuedCall.get_collection().drop()
self.queue = None
def setUp(self):
QueuedCall.get_collection().remove()
dispatch_factory.initialize()
def setUp(self):
QueuedCall.get_collection().remove()
