def coordinator_dequeue_callback(call_request, call_report):
"""
Callback to be executed upon call completion that will clean up the
coordinator's accounting data pertaining to the call.
@param call_request: call request for the call
@type call_request: L{call.CallRequest} instance
@param call_report: call report for the call
@type call_report: L{call.CallReport} instance
"""
# yes, I know that the call_request is not being used
task_id = call_report.task_id
collection = TaskResource.get_collection()
collection.remove({'task_id': task_id}, safe=True)
def _run_task(self, task, synchronous=None, timeout=None):
"""
Run a task.
@param task: task to run
@type task: L{Task} instance
@param synchronous: whether or not to run the task synchronously,
None means dependent on what the conflict response is
@type synchronous: None or bool
@param timeout: how much time to wait for a synchronous task to start
None means indefinitely
@type timeout: None or datetime.timedelta
"""
# we have to lock the task queue here as there is a race condition
# between calculating the blocking/postponing tasks and enqueueing the
# task when 2 or more tasks are being run that may have
# interdependencies
task_queue = dispatch_factory._task_queue()
task_queue.lock()
task_resource_collection = TaskResource.get_collection()
try:
response, blocking, reasons, task_resources = self._find_conflicts(task.call_request.resources)
task.call_report.response = response
task.call_report.reasons = reasons
if response is dispatch_constants.CALL_REJECTED_RESPONSE:
return
task.blocking_tasks.update(blocking)
task.call_request.add_life_cycle_callback(dispatch_constants.CALL_ENQUEUE_LIFE_CYCLE_CALLBACK, GrantPermmissionsForTaskV2())
task.call_request.add_life_cycle_callback(dispatch_constants.CALL_DEQUEUE_LIFE_CYCLE_CALLBACK, RevokePermissionsForTaskV2())
task.call_request.add_life_cycle_callback(dispatch_constants.CALL_DEQUEUE_LIFE_CYCLE_CALLBACK, coordinator_dequeue_callback)
if task_resources:
set_task_id_on_task_resources(task.id, task_resources)
task_resource_collection.insert(task_resources, safe=True)
task_queue.enqueue(task)
finally:
task_queue.unlock()
# if the call has requested synchronous execution or can be
# synchronously executed, do so
if synchronous or (synchronous is None and response is dispatch_constants.CALL_ACCEPTED_RESPONSE):
try:
# it's perfectly legitimate for the call to complete before the fist poll
running_states = [dispatch_constants.CALL_RUNNING_STATE]
running_states.extend(dispatch_constants.CALL_COMPLETE_STATES)
wait_for_task(task, running_states, poll_interval=self.task_state_poll_interval, timeout=timeout)
except OperationTimedOut:
task_queue.dequeue(task)
raise
else:
wait_for_task(task, dispatch_constants.CALL_COMPLETE_STATES,
poll_interval=self.task_state_poll_interval)
def start(self):
"""
Start the coordinator by clearing conflict metadata and restarting any
interrupted tasks.
"""
# drop all previous knowledge of running tasks
task_resource_collection = TaskResource.get_collection()
task_resource_collection.remove(safe=True)
# re-start interrupted tasks
queued_call_collection = QueuedCall.get_collection()
queued_call_list = list(queued_call_collection.find().sort('timestamp'))
queued_call_collection.remove(safe=True)
for queued_call in queued_call_list:
call_request = CallRequest.deserialize(queued_call['serialized_call_request'])
call_report = self.execute_call_asynchronously(call_request)
def _find_conflicts(self, resources):
"""
Find conflicting tasks, if any, and provide the following:
* a task response, (accepted, postponed, rejected)
* a (possibly empty) set of blocking task ids
* a list of blocking "reasons" in the form of TaskResource instances
* a list of task resources corresponding to the given resources
@param resources: dictionary of resources and their proposed operations
@type resources: dict
@return: tuple of objects described above
@rtype: tuple
"""
if not resources:
return dispatch_constants.CALL_ACCEPTED_RESPONSE, set(), [], []
postponing_tasks = set()
postponing_reasons = []
rejecting_tasks = set()
rejecting_reasons = []
task_resource_collection = TaskResource.get_collection()
task_resources = resource_dict_to_task_resources(resources)
or_query = filter_dicts(task_resources, ('resource_type', 'resource_id'))
cursor = task_resource_collection.find({'$or': or_query})
for task_resource in cursor:
proposed_operation = resources[task_resource['resource_type']][task_resource['resource_id']]
postponing_operations = get_postponing_operations(proposed_operation)
rejecting_operations = get_rejecting_operations(proposed_operation)
current_operation = task_resource['operation']
if current_operation in postponing_operations:
postponing_tasks.add(task_resource['task_id'])
reason = filter_dicts([task_resource], ('resource_type', 'resource_id'))[0]
reason['operation'] = current_operation
if reason not in postponing_reasons:
postponing_reasons.append(reason)
if current_operation in rejecting_operations:
rejecting_tasks.add(task_resource['task_id'])
reason = filter_dicts([task_resource], ('resource_type', 'resource_id'))[0]
reason['operation'] = current_operation
if reason not in rejecting_reasons:
rejecting_reasons.append(reason)
if rejecting_tasks:
return dispatch_constants.CALL_REJECTED_RESPONSE, rejecting_tasks, rejecting_reasons, task_resources
if postponing_tasks:
return dispatch_constants.CALL_POSTPONED_RESPONSE, postponing_tasks, postponing_reasons, task_resources
return dispatch_constants.CALL_ACCEPTED_RESPONSE, set(), [], task_resources
def setUp(self):
super(CoordinatorTests, self).setUp()
self.coordinator = coordinator.Coordinator()
self._task_queue_factory = dispatch_factory._task_queue
dispatch_factory._task_queue = mock.Mock() # replace the task queue
self.collection = TaskResource.get_collection()
