def coordinator_dequeue_callback(call_request, call_report): """ Callback to be executed upon call completion that will clean up the coordinator's accounting data pertaining to the call. @param call_request: call request for the call @type call_request: L{call.CallRequest} instance @param call_report: call report for the call @type call_report: L{call.CallReport} instance """ # yes, I know that the call_request is not being used task_id = call_report.task_id collection = TaskResource.get_collection() collection.remove({'task_id': task_id}, safe=True)
def _run_task(self, task, synchronous=None, timeout=None): """ Run a task. @param task: task to run @type task: L{Task} instance @param synchronous: whether or not to run the task synchronously, None means dependent on what the conflict response is @type synchronous: None or bool @param timeout: how much time to wait for a synchronous task to start None means indefinitely @type timeout: None or datetime.timedelta """ # we have to lock the task queue here as there is a race condition # between calculating the blocking/postponing tasks and enqueueing the # task when 2 or more tasks are being run that may have # interdependencies task_queue = dispatch_factory._task_queue() task_queue.lock() task_resource_collection = TaskResource.get_collection() try: response, blocking, reasons, task_resources = self._find_conflicts(task.call_request.resources) task.call_report.response = response task.call_report.reasons = reasons if response is dispatch_constants.CALL_REJECTED_RESPONSE: return task.blocking_tasks.update(blocking) task.call_request.add_life_cycle_callback(dispatch_constants.CALL_ENQUEUE_LIFE_CYCLE_CALLBACK, GrantPermmissionsForTaskV2()) task.call_request.add_life_cycle_callback(dispatch_constants.CALL_DEQUEUE_LIFE_CYCLE_CALLBACK, RevokePermissionsForTaskV2()) task.call_request.add_life_cycle_callback(dispatch_constants.CALL_DEQUEUE_LIFE_CYCLE_CALLBACK, coordinator_dequeue_callback) if task_resources: set_task_id_on_task_resources(task.id, task_resources) task_resource_collection.insert(task_resources, safe=True) task_queue.enqueue(task) finally: task_queue.unlock() # if the call has requested synchronous execution or can be # synchronously executed, do so if synchronous or (synchronous is None and response is dispatch_constants.CALL_ACCEPTED_RESPONSE): try: # it's perfectly legitimate for the call to complete before the fist poll running_states = [dispatch_constants.CALL_RUNNING_STATE] running_states.extend(dispatch_constants.CALL_COMPLETE_STATES) wait_for_task(task, running_states, poll_interval=self.task_state_poll_interval, timeout=timeout) except OperationTimedOut: task_queue.dequeue(task) raise else: wait_for_task(task, dispatch_constants.CALL_COMPLETE_STATES, poll_interval=self.task_state_poll_interval)
def start(self): """ Start the coordinator by clearing conflict metadata and restarting any interrupted tasks. """ # drop all previous knowledge of running tasks task_resource_collection = TaskResource.get_collection() task_resource_collection.remove(safe=True) # re-start interrupted tasks queued_call_collection = QueuedCall.get_collection() queued_call_list = list(queued_call_collection.find().sort('timestamp')) queued_call_collection.remove(safe=True) for queued_call in queued_call_list: call_request = CallRequest.deserialize(queued_call['serialized_call_request']) call_report = self.execute_call_asynchronously(call_request)
def _find_conflicts(self, resources): """ Find conflicting tasks, if any, and provide the following: * a task response, (accepted, postponed, rejected) * a (possibly empty) set of blocking task ids * a list of blocking "reasons" in the form of TaskResource instances * a list of task resources corresponding to the given resources @param resources: dictionary of resources and their proposed operations @type resources: dict @return: tuple of objects described above @rtype: tuple """ if not resources: return dispatch_constants.CALL_ACCEPTED_RESPONSE, set(), [], [] postponing_tasks = set() postponing_reasons = [] rejecting_tasks = set() rejecting_reasons = [] task_resource_collection = TaskResource.get_collection() task_resources = resource_dict_to_task_resources(resources) or_query = filter_dicts(task_resources, ('resource_type', 'resource_id')) cursor = task_resource_collection.find({'$or': or_query}) for task_resource in cursor: proposed_operation = resources[task_resource['resource_type']][task_resource['resource_id']] postponing_operations = get_postponing_operations(proposed_operation) rejecting_operations = get_rejecting_operations(proposed_operation) current_operation = task_resource['operation'] if current_operation in postponing_operations: postponing_tasks.add(task_resource['task_id']) reason = filter_dicts([task_resource], ('resource_type', 'resource_id'))[0] reason['operation'] = current_operation if reason not in postponing_reasons: postponing_reasons.append(reason) if current_operation in rejecting_operations: rejecting_tasks.add(task_resource['task_id']) reason = filter_dicts([task_resource], ('resource_type', 'resource_id'))[0] reason['operation'] = current_operation if reason not in rejecting_reasons: rejecting_reasons.append(reason) if rejecting_tasks: return dispatch_constants.CALL_REJECTED_RESPONSE, rejecting_tasks, rejecting_reasons, task_resources if postponing_tasks: return dispatch_constants.CALL_POSTPONED_RESPONSE, postponing_tasks, postponing_reasons, task_resources return dispatch_constants.CALL_ACCEPTED_RESPONSE, set(), [], task_resources
def setUp(self): super(CoordinatorTests, self).setUp() self.coordinator = coordinator.Coordinator() self._task_queue_factory = dispatch_factory._task_queue dispatch_factory._task_queue = mock.Mock() # replace the task queue self.collection = TaskResource.get_collection()