def test_invalid_revert_before(self): task = setup_complete_task(self) task.status = Task.Status.ABORTED task.save() # Ensure reviewer assignment has more than one iteration reviewer_assignment = assignment_history(task).last() self.assertGreater(reviewer_assignment.iterations.count(), 1) # Attempt to revert before an iteration that isn't the assignment's # first with self.assertRaises(InvalidRevertError): self._revert_task( task, get_latest_iteration(reviewer_assignment), revert_before=True, commit=True) task.delete()
def _revert_assignment_from_audit(assignment_audit): assignment = TaskAssignment.objects.get( id=assignment_audit['assignment']['id']) if assignment_audit['change'] == RevertChange.DELETED.value: assignment.delete() elif assignment_audit['change'] == RevertChange.REVERTED.value: for iteration_audit in assignment_audit['iterations']: _revert_iteration_from_audit(iteration_audit) assignment.refresh_from_db() if get_latest_iteration( assignment).status == Iteration.Status.PROCESSING: assignment.status = TaskAssignment.Status.PROCESSING else: assignment.status = TaskAssignment.Status.SUBMITTED assignment.save() # TODO(jrbotros): Right now, we're leaving the latest assignment # `in_progress_task_data` alone since we would otherwise lose it; # when we implement archival rather than deletion, we should update # the latest assignment data to the submitted data of the iteration # reverted to. return assignment
def submit_task(task_id, task_data, iteration_status, worker): """ Returns a dict mapping task prerequisites onto their latest task assignment information. The dict is of the form: {'previous-slug': {task_assignment_data}, ...} Args: task_id (int): The ID of the task to submit. task_data (str): A JSON blob of task data to submit. iteration_status (orchestra.models.Iteration.Status): The action taken upon task submission (i.e., REQUESTED_REVIEW or PROVIDED_REVIEW). worker (orchestra.models.Worker): The worker submitting the task. Returns: task (orchestra.models.Task): The modified task object. Raises: orchestra.core.errors.IllegalTaskSubmission: Submission prerequisites for the task are incomplete or the assignment is in a non-processing state. orchestra.core.errors.TaskAssignmentError: Worker belongs to more than one assignment for the given task. orchestra.core.errors.TaskStatusError: Task has already been completed. """ submit_datetime = timezone.now() task = Task.objects.select_related('step', 'project').get(id=task_id) step = task.step if not _are_desired_steps_completed_on_project(step.submission_depends_on, project=task.project): raise IllegalTaskSubmission('Submission prerequisites are not ' 'complete.') if task.status == Task.Status.COMPLETE: raise TaskStatusError('Task already completed') # Use select_for_update to prevent concurrency issues with save_task. # See https://github.com/b12io/orchestra/issues/2. assignments = (TaskAssignment.objects.select_for_update().filter( worker=worker, task=task)) # Worker can belong to only one assignment for a given task. if not assignments.count() == 1: raise TaskAssignmentError( 'Task assignment with worker is in broken state.') assignment = assignments[0] if assignment.status != TaskAssignment.Status.PROCESSING: raise IllegalTaskSubmission('Worker is not allowed to submit') next_status = get_next_task_status(task, iteration_status) assignment.in_progress_task_data = task_data # Submit latest iteration latest_iteration = get_latest_iteration(assignment) latest_iteration.status = iteration_status latest_iteration.submitted_data = assignment.in_progress_task_data latest_iteration.end_datetime = submit_datetime latest_iteration.save() assignment.status = TaskAssignment.Status.SUBMITTED assignment.save() previous_status = task.status task.status = next_status task.save() if task.status == Task.Status.PENDING_REVIEW: # Check the assignment policy to try to assign a reviewer automatically task = _preassign_workers(task, AssignmentPolicyType.REVIEWER) elif task.status == Task.Status.REVIEWING: update_related_assignment_status(task, assignment.assignment_counter + 1, assignment.in_progress_task_data, submit_datetime) elif task.status == Task.Status.POST_REVIEW_PROCESSING: update_related_assignment_status(task, assignment.assignment_counter - 1, assignment.in_progress_task_data, submit_datetime) elif task.status == Task.Status.COMPLETE: create_subsequent_tasks(task.project) notify_status_change(task, previous_status) return task
def submit_task(task_id, task_data, iteration_status, worker): """ Returns a dict mapping task prerequisites onto their latest task assignment information. The dict is of the form: {'previous-slug': {task_assignment_data}, ...} Args: task_id (int): The ID of the task to submit. task_data (str): A JSON blob of task data to submit. iteration_status (orchestra.models.Iteration.Status): The action taken upon task submission (i.e., REQUESTED_REVIEW or PROVIDED_REVIEW). worker (orchestra.models.Worker): The worker submitting the task. Returns: task (orchestra.models.Task): The modified task object. Raises: orchestra.core.errors.IllegalTaskSubmission: Submission prerequisites for the task are incomplete or the assignment is in a non-processing state. orchestra.core.errors.TaskAssignmentError: Worker belongs to more than one assignment for the given task. orchestra.core.errors.TaskStatusError: Task has already been completed. """ submit_datetime = timezone.now() task = Task.objects.select_related('step', 'project').get(id=task_id) step = task.step if not _are_desired_steps_completed_on_project(step.submission_depends_on, project=task.project): raise IllegalTaskSubmission('Submission prerequisites are not ' 'complete.') if task.status == Task.Status.COMPLETE: raise TaskStatusError('Task already completed') # Use select_for_update to prevent concurrency issues with save_task. # See https://github.com/b12io/orchestra/issues/2. assignments = (TaskAssignment.objects.select_for_update() .filter(worker=worker, task=task)) # Worker can belong to only one assignment for a given task. if not assignments.count() == 1: raise TaskAssignmentError( 'Task assignment with worker is in broken state.') assignment = assignments[0] if assignment.status != TaskAssignment.Status.PROCESSING: raise IllegalTaskSubmission('Worker is not allowed to submit') next_status = get_next_task_status(task, iteration_status) assignment.in_progress_task_data = task_data # Submit latest iteration latest_iteration = get_latest_iteration(assignment) latest_iteration.status = iteration_status latest_iteration.submitted_data = assignment.in_progress_task_data latest_iteration.end_datetime = submit_datetime latest_iteration.save() assignment.status = TaskAssignment.Status.SUBMITTED assignment.save() previous_status = task.status task.status = next_status task.save() if task.status == Task.Status.PENDING_REVIEW: # Check the assignment policy to try to assign a reviewer automatically task = _preassign_workers(task, AssignmentPolicyType.REVIEWER) elif task.status == Task.Status.REVIEWING: update_related_assignment_status(task, assignment.assignment_counter + 1, assignment.in_progress_task_data, submit_datetime) elif task.status == Task.Status.POST_REVIEW_PROCESSING: update_related_assignment_status(task, assignment.assignment_counter - 1, assignment.in_progress_task_data, submit_datetime) elif task.status == Task.Status.COMPLETE: create_subsequent_tasks(task.project) notify_status_change(task, previous_status) return task
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. with transaction.atomic(): # Uniqueness constraint on assignnment_counter and task prevents # concurrent creation of more than one assignment task_assignment, created = TaskAssignment.objects.get_or_create( assignment_counter=0, task=task, defaults={ 'status': TaskAssignment.Status.PROCESSING, 'in_progress_task_data': {}}) if created: task.status = Task.Status.PROCESSING task.save() Iteration.objects.create( assignment=task_assignment, start_datetime=task_assignment.start_datetime) else: # Task assignment already exists if task_assignment.status == TaskAssignment.Status.FAILED: # Pick up failed task for reprocessing task_assignment.status = TaskAssignment.Status.PROCESSING task_assignment.save() else: # Task assignment already processing raise MachineExecutionError( 'Task already picked up by another machine') prerequisites = previously_completed_task_data(task) function = locate(step.execution_function['path']) kwargs = step.execution_function.get('kwargs', {}) try: project_data = project.project_data project_data['project_id'] = project_id task_data = function(project_data, prerequisites, **kwargs) except: task_assignment.status = TaskAssignment.Status.FAILED logger.exception('Machine task has failed') task_assignment.save() return task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() if task.project.status == Project.Status.ABORTED: # If a long-running task's project was aborted while running, we ensure # the aborted state on the task. task.status = Task.Status.ABORTED task.save() else: task.status = Task.Status.COMPLETE task.save() iteration = get_latest_iteration(task_assignment) iteration.status = Iteration.Status.REQUESTED_REVIEW iteration.submitted_data = task_data iteration.end_datetime = timezone.now() iteration.save() create_subsequent_tasks(project)
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. with transaction.atomic(): # Uniqueness constraint on assignnment_counter and task prevents # concurrent creation of more than one assignment task_assignment, created = TaskAssignment.objects.get_or_create( assignment_counter=0, task=task, defaults={ 'status': TaskAssignment.Status.PROCESSING, 'in_progress_task_data': {}}) if created: task.status = Task.Status.PROCESSING task.save() Iteration.objects.create( assignment=task_assignment, start_datetime=task_assignment.start_datetime) else: # Task assignment already exists if task_assignment.status == TaskAssignment.Status.FAILED: # Pick up failed task for reprocessing task_assignment.status = TaskAssignment.Status.PROCESSING task_assignment.save() else: # Task assignment already processing raise MachineExecutionError( 'Task already picked up by another machine') prerequisites = get_previously_completed_task_data(task.step, task.project) function = locate(step.execution_function['path']) kwargs = step.execution_function.get('kwargs', {}) try: project_data = project.project_data project_data['project_id'] = project_id task_data = function(project_data, prerequisites, **kwargs) except: task_assignment.status = TaskAssignment.Status.FAILED logger.exception('Machine task has failed') task_assignment.save() return task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() if task.project.status == Project.Status.ABORTED: # If a long-running task's project was aborted while running, we ensure # the aborted state on the task. task.status = Task.Status.ABORTED task.save() else: task.status = Task.Status.COMPLETE task.save() iteration = get_latest_iteration(task_assignment) iteration.status = Iteration.Status.REQUESTED_REVIEW iteration.submitted_data = task_data iteration.end_datetime = timezone.now() iteration.save() create_subsequent_tasks(project)