def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. count = task.assignments.count() if count > 1: raise MachineExecutionError('At most 1 assignment per machine task') elif count == 1: task_assignment = task.assignments.first() if task_assignment.status == TaskAssignment.Status.SUBMITTED: raise MachineExecutionError('Task assignment completed ' 'but task is not!') else: task_assignment = ( TaskAssignment.objects .create(task=task, status=TaskAssignment.Status.PROCESSING, in_progress_task_data={}, snapshots={})) prerequisites = previously_completed_task_data(task) function_module = import_module(step.execution_function['module']) function = getattr(function_module, step.execution_function['name']) task_data = function(project.project_data, prerequisites) task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() task.status = Task.Status.COMPLETE task.save() create_subsequent_tasks(project)
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) workflow = get_workflow_by_slug(project.workflow_slug) step = workflow.get_step(step_slug) task = Task.objects.get(project=project, step_slug=step_slug) # Run machine function if step.worker_type != Step.WorkerType.MACHINE: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. count = task.assignments.count() if count > 1: raise MachineExecutionError('At most 1 assignment per machine task') elif count == 1: task_assignment = task.assignments.first() if task_assignment.status == TaskAssignment.Status.SUBMITTED: raise MachineExecutionError('Task assignment completed ' 'but task is not!') else: task_assignment = ( TaskAssignment.objects .create(task=task, status=TaskAssignment.Status.PROCESSING, in_progress_task_data={}, snapshots={})) prerequisites = previously_completed_task_data(task) task_data = step.function(project.project_data, prerequisites) task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() task.status = Task.Status.COMPLETE task.save() create_subsequent_tasks(project)
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError("Step worker type is not machine") if task.status == Task.Status.COMPLETE: raise MachineExecutionError("Task assignment already completed") # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. count = task.assignments.count() if count > 1: raise MachineExecutionError("At most 1 assignment per machine task") elif count == 1: task_assignment = task.assignments.first() if task_assignment.status == TaskAssignment.Status.SUBMITTED: raise MachineExecutionError("Task assignment completed " "but task is not!") else: task_assignment = TaskAssignment.objects.create( task=task, status=TaskAssignment.Status.PROCESSING, in_progress_task_data={}, snapshots={} ) prerequisites = previously_completed_task_data(task) function_module = import_module(step.execution_function["module"]) function = getattr(function_module, step.execution_function["name"]) task_data = function(project.project_data, prerequisites) task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() task.status = Task.Status.COMPLETE task.save() create_subsequent_tasks(project)
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. with transaction.atomic(): # Uniqueness constraint on assignnment_counter and task prevents # concurrent creation of more than one assignment task_assignment, created = TaskAssignment.objects.get_or_create( assignment_counter=0, task=task, defaults={ 'status': TaskAssignment.Status.PROCESSING, 'in_progress_task_data': {}}) if created: task.status = Task.Status.PROCESSING task.save() Iteration.objects.create( assignment=task_assignment, start_datetime=task_assignment.start_datetime) else: # Task assignment already exists if task_assignment.status == TaskAssignment.Status.FAILED: # Pick up failed task for reprocessing task_assignment.status = TaskAssignment.Status.PROCESSING task_assignment.save() else: # Task assignment already processing raise MachineExecutionError( 'Task already picked up by another machine') prerequisites = previously_completed_task_data(task) function = locate(step.execution_function['path']) kwargs = step.execution_function.get('kwargs', {}) try: project_data = project.project_data project_data['project_id'] = project_id task_data = function(project_data, prerequisites, **kwargs) except: task_assignment.status = TaskAssignment.Status.FAILED logger.exception('Machine task has failed') task_assignment.save() return task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() if task.project.status == Project.Status.ABORTED: # If a long-running task's project was aborted while running, we ensure # the aborted state on the task. task.status = Task.Status.ABORTED task.save() else: task.status = Task.Status.COMPLETE task.save() iteration = get_latest_iteration(task_assignment) iteration.status = Iteration.Status.REQUESTED_REVIEW iteration.submitted_data = task_data iteration.end_datetime = timezone.now() iteration.save() create_subsequent_tasks(project)
def execute(project_id, step_slug): project = Project.objects.get(id=project_id) step = Step.objects.get(slug=step_slug, workflow_version=project.workflow_version) task = Task.objects.get(project=project, step=step) # Run machine function if step.is_human: raise MachineExecutionError('Step worker type is not machine') if task.status == Task.Status.COMPLETE: raise MachineExecutionError('Task assignment already completed') # Machine tasks are only assigned to one worker/machine, # so they should only have one task assignment, # and should never be submitted for review. with transaction.atomic(): # Uniqueness constraint on assignnment_counter and task prevents # concurrent creation of more than one assignment task_assignment, created = TaskAssignment.objects.get_or_create( assignment_counter=0, task=task, defaults={ 'status': TaskAssignment.Status.PROCESSING, 'in_progress_task_data': {} }) if created: task.status = Task.Status.PROCESSING task.save() Iteration.objects.create( assignment=task_assignment, start_datetime=task_assignment.start_datetime) else: # Task assignment already exists if task_assignment.status == TaskAssignment.Status.FAILED: # Pick up failed task for reprocessing task_assignment.status = TaskAssignment.Status.PROCESSING task_assignment.save() else: # Task assignment already processing raise MachineExecutionError( 'Task already picked up by another machine') prerequisites = previously_completed_task_data(task) function = locate(step.execution_function['path']) kwargs = step.execution_function.get('kwargs', {}) try: project_data = project.project_data project_data['project_id'] = project_id task_data = function(project_data, prerequisites, **kwargs) except: task_assignment.status = TaskAssignment.Status.FAILED logger.exception('Machine task has failed') task_assignment.save() return task_assignment.status = TaskAssignment.Status.SUBMITTED task_assignment.in_progress_task_data = task_data task_assignment.save() if task.project.status == Project.Status.ABORTED: # If a long-running task's project was aborted while running, we ensure # the aborted state on the task. task.status = Task.Status.ABORTED task.save() else: task.status = Task.Status.COMPLETE task.save() iteration = get_latest_iteration(task_assignment) iteration.status = Iteration.Status.REQUESTED_REVIEW iteration.submitted_data = task_data iteration.end_datetime = timezone.now() iteration.save() create_subsequent_tasks(project)