def get_task_transition_contexts(self, task_id, route): contexts = {} task_state_entry = self.get_task_state_entry(task_id, route) if not task_state_entry: raise exc.InvalidTaskStateEntry(task_id) for t in self.graph.get_next_transitions(task_id): task_transition_id = constants.TASK_STATE_TRANSITION_FORMAT % (t[1], str(t[2])) if (task_transition_id in task_state_entry['next'] and task_state_entry['next'][task_transition_id]): contexts[task_transition_id] = self.get_task_initial_context(t[1], route) return contexts
def update_task_state(self, task_id, route, event): engine_event_queue = queue.Queue() # Throw exception if not expected event type. if not issubclass(type(event), events.ExecutionEvent): raise TypeError('Event is not type of ExecutionEvent.') # Throw exception if task does not exist in the workflow graph. if not self.graph.has_task(task_id): raise exc.InvalidTask(task_id) # Try to get the task metadata from staging or task state. staged_task = self.workflow_state.get_staged_task(task_id, route) task_state_entry = self.get_task_state_entry(task_id, route) # Throw exception if task is not staged and there is no task state entry. if not staged_task and not task_state_entry: raise exc.InvalidTaskStateEntry(task_id) # Create new task state entry if it does not exist. if not task_state_entry: task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev']) # Identify the index for the task state object for later use. task_state_idx = self._get_task_state_idx(task_id, route) # If task is already completed and in cycle, then create new task state entry. if (self.graph.in_cycle(task_id) and task_state_entry.get('status') in statuses.COMPLETED_STATUSES): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev']) # Update the index value since a new entry is created. task_state_idx = self._get_task_state_idx(task_id, route) # Remove task from staging if task is not with items. if event.status and staged_task and 'items' not in staged_task: self.workflow_state.remove_staged_task(task_id, route) # If action execution is for a task item, then store the execution status for the item. if (staged_task and event.status and event.context and 'item_id' in event.context and event.context['item_id'] is not None): item_result = {'status': event.status, 'result': event.result} staged_task['items'][event.context['item_id']] = item_result # Log the error if it is a failed execution event. if event.status == statuses.FAILED: message = 'Execution failed. See result for details.' self.log_entry('error', message, task_id=task_id, result=event.result) # Process the action execution event using the # task state machine and update the task status. old_task_status = task_state_entry.get('status', statuses.UNSET) machines.TaskStateMachine.process_event(self.workflow_state, task_state_entry, event) new_task_status = task_state_entry.get('status', statuses.UNSET) # Get task result and set current context if task is completed. if new_task_status in statuses.COMPLETED_STATUSES: # Get task details required for updating outgoing context. task_spec = self.spec.tasks.get_task(task_id) # Get task result. task_result = ([ item.get('result') for item in staged_task.get('items', []) ] if staged_task and task_spec.has_items() else event.result) # Remove remaining task from staging. self.workflow_state.remove_staged_task(task_id, route) # Set current task in the context. in_ctx_idxs = task_state_entry['ctxs']['in'] in_ctx_val = self.get_task_context(in_ctx_idxs) current_task = { 'id': task_id, 'route': route, 'result': task_result } current_ctx = ctx_util.set_current_task(in_ctx_val, current_task) # Setup context for evaluating expressions in task transition criteria. state_ctx = {'__state': self.workflow_state.serialize()} current_ctx = dict_util.merge_dicts(current_ctx, state_ctx, True) # Evaluate task transitions if task is completed and status change is not processed. if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status: has_manual_fail = False staged_next_tasks = [] # Identify task transitions for the current completed task. task_transitions = self.graph.get_next_transitions(task_id) # Mark task as terminal when there is no transitions. if not task_transitions: task_state_entry['term'] = True # Iterate thru each outbound task transitions. for task_transition in task_transitions: task_transition_id = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_transition[1], str(task_transition[2]))) # Evaluate the criteria for task transition. If there is a failure while # evaluating expression(s), fail the workflow. try: criteria = task_transition[3].get('criteria') or [] evaluated_criteria = [ expr_base.evaluate(c, current_ctx) for c in criteria ] task_state_entry['next'][task_transition_id] = all( evaluated_criteria) except Exception as e: self.log_error(e, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue # If criteria met, then mark the next task staged and calculate outgoing context. if task_state_entry['next'][task_transition_id]: next_task_node = self.graph.get_task(task_transition[1]) next_task_id = next_task_node['id'] new_ctx_idx = None # Get and process new context for the task transition. out_ctx, new_ctx, errors = task_spec.finalize_context( next_task_id, task_transition, copy.deepcopy(current_ctx)) if errors: self.log_errors(errors, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue out_ctx_idxs = copy.deepcopy( task_state_entry['ctxs']['in']) if new_ctx: self.workflow_state.contexts.append(new_ctx) new_ctx_idx = len(self.workflow_state.contexts) - 1 # Add to the list of contexts for the next task in this transition. out_ctx_idxs.append(new_ctx_idx) # Record the outgoing context for this task transition. if 'out' not in task_state_entry['ctxs']: task_state_entry['ctxs']['out'] = {} task_state_entry['ctxs']['out'] = { task_transition_id: new_ctx_idx } # Stage the next task if it is not in staging. next_task_route = self._evaluate_route( task_transition, route) staged_next_task = self.workflow_state.get_staged_task( next_task_id, next_task_route) backref = (constants.TASK_STATE_TRANSITION_FORMAT % (task_id, str(task_transition[2]))) # If the next task is already staged. if staged_next_task: # Remove the root context to avoid overwriting vars. out_ctx_idxs.remove(0) # Extend the outgoing context from this task. staged_next_task['ctxs']['in'].extend(out_ctx_idxs) # Add a backref for the current task in the next task. staged_next_task['prev'][backref] = task_state_idx else: # Otherwise create a new entry in staging for the next task. staged_next_task = self.workflow_state.add_staged_task( next_task_id, next_task_route, ctxs=out_ctx_idxs, prev={backref: task_state_idx}, ready=False) # Check if inbound criteria are met. Must use the original route # to identify the inbound task transitions. staged_next_task[ 'ready'] = self._inbound_criteria_satisfied( next_task_id, route) # Put the next task in the engine event queue if it is an engine command. if next_task_id in events.ENGINE_EVENT_MAP.keys(): queue_entry = (staged_next_task['id'], staged_next_task['route']) engine_event_queue.put(queue_entry) # Flag if there is at least one fail command in the task transition. if not has_manual_fail: has_manual_fail = (next_task_id == 'fail') else: # If not an engine command and the next task is ready, then # make a record of it for processing manual fail below. if staged_next_task['ready']: staged_next_tasks.append(staged_next_task) # Task failure is remediable. For example, there may be workflow that wants # to run a cleanup task on failure. In certain cases, we still want to fail # the workflow after the remediation. The fail command can be in the # task transition under the cleanup task. However, the cleanup task may be # reusable when either workflow succeed or fail. It does not make sense to # put the fail command in the task transition of the cleanup task. So we # want to allow user to be able to define both the cleanup and fail in the # task transition under the current task but still return the cleanup task # even when the workflow is set to failed status. if has_manual_fail: for staged_next_task in staged_next_tasks: staged_next_task['run_on_fail'] = True # Process the task event using the workflow state machine and update the workflow status. task_ex_event = events.TaskExecutionEvent(task_id, route, task_state_entry['status']) machines.WorkflowStateMachine.process_event(self.workflow_state, task_ex_event) # Process any engine commands in the queue. while not engine_event_queue.empty(): next_task_id, next_task_route = engine_event_queue.get() engine_event = events.ENGINE_EVENT_MAP[next_task_id] self.update_task_state(next_task_id, next_task_route, engine_event()) # Render workflow output if workflow is completed. if self.get_workflow_status() in statuses.COMPLETED_STATUSES: task_state_entry['term'] = True self._render_workflow_outputs() return task_state_entry
def update_task_state(self, task_id, route, event): engine_event_queue = queue.Queue() # Throw exception if not expected event type. if not issubclass(type(event), events.ExecutionEvent): raise TypeError('Event is not type of ExecutionEvent.') # Throw exception if task does not exist in the workflow graph. if not self.graph.has_task(task_id): raise exc.InvalidTask(task_id) # Try to get the task metadata from staging or task state. staged_task = self.workflow_state.get_staged_task(task_id, route) task_state_entry = self.get_task_state_entry(task_id, route) # Get the task spec for the task which contains additional meta data. task_spec = self.spec.tasks.get_task(task_id) # Throw exception if task is not staged and there is no task state entry. if not staged_task and not task_state_entry: raise exc.InvalidTaskStateEntry(task_id) # Create new task state entry if it does not exist or if it is an engine command. if not task_state_entry or task_id in events.ENGINE_EVENT_MAP.keys(): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev'] ) # Identify the index for the task state object for later use. task_state_idx = self._get_task_state_idx(task_id, route) # If task is already completed and in cycle, then create new task state entry. # Unfortunately, the method in the graph to check for cycle is too simple and # misses forks that extends from the cycle. The check here assumes that the # last task entry is already completed and the new task status is one of the # starting statuses, then there is high likelihood that this is a cycle. if (task_state_entry.get('status') in statuses.COMPLETED_STATUSES and event.status and event.status in statuses.STARTING_STATUSES): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev'] ) # Update the index value since a new entry is created. task_state_idx = self._get_task_state_idx(task_id, route) # Remove task from staging if task is not with items. if event.status and staged_task and 'items' not in staged_task: self.workflow_state.remove_staged_task(task_id, route) # If action execution is for a task item, then record the execution status for the item. # Result for each item is not recorded in the staged_task because it impacts database # write performance if there are a lot of items and/or item result size is huge. if staged_task and isinstance(event, events.TaskItemActionExecutionEvent): staged_task['items'][event.item_id] = {'status': event.status} # Log the error if it is a failed execution event. if event.status == statuses.FAILED: message = 'Execution failed. See result for details.' self.log_entry('error', message, task_id=task_id, result=event.result) # Process the action execution event using the # task state machine and update the task status. old_task_status = task_state_entry.get('status', statuses.UNSET) machines.TaskStateMachine.process_event(self.workflow_state, task_state_entry, event) new_task_status = task_state_entry.get('status', statuses.UNSET) # If retrying, staged the task to be returned in get_next_tasks. if new_task_status == statuses.RETRYING: # Increment the number of times that the task has retried. task_state_entry['retry']['tally'] += 1 # Reset the staged task to be returned in get_next_tasks self.workflow_state.remove_staged_task(task_id, route) self.workflow_state.add_staged_task( task_id, route, ctxs=task_state_entry['ctxs']['in'], prev=task_state_entry['prev'], retry=task_state_entry['retry'], ready=True ) # Get task result and set current context if task is completed. if new_task_status in statuses.COMPLETED_STATUSES: # Remove task from staging if exists but keep entry # if task has items and failed for manual rerun. if not (task_spec.has_items() and new_task_status in statuses.ABENDED_STATUSES): self.workflow_state.remove_staged_task(task_id, route) # Format task result depending on the type of task. task_result = self.make_task_result(task_spec, event) # Set current task in the context. current_ctx = self.make_task_context(task_state_entry, task_result=task_result) # Evaluate if there is a task retry. The task retry can only be evaluated after # the state machine has determined the status for the task execution. If the task # is completed, get the task result and context which is required to evaluate the # the condition if a retry for the task is required. if (self.get_workflow_status() in statuses.ACTIVE_STATUSES and self._evaluate_task_retry(task_state_entry, current_ctx)): self.update_task_state(task_id, route, events.TaskRetryEvent()) # Evaluate task transitions if task is completed and status change is not processed. if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status: has_manual_fail = False staged_next_tasks = [] # Identify task transitions for the current completed task. task_transitions = self.graph.get_next_transitions(task_id) # Mark task as terminal when there is no transitions. if not task_transitions: task_state_entry['term'] = True # Iterate thru each outbound task transitions. for task_transition in task_transitions: task_transition_id = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_transition[1], str(task_transition[2])) ) # Evaluate the criteria for task transition. If there is a failure while # evaluating expression(s), fail the workflow. try: criteria = task_transition[3].get('criteria') or [] evaluated_criteria = [expr_base.evaluate(c, current_ctx) for c in criteria] task_state_entry['next'][task_transition_id] = all(evaluated_criteria) except Exception as e: self.log_error(e, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue # If criteria met, then mark the next task staged and calculate outgoing context. if task_state_entry['next'][task_transition_id]: next_task_node = self.graph.get_task(task_transition[1]) next_task_id = next_task_node['id'] new_ctx_idx = None # Get and process new context for the task transition. out_ctx, new_ctx, errors = task_spec.finalize_context( next_task_id, task_transition, json_util.deepcopy(current_ctx) ) if errors: self.log_errors(errors, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue out_ctx_idxs = json_util.deepcopy(task_state_entry['ctxs']['in']) if new_ctx: self.workflow_state.contexts.append(new_ctx) new_ctx_idx = len(self.workflow_state.contexts) - 1 # Add to the list of contexts for the next task in this transition. out_ctx_idxs.append(new_ctx_idx) # Record the outgoing context for this task transition. if 'out' not in task_state_entry['ctxs']: task_state_entry['ctxs']['out'] = {} task_state_entry['ctxs']['out'] = {task_transition_id: new_ctx_idx} # Stage the next task if it is not in staging. next_task_route = self._evaluate_route(task_transition, route) staged_next_task = self.workflow_state.get_staged_task( next_task_id, next_task_route ) backref = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_id, str(task_transition[2])) ) # If the next task is already staged. if staged_next_task: # Remove the root context to avoid overwriting vars. out_ctx_idxs.remove(0) # Extend the outgoing context from this task. staged_next_task['ctxs']['in'].extend(out_ctx_idxs) # Add a backref for the current task in the next task. staged_next_task['prev'][backref] = task_state_idx # Clear list of items for with items task. staged_next_task.pop('items', None) else: # Otherwise create a new entry in staging for the next task. staged_next_task = self.workflow_state.add_staged_task( next_task_id, next_task_route, ctxs=out_ctx_idxs, prev={backref: task_state_idx}, ready=False ) # Check if inbound criteria are met. Must use the original route # to identify the inbound task transitions. staged_next_task['ready'] = self._inbound_criteria_satisfied( next_task_id, route ) # Put the next task in the engine event queue if it is an engine command. if next_task_id in events.ENGINE_EVENT_MAP.keys(): queue_entry = (staged_next_task['id'], staged_next_task['route']) engine_event_queue.put(queue_entry) # Flag if there is at least one fail command in the task transition. if not has_manual_fail: has_manual_fail = (next_task_id == 'fail') else: # If not an engine command and the next task is ready, then # make a record of it for processing manual fail below. if staged_next_task['ready']: staged_next_tasks.append(staged_next_task) # Task failure is remediable. For example, there may be workflow that wants # to run a cleanup task on failure. In certain cases, we still want to fail # the workflow after the remediation. The fail command can be in the # task transition under the cleanup task. However, the cleanup task may be # reusable when either workflow succeed or fail. It does not make sense to # put the fail command in the task transition of the cleanup task. So we # want to allow user to be able to define both the cleanup and fail in the # task transition under the current task but still return the cleanup task # even when the workflow is set to failed status. if has_manual_fail: for staged_next_task in staged_next_tasks: staged_next_task['run_on_fail'] = True # Process the task event using the workflow state machine and update the workflow status. task_ex_event = events.TaskExecutionEvent(task_id, route, task_state_entry['status']) machines.WorkflowStateMachine.process_event(self.workflow_state, task_ex_event) # Process any engine commands in the queue. while not engine_event_queue.empty(): next_task_id, next_task_route = engine_event_queue.get() engine_event = events.ENGINE_EVENT_MAP[next_task_id] self.update_task_state(next_task_id, next_task_route, engine_event()) # Mark the task as a terminal task if workflow execution is completed. if self.get_workflow_status() in statuses.COMPLETED_STATUSES: task_state_entry['term'] = True return task_state_entry