def test_workflow_state_transition(self): conductor = self._prep_conductor(states.RUNNING) tk_ex_event = events.TaskExecutionEvent('task1', states.RUNNING) machines.WorkflowStateMachine.process_event(conductor, tk_ex_event) self.assertEqual(conductor.get_workflow_state(), states.RUNNING) tk_ex_event = events.TaskExecutionEvent('task1', states.PAUSED) machines.WorkflowStateMachine.process_event(conductor, tk_ex_event) self.assertEqual(conductor.get_workflow_state(), states.PAUSED)
def test_bad_current_workflow_state_to_event_mapping(self): conductor = self._prep_conductor(states.REQUESTED) tk_ex_event = events.TaskExecutionEvent('task1', states.RUNNING) # If transition is not supported, then workflow state will not change. machines.WorkflowStateMachine.process_event(conductor, tk_ex_event) self.assertEqual(conductor.get_workflow_state(), states.REQUESTED)
def test_bad_current_workflow_status(self): conductor = self._prep_conductor() conductor.workflow_state.status = statuses.ABANDONED tk_ex_event = events.TaskExecutionEvent('task1', 0, statuses.RUNNING) self.assertRaises(exc.InvalidWorkflowStatusTransition, machines.WorkflowStateMachine.process_event, conductor.workflow_state, tk_ex_event)
def test_bad_event_name(self): conductor = self._prep_conductor(statuses.RUNNING) tk_ex_event = events.TaskExecutionEvent('task1', 0, statuses.RUNNING) setattr(tk_ex_event, 'name', 'foobar') self.assertRaises(exc.InvalidEvent, machines.WorkflowStateMachine.process_event, conductor.workflow_state, tk_ex_event)
def update_task_flow(self, task_id, event): in_ctx_idx = 0 engine_event_queue = queue.Queue() # Throw exception if not expected event type. if not issubclass(type(event), events.ExecutionEvent): raise TypeError('Event is not type of ExecutionEvent.') # Throw exception if task does not exist in the workflow graph. if not self.graph.has_task(task_id): raise exc.InvalidTask(task_id) # Try to get the task flow entry. task_flow_entry = self.get_task_flow_entry(task_id) # Throw exception if task is not staged and there is no task flow entry. if task_id not in self.flow.staged and not task_flow_entry: raise exc.InvalidTaskFlowEntry(task_id) # Get the incoming context from the staged task. if task_id in self.flow.staged: in_ctx_idxs = self.flow.staged[task_id]['ctxs'] if len(in_ctx_idxs) <= 0 or all(x == in_ctx_idxs[0] for x in in_ctx_idxs): in_ctx_idx = in_ctx_idxs[0] else: new_ctx_entry = self._converge_task_contexts(in_ctx_idxs) self.flow.contexts.append(new_ctx_entry) in_ctx_idx = len(self.flow.contexts) - 1 # Create new task flow entry if it does not exist. if not task_flow_entry: task_flow_entry = self.add_task_flow(task_id, in_ctx_idx=in_ctx_idx) # If task is already completed and in cycle, then create new task flow entry. if self.graph.in_cycle(task_id) and task_flow_entry.get('state') in states.COMPLETED_STATES: task_flow_entry = self.add_task_flow(task_id, in_ctx_idx=in_ctx_idx) # Remove task from staging if task is not with items. if event.state and task_id in self.flow.staged and 'items' not in self.flow.staged[task_id]: del self.flow.staged[task_id] # If action execution is for a task item, then store the execution state for the item. if (event.state and event.context and 'item_id' in event.context and event.context['item_id'] is not None): item_result = {'state': event.state, 'result': event.result} self.flow.staged[task_id]['items'][event.context['item_id']] = item_result # Log the error if it is a failed execution event. if event.state == states.FAILED: message = 'Execution failed. See result for details.' self.log_entry('error', message, task_id=task_id, result=event.result) # Process the action execution event using the task state machine and update the task state. old_task_state = task_flow_entry.get('state', states.UNSET) machines.TaskStateMachine.process_event(self, task_flow_entry, event) new_task_state = task_flow_entry.get('state', states.UNSET) # Get task result and set current context if task is completed. if new_task_state in states.COMPLETED_STATES: # Get task details required for updating outgoing context. task_node = self.graph.get_task(task_id) task_name = task_node['name'] task_spec = self.spec.tasks.get_task(task_name) task_flow_idx = self._get_task_flow_idx(task_id) # Get task result. task_result = ( [item.get('result') for item in self.flow.staged[task_id]['items']] if task_spec.has_items() else event.result ) # Remove remaining task from staging. self.flow.remove_staged_task(task_id) # Set current task in the context. in_ctx_idx = task_flow_entry['ctx'] in_ctx_val = self.flow.contexts[in_ctx_idx]['value'] current_task = {'id': task_id, 'name': task_name, 'result': task_result} current_ctx = ctx.set_current_task(in_ctx_val, current_task) # Setup context for evaluating expressions in task transition criteria. flow_ctx = {'__flow': self.flow.serialize()} current_ctx = dx.merge_dicts(current_ctx, flow_ctx, True) # Evaluate task transitions if task is completed and state change is not processed. if new_task_state in states.COMPLETED_STATES and new_task_state != old_task_state: # Identify task transitions for the current completed task. task_transitions = self.graph.get_next_transitions(task_id) # Update workflow context when there is no transitions. if not task_transitions: self._update_workflow_terminal_context(in_ctx_val, task_flow_idx) # Iterate thru each outbound task transitions. for task_transition in task_transitions: task_transition_id = task_transition[1] + '__' + str(task_transition[2]) # Evaluate the criteria for task transition. If there is a failure while # evaluating expression(s), fail the workflow. try: criteria = task_transition[3].get('criteria') or [] evaluated_criteria = [expr.evaluate(c, current_ctx) for c in criteria] task_flow_entry[task_transition_id] = all(evaluated_criteria) except Exception as e: self.log_error(e, task_id, task_transition_id) self.request_workflow_state(states.FAILED) continue # If criteria met, then mark the next task staged and calculate outgoing context. if task_flow_entry[task_transition_id]: next_task_node = self.graph.get_task(task_transition[1]) next_task_name = next_task_node['name'] next_task_id = next_task_node['id'] out_ctx_val, errors = task_spec.finalize_context( next_task_name, task_transition, copy.deepcopy(current_ctx) ) if errors: self.log_errors(errors, task_id, task_transition_id) self.request_workflow_state(states.FAILED) continue if out_ctx_val != in_ctx_val: task_flow_idx = self._get_task_flow_idx(task_id) self.flow.contexts.append({'srcs': [task_flow_idx], 'value': out_ctx_val}) out_ctx_idx = len(self.flow.contexts) - 1 else: out_ctx_idx = in_ctx_idx # Check if inbound criteria are met. ready = self._inbound_criteria_satisfied(task_transition[1]) if (task_transition[1] in self.flow.staged and 'ctxs' in self.flow.staged[task_transition[1]]): self.flow.staged[task_transition[1]]['ctxs'].append(out_ctx_idx) self.flow.staged[task_transition[1]]['ready'] = ready else: staging_data = {'ctxs': [out_ctx_idx], 'ready': ready} self.flow.staged[task_transition[1]] = staging_data # If the next task is noop, then mark the task as completed. if next_task_name in events.ENGINE_EVENT_MAP.keys(): engine_event_queue.put((next_task_id, next_task_name)) # Process the task event using the workflow state machine and update the workflow state. task_ex_event = events.TaskExecutionEvent(task_id, task_flow_entry['state']) machines.WorkflowStateMachine.process_event(self, task_ex_event) # Process any engine commands in the queue. while not engine_event_queue.empty(): next_task_id, next_task_name = engine_event_queue.get() engine_event = events.ENGINE_EVENT_MAP[next_task_name] self.update_task_flow(next_task_id, engine_event()) # Render workflow output if workflow is completed. if self.get_workflow_state() in states.COMPLETED_STATES: in_ctx_idx = task_flow_entry['ctx'] in_ctx_val = self.flow.contexts[in_ctx_idx]['value'] task_flow_idx = self._get_task_flow_idx(task_id) self._update_workflow_terminal_context(in_ctx_val, task_flow_idx) self._render_workflow_outputs() return task_flow_entry
def update_task_state(self, task_id, route, event): engine_event_queue = queue.Queue() # Throw exception if not expected event type. if not issubclass(type(event), events.ExecutionEvent): raise TypeError('Event is not type of ExecutionEvent.') # Throw exception if task does not exist in the workflow graph. if not self.graph.has_task(task_id): raise exc.InvalidTask(task_id) # Try to get the task metadata from staging or task state. staged_task = self.workflow_state.get_staged_task(task_id, route) task_state_entry = self.get_task_state_entry(task_id, route) # Throw exception if task is not staged and there is no task state entry. if not staged_task and not task_state_entry: raise exc.InvalidTaskStateEntry(task_id) # Create new task state entry if it does not exist. if not task_state_entry: task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev']) # Identify the index for the task state object for later use. task_state_idx = self._get_task_state_idx(task_id, route) # If task is already completed and in cycle, then create new task state entry. if (self.graph.in_cycle(task_id) and task_state_entry.get('status') in statuses.COMPLETED_STATUSES): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev']) # Update the index value since a new entry is created. task_state_idx = self._get_task_state_idx(task_id, route) # Remove task from staging if task is not with items. if event.status and staged_task and 'items' not in staged_task: self.workflow_state.remove_staged_task(task_id, route) # If action execution is for a task item, then store the execution status for the item. if (staged_task and event.status and event.context and 'item_id' in event.context and event.context['item_id'] is not None): item_result = {'status': event.status, 'result': event.result} staged_task['items'][event.context['item_id']] = item_result # Log the error if it is a failed execution event. if event.status == statuses.FAILED: message = 'Execution failed. See result for details.' self.log_entry('error', message, task_id=task_id, result=event.result) # Process the action execution event using the # task state machine and update the task status. old_task_status = task_state_entry.get('status', statuses.UNSET) machines.TaskStateMachine.process_event(self.workflow_state, task_state_entry, event) new_task_status = task_state_entry.get('status', statuses.UNSET) # Get task result and set current context if task is completed. if new_task_status in statuses.COMPLETED_STATUSES: # Get task details required for updating outgoing context. task_spec = self.spec.tasks.get_task(task_id) # Get task result. task_result = ([ item.get('result') for item in staged_task.get('items', []) ] if staged_task and task_spec.has_items() else event.result) # Remove remaining task from staging. self.workflow_state.remove_staged_task(task_id, route) # Set current task in the context. in_ctx_idxs = task_state_entry['ctxs']['in'] in_ctx_val = self.get_task_context(in_ctx_idxs) current_task = { 'id': task_id, 'route': route, 'result': task_result } current_ctx = ctx_util.set_current_task(in_ctx_val, current_task) # Setup context for evaluating expressions in task transition criteria. state_ctx = {'__state': self.workflow_state.serialize()} current_ctx = dict_util.merge_dicts(current_ctx, state_ctx, True) # Evaluate task transitions if task is completed and status change is not processed. if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status: has_manual_fail = False staged_next_tasks = [] # Identify task transitions for the current completed task. task_transitions = self.graph.get_next_transitions(task_id) # Mark task as terminal when there is no transitions. if not task_transitions: task_state_entry['term'] = True # Iterate thru each outbound task transitions. for task_transition in task_transitions: task_transition_id = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_transition[1], str(task_transition[2]))) # Evaluate the criteria for task transition. If there is a failure while # evaluating expression(s), fail the workflow. try: criteria = task_transition[3].get('criteria') or [] evaluated_criteria = [ expr_base.evaluate(c, current_ctx) for c in criteria ] task_state_entry['next'][task_transition_id] = all( evaluated_criteria) except Exception as e: self.log_error(e, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue # If criteria met, then mark the next task staged and calculate outgoing context. if task_state_entry['next'][task_transition_id]: next_task_node = self.graph.get_task(task_transition[1]) next_task_id = next_task_node['id'] new_ctx_idx = None # Get and process new context for the task transition. out_ctx, new_ctx, errors = task_spec.finalize_context( next_task_id, task_transition, copy.deepcopy(current_ctx)) if errors: self.log_errors(errors, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue out_ctx_idxs = copy.deepcopy( task_state_entry['ctxs']['in']) if new_ctx: self.workflow_state.contexts.append(new_ctx) new_ctx_idx = len(self.workflow_state.contexts) - 1 # Add to the list of contexts for the next task in this transition. out_ctx_idxs.append(new_ctx_idx) # Record the outgoing context for this task transition. if 'out' not in task_state_entry['ctxs']: task_state_entry['ctxs']['out'] = {} task_state_entry['ctxs']['out'] = { task_transition_id: new_ctx_idx } # Stage the next task if it is not in staging. next_task_route = self._evaluate_route( task_transition, route) staged_next_task = self.workflow_state.get_staged_task( next_task_id, next_task_route) backref = (constants.TASK_STATE_TRANSITION_FORMAT % (task_id, str(task_transition[2]))) # If the next task is already staged. if staged_next_task: # Remove the root context to avoid overwriting vars. out_ctx_idxs.remove(0) # Extend the outgoing context from this task. staged_next_task['ctxs']['in'].extend(out_ctx_idxs) # Add a backref for the current task in the next task. staged_next_task['prev'][backref] = task_state_idx else: # Otherwise create a new entry in staging for the next task. staged_next_task = self.workflow_state.add_staged_task( next_task_id, next_task_route, ctxs=out_ctx_idxs, prev={backref: task_state_idx}, ready=False) # Check if inbound criteria are met. Must use the original route # to identify the inbound task transitions. staged_next_task[ 'ready'] = self._inbound_criteria_satisfied( next_task_id, route) # Put the next task in the engine event queue if it is an engine command. if next_task_id in events.ENGINE_EVENT_MAP.keys(): queue_entry = (staged_next_task['id'], staged_next_task['route']) engine_event_queue.put(queue_entry) # Flag if there is at least one fail command in the task transition. if not has_manual_fail: has_manual_fail = (next_task_id == 'fail') else: # If not an engine command and the next task is ready, then # make a record of it for processing manual fail below. if staged_next_task['ready']: staged_next_tasks.append(staged_next_task) # Task failure is remediable. For example, there may be workflow that wants # to run a cleanup task on failure. In certain cases, we still want to fail # the workflow after the remediation. The fail command can be in the # task transition under the cleanup task. However, the cleanup task may be # reusable when either workflow succeed or fail. It does not make sense to # put the fail command in the task transition of the cleanup task. So we # want to allow user to be able to define both the cleanup and fail in the # task transition under the current task but still return the cleanup task # even when the workflow is set to failed status. if has_manual_fail: for staged_next_task in staged_next_tasks: staged_next_task['run_on_fail'] = True # Process the task event using the workflow state machine and update the workflow status. task_ex_event = events.TaskExecutionEvent(task_id, route, task_state_entry['status']) machines.WorkflowStateMachine.process_event(self.workflow_state, task_ex_event) # Process any engine commands in the queue. while not engine_event_queue.empty(): next_task_id, next_task_route = engine_event_queue.get() engine_event = events.ENGINE_EVENT_MAP[next_task_id] self.update_task_state(next_task_id, next_task_route, engine_event()) # Render workflow output if workflow is completed. if self.get_workflow_status() in statuses.COMPLETED_STATUSES: task_state_entry['term'] = True self._render_workflow_outputs() return task_state_entry
def update_task_state(self, task_id, route, event): engine_event_queue = queue.Queue() # Throw exception if not expected event type. if not issubclass(type(event), events.ExecutionEvent): raise TypeError('Event is not type of ExecutionEvent.') # Throw exception if task does not exist in the workflow graph. if not self.graph.has_task(task_id): raise exc.InvalidTask(task_id) # Try to get the task metadata from staging or task state. staged_task = self.workflow_state.get_staged_task(task_id, route) task_state_entry = self.get_task_state_entry(task_id, route) # Get the task spec for the task which contains additional meta data. task_spec = self.spec.tasks.get_task(task_id) # Throw exception if task is not staged and there is no task state entry. if not staged_task and not task_state_entry: raise exc.InvalidTaskStateEntry(task_id) # Create new task state entry if it does not exist or if it is an engine command. if not task_state_entry or task_id in events.ENGINE_EVENT_MAP.keys(): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev'] ) # Identify the index for the task state object for later use. task_state_idx = self._get_task_state_idx(task_id, route) # If task is already completed and in cycle, then create new task state entry. # Unfortunately, the method in the graph to check for cycle is too simple and # misses forks that extends from the cycle. The check here assumes that the # last task entry is already completed and the new task status is one of the # starting statuses, then there is high likelihood that this is a cycle. if (task_state_entry.get('status') in statuses.COMPLETED_STATUSES and event.status and event.status in statuses.STARTING_STATUSES): task_state_entry = self.add_task_state( task_id, staged_task['route'], in_ctx_idxs=staged_task['ctxs']['in'], prev=staged_task['prev'] ) # Update the index value since a new entry is created. task_state_idx = self._get_task_state_idx(task_id, route) # Remove task from staging if task is not with items. if event.status and staged_task and 'items' not in staged_task: self.workflow_state.remove_staged_task(task_id, route) # If action execution is for a task item, then record the execution status for the item. # Result for each item is not recorded in the staged_task because it impacts database # write performance if there are a lot of items and/or item result size is huge. if staged_task and isinstance(event, events.TaskItemActionExecutionEvent): staged_task['items'][event.item_id] = {'status': event.status} # Log the error if it is a failed execution event. if event.status == statuses.FAILED: message = 'Execution failed. See result for details.' self.log_entry('error', message, task_id=task_id, result=event.result) # Process the action execution event using the # task state machine and update the task status. old_task_status = task_state_entry.get('status', statuses.UNSET) machines.TaskStateMachine.process_event(self.workflow_state, task_state_entry, event) new_task_status = task_state_entry.get('status', statuses.UNSET) # If retrying, staged the task to be returned in get_next_tasks. if new_task_status == statuses.RETRYING: # Increment the number of times that the task has retried. task_state_entry['retry']['tally'] += 1 # Reset the staged task to be returned in get_next_tasks self.workflow_state.remove_staged_task(task_id, route) self.workflow_state.add_staged_task( task_id, route, ctxs=task_state_entry['ctxs']['in'], prev=task_state_entry['prev'], retry=task_state_entry['retry'], ready=True ) # Get task result and set current context if task is completed. if new_task_status in statuses.COMPLETED_STATUSES: # Remove task from staging if exists but keep entry # if task has items and failed for manual rerun. if not (task_spec.has_items() and new_task_status in statuses.ABENDED_STATUSES): self.workflow_state.remove_staged_task(task_id, route) # Format task result depending on the type of task. task_result = self.make_task_result(task_spec, event) # Set current task in the context. current_ctx = self.make_task_context(task_state_entry, task_result=task_result) # Evaluate if there is a task retry. The task retry can only be evaluated after # the state machine has determined the status for the task execution. If the task # is completed, get the task result and context which is required to evaluate the # the condition if a retry for the task is required. if (self.get_workflow_status() in statuses.ACTIVE_STATUSES and self._evaluate_task_retry(task_state_entry, current_ctx)): self.update_task_state(task_id, route, events.TaskRetryEvent()) # Evaluate task transitions if task is completed and status change is not processed. if new_task_status in statuses.COMPLETED_STATUSES and new_task_status != old_task_status: has_manual_fail = False staged_next_tasks = [] # Identify task transitions for the current completed task. task_transitions = self.graph.get_next_transitions(task_id) # Mark task as terminal when there is no transitions. if not task_transitions: task_state_entry['term'] = True # Iterate thru each outbound task transitions. for task_transition in task_transitions: task_transition_id = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_transition[1], str(task_transition[2])) ) # Evaluate the criteria for task transition. If there is a failure while # evaluating expression(s), fail the workflow. try: criteria = task_transition[3].get('criteria') or [] evaluated_criteria = [expr_base.evaluate(c, current_ctx) for c in criteria] task_state_entry['next'][task_transition_id] = all(evaluated_criteria) except Exception as e: self.log_error(e, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue # If criteria met, then mark the next task staged and calculate outgoing context. if task_state_entry['next'][task_transition_id]: next_task_node = self.graph.get_task(task_transition[1]) next_task_id = next_task_node['id'] new_ctx_idx = None # Get and process new context for the task transition. out_ctx, new_ctx, errors = task_spec.finalize_context( next_task_id, task_transition, json_util.deepcopy(current_ctx) ) if errors: self.log_errors(errors, task_id, route, task_transition_id) self.request_workflow_status(statuses.FAILED) continue out_ctx_idxs = json_util.deepcopy(task_state_entry['ctxs']['in']) if new_ctx: self.workflow_state.contexts.append(new_ctx) new_ctx_idx = len(self.workflow_state.contexts) - 1 # Add to the list of contexts for the next task in this transition. out_ctx_idxs.append(new_ctx_idx) # Record the outgoing context for this task transition. if 'out' not in task_state_entry['ctxs']: task_state_entry['ctxs']['out'] = {} task_state_entry['ctxs']['out'] = {task_transition_id: new_ctx_idx} # Stage the next task if it is not in staging. next_task_route = self._evaluate_route(task_transition, route) staged_next_task = self.workflow_state.get_staged_task( next_task_id, next_task_route ) backref = ( constants.TASK_STATE_TRANSITION_FORMAT % (task_id, str(task_transition[2])) ) # If the next task is already staged. if staged_next_task: # Remove the root context to avoid overwriting vars. out_ctx_idxs.remove(0) # Extend the outgoing context from this task. staged_next_task['ctxs']['in'].extend(out_ctx_idxs) # Add a backref for the current task in the next task. staged_next_task['prev'][backref] = task_state_idx # Clear list of items for with items task. staged_next_task.pop('items', None) else: # Otherwise create a new entry in staging for the next task. staged_next_task = self.workflow_state.add_staged_task( next_task_id, next_task_route, ctxs=out_ctx_idxs, prev={backref: task_state_idx}, ready=False ) # Check if inbound criteria are met. Must use the original route # to identify the inbound task transitions. staged_next_task['ready'] = self._inbound_criteria_satisfied( next_task_id, route ) # Put the next task in the engine event queue if it is an engine command. if next_task_id in events.ENGINE_EVENT_MAP.keys(): queue_entry = (staged_next_task['id'], staged_next_task['route']) engine_event_queue.put(queue_entry) # Flag if there is at least one fail command in the task transition. if not has_manual_fail: has_manual_fail = (next_task_id == 'fail') else: # If not an engine command and the next task is ready, then # make a record of it for processing manual fail below. if staged_next_task['ready']: staged_next_tasks.append(staged_next_task) # Task failure is remediable. For example, there may be workflow that wants # to run a cleanup task on failure. In certain cases, we still want to fail # the workflow after the remediation. The fail command can be in the # task transition under the cleanup task. However, the cleanup task may be # reusable when either workflow succeed or fail. It does not make sense to # put the fail command in the task transition of the cleanup task. So we # want to allow user to be able to define both the cleanup and fail in the # task transition under the current task but still return the cleanup task # even when the workflow is set to failed status. if has_manual_fail: for staged_next_task in staged_next_tasks: staged_next_task['run_on_fail'] = True # Process the task event using the workflow state machine and update the workflow status. task_ex_event = events.TaskExecutionEvent(task_id, route, task_state_entry['status']) machines.WorkflowStateMachine.process_event(self.workflow_state, task_ex_event) # Process any engine commands in the queue. while not engine_event_queue.empty(): next_task_id, next_task_route = engine_event_queue.get() engine_event = events.ENGINE_EVENT_MAP[next_task_id] self.update_task_state(next_task_id, next_task_route, engine_event()) # Mark the task as a terminal task if workflow execution is completed. if self.get_workflow_status() in statuses.COMPLETED_STATUSES: task_state_entry['term'] = True return task_state_entry