def _run_chain(self, action_parameters, resuming=False): # Set chain status to fail unless explicitly set to succeed. chain_status = action_constants.LIVEACTION_STATUS_FAILED # Result holds the final result that the chain store in the database. result = {'tasks': []} # Save published variables into the result if specified. if self._display_published: result[PUBLISHED_VARS_KEY] = {} context_result = { } # Holds result which is used for the template context purposes top_level_error = None # Stores a reference to a top level error action_node = None last_task = None try: # Initialize vars with the action parameters. # This allows action parameers to be referenced from vars. self.chain_holder.init_vars(action_parameters) except Exception as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed initializing ``vars`` in chain.' LOG.exception(m) top_level_error = self._format_error(e, m) result.update(top_level_error) return (chain_status, result, None) # Restore state on resuming an existing chain execution. if resuming: # Restore vars is any from the liveaction. ctx_vars = self.liveaction.context.pop('vars', {}) self.chain_holder.restore_vars(ctx_vars) # Restore result if any from the liveaction. if self.liveaction and hasattr( self.liveaction, 'result') and self.liveaction.result: result = self.liveaction.result # Initialize or rebuild existing context_result from liveaction # which holds the result used for resolving context in Jinja template. for task in result.get('tasks', []): context_result[task['name']] = task['result'] # Restore or initialize the top_level_error # that stores a reference to a top level error. if 'error' in result or 'traceback' in result: top_level_error = { 'error': result.get('error'), 'traceback': result.get('traceback') } # If there are no executed tasks in the chain, then get the first node. if len(result['tasks']) <= 0: try: action_node = self.chain_holder.get_next_node() except Exception as e: m = 'Failed to get starting node "%s".', action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) # If there are no action node to run next, then mark the chain successful. if not action_node: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED # Otherwise, figure out the last task executed and # its state to determine where to begin executing. else: last_task = result['tasks'][-1] action_node = self.chain_holder.get_node(last_task['name']) liveaction = action_db_util.get_liveaction_by_id( last_task['liveaction_id']) # If the liveaction of the last task has changed, update the result entry. if liveaction.status != last_task['state']: updated_task_result = self._get_updated_action_exec_result( action_node, liveaction, last_task) del result['tasks'][-1] result['tasks'].append(updated_task_result) # Also need to update context_result so the updated result # is available to Jinja expressions updated_task_name = updated_task_result['name'] context_result[updated_task_name][ 'result'] = updated_task_result['result'] # If the last task was canceled, then canceled the chain altogether. if liveaction.status == action_constants.LIVEACTION_STATUS_CANCELED: chain_status = action_constants.LIVEACTION_STATUS_CANCELED return (chain_status, result, None) # If the last task was paused, then stay on this action node. # This is explicitly put here for clarity. if liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED: pass # If the last task succeeded, then get the next on-success action node. if liveaction.status == action_constants.LIVEACTION_STATUS_SUCCEEDED: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_node = self.chain_holder.get_next_node( last_task['name'], condition='on-success') # If the last task failed, then get the next on-failure action node. if liveaction.status in action_constants.LIVEACTION_FAILED_STATES: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( last_task['name'], condition='on-failure') # Setup parent context. parent_context = {'execution_id': self.execution_id} if getattr(self.liveaction, 'context', None): parent_context.update(self.liveaction.context) # Run the action chain until there are no more tasks. while action_node: error = None liveaction = None last_task = result['tasks'][-1] if len( result['tasks']) > 0 else None created_at = date_utils.get_datetime_utc_now() try: # If last task was paused, then fetch the liveaction and resume it first. if last_task and last_task[ 'state'] == action_constants.LIVEACTION_STATUS_PAUSED: liveaction = action_db_util.get_liveaction_by_id( last_task['liveaction_id']) del result['tasks'][-1] else: liveaction = self._get_next_action( action_node=action_node, parent_context=parent_context, action_params=action_parameters, context_result=context_result) except action_exc.InvalidActionReferencedException as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = ( 'Failed to run task "%s". Action with reference "%s" doesn\'t exist.' % (action_node.name, action_node.ref)) LOG.exception(m) top_level_error = self._format_error(e, m) break except action_exc.ParameterRenderingFailedException as e: # Rendering parameters failed before we even got to running this action, # abort and fail the whole action chain chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to run task "%s". Parameter rendering failed.' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) break except db_exc.StackStormDBObjectNotFoundError as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to resume task "%s". Unable to find liveaction.' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) break try: # If last task was paused, then fetch the liveaction and resume it first. if last_task and last_task[ 'state'] == action_constants.LIVEACTION_STATUS_PAUSED: LOG.info('Resume task %s for chain %s.', action_node.name, self.liveaction.id) liveaction = self._resume_action(liveaction) else: LOG.info('Run task %s for chain %s.', action_node.name, self.liveaction.id) liveaction = self._run_action(liveaction) except Exception as e: # Save the traceback and error message m = 'Failed running task "%s".' % action_node.name LOG.exception(m) error = self._format_error(e, m) context_result[action_node.name] = error else: # Update context result context_result[action_node.name] = liveaction.result # Render and publish variables rendered_publish_vars = ActionChainRunner._render_publish_vars( action_node=action_node, action_parameters=action_parameters, execution_result=liveaction.result, previous_execution_results=context_result, chain_vars=self.chain_holder.vars) if rendered_publish_vars: self.chain_holder.vars.update(rendered_publish_vars) if self._display_published: result[PUBLISHED_VARS_KEY].update( rendered_publish_vars) finally: # Record result and resolve a next node based on the task success or failure updated_at = date_utils.get_datetime_utc_now() task_result = self._format_action_exec_result(action_node, liveaction, created_at, updated_at, error=error) result['tasks'].append(task_result) try: if not liveaction: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_TIMED_OUT: chain_status = action_constants.LIVEACTION_STATUS_TIMED_OUT action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_CANCELED: LOG.info( 'Chain execution (%s) canceled because task "%s" is canceled.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_CANCELED action_node = None elif liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED: LOG.info( 'Chain execution (%s) paused because task "%s" is paused.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() action_node = None elif liveaction.status == action_constants.LIVEACTION_STATUS_PENDING: LOG.info( 'Chain execution (%s) paused because task "%s" is pending.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() action_node = None elif liveaction.status in action_constants.LIVEACTION_FAILED_STATES: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_SUCCEEDED: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-success') else: action_node = None except Exception as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to get next node "%s".' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) action_node = None break if action_service.is_action_canceled_or_canceling( self.liveaction.id): LOG.info('Chain execution (%s) canceled by user.', self.liveaction.id) chain_status = action_constants.LIVEACTION_STATUS_CANCELED return (chain_status, result, None) if action_service.is_action_paused_or_pausing(self.liveaction.id): LOG.info('Chain execution (%s) paused by user.', self.liveaction.id) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() return (chain_status, result, self.liveaction.context) if top_level_error and isinstance(top_level_error, dict): result.update(top_level_error) return (chain_status, result, self.liveaction.context)
def _run_chain(self, action_parameters, resuming=False): # Set chain status to fail unless explicitly set to succeed. chain_status = action_constants.LIVEACTION_STATUS_FAILED # Result holds the final result that the chain store in the database. result = {'tasks': []} # Save published variables into the result if specified. if self._display_published: result[PUBLISHED_VARS_KEY] = {} context_result = {} # Holds result which is used for the template context purposes top_level_error = None # Stores a reference to a top level error action_node = None last_task = None try: # Initialize vars with the action parameters. # This allows action parameers to be referenced from vars. self.chain_holder.init_vars(action_parameters) except Exception as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed initializing ``vars`` in chain.' LOG.exception(m) top_level_error = self._format_error(e, m) result.update(top_level_error) return (chain_status, result, None) # Restore state on resuming an existing chain execution. if resuming: # Restore vars is any from the liveaction. ctx_vars = self.liveaction.context.pop('vars', {}) self.chain_holder.restore_vars(ctx_vars) # Restore result if any from the liveaction. if self.liveaction and hasattr(self.liveaction, 'result') and self.liveaction.result: result = self.liveaction.result # Initialize or rebuild existing context_result from liveaction # which holds the result used for resolving context in Jinja template. for task in result.get('tasks', []): context_result[task['name']] = task['result'] # Restore or initialize the top_level_error # that stores a reference to a top level error. if 'error' in result or 'traceback' in result: top_level_error = { 'error': result.get('error'), 'traceback': result.get('traceback') } # If there are no executed tasks in the chain, then get the first node. if len(result['tasks']) <= 0: try: action_node = self.chain_holder.get_next_node() except Exception as e: m = 'Failed to get starting node "%s".', action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) # If there are no action node to run next, then mark the chain successful. if not action_node: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED # Otherwise, figure out the last task executed and # its state to determine where to begin executing. else: last_task = result['tasks'][-1] action_node = self.chain_holder.get_node(last_task['name']) liveaction = action_db_util.get_liveaction_by_id(last_task['liveaction_id']) # If the liveaction of the last task has changed, update the result entry. if liveaction.status != last_task['state']: updated_task_result = self._get_updated_action_exec_result( action_node, liveaction, last_task) del result['tasks'][-1] result['tasks'].append(updated_task_result) # Also need to update context_result so the updated result # is available to Jinja expressions updated_task_name = updated_task_result['name'] context_result[updated_task_name]['result'] = updated_task_result['result'] # If the last task was canceled, then canceled the chain altogether. if liveaction.status == action_constants.LIVEACTION_STATUS_CANCELED: chain_status = action_constants.LIVEACTION_STATUS_CANCELED return (chain_status, result, None) # If the last task was paused, then stay on this action node. # This is explicitly put here for clarity. if liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED: pass # If the last task succeeded, then get the next on-success action node. if liveaction.status == action_constants.LIVEACTION_STATUS_SUCCEEDED: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_node = self.chain_holder.get_next_node( last_task['name'], condition='on-success') # If the last task failed, then get the next on-failure action node. if liveaction.status in action_constants.LIVEACTION_FAILED_STATES: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( last_task['name'], condition='on-failure') # Setup parent context. parent_context = { 'execution_id': self.execution_id } if getattr(self.liveaction, 'context', None): parent_context.update(self.liveaction.context) # Run the action chain until there are no more tasks. while action_node: error = None liveaction = None last_task = result['tasks'][-1] if len(result['tasks']) > 0 else None created_at = date_utils.get_datetime_utc_now() try: # If last task was paused, then fetch the liveaction and resume it first. if last_task and last_task['state'] == action_constants.LIVEACTION_STATUS_PAUSED: liveaction = action_db_util.get_liveaction_by_id(last_task['liveaction_id']) del result['tasks'][-1] else: liveaction = self._get_next_action( action_node=action_node, parent_context=parent_context, action_params=action_parameters, context_result=context_result) except action_exc.InvalidActionReferencedException as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = ('Failed to run task "%s". Action with reference "%s" doesn\'t exist.' % (action_node.name, action_node.ref)) LOG.exception(m) top_level_error = self._format_error(e, m) break except action_exc.ParameterRenderingFailedException as e: # Rendering parameters failed before we even got to running this action, # abort and fail the whole action chain chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to run task "%s". Parameter rendering failed.' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) break except db_exc.StackStormDBObjectNotFoundError as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to resume task "%s". Unable to find liveaction.' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) break try: # If last task was paused, then fetch the liveaction and resume it first. if last_task and last_task['state'] == action_constants.LIVEACTION_STATUS_PAUSED: LOG.info('Resume task %s for chain %s.', action_node.name, self.liveaction.id) liveaction = self._resume_action(liveaction) else: LOG.info('Run task %s for chain %s.', action_node.name, self.liveaction.id) liveaction = self._run_action(liveaction) except Exception as e: # Save the traceback and error message m = 'Failed running task "%s".' % action_node.name LOG.exception(m) error = self._format_error(e, m) context_result[action_node.name] = error else: # Update context result context_result[action_node.name] = liveaction.result # Render and publish variables rendered_publish_vars = ActionChainRunner._render_publish_vars( action_node=action_node, action_parameters=action_parameters, execution_result=liveaction.result, previous_execution_results=context_result, chain_vars=self.chain_holder.vars) if rendered_publish_vars: self.chain_holder.vars.update(rendered_publish_vars) if self._display_published: result[PUBLISHED_VARS_KEY].update(rendered_publish_vars) finally: # Record result and resolve a next node based on the task success or failure updated_at = date_utils.get_datetime_utc_now() task_result = self._format_action_exec_result( action_node, liveaction, created_at, updated_at, error=error ) result['tasks'].append(task_result) try: if not liveaction: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_TIMED_OUT: chain_status = action_constants.LIVEACTION_STATUS_TIMED_OUT action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_CANCELED: LOG.info('Chain execution (%s) canceled because task "%s" is canceled.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_CANCELED action_node = None elif liveaction.status == action_constants.LIVEACTION_STATUS_PAUSED: LOG.info('Chain execution (%s) paused because task "%s" is paused.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() action_node = None elif liveaction.status == action_constants.LIVEACTION_STATUS_PENDING: LOG.info('Chain execution (%s) paused because task "%s" is pending.', self.liveaction_id, action_node.name) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() action_node = None elif liveaction.status in action_constants.LIVEACTION_FAILED_STATES: chain_status = action_constants.LIVEACTION_STATUS_FAILED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-failure') elif liveaction.status == action_constants.LIVEACTION_STATUS_SUCCEEDED: chain_status = action_constants.LIVEACTION_STATUS_SUCCEEDED action_node = self.chain_holder.get_next_node( action_node.name, condition='on-success') else: action_node = None except Exception as e: chain_status = action_constants.LIVEACTION_STATUS_FAILED m = 'Failed to get next node "%s".' % action_node.name LOG.exception(m) top_level_error = self._format_error(e, m) action_node = None break if action_service.is_action_canceled_or_canceling(self.liveaction.id): LOG.info('Chain execution (%s) canceled by user.', self.liveaction.id) chain_status = action_constants.LIVEACTION_STATUS_CANCELED return (chain_status, result, None) if action_service.is_action_paused_or_pausing(self.liveaction.id): LOG.info('Chain execution (%s) paused by user.', self.liveaction.id) chain_status = action_constants.LIVEACTION_STATUS_PAUSED self._save_vars() return (chain_status, result, self.liveaction.context) if top_level_error and isinstance(top_level_error, dict): result.update(top_level_error) return (chain_status, result, self.liveaction.context)