def on_action_complete(self, action_ex_id, result): wf_ex_id = None try: with db_api.transaction(): action_ex = db_api.get_action_execution(action_ex_id) wf_ex_id = action_ex.task_execution.workflow_execution_id # Must be before loading the object itself (see method doc). self._lock_workflow_execution(wf_ex_id) wf_ex = action_ex.task_execution.workflow_execution task_ex = task_handler.on_action_complete(action_ex, result) # If workflow is on pause or completed then there's no # need to continue workflow. if states.is_paused_or_completed(wf_ex.state): return action_ex self._on_task_state_change(task_ex, wf_ex, action_ex) return action_ex.get_clone() except Exception as e: # TODO(dzimine): try to find out which command caused failure. # TODO(rakhmerov): Need to refactor logging in a more elegant way. LOG.error( "Failed to handle action execution result [id=%s]: %s\n%s", action_ex_id, e, traceback.format_exc() ) self._fail_workflow(wf_ex_id, e) raise e
def check_workflow_completion(wf_ex): if states.is_paused_or_completed(wf_ex.state): return # Workflow is not completed if there are any incomplete task # executions that are not in WAITING state. If all incomplete # tasks are waiting and there are no unhandled errors, then these # tasks will not reach completion. In this case, mark the # workflow complete. incomplete_tasks = wf_utils.find_incomplete_task_executions(wf_ex) if any(not states.is_waiting(t.state) for t in incomplete_tasks): return wf_spec = spec_parser.get_workflow_spec(wf_ex.spec) wf_ctrl = wf_base.get_controller(wf_ex, wf_spec) if wf_ctrl.all_errors_handled(): succeed_workflow(wf_ex, wf_ctrl.evaluate_workflow_final_context(), wf_spec) else: state_info = wf_utils.construct_fail_info_message(wf_ctrl, wf_ex) fail_workflow(wf_ex, state_info)
def _fail_workflow(self, final_context, msg): if states.is_paused_or_completed(self.wf_ex.state): return output_on_error = {} try: output_on_error = data_flow.evaluate_workflow_output( self.wf_ex, self.wf_spec.get_output_on_error(), final_context) except exc.MistralException as e: msg = ("Failed to evaluate expression in output-on-error! " "(output-on-error: '%s', exception: '%s' Cause: '%s'" % (self.wf_spec.get_output_on_error(), e, msg)) LOG.error(msg) self.set_state(states.ERROR, state_info=msg) # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. length_output_on_error = len(str(output_on_error).encode("utf-8")) total_output_length = utils.get_number_of_chars_from_kilobytes( cfg.CONF.engine.execution_field_size_limit_kb) if length_output_on_error < total_output_length: msg = utils.cut_by_char( msg, total_output_length - length_output_on_error) else: msg = utils.cut_by_kb( msg, cfg.CONF.engine.execution_field_size_limit_kb) self.wf_ex.output = merge_dicts({'result': msg}, output_on_error) if self.wf_ex.task_execution_id: self._send_result_to_parent_workflow()
def _fail_workflow(self, final_context, msg): if states.is_paused_or_completed(self.wf_ex.state): return output_on_error = {} try: output_on_error = data_flow.evaluate_workflow_output( self.wf_ex, self.wf_spec.get_output_on_error(), final_context ) except exc.MistralException as e: msg = ( "Failed to evaluate expression in output-on-error! " "(output-on-error: '%s', exception: '%s' Cause: '%s'" % (self.wf_spec.get_output_on_error(), e, msg) ) LOG.error(msg) self.set_state(states.ERROR, state_info=msg) # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. msg = utils.cut_by_kb( msg, cfg.CONF.engine.execution_field_size_limit_kb ) self.wf_ex.output = merge_dicts({'result': msg}, output_on_error) if self.wf_ex.task_execution_id: self._schedule_send_result_to_parent_workflow()
def fail_workflow(wf_ex, state_info): if states.is_paused_or_completed(wf_ex.state): return set_execution_state(wf_ex, states.ERROR, state_info) if wf_ex.task_execution_id: _schedule_send_result_to_parent_workflow(wf_ex)
def fail_workflow(wf_ex, state_info): if states.is_paused_or_completed(wf_ex.state): return wf_ex set_execution_state(wf_ex, states.ERROR, state_info) if wf_ex.task_execution_id: _schedule_send_result_to_parent_workflow(wf_ex) return wf_ex
def on_action_complete(self, action_ex_id, result): wf_ex_id = None try: with db_api.transaction(): action_ex = db_api.get_action_execution(action_ex_id) # In case of single action execution there is no # assigned task execution. if not action_ex.task_execution: return action_handler.store_action_result( action_ex, result).get_clone() wf_ex_id = action_ex.task_execution.workflow_execution_id wf_ex = wf_handler.lock_workflow_execution(wf_ex_id) task_ex = task_handler.on_action_complete(action_ex, result) # If workflow is on pause or completed then there's no # need to continue workflow. if states.is_paused_or_completed(wf_ex.state): return action_ex.get_clone() prev_task_state = task_ex.state # Separate the task transition in a separate transaction. The task # has already completed for better or worst. The task state should # not be affected by errors during transition on conditions such as # on-success and on-error. with db_api.transaction(): wf_ex = wf_handler.lock_workflow_execution(wf_ex_id) action_ex = db_api.get_action_execution(action_ex_id) task_ex = action_ex.task_execution self._on_task_state_change(task_ex, wf_ex, task_state=prev_task_state) return action_ex.get_clone() except Exception as e: # TODO(dzimine): try to find out which command caused failure. # TODO(rakhmerov): Need to refactor logging in a more elegant way. LOG.error( "Failed to handle action execution result [id=%s]: %s\n%s", action_ex_id, e, traceback.format_exc()) # If an exception was thrown after we got the wf_ex_id if wf_ex_id: self._fail_workflow(wf_ex_id, e) raise e
def _fail_workflow(self, msg): if states.is_paused_or_completed(self.wf_ex.state): return self.set_state(states.ERROR, state_info=msg) # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. msg = utils.cut_by_kb(msg, cfg.CONF.engine.execution_field_size_limit_kb) self.wf_ex.output = {'result': msg} if self.wf_ex.task_execution_id: self._schedule_send_result_to_parent_workflow()
def check_and_complete(self): """Completes the workflow if it needs to be completed. The method simply checks if there are any tasks that are not in a terminal state. If there aren't any then it performs all necessary logic to finalize the workflow (calculate output etc.). :return: Number of incomplete tasks. """ if states.is_paused_or_completed(self.wf_ex.state): return 0 # Workflow is not completed if there are any incomplete task # executions. incomplete_tasks_count = db_api.get_incomplete_task_executions_count( workflow_execution_id=self.wf_ex.id, ) if incomplete_tasks_count > 0: return incomplete_tasks_count LOG.debug("Workflow completed [id=%s]", self.wf_ex.id) # NOTE(rakhmerov): Once we know that the workflow has completed, # we need to expire all the objects in the DB session to make sure # to read the most relevant data from the DB (that's already been # committed in parallel transactions). Otherwise, some data like # workflow context may be stale and decisions made upon it will be # wrong. db_api.expire_all() wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) if wf_ctrl.any_cancels(): msg = _build_cancel_info_message(wf_ctrl, self.wf_ex) self._cancel_workflow(msg) elif wf_ctrl.all_errors_handled(): ctx = wf_ctrl.evaluate_workflow_final_context() self._succeed_workflow(ctx) else: msg = _build_fail_info_message(wf_ctrl, self.wf_ex) final_context = wf_ctrl.evaluate_workflow_final_context() self._fail_workflow(final_context, msg) return 0
def _fail_workflow(self, final_context, msg): if states.is_paused_or_completed(self.wf_ex.state): return output_on_error = {} try: output_on_error = data_flow.evaluate_workflow_output( self.wf_ex, self.wf_spec.get_output_on_error(), final_context ) except exc.MistralException as e: msg = ( "Failed to evaluate expression in output-on-error! " "(output-on-error: '%s', exception: '%s' Cause: '%s'" % (self.wf_spec.get_output_on_error(), e, msg) ) LOG.error(msg) if not self.set_state(states.ERROR, state_info=msg): return # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. length_output_on_error = len(str(output_on_error).encode("utf-8")) total_output_length = utils.get_number_of_chars_from_kilobytes( cfg.CONF.engine.execution_field_size_limit_kb) if length_output_on_error < total_output_length: msg = utils.cut_by_char( msg, total_output_length - length_output_on_error ) else: msg = utils.cut_by_kb( msg, cfg.CONF.engine.execution_field_size_limit_kb ) self.wf_ex.output = merge_dicts({'result': msg}, output_on_error) # Publish event. self.notify(events.WORKFLOW_FAILED) if self.wf_ex.task_execution_id: self._send_result_to_parent_workflow()
def _check_workflow_completion(wf_ex, wf_ctrl): if states.is_paused_or_completed(wf_ex.state): return if wf_utils.find_incomplete_task_executions(wf_ex): return if wf_ctrl.all_errors_handled(): wf_handler.succeed_workflow( wf_ex, wf_ctrl.evaluate_workflow_final_context() ) else: state_info = wf_utils.construct_fail_info_message(wf_ctrl, wf_ex) wf_handler.fail_workflow(wf_ex, state_info)
def fail_workflow(wf_ex, state_info): if states.is_paused_or_completed(wf_ex.state): return wf_ex set_workflow_state(wf_ex, states.ERROR, state_info) # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. state_info = utils.cut_by_kb(state_info, cfg.CONF.engine.execution_field_size_limit_kb) wf_ex.output = {'result': state_info} if wf_ex.task_execution_id: _schedule_send_result_to_parent_workflow(wf_ex) return wf_ex
def on_action_complete(self, action_ex_id, result): wf_ex_id = None try: with db_api.transaction(): action_ex = db_api.get_action_execution(action_ex_id) # In case of single action execution there is no # assigned task execution. if not action_ex.task_execution: return action_handler.store_action_result( action_ex, result ).get_clone() wf_ex_id = action_ex.task_execution.workflow_execution_id wf_ex = wf_handler.lock_workflow_execution(wf_ex_id) wf_spec = spec_parser.get_workflow_spec(wf_ex.spec) task_ex = task_handler.on_action_complete( action_ex, wf_spec, result ) # If workflow is on pause or completed then there's no # need to continue workflow. if states.is_paused_or_completed(wf_ex.state): return action_ex.get_clone() self._on_task_state_change(task_ex, wf_ex, wf_spec) return action_ex.get_clone() except Exception as e: # TODO(rakhmerov): Need to refactor logging in a more elegant way. LOG.error( 'Failed to handle action execution result [id=%s]: %s\n%s', action_ex_id, e, traceback.format_exc() ) # If an exception was thrown after we got the wf_ex_id if wf_ex_id: self._fail_workflow(wf_ex_id, e) raise e
def _fail_workflow(self, msg): if states.is_paused_or_completed(self.wf_ex.state): return self.set_state(states.ERROR, state_info=msg) # When we set an ERROR state we should safely set output value getting # w/o exceptions due to field size limitations. msg = utils.cut_by_kb( msg, cfg.CONF.engine.execution_field_size_limit_kb ) self.wf_ex.output = {'result': msg} if self.wf_ex.task_execution_id: self._schedule_send_result_to_parent_workflow()
def _check_workflow_completion(wf_ex, action_ex, wf_ctrl): if states.is_paused_or_completed(wf_ex.state): return if wf_utils.find_incomplete_tasks(wf_ex): return if wf_ctrl.all_errors_handled(): wf_handler.succeed_workflow( wf_ex, wf_ctrl.evaluate_workflow_final_context()) else: result_str = (str(action_ex.output.get('result', 'Unknown')) if action_ex.output else 'Unknown') state_info = ("Failure caused by error in task '%s': %s" % (action_ex.task_execution.name, result_str)) wf_handler.fail_workflow(wf_ex, state_info)
def _fail_workflow(wf_ex_id, exc): """Private helper to fail workflow on exceptions.""" with db_api.transaction(): wf_ex = db_api.load_workflow_execution(wf_ex_id) if wf_ex is None: LOG.error( "Can't fail workflow execution with id='%s': not found.", wf_ex_id ) return None wf_ex = wf_handler.lock_workflow_execution(wf_ex_id) if not states.is_paused_or_completed(wf_ex.state): wf_handler.set_execution_state(wf_ex, states.ERROR, str(exc)) return wf_ex
def _check_and_complete(self): if states.is_paused_or_completed(self.wf_ex.state): return # Workflow is not completed if there are any incomplete task # executions. incomplete_tasks = wf_utils.find_incomplete_task_executions(self.wf_ex) if incomplete_tasks: return wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) if wf_ctrl.any_cancels(): self._cancel_workflow( _build_cancel_info_message(wf_ctrl, self.wf_ex)) elif wf_ctrl.all_errors_handled(): self._succeed_workflow(wf_ctrl.evaluate_workflow_final_context()) else: self._fail_workflow(_build_fail_info_message(wf_ctrl, self.wf_ex))
def _check_and_complete(self): if states.is_paused_or_completed(self.wf_ex.state): return # Workflow is not completed if there are any incomplete task # executions that are not in WAITING state. If all incomplete # tasks are waiting and there are unhandled errors, then these # tasks will not reach completion. In this case, mark the # workflow complete. incomplete_tasks = wf_utils.find_incomplete_task_executions(self.wf_ex) if any(not states.is_waiting(t.state) for t in incomplete_tasks): return wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) if wf_ctrl.all_errors_handled(): self._succeed_workflow(wf_ctrl.evaluate_workflow_final_context()) else: self._fail_workflow(_build_fail_info_message(wf_ctrl, self.wf_ex))
def _check_workflow_completion(wf_ex, action_ex, wf_ctrl): if states.is_paused_or_completed(wf_ex.state): return if wf_utils.find_incomplete_tasks(wf_ex): return if wf_ctrl.all_errors_handled(): wf_handler.succeed_workflow( wf_ex, wf_ctrl.evaluate_workflow_final_context() ) else: result_str = str(action_ex.output.get('result', "Unknown")) state_info = ( "Failure caused by error in task '%s': %s" % (action_ex.task_execution.name, result_str) ) wf_handler.fail_workflow(wf_ex, state_info)
def check_and_complete(self): """Completes the workflow if it needs to be completed. The method simply checks if there are any tasks that are not in a terminal state. If there aren't any then it performs all necessary logic to finalize the workflow (calculate output etc.). :return: Number of incomplete tasks. """ if states.is_paused_or_completed(self.wf_ex.state): return 0 # Workflow is not completed if there are any incomplete task # executions. incomplete_tasks_count = db_api.get_incomplete_task_executions_count( workflow_execution_id=self.wf_ex.id, ) if incomplete_tasks_count > 0: return incomplete_tasks_count wf_ctrl = wf_base.get_controller(self.wf_ex, self.wf_spec) if wf_ctrl.any_cancels(): msg = _build_cancel_info_message(wf_ctrl, self.wf_ex) self._cancel_workflow(msg) elif wf_ctrl.all_errors_handled(): ctx = wf_ctrl.evaluate_workflow_final_context() self._succeed_workflow(ctx) else: msg = _build_fail_info_message(wf_ctrl, self.wf_ex) final_context = wf_ctrl.evaluate_workflow_final_context() self._fail_workflow(final_context, msg) return 0
def on_action_complete(self, action_ex_id, result): wf_ex_id = None try: with db_api.transaction(): action_ex = db_api.get_action_execution(action_ex_id) # In case of single action execution there is no # assigned task execution. if not action_ex.task_execution: return action_handler.store_action_result( action_ex, result).get_clone() wf_ex_id = action_ex.task_execution.workflow_execution_id # Must be before loading the object itself (see method doc). self._lock_workflow_execution(wf_ex_id) wf_ex = action_ex.task_execution.workflow_execution task_ex = task_handler.on_action_complete(action_ex, result) # If workflow is on pause or completed then there's no # need to continue workflow. if states.is_paused_or_completed(wf_ex.state): return action_ex self._on_task_state_change(task_ex, wf_ex, action_ex) return action_ex.get_clone() except Exception as e: # TODO(dzimine): try to find out which command caused failure. # TODO(rakhmerov): Need to refactor logging in a more elegant way. LOG.error( "Failed to handle action execution result [id=%s]: %s\n%s", action_ex_id, e, traceback.format_exc()) self._fail_workflow(wf_ex_id, e) raise e
def _check_workflow_completion(wf_ex, wf_ctrl, wf_spec): if states.is_paused_or_completed(wf_ex.state): return # Workflow is not completed if there are any incomplete task # executions that are not in WAITING state. If all incomplete # tasks are waiting and there are unhandled errors, then these # tasks will not reach completion. In this case, mark the # workflow complete. incomplete_tasks = wf_utils.find_incomplete_task_executions(wf_ex) if any(not states.is_waiting(t.state) for t in incomplete_tasks): return if wf_ctrl.all_errors_handled(): wf_handler.succeed_workflow( wf_ex, wf_ctrl.evaluate_workflow_final_context(), wf_spec ) else: state_info = wf_utils.construct_fail_info_message(wf_ctrl, wf_ex) wf_handler.fail_workflow(wf_ex, state_info)
def _is_paused_or_completed(self): return states.is_paused_or_completed(self.wf_ex.state)
def on_workflow_status_update(ex_id, data, event, timestamp, **kwargs): root_id = data.get('root_execution_id') or ex_id if root_id != ex_id: LOG.info( '[%s] The workflow event %s for subworkflow %s is ' 'not published to st2. This is expected because it ' 'does not have a corresponding execution record in st2.', root_id, event, ex_id ) return wf_ex_env = data['params']['env'] st2_ex_id = wf_ex_env['st2_execution_id'] st2_ctx = wf_ex_env['__actions']['st2.action']['st2_context'] st2_token = st2_ctx.get('auth_token') endpoint = st2_ctx['api_url'] + '/executions/' + st2_ex_id body = { 'status': WF_EX_STATUS_MAP[data['state']], 'result': data.get('output', {}) } body['result']['extra'] = { 'state': data['state'], 'state_info': data['state_info'] } # If workflow is in completed state, then # include task details in the result. if states.is_paused_or_completed(data['state']): with db_api.transaction(): wf_ex = db_api.get_workflow_execution(ex_id) task_exs = [ task_ex.to_dict() for task_ex in wf_ex.task_executions ] task_exs_result = [ data_flow.get_task_execution_result(task_ex) for task_ex in wf_ex.task_executions ] if task_exs: body['result']['tasks'] = [] for task_ex, task_ex_result in zip(task_exs, task_exs_result): parent_wf_ex_id = task_ex.get('workflow_execution_id', None) task_input = try_json_loads(task_ex.get('input', None)) task_publish = try_json_loads(task_ex.get('published', None)) task_result = { 'id': task_ex['id'], 'name': task_ex['name'], 'workflow_execution_id': parent_wf_ex_id, 'workflow_name': task_ex['workflow_name'], 'created_at': task_ex.get('created_at', None), 'updated_at': task_ex.get('updated_at', None), 'state': task_ex.get('state', None), 'state_info': task_ex.get('state_info', None), 'input': task_input, 'published': task_publish, 'result': task_ex_result } body['result']['tasks'].append(task_result) LOG.info( '[%s] The workflow event %s for %s will be published to st2.', root_id, event, ex_id ) resp = http.put(endpoint, body, token=st2_token) if resp.status_code == http_client.OK: LOG.info( '[%s] The workflow event %s for %s is published to st2.', root_id, event, ex_id ) else: raise Exception( '[%s] Unable to publish event because st2 returned ' 'status code %s. %s' % (root_id, resp.status_code, resp.text) )
def on_workflow_status_update(ex_id, data, event, timestamp, **kwargs): root_id = data.get('root_execution_id') or ex_id if root_id != ex_id: LOG.info( '[%s] The workflow event %s for subworkflow %s is ' 'not published to st2. This is expected because it ' 'does not have a corresponding execution record in st2.', root_id, event, ex_id) return wf_ex_env = data['params']['env'] st2_ex_id = wf_ex_env['st2_execution_id'] st2_ctx = wf_ex_env['__actions']['st2.action']['st2_context'] st2_token = st2_ctx.get('auth_token') endpoint = st2_ctx['api_url'] + '/executions/' + st2_ex_id body = { 'status': WF_EX_STATUS_MAP[data['state']], 'result': data.get('output', {}) } body['result']['extra'] = { 'state': data['state'], 'state_info': data['state_info'] } # If workflow is in completed state, then # include task details in the result. if states.is_paused_or_completed(data['state']): with db_api.transaction(): wf_ex = db_api.get_workflow_execution(ex_id) task_exs = [task_ex.to_dict() for task_ex in wf_ex.task_executions] task_exs_result = [ data_flow.get_task_execution_result(task_ex) for task_ex in wf_ex.task_executions ] if task_exs: body['result']['tasks'] = [] for task_ex, task_ex_result in zip(task_exs, task_exs_result): parent_wf_ex_id = task_ex.get('workflow_execution_id', None) task_input = try_json_loads(task_ex.get('input', None)) task_publish = try_json_loads(task_ex.get('published', None)) task_result = { 'id': task_ex['id'], 'name': task_ex['name'], 'workflow_execution_id': parent_wf_ex_id, 'workflow_name': task_ex['workflow_name'], 'created_at': task_ex.get('created_at', None), 'updated_at': task_ex.get('updated_at', None), 'state': task_ex.get('state', None), 'state_info': task_ex.get('state_info', None), 'input': task_input, 'published': task_publish, 'result': task_ex_result } body['result']['tasks'].append(task_result) LOG.info('[%s] The workflow event %s for %s will be published to st2.', root_id, event, ex_id) resp = http.put(endpoint, body, token=st2_token) if resp.status_code == http_client.OK: LOG.info('[%s] The workflow event %s for %s is published to st2.', root_id, event, ex_id) else: raise Exception('[%s] Unable to publish event because st2 returned ' 'status code %s. %s' % (root_id, resp.status_code, resp.text))