def _validate_dependencies(renderable_params, context): ''' Validates dependencies between the parameters. e.g. { 'a': '{{b}}', 'b': '{{a}}' } In this example 'a' requires 'b' for template rendering and vice-versa. There is no way for these templates to be rendered and will be flagged with an ActionRunnerException. ''' env = Environment(undefined=StrictUndefined) dependencies = {} for k, v in six.iteritems(renderable_params): template_ast = env.parse(v) dependencies[k] = meta.find_undeclared_variables(template_ast) for k, v in six.iteritems(dependencies): if not _check_availability(k, v, renderable_params, context): msg = 'Dependecy unsatisfied - %s: %s.' % (k, v) raise actionrunner.ActionRunnerException(msg) dep_chain = [] dep_chain.append(k) if not _check_cyclic(dep_chain, dependencies): msg = 'Cyclic dependecy found - %s.' % dep_chain raise actionrunner.ActionRunnerException(msg)
def _do_render_params(renderable_params, context): ''' Will render the params per the context and will return best attempt to render. Render attempts with missing params will leave blanks. ''' if not renderable_params: return renderable_params _validate_dependencies(renderable_params, context) env = Environment(undefined=StrictUndefined) rendered_params = {} rendered_params.update(context) # Maps parameter key to render exception # We save the exception so we can throw a more meaningful exception at the end if rendering of # some parameter fails parameter_render_exceptions = {} num_parameters = len(renderable_params) + len(context) # After how many attempts at failing to render parameter we should bail out max_rendered_parameters_unchanged_count = num_parameters rendered_params_unchanged_count = 0 while len(renderable_params) != 0: renderable_params_pre_loop = renderable_params.copy() for k, v in six.iteritems(renderable_params): template = env.from_string(v) try: rendered = template.render(rendered_params) rendered_params[k] = rendered if k in parameter_render_exceptions: del parameter_render_exceptions[k] except Exception as e: # Note: This sucks, but because we support multi level and out of order # rendering, we can't throw an exception here yet since the parameter could get # rendered in future iteration LOG.debug('Failed to render %s: %s', k, v, exc_info=True) parameter_render_exceptions[k] = e for k in rendered_params: if k in renderable_params: del renderable_params[k] if renderable_params_pre_loop == renderable_params: rendered_params_unchanged_count += 1 # Make sure we terminate and don't end up in an infinite loop if we # tried to render all the parameters but rendering of some parameters # still fails if rendered_params_unchanged_count >= max_rendered_parameters_unchanged_count: k = parameter_render_exceptions.keys()[0] e = parameter_render_exceptions[k] msg = 'Failed to render parameter "%s": %s' % (k, str(e)) raise actionrunner.ActionRunnerException(msg) return rendered_params
def get_next_node(self, curr_node_name=None, condition='on-success'): if not curr_node_name: return self.get_node(self.actionchain.default) current_node = self.get_node(curr_node_name) if condition == 'on-success': return self.get_node(current_node.on_success, raise_on_failure=True) elif condition == 'on-failure': return self.get_node(current_node.on_failure, raise_on_failure=True) raise runnerexceptions.ActionRunnerException('Unknown condition %s.' % condition)
def get_node(self, node_name=None, raise_on_failure=False): if not node_name: return None for node in self.actionchain.chain: if node.name == node_name: return node if raise_on_failure: raise runnerexceptions.ActionRunnerException( 'Unable to find node with name "%s".' % (node_name)) return None
def _do_run(self, runner, runnertype_db, action_db, liveaction_db): # Create a temporary auth token which will be available # for the duration of the action execution. runner.auth_token = self._create_auth_token( context=runner.context, action_db=action_db, liveaction_db=liveaction_db) try: # Finalized parameters are resolved and then rendered. This process could # fail. Handle the exception and report the error correctly. try: runner_params, action_params = param_utils.render_final_params( runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters, liveaction_db.context) runner.runner_parameters = runner_params except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) LOG.debug('Performing pre-run for runner: %s', runner.runner_id) runner.pre_run() # Mask secret parameters in the log context resolved_action_params = ResolvedActionParameters( action_db=action_db, runner_type_db=runnertype_db, runner_parameters=runner_params, action_parameters=action_params) extra = {'runner': runner, 'parameters': resolved_action_params} LOG.debug('Performing run for runner: %s' % (runner.runner_id), extra=extra) (status, result, context) = runner.run(action_params) try: result = json.loads(result) except: pass action_completed = status in action_constants.LIVEACTION_COMPLETED_STATES if isinstance(runner, AsyncActionRunner) and not action_completed: self._setup_async_query(liveaction_db.id, runnertype_db, context) except: LOG.exception('Failed to run action.') _, ex, tb = sys.exc_info() # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED # include the error message and traceback to try and provide some hints. result = { 'error': str(ex), 'traceback': ''.join(traceback.format_tb(tb, 20)) } context = None finally: # Log action completion extra = {'result': result, 'status': status} LOG.debug('Action "%s" completed.' % (action_db.name), extra=extra) # Update the final status of liveaction and corresponding action execution. liveaction_db = self._update_status(liveaction_db.id, status, result, context) # Always clean-up the auth_token # This method should be called in the finally block to ensure post_run is not impacted. self._clean_up_auth_token(runner=runner, status=status) LOG.debug('Performing post_run for runner: %s', runner.runner_id) runner.post_run(status=status, result=result) LOG.debug('Runner do_run result', extra={'result': liveaction_db.result}) LOG.audit('Liveaction completed', extra={'liveaction_db': liveaction_db}) return liveaction_db
def base(event, context, passthrough=False): # Set up logging logger = logging.getLogger() # Read DEBUG value from the environment variable debug = os.environ.get('ST2_DEBUG', False) if str(debug).lower() in ['true', '1']: debug = True if debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) if isinstance(event, basestring): try: event = json.loads(event) except ValueError as e: LOG.error("ERROR: Can not parse `event`: '{}'\n{}".format( str(event), str(e))) raise e LOG.info("Received event: " + json.dumps(event, indent=2)) # Special case for Lambda function being called over HTTP via API gateway # See # https://serverless.com/framework/docs/providers/aws/events/apigateway # #example-lambda-proxy-event-default # for details is_event_body_string = (isinstance(event.get('body'), basestring) is True) content_type = event.get('headers', {}).get('content-type', '').lower() if is_event_body_string: if content_type == 'application/json': try: event['body'] = json.loads(event['body']) except Exception as e: LOG.warn('`event` has `body` which is not JSON: %s', str(e.message)) elif content_type == 'application/x-www-form-urlencoded': try: event['body'] = dict( parse_qsl(['body'], keep_blank_values=True)) except Exception as e: LOG.warn('`event` has `body` which is not `%s`: %s', content_type, str(e.message)) else: LOG.warn('Unsupported event content type: %s' % (content_type)) action_name = os.environ['ST2_ACTION'] try: action_db = ACTIONS[action_name] except KeyError: raise ValueError('No action named "%s" has been installed.' % (action_name)) manager = DriverManager(namespace='st2common.runners.runner', invoke_on_load=False, name=action_db.runner_type['name']) runnertype_db = RunnerTypeAPI.to_model( RunnerTypeAPI(**manager.driver.get_metadata()[0])) if passthrough: runner = PassthroughRunner() else: runner = manager.driver.get_runner() runner._sandbox = False runner.runner_type_db = runnertype_db runner.action = action_db runner.action_name = action_db.name # runner.liveaction = liveaction_db # runner.liveaction_id = str(liveaction_db.id) # runner.execution = ActionExecution.get(liveaction__id=runner.liveaction_id) # runner.execution_id = str(runner.execution.id) runner.entry_point = content_utils.get_entry_point_abs_path( pack=action_db.pack, entry_point=action_db.entry_point) runner.context = {} # getattr(liveaction_db, 'context', dict()) # runner.callback = getattr(liveaction_db, 'callback', dict()) runner.libs_dir_path = content_utils.get_action_libs_abs_path( pack=action_db.pack, entry_point=action_db.entry_point) # For re-run, get the ActionExecutionDB in which the re-run is based on. # rerun_ref_id = runner.context.get('re-run', {}).get('ref') # runner.rerun_ex_ref = ActionExecution.get(id=rerun_ref_id) if rerun_ref_id else None config_schema = CONFIG_SCHEMAS.get(action_db.pack, None) config_values = os.environ.get('ST2_CONFIG', None) if config_schema and config_values: runner._config = validate_config_against_schema( config_schema=config_schema, config_object=json.loads(config_values), config_path=None, pack_name=action_db.pack) param_values = os.environ.get('ST2_PARAMETERS', None) try: if param_values: live_params = param_utils.render_live_params( runner_parameters=runnertype_db.runner_parameters, action_parameters=action_db.parameters, params=json.loads(param_values), action_context={}, additional_contexts={'input': event}) else: live_params = event if debug and 'log_level' not in live_params: # Set log_level runner parameter live_params['log_level'] = 'DEBUG' runner_params, action_params = param_utils.render_final_params( runner_parameters=runnertype_db.runner_parameters, action_parameters=action_db.parameters, params=live_params, action_context={}) except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) runner.runner_parameters = runner_params LOG.debug('Performing pre-run for runner: %s', runner.runner_id) runner.pre_run() (status, output, context) = runner.run(action_params) output_values = os.environ.get('ST2_OUTPUT', None) if output_values: try: result = param_utils.render_live_params( runner_parameters=runnertype_db.runner_parameters, action_parameters=action_db.parameters, params=json.loads(output_values), action_context={}, additional_contexts={ 'input': event, 'output': output }) except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) else: result = output # Log the logs generated by the action. We do that so the actual action logs # (action stderr) end up in CloudWatch output = output or {} if output.get('stdout', None): LOG.info('Action stdout: %s' % (output['stdout'])) if output.get('stderr', None): LOG.info('Action stderr and logs: %s' % (output['stderr'])) return { 'event': event, 'live_params': live_params, 'output': output, 'result': result }
def _do_run(self, runner): # Create a temporary auth token which will be available # for the duration of the action execution. runner.auth_token = self._create_auth_token( context=runner.context, action_db=runner.action, liveaction_db=runner.liveaction, ) try: # Finalized parameters are resolved and then rendered. This process could # fail. Handle the exception and report the error correctly. try: runner_params, action_params = param_utils.render_final_params( runner.runner_type.runner_parameters, runner.action.parameters, runner.liveaction.parameters, runner.liveaction.context, ) runner.runner_parameters = runner_params except ParamException as e: raise actionrunner.ActionRunnerException(six.text_type(e)) LOG.debug("Performing pre-run for runner: %s", runner.runner_id) runner.pre_run() # Mask secret parameters in the log context resolved_action_params = ResolvedActionParameters( action_db=runner.action, runner_type_db=runner.runner_type, runner_parameters=runner_params, action_parameters=action_params, ) extra = {"runner": runner, "parameters": resolved_action_params} LOG.debug("Performing run for runner: %s" % (runner.runner_id), extra=extra) with CounterWithTimer(key="action.executions"): with CounterWithTimer(key="action.%s.executions" % (runner.action.ref)): (status, result, context) = runner.run(action_params) result = jsonify.try_loads(result) action_completed = status in action_constants.LIVEACTION_COMPLETED_STATES if (isinstance(runner, PollingAsyncActionRunner) and runner.is_polling_enabled() and not action_completed): queries.setup_query(runner.liveaction.id, runner.runner_type, context) except: LOG.exception("Failed to run action.") _, ex, tb = sys.exc_info() # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED # include the error message and traceback to try and provide some hints. result = { "error": str(ex), "traceback": "".join(traceback.format_tb(tb, 20)), } context = None finally: # Log action completion extra = {"result": result, "status": status} LOG.debug('Action "%s" completed.' % (runner.action.name), extra=extra) # Update the final status of liveaction and corresponding action execution. with Timer(key="action.executions.update_status"): runner.liveaction = self._update_status( runner.liveaction.id, status, result, context) # Always clean-up the auth_token # This method should be called in the finally block to ensure post_run is not impacted. self._clean_up_auth_token(runner=runner, status=status) LOG.debug("Performing post_run for runner: %s", runner.runner_id) runner.post_run(status=status, result=result) LOG.debug("Runner do_run result", extra={"result": runner.liveaction.result}) LOG.audit("Liveaction completed", extra={"liveaction_db": runner.liveaction}) return runner.liveaction
def main(): # Read DEBUG value from the environment variable debug = os.environ.get('ST2_DEBUG', False) if str(debug).lower() in ['true', '1']: debug = True if debug: LOG.setLevel(logging.DEBUG) else: LOG.setLevel(logging.INFO) # Read input = os.environ.get('ST2_INPUT', {}) if isinstance(input, six.string_types): try: input = json.loads(input) except ValueError as e: LOG.error("ERROR: Can not parse `input`: '{}'\n{}".format(str(input), str(e))) raise e LOG.debug("Received input: " + json.dumps(input, indent=2)) # Read action name from environment variable action_name = os.environ['ST2_ACTION'] try: action_db = ACTIONS[action_name] except KeyError: raise ValueError('No action named "%s" has been installed.' % (action_name)) # Initialize runner manager = DriverManager(namespace='st2common.runners.runner', invoke_on_load=False, name=action_db.runner_type['name']) runnertype_db = RunnerTypeAPI.to_model(RunnerTypeAPI(**manager.driver.get_metadata())) runner = manager.driver.get_runner() runner._sandbox = False runner.runner_type_db = runnertype_db runner.action = action_db runner.action_name = action_db.name runner.entry_point = content_utils.get_entry_point_abs_path(pack=action_db.pack, entry_point=action_db.entry_point) runner.context = {} runner.libs_dir_path = content_utils.get_action_libs_abs_path(pack=action_db.pack, entry_point=action_db.entry_point) config_schema = CONFIG_SCHEMAS.get(action_db.pack, None) config_values = os.environ.get('ST2_CONFIG', None) if config_schema and config_values: runner._config = validate_config_against_schema(config_schema=config_schema, config_object=json.loads(config_values), config_path=None, pack_name=action_db.pack) param_values = os.environ.get('ST2_PARAMETERS', None) try: if param_values: live_params = param_utils.render_live_params( runner_parameters=runnertype_db.runner_parameters, action_parameters=action_db.parameters, params=json.loads(param_values), action_context={}, additional_contexts={ 'input': input }) else: live_params = input if debug and 'log_level' not in live_params: # Set log_level runner parameter live_params['log_level'] = 'DEBUG' runner_params, action_params = param_utils.render_final_params( runner_parameters=runnertype_db.runner_parameters, action_parameters=action_db.parameters, params=live_params, action_context={}) except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) runner.runner_parameters = runner_params LOG.debug('Performing pre-run for runner: %s', runner.runner_id) runner.pre_run() (status, output, context) = runner.run(action_params) try: output['result'] = json.loads(output['result']) except Exception: pass output_values = os.environ.get('ST2_OUTPUT', None) if output_values: try: result = param_utils.render_live_params( runner_parameters={}, action_parameters={}, params=json.loads(output_values), action_context={}, additional_contexts={ 'input': input, 'output': output }) except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) else: result = output output = output or {} if output.get('stdout', None): LOG.info('Action stdout: %s' % (output['stdout'])) if output.get('stderr', None): LOG.info('Action stderr and logs: %s' % (output['stderr'])) print(json.dumps(result))
def _do_run(self, runner, runnertype_db, action_db, liveaction_db): # Create a temporary auth token which will be available # for the duration of the action execution. runner.auth_token = self._create_auth_token(runner.context) updated_liveaction_db = None try: # Finalized parameters are resolved and then rendered. This process could # fail. Handle the exception and report the error correctly. try: runner_params, action_params = param_utils.render_final_params( runnertype_db.runner_parameters, action_db.parameters, liveaction_db.parameters, liveaction_db.context) runner.runner_parameters = runner_params except ParamException as e: raise actionrunner.ActionRunnerException(str(e)) LOG.debug('Performing pre-run for runner: %s', runner.runner_id) runner.pre_run() # Mask secret parameters in the log context resolved_action_params = ResolvedActionParameters( action_db=action_db, runner_type_db=runnertype_db, runner_parameters=runner_params, action_parameters=action_params) extra = {'runner': runner, 'parameters': resolved_action_params} LOG.debug('Performing run for runner: %s' % (runner.runner_id), extra=extra) (status, result, context) = runner.run(action_params) try: result = json.loads(result) except: pass action_completed = status in action_constants.LIVEACTION_COMPLETED_STATES if isinstance(runner, AsyncActionRunner) and not action_completed: self._setup_async_query(liveaction_db.id, runnertype_db, context) except: LOG.exception('Failed to run action.') _, ex, tb = sys.exc_info() # mark execution as failed. status = action_constants.LIVEACTION_STATUS_FAILED # include the error message and traceback to try and provide some hints. result = { 'error': str(ex), 'traceback': ''.join(traceback.format_tb(tb, 20)) } context = None finally: # Log action completion extra = {'result': result, 'status': status} LOG.debug('Action "%s" completed.' % (action_db.name), extra=extra) # Always clean-up the auth_token try: LOG.debug('Setting status: %s for liveaction: %s', status, liveaction_db.id) updated_liveaction_db = self._update_live_action_db( liveaction_db.id, status, result, context) except: error = 'Cannot update LiveAction object for id: %s, status: %s, result: %s.' % ( liveaction_db.id, status, result) LOG.exception(error) raise executions.update_execution(updated_liveaction_db) extra = {'liveaction_db': updated_liveaction_db} LOG.debug('Updated liveaction after run', extra=extra) # Deletion of the runner generated auth token is delayed until the token expires. # Async actions such as Mistral workflows uses the auth token to launch other # actions in the workflow. If the auth token is deleted here, then the actions # in the workflow will fail with unauthorized exception. is_async_runner = isinstance(runner, AsyncActionRunner) action_completed = status in action_constants.LIVEACTION_COMPLETED_STATES if not is_async_runner or (is_async_runner and action_completed): try: self._delete_auth_token(runner.auth_token) except: LOG.exception('Unable to clean-up auth_token.') LOG.debug('Performing post_run for runner: %s', runner.runner_id) runner.post_run(status, result) runner.container_service = None LOG.debug('Runner do_run result', extra={'result': updated_liveaction_db.result}) LOG.audit('Liveaction completed', extra={'liveaction_db': updated_liveaction_db}) return updated_liveaction_db