def test_filter_from_yaml_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} obj_yaml_str = ("---\n" "a: b\n" "c:\n" " d: e\n" " f: 1\n" " g: true\n") template = '{{k1 | from_yaml_string}}' obj_str = env.from_string(template).render({'k1': obj_yaml_str}) obj = eval(obj_str) self.assertDictEqual(obj, expected_obj) # With KeyValueLookup object env = jinja_utils.get_jinja_environment() obj_yaml_str = ("---\n" "- a\n" "- b\n" "- c\n") expected_obj = ['a', 'b', 'c'] template = '{{ k1 | from_yaml_string }}' lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix='b') lookup._value_cache['b'] = obj_yaml_str obj_str = env.from_string(template).render({'k1': lookup}) obj = eval(obj_str) self.assertEqual(obj, expected_obj)
def test_hierarchical_lookup_dotted(self): k1 = KeyValuePair.add_or_update(KeyValuePairDB(name="a.b", value="v1")) k2 = KeyValuePair.add_or_update( KeyValuePairDB(name="a.b.c", value="v2")) k3 = KeyValuePair.add_or_update(KeyValuePairDB(name="b.c", value="v3")) k4 = KeyValuePair.add_or_update( KeyValuePairDB(name="stanley:r.i.p", value="v4", scope=FULL_USER_SCOPE)) lookup = KeyValueLookup() self.assertEqual(str(lookup.a.b), k1.value) self.assertEqual(str(lookup.a.b.c), k2.value) self.assertEqual(str(lookup.b.c), k3.value) self.assertEqual(str(lookup.a), "") # Scoped lookup lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE) self.assertEqual(str(lookup.r.i.p), "") user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user="******") self.assertEqual(str(user_lookup.r.i.p), k4.value)
def test_lookups_older_scope_names_backward_compatibility(self): k1 = KeyValuePair.add_or_update( KeyValuePairDB(name="a.b", value="v1", scope=FULL_SYSTEM_SCOPE)) lookup = KeyValueLookup(scope=SYSTEM_SCOPE) self.assertEqual(str(lookup["a"]["b"]), k1.value) k2 = KeyValuePair.add_or_update( KeyValuePairDB(name="stanley:r.i.p", value="v4", scope=FULL_USER_SCOPE)) user_lookup = UserKeyValueLookup(scope=USER_SCOPE, user="******") self.assertEqual(str(user_lookup["r"]["i"]["p"]), k2.value)
def _construct_context(prefix, data, context): if data is None: return context # setup initial context as system context to help resolve the original context # which may itself contain references to system variables. context = {SYSTEM_KV_PREFIX: KeyValueLookup()} template = jinja2.Template(json.dumps(data)) resolved_data = json.loads(template.render(context)) if resolved_data: if prefix not in context: context[prefix] = {} context[prefix].update(resolved_data) return context
def _get_rendered_vars(vars, action_parameters): if not vars: return {} context = {} context.update({ DATASTORE_PARENT_SCOPE: { SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE) } }) context.update(action_parameters) LOG.info('Rendering action chain vars. Mapping = %s; Context = %s', vars, context) return jinja_utils.render_values(mapping=vars, context=context)
def _resolve_params(action_node, original_parameters, results, chain_vars, chain_context): # setup context with original parameters and the intermediate results. chain_parent = chain_context.get('parent', {}) pack = chain_parent.get('pack') user = chain_parent.get('user') config = get_config(pack, user) context = {} context.update(original_parameters) context.update(results) context.update(chain_vars) context.update({RESULTS_KEY: results}) context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}) context.update({ DATASTORE_PARENT_SCOPE: { SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE) } }) context.update({ACTION_CONTEXT_KV_PREFIX: chain_context}) context.update({PACK_CONFIG_CONTEXT_KV_PREFIX: config}) try: rendered_params = jinja_utils.render_values( mapping=action_node.get_parameters(), context=context) except Exception as e: LOG.exception('Jinja rendering for parameter "%s" failed.' % (e.key)) key = getattr(e, 'key', None) value = getattr(e, 'value', None) msg = ( 'Failed rendering value for action parameter "%s" in task "%s" ' '(template string=%s): %s') % (key, action_node.name, value, str(e)) raise ParameterRenderingFailedException(msg) LOG.debug('Rendered params: %s: Type: %s', rendered_params, type(rendered_params)) return rendered_params
def test_filter_from_yaml_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}} obj_yaml_str = "---\n" "a: b\n" "c:\n" " d: e\n" " f: 1\n" " g: true\n" template = "{{k1 | from_yaml_string}}" obj_str = env.from_string(template).render({"k1": obj_yaml_str}) obj = eval(obj_str) self.assertDictEqual(obj, expected_obj) # With KeyValueLookup object env = jinja_utils.get_jinja_environment() obj_yaml_str = "---\n" "- a\n" "- b\n" "- c\n" expected_obj = ["a", "b", "c"] template = "{{ k1 | from_yaml_string }}" lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix="b") lookup._value_cache["b"] = obj_yaml_str obj_str = env.from_string(template).render({"k1": lookup}) obj = eval(obj_str) self.assertEqual(obj, expected_obj)
def _resolve_params(action_node, original_parameters, results, chain_vars): # setup context with original parameters and the intermediate results. context = {} context.update(original_parameters) context.update(results) context.update(chain_vars) context.update({RESULTS_KEY: results}) context.update({SYSTEM_KV_PREFIX: KeyValueLookup()}) rendered_params = jinja_utils.render_values(mapping=action_node.params, context=context) LOG.debug('Rendered params: %s: Type: %s', rendered_params, type(rendered_params)) return rendered_params
def _render_publish_vars(action_node, action_parameters, execution_result, previous_execution_results, chain_vars): """ If no output is specified on the action_node the output is the entire execution_result. If any output is specified then only those variables are published as output of an execution of this action_node. The output variable can refer to a variable from the execution_result, previous_execution_results or chain_vars. """ if not action_node.publish: return {} context = {} context.update(action_parameters) context.update({action_node.name: execution_result}) context.update(previous_execution_results) context.update(chain_vars) context.update({RESULTS_KEY: previous_execution_results}) context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}) context.update({ DATASTORE_PARENT_SCOPE: { SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE) } }) try: rendered_result = jinja_utils.render_values( mapping=action_node.publish, context=context) except Exception as e: key = getattr(e, 'key', None) value = getattr(e, 'value', None) msg = ( 'Failed rendering value for publish parameter "%s" in task "%s" ' '(template string=%s): %s' % (key, action_node.name, value, str(e))) raise ParameterRenderingFailedException(msg) return rendered_result
def render_template_with_system_context(value, context=None, prefix=None): """ Render provided template with a default system context. :param value: Template string. :type value: ``str`` :param context: Template context (optional). :type context: ``dict`` :param prefix: Datastore key prefix (optional). :type prefix: ``str`` :rtype: ``str`` """ context = context or {} context[SYSTEM_SCOPE] = KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE) context[DATASTORE_PARENT_SCOPE] = { SYSTEM_SCOPE: KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE) } rendered = render_template(value=value, context=context) return rendered
def test_filter_from_json_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}} obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}' template = "{{k1 | from_json_string}}" obj_str = env.from_string(template).render({"k1": obj_json_str}) obj = eval(obj_str) self.assertDictEqual(obj, expected_obj) # With KeyValueLookup object env = jinja_utils.get_jinja_environment() obj_json_str = '["a", "b", "c"]' expected_obj = ["a", "b", "c"] template = "{{ k1 | from_json_string}}" lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix="a") lookup._value_cache["a"] = obj_json_str obj_str = env.from_string(template).render({"k1": lookup}) obj = eval(obj_str) self.assertEqual(obj, expected_obj)
def test_filter_from_json_string(self): env = jinja_utils.get_jinja_environment() expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}} obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}' template = '{{k1 | from_json_string}}' obj_str = env.from_string(template).render({'k1': obj_json_str}) obj = eval(obj_str) self.assertDictEqual(obj, expected_obj) # With KeyValueLookup object env = jinja_utils.get_jinja_environment() obj_json_str = '["a", "b", "c"]' expected_obj = ['a', 'b', 'c'] template = '{{ k1 | from_json_string}}' lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix='a') lookup._value_cache['a'] = obj_json_str obj_str = env.from_string(template).render({'k1': lookup}) obj = eval(obj_str) self.assertEqual(obj, expected_obj)
def test_filter_decrypt_kv(self): secret = 'Build a wall' crypto_key_path = cfg.CONF.keyvalue.encryption_key_path crypto_key = read_crypto_key(key_path=crypto_key_path) secret_value = symmetric_encrypt(encrypt_key=crypto_key, plaintext=secret) KeyValuePair.add_or_update( KeyValuePairDB(name='k8', value=secret_value, scope=FULL_SYSTEM_SCOPE, secret=True)) env = jinja_utils.get_jinja_environment() context = {} context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}) context.update({ DATASTORE_PARENT_SCOPE: { SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE) } }) template = '{{st2kv.system.k8 | decrypt_kv}}' actual = env.from_string(template).render(context) self.assertEqual(actual, secret)
def test_filter_decrypt_kv_with_user_scope_value_datastore_value_doesnt_exist(self): context = {} context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}) context.update({ DATASTORE_PARENT_SCOPE: { USER_SCOPE: UserKeyValueLookup(user='******', scope=FULL_USER_SCOPE) } }) template = '{{st2kv.user.doesnt_exist | decrypt_kv}}' expected_msg = ('Referenced datastore item "st2kv.user.doesnt_exist" doesn\'t exist or ' 'it contains an empty string') self.assertRaisesRegexp(ValueError, expected_msg, self.env.from_string(template).render, context)
def render_template_with_system_context(value): """ Render provided template with a default system context. :param value: Template string. :type value: ``str`` :param context: Template context. :type context: ``dict`` """ context = { SYSTEM_KV_PREFIX: KeyValueLookup(), } rendered = render_template(value=value, context=context) return rendered
def _resolve_params(action_node, original_parameters, results, chain_vars, chain_context): # setup context with original parameters and the intermediate results. context = {} context.update(original_parameters) context.update(results) context.update(chain_vars) context.update({RESULTS_KEY: results}) context.update({SYSTEM_KV_PREFIX: KeyValueLookup()}) context.update({ACTION_KV_PREFIX: chain_context}) try: rendered_params = jinja_utils.render_values(mapping=action_node.params, context=context) except Exception as e: LOG.exception('Jinja rendering failed.') raise ParameterRenderingFailedException(e) LOG.debug('Rendered params: %s: Type: %s', rendered_params, type(rendered_params)) return rendered_params
def _construct_context(prefix, data, context): if data is None: return context context = context or {} context.update({ DATASTORE_PARENT_SCOPE: { SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE) } }) # add in the data in the context without any processing. Payload may # contain renderable keys however those are often due to nature of the # events being posted e.g. ActionTrigger with template variables. Rendering # these values would lead to bugs in the data so best to avoid. if prefix not in context: context[prefix] = {} context[prefix].update(data) return context
def _render_publish_vars(action_node, execution_result, previous_execution_results, chain_vars): """ If no output is specified on the action_node the output is the entire execution_result. If any output is specified then only those variables are published as output of an execution of this action_node. The output variable can refer to a variable from the execution_result, previous_execution_results or chain_vars. """ if not action_node.publish: return {} context = {} context.update({action_node.name: execution_result}) context.update(previous_execution_results) context.update(chain_vars) context.update({RESULTS_KEY: previous_execution_results}) context.update({SYSTEM_KV_PREFIX: KeyValueLookup()}) rendered_result = render_values(action_node.publish, context) return rendered_result
def test_secret_lookup(self): secret_value = '0055A2D9A09E1071931925933744965EEA7E23DCF59A8D1D7A3' + \ '64338294916D37E83C4796283C584751750E39844E2FD97A3727DB5D553F638' k1 = KeyValuePair.add_or_update( KeyValuePairDB(name='k1', value=secret_value, secret=True)) k2 = KeyValuePair.add_or_update(KeyValuePairDB(name='k2', value='v2')) k3 = KeyValuePair.add_or_update( KeyValuePairDB(name='stanley:k3', value=secret_value, scope=FULL_USER_SCOPE, secret=True)) lookup = KeyValueLookup() self.assertEqual(str(lookup.k1), k1.value) self.assertEqual(str(lookup.k2), k2.value) self.assertEqual(str(lookup.k3), '') user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user='******') self.assertEqual(str(user_lookup.k3), k3.value)
def _create_graph(action_context, config): ''' Creates a generic directed graph for depencency tree and fills it with basic context variables ''' G = nx.DiGraph() system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)} # If both 'user' and 'api_user' are specified, this prioritize 'api_user' user = action_context['user'] if 'user' in action_context else None user = action_context['api_user'] if 'api_user' in action_context else user if not user: # When no user is not specified, this selects system-user's scope by default. user = cfg.CONF.system_user.user LOG.info('Unable to retrieve user / api_user value from action_context. Falling back ' 'to and using system_user (%s).' % (user)) system_keyvalue_context[USER_SCOPE] = UserKeyValueLookup(scope=FULL_USER_SCOPE, user=user) G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context) G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context) G.add_node(PACK_CONFIG_CONTEXT_KV_PREFIX, value=config) return G
def test_secret_lookup(self): secret_value = ( "0055A2D9A09E1071931925933744965EEA7E23DCF59A8D1D7A3" + "64338294916D37E83C4796283C584751750E39844E2FD97A3727DB5D553F638") k1 = KeyValuePair.add_or_update( KeyValuePairDB(name="k1", value=secret_value, secret=True)) k2 = KeyValuePair.add_or_update(KeyValuePairDB(name="k2", value="v2")) k3 = KeyValuePair.add_or_update( KeyValuePairDB( name="stanley:k3", value=secret_value, scope=FULL_USER_SCOPE, secret=True, )) lookup = KeyValueLookup() self.assertEqual(str(lookup.k1), k1.value) self.assertEqual(str(lookup.k2), k2.value) self.assertEqual(str(lookup.k3), "") user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user="******") self.assertEqual(str(user_lookup.k3), k3.value)
def get_rendered_params(runner_parameters, action_parameters, runnertype_parameter_info, action_parameter_info): ''' Renders the templates in runner_parameters and action_parameters. Using the type information from *_parameter_info will appropriately cast the parameters. ''' # To render the params it is necessary to combine the params together so that cross # parameter category references are also rendered correctly. Particularly in the cases where # a runner parameter is overridden in an action it is likely that a runner parameter could # depend on an action parameter. system_context = {SYSTEM_KV_PREFIX: KeyValueLookup()} renderable_params, context = _renderable_context_param_split( action_parameters, runner_parameters, system_context) rendered_params = _do_render_params(renderable_params, context) template_free_params = {} template_free_params.update(rendered_params) template_free_params.update(context) r_runner_parameters, r_action_parameters = _split_params( runnertype_parameter_info, action_parameter_info, template_free_params) return (_cast_params(r_runner_parameters, runnertype_parameter_info), _cast_params(r_action_parameters, action_parameter_info))
def render_template_with_system_and_user_context(value, user, context=None, prefix=None): """ Render provided template with a default system context and user context for the provided user. :param value: Template string. :type value: ``str`` :param user: Name (PK) of the user for the user scope. :type user: ``str`` :param context: Template context (optional). :type context: ``dict`` :param prefix: Datastore key prefix (optional). :type prefix: ``str`` :rtype: ``str`` """ context = context or {} context[SYSTEM_SCOPE] = KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE) context[USER_SCOPE] = UserKeyValueLookup(prefix=prefix, user=user, scope=USER_SCOPE) rendered = render_template(value=value, context=context) return rendered
def _get_rendered_vars(vars): if not vars: return {} context = {SYSTEM_KV_PREFIX: KeyValueLookup()} return render_values(vars, context)
def __init__(self, payload): self._context = {TRIGGER_PAYLOAD_PREFIX: payload} for system_scope in SYSTEM_SCOPES: self._context[system_scope] = KeyValueLookup(scope=system_scope)
def __init__(self, payload): self._context = { SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE), TRIGGER_PAYLOAD_PREFIX: payload }
def __init__(self, payload): self._context = { SYSTEM_KV_PREFIX: KeyValueLookup(), TRIGGER_PAYLOAD_PREFIX: payload }
def _get_rendered_vars(vars, action_parameters): if not vars: return {} context = {SYSTEM_KV_PREFIX: KeyValueLookup()} context.update(action_parameters) return jinja_utils.render_values(mapping=vars, context=context)
def _get_rendered_vars(vars): if not vars: return {} context = {SYSTEM_KV_PREFIX: KeyValueLookup()} return jinja_utils.render_values(mapping=vars, context=context)
def _build_jinja_context(self, liveaction, execution): context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)} context.update({ACTION_PARAMETERS_KV_PREFIX: liveaction.parameters}) context.update({ACTION_CONTEXT_KV_PREFIX: liveaction.context}) context.update({ACTION_RESULTS_KV_PREFIX: execution.result}) return context
def __call__(self, mapping): context = copy.copy(self._payload_context) context[SYSTEM_SCOPE] = KeyValueLookup(scope=SYSTEM_SCOPE) return jinja_utils.render_values(mapping=mapping, context=context)