def test_filter_from_yaml_string(self):
env = jinja_utils.get_jinja_environment()
expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
obj_yaml_str = ("---\n"
"a: b\n"
"c:\n"
" d: e\n"
" f: 1\n"
" g: true\n")
template = '{{k1 | from_yaml_string}}'
obj_str = env.from_string(template).render({'k1': obj_yaml_str})
obj = eval(obj_str)
self.assertDictEqual(obj, expected_obj)
# With KeyValueLookup object
env = jinja_utils.get_jinja_environment()
obj_yaml_str = ("---\n"
"- a\n"
"- b\n"
"- c\n")
expected_obj = ['a', 'b', 'c']
template = '{{ k1 | from_yaml_string }}'
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix='b')
lookup._value_cache['b'] = obj_yaml_str
obj_str = env.from_string(template).render({'k1': lookup})
obj = eval(obj_str)
self.assertEqual(obj, expected_obj)
def test_hierarchical_lookup_dotted(self):
k1 = KeyValuePair.add_or_update(KeyValuePairDB(name="a.b", value="v1"))
k2 = KeyValuePair.add_or_update(
KeyValuePairDB(name="a.b.c", value="v2"))
k3 = KeyValuePair.add_or_update(KeyValuePairDB(name="b.c", value="v3"))
k4 = KeyValuePair.add_or_update(
KeyValuePairDB(name="stanley:r.i.p",
value="v4",
scope=FULL_USER_SCOPE))
lookup = KeyValueLookup()
self.assertEqual(str(lookup.a.b), k1.value)
self.assertEqual(str(lookup.a.b.c), k2.value)
self.assertEqual(str(lookup.b.c), k3.value)
self.assertEqual(str(lookup.a), "")
# Scoped lookup
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
self.assertEqual(str(lookup.r.i.p), "")
user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user="******")
self.assertEqual(str(user_lookup.r.i.p), k4.value)
def test_lookups_older_scope_names_backward_compatibility(self):
k1 = KeyValuePair.add_or_update(
KeyValuePairDB(name="a.b", value="v1", scope=FULL_SYSTEM_SCOPE))
lookup = KeyValueLookup(scope=SYSTEM_SCOPE)
self.assertEqual(str(lookup["a"]["b"]), k1.value)
k2 = KeyValuePair.add_or_update(
KeyValuePairDB(name="stanley:r.i.p",
value="v4",
scope=FULL_USER_SCOPE))
user_lookup = UserKeyValueLookup(scope=USER_SCOPE, user="******")
self.assertEqual(str(user_lookup["r"]["i"]["p"]), k2.value)
def _construct_context(prefix, data, context):
if data is None:
return context
# setup initial context as system context to help resolve the original context
# which may itself contain references to system variables.
context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
template = jinja2.Template(json.dumps(data))
resolved_data = json.loads(template.render(context))
if resolved_data:
if prefix not in context:
context[prefix] = {}
context[prefix].update(resolved_data)
return context
def _get_rendered_vars(vars, action_parameters):
if not vars:
return {}
context = {}
context.update({
DATASTORE_PARENT_SCOPE: {
SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
}
})
context.update(action_parameters)
LOG.info('Rendering action chain vars. Mapping = %s; Context = %s',
vars, context)
return jinja_utils.render_values(mapping=vars, context=context)
def _resolve_params(action_node, original_parameters, results, chain_vars,
chain_context):
# setup context with original parameters and the intermediate results.
chain_parent = chain_context.get('parent', {})
pack = chain_parent.get('pack')
user = chain_parent.get('user')
config = get_config(pack, user)
context = {}
context.update(original_parameters)
context.update(results)
context.update(chain_vars)
context.update({RESULTS_KEY: results})
context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
context.update({
DATASTORE_PARENT_SCOPE: {
SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
}
})
context.update({ACTION_CONTEXT_KV_PREFIX: chain_context})
context.update({PACK_CONFIG_CONTEXT_KV_PREFIX: config})
try:
rendered_params = jinja_utils.render_values(
mapping=action_node.get_parameters(), context=context)
except Exception as e:
LOG.exception('Jinja rendering for parameter "%s" failed.' %
(e.key))
key = getattr(e, 'key', None)
value = getattr(e, 'value', None)
msg = (
'Failed rendering value for action parameter "%s" in task "%s" '
'(template string=%s): %s') % (key, action_node.name, value,
str(e))
raise ParameterRenderingFailedException(msg)
LOG.debug('Rendered params: %s: Type: %s', rendered_params,
type(rendered_params))
return rendered_params
def test_filter_from_yaml_string(self):
env = jinja_utils.get_jinja_environment()
expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}}
obj_yaml_str = "---\n" "a: b\n" "c:\n" " d: e\n" " f: 1\n" " g: true\n"
template = "{{k1 | from_yaml_string}}"
obj_str = env.from_string(template).render({"k1": obj_yaml_str})
obj = eval(obj_str)
self.assertDictEqual(obj, expected_obj)
# With KeyValueLookup object
env = jinja_utils.get_jinja_environment()
obj_yaml_str = "---\n" "- a\n" "- b\n" "- c\n"
expected_obj = ["a", "b", "c"]
template = "{{ k1 | from_yaml_string }}"
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix="b")
lookup._value_cache["b"] = obj_yaml_str
obj_str = env.from_string(template).render({"k1": lookup})
obj = eval(obj_str)
self.assertEqual(obj, expected_obj)
def _resolve_params(action_node, original_parameters, results, chain_vars):
# setup context with original parameters and the intermediate results.
context = {}
context.update(original_parameters)
context.update(results)
context.update(chain_vars)
context.update({RESULTS_KEY: results})
context.update({SYSTEM_KV_PREFIX: KeyValueLookup()})
rendered_params = jinja_utils.render_values(mapping=action_node.params,
context=context)
LOG.debug('Rendered params: %s: Type: %s', rendered_params,
type(rendered_params))
return rendered_params
def _render_publish_vars(action_node, action_parameters, execution_result,
previous_execution_results, chain_vars):
"""
If no output is specified on the action_node the output is the entire execution_result.
If any output is specified then only those variables are published as output of an
execution of this action_node.
The output variable can refer to a variable from the execution_result,
previous_execution_results or chain_vars.
"""
if not action_node.publish:
return {}
context = {}
context.update(action_parameters)
context.update({action_node.name: execution_result})
context.update(previous_execution_results)
context.update(chain_vars)
context.update({RESULTS_KEY: previous_execution_results})
context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
context.update({
DATASTORE_PARENT_SCOPE: {
SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
}
})
try:
rendered_result = jinja_utils.render_values(
mapping=action_node.publish, context=context)
except Exception as e:
key = getattr(e, 'key', None)
value = getattr(e, 'value', None)
msg = (
'Failed rendering value for publish parameter "%s" in task "%s" '
'(template string=%s): %s' %
(key, action_node.name, value, str(e)))
raise ParameterRenderingFailedException(msg)
return rendered_result
def render_template_with_system_context(value, context=None, prefix=None):
"""
Render provided template with a default system context.
:param value: Template string.
:type value: ``str``
:param context: Template context (optional).
:type context: ``dict``
:param prefix: Datastore key prefix (optional).
:type prefix: ``str``
:rtype: ``str``
"""
context = context or {}
context[SYSTEM_SCOPE] = KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE)
context[DATASTORE_PARENT_SCOPE] = {
SYSTEM_SCOPE: KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE)
}
rendered = render_template(value=value, context=context)
return rendered
def test_filter_from_json_string(self):
env = jinja_utils.get_jinja_environment()
expected_obj = {"a": "b", "c": {"d": "e", "f": 1, "g": True}}
obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}'
template = "{{k1 | from_json_string}}"
obj_str = env.from_string(template).render({"k1": obj_json_str})
obj = eval(obj_str)
self.assertDictEqual(obj, expected_obj)
# With KeyValueLookup object
env = jinja_utils.get_jinja_environment()
obj_json_str = '["a", "b", "c"]'
expected_obj = ["a", "b", "c"]
template = "{{ k1 | from_json_string}}"
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix="a")
lookup._value_cache["a"] = obj_json_str
obj_str = env.from_string(template).render({"k1": lookup})
obj = eval(obj_str)
self.assertEqual(obj, expected_obj)
def test_filter_from_json_string(self):
env = jinja_utils.get_jinja_environment()
expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}'
template = '{{k1 | from_json_string}}'
obj_str = env.from_string(template).render({'k1': obj_json_str})
obj = eval(obj_str)
self.assertDictEqual(obj, expected_obj)
# With KeyValueLookup object
env = jinja_utils.get_jinja_environment()
obj_json_str = '["a", "b", "c"]'
expected_obj = ['a', 'b', 'c']
template = '{{ k1 | from_json_string}}'
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix='a')
lookup._value_cache['a'] = obj_json_str
obj_str = env.from_string(template).render({'k1': lookup})
obj = eval(obj_str)
self.assertEqual(obj, expected_obj)
def test_filter_from_json_string(self):
env = jinja_utils.get_jinja_environment()
expected_obj = {'a': 'b', 'c': {'d': 'e', 'f': 1, 'g': True}}
obj_json_str = '{"a": "b", "c": {"d": "e", "f": 1, "g": true}}'
template = '{{k1 | from_json_string}}'
obj_str = env.from_string(template).render({'k1': obj_json_str})
obj = eval(obj_str)
self.assertDictEqual(obj, expected_obj)
# With KeyValueLookup object
env = jinja_utils.get_jinja_environment()
obj_json_str = '["a", "b", "c"]'
expected_obj = ['a', 'b', 'c']
template = '{{ k1 | from_json_string}}'
lookup = KeyValueLookup(scope=FULL_SYSTEM_SCOPE, key_prefix='a')
lookup._value_cache['a'] = obj_json_str
obj_str = env.from_string(template).render({'k1': lookup})
obj = eval(obj_str)
self.assertEqual(obj, expected_obj)
def test_filter_decrypt_kv(self):
secret = 'Build a wall'
crypto_key_path = cfg.CONF.keyvalue.encryption_key_path
crypto_key = read_crypto_key(key_path=crypto_key_path)
secret_value = symmetric_encrypt(encrypt_key=crypto_key,
plaintext=secret)
KeyValuePair.add_or_update(
KeyValuePairDB(name='k8',
value=secret_value,
scope=FULL_SYSTEM_SCOPE,
secret=True))
env = jinja_utils.get_jinja_environment()
context = {}
context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
context.update({
DATASTORE_PARENT_SCOPE: {
SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
}
})
template = '{{st2kv.system.k8 | decrypt_kv}}'
actual = env.from_string(template).render(context)
self.assertEqual(actual, secret)
def test_filter_decrypt_kv_with_user_scope_value_datastore_value_doesnt_exist(self):
context = {}
context.update({SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)})
context.update({
DATASTORE_PARENT_SCOPE: {
USER_SCOPE: UserKeyValueLookup(user='******', scope=FULL_USER_SCOPE)
}
})
template = '{{st2kv.user.doesnt_exist | decrypt_kv}}'
expected_msg = ('Referenced datastore item "st2kv.user.doesnt_exist" doesn\'t exist or '
'it contains an empty string')
self.assertRaisesRegexp(ValueError, expected_msg, self.env.from_string(template).render,
context)
def render_template_with_system_context(value):
"""
Render provided template with a default system context.
:param value: Template string.
:type value: ``str``
:param context: Template context.
:type context: ``dict``
"""
context = {
SYSTEM_KV_PREFIX: KeyValueLookup(),
}
rendered = render_template(value=value, context=context)
return rendered
def _resolve_params(action_node, original_parameters, results, chain_vars, chain_context):
# setup context with original parameters and the intermediate results.
context = {}
context.update(original_parameters)
context.update(results)
context.update(chain_vars)
context.update({RESULTS_KEY: results})
context.update({SYSTEM_KV_PREFIX: KeyValueLookup()})
context.update({ACTION_KV_PREFIX: chain_context})
try:
rendered_params = jinja_utils.render_values(mapping=action_node.params,
context=context)
except Exception as e:
LOG.exception('Jinja rendering failed.')
raise ParameterRenderingFailedException(e)
LOG.debug('Rendered params: %s: Type: %s', rendered_params, type(rendered_params))
return rendered_params
def _construct_context(prefix, data, context):
if data is None:
return context
context = context or {}
context.update({
DATASTORE_PARENT_SCOPE: {
SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)
}
})
# add in the data in the context without any processing. Payload may
# contain renderable keys however those are often due to nature of the
# events being posted e.g. ActionTrigger with template variables. Rendering
# these values would lead to bugs in the data so best to avoid.
if prefix not in context:
context[prefix] = {}
context[prefix].update(data)
return context
def _render_publish_vars(action_node, execution_result,
previous_execution_results, chain_vars):
"""
If no output is specified on the action_node the output is the entire execution_result.
If any output is specified then only those variables are published as output of an
execution of this action_node.
The output variable can refer to a variable from the execution_result,
previous_execution_results or chain_vars.
"""
if not action_node.publish:
return {}
context = {}
context.update({action_node.name: execution_result})
context.update(previous_execution_results)
context.update(chain_vars)
context.update({RESULTS_KEY: previous_execution_results})
context.update({SYSTEM_KV_PREFIX: KeyValueLookup()})
rendered_result = render_values(action_node.publish, context)
return rendered_result
def test_secret_lookup(self):
secret_value = '0055A2D9A09E1071931925933744965EEA7E23DCF59A8D1D7A3' + \
'64338294916D37E83C4796283C584751750E39844E2FD97A3727DB5D553F638'
k1 = KeyValuePair.add_or_update(
KeyValuePairDB(name='k1', value=secret_value, secret=True))
k2 = KeyValuePair.add_or_update(KeyValuePairDB(name='k2', value='v2'))
k3 = KeyValuePair.add_or_update(
KeyValuePairDB(name='stanley:k3',
value=secret_value,
scope=FULL_USER_SCOPE,
secret=True))
lookup = KeyValueLookup()
self.assertEqual(str(lookup.k1), k1.value)
self.assertEqual(str(lookup.k2), k2.value)
self.assertEqual(str(lookup.k3), '')
user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user='******')
self.assertEqual(str(user_lookup.k3), k3.value)
def _create_graph(action_context, config):
'''
Creates a generic directed graph for depencency tree and fills it with basic context variables
'''
G = nx.DiGraph()
system_keyvalue_context = {SYSTEM_SCOPE: KeyValueLookup(scope=FULL_SYSTEM_SCOPE)}
# If both 'user' and 'api_user' are specified, this prioritize 'api_user'
user = action_context['user'] if 'user' in action_context else None
user = action_context['api_user'] if 'api_user' in action_context else user
if not user:
# When no user is not specified, this selects system-user's scope by default.
user = cfg.CONF.system_user.user
LOG.info('Unable to retrieve user / api_user value from action_context. Falling back '
'to and using system_user (%s).' % (user))
system_keyvalue_context[USER_SCOPE] = UserKeyValueLookup(scope=FULL_USER_SCOPE, user=user)
G.add_node(DATASTORE_PARENT_SCOPE, value=system_keyvalue_context)
G.add_node(ACTION_CONTEXT_KV_PREFIX, value=action_context)
G.add_node(PACK_CONFIG_CONTEXT_KV_PREFIX, value=config)
return G
def test_secret_lookup(self):
secret_value = (
"0055A2D9A09E1071931925933744965EEA7E23DCF59A8D1D7A3" +
"64338294916D37E83C4796283C584751750E39844E2FD97A3727DB5D553F638")
k1 = KeyValuePair.add_or_update(
KeyValuePairDB(name="k1", value=secret_value, secret=True))
k2 = KeyValuePair.add_or_update(KeyValuePairDB(name="k2", value="v2"))
k3 = KeyValuePair.add_or_update(
KeyValuePairDB(
name="stanley:k3",
value=secret_value,
scope=FULL_USER_SCOPE,
secret=True,
))
lookup = KeyValueLookup()
self.assertEqual(str(lookup.k1), k1.value)
self.assertEqual(str(lookup.k2), k2.value)
self.assertEqual(str(lookup.k3), "")
user_lookup = UserKeyValueLookup(scope=FULL_USER_SCOPE, user="******")
self.assertEqual(str(user_lookup.k3), k3.value)
def get_rendered_params(runner_parameters, action_parameters,
runnertype_parameter_info, action_parameter_info):
'''
Renders the templates in runner_parameters and action_parameters. Using the type information
from *_parameter_info will appropriately cast the parameters.
'''
# To render the params it is necessary to combine the params together so that cross
# parameter category references are also rendered correctly. Particularly in the cases where
# a runner parameter is overridden in an action it is likely that a runner parameter could
# depend on an action parameter.
system_context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
renderable_params, context = _renderable_context_param_split(
action_parameters, runner_parameters, system_context)
rendered_params = _do_render_params(renderable_params, context)
template_free_params = {}
template_free_params.update(rendered_params)
template_free_params.update(context)
r_runner_parameters, r_action_parameters = _split_params(
runnertype_parameter_info, action_parameter_info, template_free_params)
return (_cast_params(r_runner_parameters, runnertype_parameter_info),
_cast_params(r_action_parameters, action_parameter_info))
def render_template_with_system_and_user_context(value, user, context=None, prefix=None):
"""
Render provided template with a default system context and user context for the provided user.
:param value: Template string.
:type value: ``str``
:param user: Name (PK) of the user for the user scope.
:type user: ``str``
:param context: Template context (optional).
:type context: ``dict``
:param prefix: Datastore key prefix (optional).
:type prefix: ``str``
:rtype: ``str``
"""
context = context or {}
context[SYSTEM_SCOPE] = KeyValueLookup(prefix=prefix, scope=SYSTEM_SCOPE)
context[USER_SCOPE] = UserKeyValueLookup(prefix=prefix, user=user, scope=USER_SCOPE)
rendered = render_template(value=value, context=context)
return rendered
def _get_rendered_vars(vars):
if not vars:
return {}
context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
return render_values(vars, context)
def __init__(self, payload):
self._context = {TRIGGER_PAYLOAD_PREFIX: payload}
for system_scope in SYSTEM_SCOPES:
self._context[system_scope] = KeyValueLookup(scope=system_scope)
def __init__(self, payload):
self._context = {
SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE),
TRIGGER_PAYLOAD_PREFIX: payload
}
def __init__(self, payload):
self._context = {
SYSTEM_KV_PREFIX: KeyValueLookup(),
TRIGGER_PAYLOAD_PREFIX: payload
}
def _get_rendered_vars(vars, action_parameters):
if not vars:
return {}
context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
context.update(action_parameters)
return jinja_utils.render_values(mapping=vars, context=context)
def _get_rendered_vars(vars):
if not vars:
return {}
context = {SYSTEM_KV_PREFIX: KeyValueLookup()}
return jinja_utils.render_values(mapping=vars, context=context)
def _build_jinja_context(self, liveaction, execution):
context = {SYSTEM_SCOPE: KeyValueLookup(scope=SYSTEM_SCOPE)}
context.update({ACTION_PARAMETERS_KV_PREFIX: liveaction.parameters})
context.update({ACTION_CONTEXT_KV_PREFIX: liveaction.context})
context.update({ACTION_RESULTS_KV_PREFIX: execution.result})
return context
def __call__(self, mapping):
context = copy.copy(self._payload_context)
context[SYSTEM_SCOPE] = KeyValueLookup(scope=SYSTEM_SCOPE)
return jinja_utils.render_values(mapping=mapping, context=context)