def _process(G, name, value):
'''
Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
'''
# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
# Instead we're just assuming every string to be a unicode string
if isinstance(value, str):
value = to_unicode(value)
complex_value_str = None
if isinstance(value, list) or isinstance(value, dict):
complex_value_str = str(value)
is_jinja_expr = (
jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(
complex_value_str
)
)
if is_jinja_expr:
G.add_node(name, template=value)
template_ast = ENV.parse(value)
LOG.debug('Template ast: %s', template_ast)
# Dependencies of the node represent jinja variables used in the template
# We're connecting nodes with an edge for every depencency to traverse them
# in the right order and also make sure that we don't have missing or cyclic
# dependencies upfront.
dependencies = meta.find_undeclared_variables(template_ast)
LOG.debug('Dependencies: %s', dependencies)
if dependencies:
for dependency in dependencies:
G.add_edge(dependency, name)
else:
G.add_node(name, value=value)
def _process(G, name, value):
'''
Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly.
'''
# Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level
# Instead we're just assuming every string to be a unicode string
if isinstance(value, str):
value = to_unicode(value)
complex_value_str = None
if isinstance(value, list) or isinstance(value, dict):
complex_value_str = str(value)
is_jinja_expr = (jinja_utils.is_jinja_expression(value)
or jinja_utils.is_jinja_expression(complex_value_str))
if is_jinja_expr:
G.add_node(name, template=value)
template_ast = ENV.parse(value)
LOG.debug('Template ast: %s', template_ast)
# Dependencies of the node represent jinja variables used in the template
# We're connecting nodes with an edge for every depencency to traverse them
# in the right order and also make sure that we don't have missing or cyclic
# dependencies upfront.
dependencies = meta.find_undeclared_variables(template_ast)
LOG.debug('Dependencies: %s', dependencies)
if dependencies:
for dependency in dependencies:
G.add_edge(dependency, name)
else:
G.add_node(name, value=value)
def _get_values_for_config(self, config_schema_db, config_db):
schema_values = getattr(config_schema_db, 'attributes', {})
result = {}
for config_item_key, config_item_value in six.iteritems(
config_db.values):
is_jinja_expression = jinja_utils.is_jinja_expression(
value=config_item_value)
if is_jinja_expression:
config_schema_item = schema_values.get(config_item_key, {})
value = self._get_datastore_value_for_expression(
value=config_item_value,
config_schema_item=config_schema_item)
result[config_item_key] = value
else:
# Static value, no resolution needed
result[config_item_key] = config_item_value
# If config_schema is available we do a second pass and set default values for required
# items which values are not provided / available in the config itself
for schema_item_key, schema_item in six.iteritems(schema_values):
default_value = schema_item.get('default', None)
is_required = schema_item.get('required', False)
if is_required and default_value and not result.get(
schema_item_key, None):
result[schema_item_key] = default_value
return result
def _get_values_for_config(self, config_schema_db, config_db):
schema_values = getattr(config_schema_db, 'attributes', {})
result = {}
for config_item_key, config_item_value in six.iteritems(config_db.values):
is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
if is_jinja_expression:
config_schema_item = schema_values.get(config_item_key, {})
value = self._get_datastore_value_for_expression(value=config_item_value,
config_schema_item=config_schema_item)
result[config_item_key] = value
else:
# Static value, no resolution needed
result[config_item_key] = config_item_value
# If config_schema is available we do a second pass and set default values for required
# items which values are not provided / available in the config itself
for schema_item_key, schema_item in six.iteritems(schema_values):
default_value = schema_item.get('default', None)
is_required = schema_item.get('required', False)
if is_required and default_value and not result.get(schema_item_key, None):
result[schema_item_key] = default_value
return result
def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
"""
Assign dynamic config value for a particular config item if the ite utilizes a Jinja
expression for dynamic config values.
Note: This method mutates config argument in place.
:rtype: ``dict``
"""
parent_keys = parent_keys or []
config_is_dict = isinstance(config, dict)
config_is_list = isinstance(config, list)
iterator = six.iteritems(config) if config_is_dict else enumerate(config)
# config_item_key - if config_is_dict then this is the key in the dictionary
# if config_is_list then this is the index of them item
# config_item_value - the value of the key/index for the current item
for config_item_key, config_item_value in iterator:
if config_is_dict:
# different schema for each key/value pair
schema_item = schema.get(config_item_key, {})
if config_is_list:
# same schema is shared between every item in the list
schema_item = schema
is_dictionary = isinstance(config_item_value, dict)
is_list = isinstance(config_item_value, list)
# Inspect nested object properties
if is_dictionary:
parent_keys += [str(config_item_key)]
self._assign_dynamic_config_values(schema=schema_item.get('properties', {}),
config=config[config_item_key],
parent_keys=parent_keys)
# Inspect nested list items
elif is_list:
parent_keys += [str(config_item_key)]
self._assign_dynamic_config_values(schema=schema_item.get('items', {}),
config=config[config_item_key],
parent_keys=parent_keys)
else:
is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
if is_jinja_expression:
# Resolve / render the Jinja template expression
full_config_item_key = '.'.join(parent_keys + [str(config_item_key)])
value = self._get_datastore_value_for_expression(key=full_config_item_key,
value=config_item_value,
config_schema_item=schema_item)
config[config_item_key] = value
else:
# Static value, no resolution needed
config[config_item_key] = config_item_value
return config
def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
"""
Assign dynamic config value for a particular config item if the ite utilizes a Jinja
expression for dynamic config values.
Note: This method mutates config argument in place.
:rtype: ``dict``
"""
parent_keys = parent_keys or []
config_is_dict = isinstance(config, dict)
config_is_list = isinstance(config, list)
iterator = six.iteritems(config) if config_is_dict else enumerate(config)
# config_item_key - if config_is_dict then this is the key in the dictionary
# if config_is_list then this is the index of them item
# config_item_value - the value of the key/index for the current item
for config_item_key, config_item_value in iterator:
if config_is_dict:
# different schema for each key/value pair
schema_item = schema.get(config_item_key, {})
if config_is_list:
# same schema is shared between every item in the list
schema_item = schema
is_dictionary = isinstance(config_item_value, dict)
is_list = isinstance(config_item_value, list)
# Inspect nested object properties
if is_dictionary:
parent_keys += [str(config_item_key)]
self._assign_dynamic_config_values(schema=schema_item.get('properties', {}),
config=config[config_item_key],
parent_keys=parent_keys)
# Inspect nested list items
elif is_list:
parent_keys += [str(config_item_key)]
self._assign_dynamic_config_values(schema=schema_item.get('items', {}),
config=config[config_item_key],
parent_keys=parent_keys)
else:
is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
if is_jinja_expression:
# Resolve / render the Jinja template expression
full_config_item_key = '.'.join(parent_keys + [str(config_item_key)])
value = self._get_datastore_value_for_expression(key=full_config_item_key,
value=config_item_value,
config_schema_item=schema_item)
config[config_item_key] = value
else:
# Static value, no resolution needed
config[config_item_key] = config_item_value
return config
def validate_config_against_schema(config_schema,
config_object,
config_path,
pack_name=None):
"""
Validate provided config dictionary against the provided config schema
dictionary.
"""
# NOTE: Lazy improt to avoid performance overhead of importing this module when it's not used
import jsonschema
pack_name = pack_name or "unknown"
schema = util_schema.get_schema_for_resource_parameters(
parameters_schema=config_schema, allow_additional_properties=True)
instance = config_object
try:
cleaned = util_schema.validate(
instance=instance,
schema=schema,
cls=util_schema.CustomValidator,
use_default=True,
allow_default_none=True,
)
for key in cleaned:
if (jinja_utils.is_jinja_expression(value=cleaned.get(key))
and "decrypt_kv" in cleaned.get(key)
and config_schema.get(key).get("secret")):
raise ValueValidationException(
'Values specified as "secret: True" in config '
"schema are automatically decrypted by default. Use "
'of "decrypt_kv" jinja filter is not allowed for '
"such values. Please check the specified values in "
"the config or the default values in the schema.")
except jsonschema.ValidationError as e:
attribute = getattr(e, "path", [])
if isinstance(attribute, (tuple, list, Iterable)):
attribute = [str(item) for item in attribute]
attribute = ".".join(attribute)
else:
attribute = str(attribute)
msg = 'Failed validating attribute "%s" in config for pack "%s" (%s): %s' % (
attribute,
pack_name,
config_path,
six.text_type(e),
)
raise jsonschema.ValidationError(msg)
return cleaned
def _cast_params_from(params, context, schemas):
"""
Pick a list of parameters from context and cast each of them according to the schemas provided
"""
result = {}
# First, cast only explicitly provided live parameters
for name in params:
param_schema = {}
for schema in schemas:
if name in schema:
param_schema = schema[name]
result[name] = _cast(context[name], param_schema)
# Now, iterate over all parameters, and add any to the live set that satisfy ALL of the
# following criteria:
#
# - Have a default value that is a Jinja template
# - Are using the default value (i.e. not being overwritten by an actual live param)
#
# We do this because the execution API controller first determines live params before
# validating params against the schema. So, we want to make sure that if the default
# value is a template, it is rendered and added to the live params before this validation.
for schema in schemas:
for param_name, param_details in schema.items():
# Skip if the parameter have immutable set to true in schema
if param_details.get("immutable"):
continue
# Skip if the parameter doesn't have a default, or if the
# value in the context is identical to the default
if (
"default" not in param_details
or param_details.get("default") == context[param_name]
):
continue
# Skip if the default value isn't a Jinja expression
if not is_jinja_expression(param_details.get("default")):
continue
# Skip if the parameter is being overridden
if param_name in params:
continue
result[param_name] = _cast(context[param_name], param_details)
return result
def _cast_params_from(params, context, schemas):
'''
Pick a list of parameters from context and cast each of them according to the schemas provided
'''
result = {}
# First, cast only explicitly provided live parameters
for name in params:
param_schema = {}
for schema in schemas:
if name in schema:
param_schema = schema[name]
result[name] = _cast(context[name], param_schema)
# Now, iterate over all parameters, and add any to the live set that satisfy ALL of the
# following criteria:
#
# - Have a default value that is a Jinja template
# - Are using the default value (i.e. not being overwritten by an actual live param)
#
# We do this because the execution API controller first determines live params before
# validating params against the schema. So, we want to make sure that if the default
# value is a template, it is rendered and added to the live params before this validation.
for schema in schemas:
for param_name, param_details in schema.items():
# Skip if the parameter have immutable set to true in schema
if param_details.get('immutable'):
continue
# Skip if the parameter doesn't have a default, or if the
# value in the context is identical to the default
if 'default' not in param_details or \
param_details.get('default') == context[param_name]:
continue
# Skip if the default value isn't a Jinja expression
if not is_jinja_expression(param_details.get('default')):
continue
# Skip if the parameter is being overridden
if param_name in params:
continue
result[param_name] = _cast(context[param_name], param_details)
return result
def _get_values_for_config(self, config_schema_db, config_db):
schema_values = getattr(config_schema_db, 'attributes', {})
result = {}
for config_item_key, config_item_value in six.iteritems(config_db.values):
is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
if is_jinja_expression:
config_schema_item = schema_values.get(config_item_key, {})
value = self._get_datastore_value_for_expression(value=config_item_value,
config_schema_item=config_schema_item)
result[config_item_key] = value
else:
# Static value, no resolution needed
result[config_item_key] = config_item_value
return result
def _is_valid_node_name(self, all_node_names, node_name):
"""
Function which validates that the provided node name is defined in the workflow definition
and it's valid.
Keep in mind that we can only perform validation for task names which don't include jinja
expressions since those are rendered at run time.
"""
if not node_name:
# This task name needs to be resolved during run time so we cant validate the name now
return True
is_jinja_expression = jinja_utils.is_jinja_expression(value=node_name)
if is_jinja_expression:
# This task name needs to be resolved during run time so we cant validate the name
# now
return True
return node_name in all_node_names
def _is_valid_node_name(self, all_node_names, node_name):
"""
Function which validates that the provided node name is defined in the workflow definition
and it's valid.
Keep in mind that we can only perform validation for task names which don't include jinja
expressions since those are rendered at run time.
"""
if not node_name:
# This task name needs to be resolved during run time so we cant validate the name now
return True
is_jinja_expression = jinja_utils.is_jinja_expression(value=node_name)
if is_jinja_expression:
# This task name needs to be resolved during run time so we cant validate the name
# now
return True
return node_name in all_node_names
def _get_values_for_config(self, config_schema_db, config_db):
schema_values = getattr(config_schema_db, 'attributes', {})
result = {}
for config_item_key, config_item_value in six.iteritems(
config_db.values):
is_jinja_expression = jinja_utils.is_jinja_expression(
value=config_item_value)
if is_jinja_expression:
config_schema_item = schema_values.get(config_item_key, {})
value = self._get_datastore_value_for_expression(
value=config_item_value,
config_schema_item=config_schema_item)
result[config_item_key] = value
else:
# Static value, no resolution needed
result[config_item_key] = config_item_value
return result
def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
"""
Assign dynamic config value for a particular config item if the ite utilizes a Jinja
expression for dynamic config values.
Note: This method mutates config argument in place.
:rtype: ``dict``
"""
parent_keys = parent_keys or []
for config_item_key, config_item_value in six.iteritems(config):
schema_item = schema.get(config_item_key, {})
is_dictionary = isinstance(config_item_value, dict)
# Inspect nested object properties
if is_dictionary:
parent_keys += [config_item_key]
self._assign_dynamic_config_values(
schema=schema_item.get('properties', {}),
config=config[config_item_key],
parent_keys=parent_keys)
else:
is_jinja_expression = jinja_utils.is_jinja_expression(
value=config_item_value)
if is_jinja_expression:
# Resolve / render the Jinja template expression
full_config_item_key = '.'.join(parent_keys +
[config_item_key])
value = self._get_datastore_value_for_expression(
key=full_config_item_key,
value=config_item_value,
config_schema_item=schema_item)
config[config_item_key] = value
else:
# Static value, no resolution needed
config[config_item_key] = config_item_value
return config
def _assign_dynamic_config_values(self, schema, config, parent_keys=None):
"""
Assign dynamic config value for a particular config item if the ite utilizes a Jinja
expression for dynamic config values.
Note: This method mutates config argument in place.
:rtype: ``dict``
"""
parent_keys = parent_keys or []
for config_item_key, config_item_value in six.iteritems(config):
schema_item = schema.get(config_item_key, {})
is_dictionary = isinstance(config_item_value, dict)
# Inspect nested object properties
if is_dictionary:
parent_keys += [config_item_key]
self._assign_dynamic_config_values(schema=schema_item.get('properties', {}),
config=config[config_item_key],
parent_keys=parent_keys)
else:
is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value)
if is_jinja_expression:
# Resolve / render the Jinja template expression
full_config_item_key = '.'.join(parent_keys + [config_item_key])
value = self._get_datastore_value_for_expression(key=full_config_item_key,
value=config_item_value,
config_schema_item=schema_item)
config[config_item_key] = value
else:
# Static value, no resolution needed
config[config_item_key] = config_item_value
return config
