def _process(G, name, value): ''' Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly. ''' # Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level # Instead we're just assuming every string to be a unicode string if isinstance(value, str): value = to_unicode(value) complex_value_str = None if isinstance(value, list) or isinstance(value, dict): complex_value_str = str(value) is_jinja_expr = ( jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression( complex_value_str ) ) if is_jinja_expr: G.add_node(name, template=value) template_ast = ENV.parse(value) LOG.debug('Template ast: %s', template_ast) # Dependencies of the node represent jinja variables used in the template # We're connecting nodes with an edge for every depencency to traverse them # in the right order and also make sure that we don't have missing or cyclic # dependencies upfront. dependencies = meta.find_undeclared_variables(template_ast) LOG.debug('Dependencies: %s', dependencies) if dependencies: for dependency in dependencies: G.add_edge(dependency, name) else: G.add_node(name, value=value)
def _process(G, name, value): ''' Determines whether parameter is a template or a value. Adds graph nodes and edges accordingly. ''' # Jinja defaults to ascii parser in python 2.x unless you set utf-8 support on per module level # Instead we're just assuming every string to be a unicode string if isinstance(value, str): value = to_unicode(value) complex_value_str = None if isinstance(value, list) or isinstance(value, dict): complex_value_str = str(value) is_jinja_expr = (jinja_utils.is_jinja_expression(value) or jinja_utils.is_jinja_expression(complex_value_str)) if is_jinja_expr: G.add_node(name, template=value) template_ast = ENV.parse(value) LOG.debug('Template ast: %s', template_ast) # Dependencies of the node represent jinja variables used in the template # We're connecting nodes with an edge for every depencency to traverse them # in the right order and also make sure that we don't have missing or cyclic # dependencies upfront. dependencies = meta.find_undeclared_variables(template_ast) LOG.debug('Dependencies: %s', dependencies) if dependencies: for dependency in dependencies: G.add_edge(dependency, name) else: G.add_node(name, value=value)
def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems( config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression( value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression( value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value # If config_schema is available we do a second pass and set default values for required # items which values are not provided / available in the config itself for schema_item_key, schema_item in six.iteritems(schema_values): default_value = schema_item.get('default', None) is_required = schema_item.get('required', False) if is_required and default_value and not result.get( schema_item_key, None): result[schema_item_key] = default_value return result
def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems(config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression(value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value # If config_schema is available we do a second pass and set default values for required # items which values are not provided / available in the config itself for schema_item_key, schema_item in six.iteritems(schema_values): default_value = schema_item.get('default', None) is_required = schema_item.get('required', False) if is_required and default_value and not result.get(schema_item_key, None): result[schema_item_key] = default_value return result
def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja expression for dynamic config values. Note: This method mutates config argument in place. :rtype: ``dict`` """ parent_keys = parent_keys or [] config_is_dict = isinstance(config, dict) config_is_list = isinstance(config, list) iterator = six.iteritems(config) if config_is_dict else enumerate(config) # config_item_key - if config_is_dict then this is the key in the dictionary # if config_is_list then this is the index of them item # config_item_value - the value of the key/index for the current item for config_item_key, config_item_value in iterator: if config_is_dict: # different schema for each key/value pair schema_item = schema.get(config_item_key, {}) if config_is_list: # same schema is shared between every item in the list schema_item = schema is_dictionary = isinstance(config_item_value, dict) is_list = isinstance(config_item_value, list) # Inspect nested object properties if is_dictionary: parent_keys += [str(config_item_key)] self._assign_dynamic_config_values(schema=schema_item.get('properties', {}), config=config[config_item_key], parent_keys=parent_keys) # Inspect nested list items elif is_list: parent_keys += [str(config_item_key)] self._assign_dynamic_config_values(schema=schema_item.get('items', {}), config=config[config_item_key], parent_keys=parent_keys) else: is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: # Resolve / render the Jinja template expression full_config_item_key = '.'.join(parent_keys + [str(config_item_key)]) value = self._get_datastore_value_for_expression(key=full_config_item_key, value=config_item_value, config_schema_item=schema_item) config[config_item_key] = value else: # Static value, no resolution needed config[config_item_key] = config_item_value return config
def validate_config_against_schema(config_schema, config_object, config_path, pack_name=None): """ Validate provided config dictionary against the provided config schema dictionary. """ # NOTE: Lazy improt to avoid performance overhead of importing this module when it's not used import jsonschema pack_name = pack_name or "unknown" schema = util_schema.get_schema_for_resource_parameters( parameters_schema=config_schema, allow_additional_properties=True) instance = config_object try: cleaned = util_schema.validate( instance=instance, schema=schema, cls=util_schema.CustomValidator, use_default=True, allow_default_none=True, ) for key in cleaned: if (jinja_utils.is_jinja_expression(value=cleaned.get(key)) and "decrypt_kv" in cleaned.get(key) and config_schema.get(key).get("secret")): raise ValueValidationException( 'Values specified as "secret: True" in config ' "schema are automatically decrypted by default. Use " 'of "decrypt_kv" jinja filter is not allowed for ' "such values. Please check the specified values in " "the config or the default values in the schema.") except jsonschema.ValidationError as e: attribute = getattr(e, "path", []) if isinstance(attribute, (tuple, list, Iterable)): attribute = [str(item) for item in attribute] attribute = ".".join(attribute) else: attribute = str(attribute) msg = 'Failed validating attribute "%s" in config for pack "%s" (%s): %s' % ( attribute, pack_name, config_path, six.text_type(e), ) raise jsonschema.ValidationError(msg) return cleaned
def _cast_params_from(params, context, schemas): """ Pick a list of parameters from context and cast each of them according to the schemas provided """ result = {} # First, cast only explicitly provided live parameters for name in params: param_schema = {} for schema in schemas: if name in schema: param_schema = schema[name] result[name] = _cast(context[name], param_schema) # Now, iterate over all parameters, and add any to the live set that satisfy ALL of the # following criteria: # # - Have a default value that is a Jinja template # - Are using the default value (i.e. not being overwritten by an actual live param) # # We do this because the execution API controller first determines live params before # validating params against the schema. So, we want to make sure that if the default # value is a template, it is rendered and added to the live params before this validation. for schema in schemas: for param_name, param_details in schema.items(): # Skip if the parameter have immutable set to true in schema if param_details.get("immutable"): continue # Skip if the parameter doesn't have a default, or if the # value in the context is identical to the default if ( "default" not in param_details or param_details.get("default") == context[param_name] ): continue # Skip if the default value isn't a Jinja expression if not is_jinja_expression(param_details.get("default")): continue # Skip if the parameter is being overridden if param_name in params: continue result[param_name] = _cast(context[param_name], param_details) return result
def _cast_params_from(params, context, schemas): ''' Pick a list of parameters from context and cast each of them according to the schemas provided ''' result = {} # First, cast only explicitly provided live parameters for name in params: param_schema = {} for schema in schemas: if name in schema: param_schema = schema[name] result[name] = _cast(context[name], param_schema) # Now, iterate over all parameters, and add any to the live set that satisfy ALL of the # following criteria: # # - Have a default value that is a Jinja template # - Are using the default value (i.e. not being overwritten by an actual live param) # # We do this because the execution API controller first determines live params before # validating params against the schema. So, we want to make sure that if the default # value is a template, it is rendered and added to the live params before this validation. for schema in schemas: for param_name, param_details in schema.items(): # Skip if the parameter have immutable set to true in schema if param_details.get('immutable'): continue # Skip if the parameter doesn't have a default, or if the # value in the context is identical to the default if 'default' not in param_details or \ param_details.get('default') == context[param_name]: continue # Skip if the default value isn't a Jinja expression if not is_jinja_expression(param_details.get('default')): continue # Skip if the parameter is being overridden if param_name in params: continue result[param_name] = _cast(context[param_name], param_details) return result
def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems(config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression(value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value return result
def _is_valid_node_name(self, all_node_names, node_name): """ Function which validates that the provided node name is defined in the workflow definition and it's valid. Keep in mind that we can only perform validation for task names which don't include jinja expressions since those are rendered at run time. """ if not node_name: # This task name needs to be resolved during run time so we cant validate the name now return True is_jinja_expression = jinja_utils.is_jinja_expression(value=node_name) if is_jinja_expression: # This task name needs to be resolved during run time so we cant validate the name # now return True return node_name in all_node_names
def _get_values_for_config(self, config_schema_db, config_db): schema_values = getattr(config_schema_db, 'attributes', {}) result = {} for config_item_key, config_item_value in six.iteritems( config_db.values): is_jinja_expression = jinja_utils.is_jinja_expression( value=config_item_value) if is_jinja_expression: config_schema_item = schema_values.get(config_item_key, {}) value = self._get_datastore_value_for_expression( value=config_item_value, config_schema_item=config_schema_item) result[config_item_key] = value else: # Static value, no resolution needed result[config_item_key] = config_item_value return result
def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja expression for dynamic config values. Note: This method mutates config argument in place. :rtype: ``dict`` """ parent_keys = parent_keys or [] for config_item_key, config_item_value in six.iteritems(config): schema_item = schema.get(config_item_key, {}) is_dictionary = isinstance(config_item_value, dict) # Inspect nested object properties if is_dictionary: parent_keys += [config_item_key] self._assign_dynamic_config_values( schema=schema_item.get('properties', {}), config=config[config_item_key], parent_keys=parent_keys) else: is_jinja_expression = jinja_utils.is_jinja_expression( value=config_item_value) if is_jinja_expression: # Resolve / render the Jinja template expression full_config_item_key = '.'.join(parent_keys + [config_item_key]) value = self._get_datastore_value_for_expression( key=full_config_item_key, value=config_item_value, config_schema_item=schema_item) config[config_item_key] = value else: # Static value, no resolution needed config[config_item_key] = config_item_value return config
def _assign_dynamic_config_values(self, schema, config, parent_keys=None): """ Assign dynamic config value for a particular config item if the ite utilizes a Jinja expression for dynamic config values. Note: This method mutates config argument in place. :rtype: ``dict`` """ parent_keys = parent_keys or [] for config_item_key, config_item_value in six.iteritems(config): schema_item = schema.get(config_item_key, {}) is_dictionary = isinstance(config_item_value, dict) # Inspect nested object properties if is_dictionary: parent_keys += [config_item_key] self._assign_dynamic_config_values(schema=schema_item.get('properties', {}), config=config[config_item_key], parent_keys=parent_keys) else: is_jinja_expression = jinja_utils.is_jinja_expression(value=config_item_value) if is_jinja_expression: # Resolve / render the Jinja template expression full_config_item_key = '.'.join(parent_keys + [config_item_key]) value = self._get_datastore_value_for_expression(key=full_config_item_key, value=config_item_value, config_schema_item=schema_item) config[config_item_key] = value else: # Static value, no resolution needed config[config_item_key] = config_item_value return config