def validate_next_value(schema_data, value): schema_name = schema_data.get('name') schema_ctx = schema_data.get('ctx') schema_info = schema_data.get('info') schema_info_dict = schema_data.get('dict') is_prop = schema_data.get('prop') is_subelement = schema_data.get('subelement') is_simple = schema_data.get('simple') required = schema_data.get('required') schema_suffix = '' if is_prop: schema_suffix = ' (prop)' elif is_subelement: schema_suffix = ' (subelement)' if schema_info is None: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: schema is not defined' ]] non_empty = schema_info.get('non_empty') if required or non_empty: if value is None: non_empty_info = ' (' + ('non_empty' if non_empty else 'required') + ')' return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('msg: value is not defined' + non_empty_info) ]] if (not is_subelement) and (not is_prop) and (not is_simple) and ( 'schema' in schema_info): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a schema definition should not have a schema property' ]] value_type = schema_info.get('type') choices = schema_info.get('choices') regex = schema_info.get('regex') minimum = to_int(schema_info.get('min')) maximum = to_int(schema_info.get('max')) next_schema = schema_info.get('schema') if (not value_type) and (not next_schema): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have either a type or schema property' ]] elif value_type and next_schema: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should not have both type and schema properties' ]] if not value_type: if choices: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have choices only when type is defined' ]] elif regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have regex only when type is defined' ]] elif minimum is not None: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have min only when type is defined' ]] elif maximum is not None: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have max only when type is defined' ]] primitive_types = [ 'primitive', 'str', 'bool', 'int', 'float', ] if choices: is_list_type = (value_type in ['list', 'simple_list']) if (value_type not in primitive_types) and not is_list_type: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: value type is not primitive nor a list but has choices', ]] elif is_list_type and (schema_info.get('elem_schema')): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: value type is a list with elem_schema defined but has choices', 'tip: a list with choices should have only elem_type defined', ]] if regex and (value_type != 'str'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: regex should only be specified for a string (str) type', ]] valid_min_max_types = ['str', 'int', 'float'] if value_type not in valid_min_max_types: if minimum is not None: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: min is specified for an invalid type', 'allowed types:', valid_min_max_types, ]] if maximum is not None: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: max is specified for an invalid type', 'allowed types:', valid_min_max_types, ]] alternative_type = schema_info.get('alternative_type') alternative_choices = schema_info.get('alternative_choices') alternative_regex = schema_info.get('alternative_regex') alternative_min = schema_info.get('alternative_min') alternative_max = schema_info.get('alternative_max') main_schema = schema_info.get('main_schema') alternative_schema = schema_info.get('alternative_schema') if (value_type == 'simple_dict') and (not alternative_type) and ( not alternative_schema): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a simple_dict must have an alternative type or alternative schema' ]] elif main_schema and (value_type != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a main schema should be defined only for simple_dict' ]] elif alternative_type and (value_type != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: an alternative type should be defined only for simple_dict' ]] elif alternative_schema and (value_type != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: an alternative schema should be defined only for simple_dict' ]] elif alternative_type and alternative_schema: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: define only an alternative type or alternative schema, not both' ]] if not alternative_type: if alternative_choices: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have alternative_choices only ' + 'when alternative_type is defined' ]] elif alternative_regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have alternative_regex only ' + 'when alternative_type is defined' ]] elif alternative_min: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have alternative_min only ' + 'when alternative_type is defined' ]] elif alternative_max: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have alternative_max only ' + 'when alternative_type is defined' ]] if next_schema: new_schema_info = schema_info_dict.get(next_schema) new_schema_data = dict(name=next_schema, ctx=schema_ctx, info=new_schema_info, dict=schema_info_dict, prop=False, subelement=False, required=True) return validate_next_value(new_schema_data, value) elem_key_regex = schema_info.get('elem_key_regex') elem_type = schema_info.get('elem_type') elem_alternative_type = schema_info.get('elem_alternative_type') elem_alternative_choices = schema_info.get('elem_alternative_choices') elem_alternative_regex = schema_info.get('elem_alternative_regex') elem_alternative_min = schema_info.get('elem_alternative_min') elem_alternative_max = schema_info.get('elem_alternative_max') elem_schema_name = schema_info.get('elem_schema') elem_main_schema = schema_info.get('elem_main_schema') elem_alternative_schema_name = schema_info.get('elem_alternative_schema') elem_required = schema_info.get('elem_required') elem_non_empty = schema_info.get('elem_non_empty') elem_choices = schema_info.get('elem_choices') elem_regex = schema_info.get('elem_regex') elem_min = schema_info.get('elem_min') elem_max = schema_info.get('elem_max') if value_type not in ['map', 'simple_map']: if elem_key_regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_key_regex only for maps' ]] if value_type not in ['map', 'simple_map', 'list', 'simple_list']: if elem_type: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_type only for lists and maps' ]] if elem_alternative_type: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_type only for lists and maps' ]] elif elem_alternative_choices: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_choices only for lists and maps' ]] elif elem_alternative_regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_regex only for lists and maps' ]] elif elem_alternative_min: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_min only for lists and maps' ]] elif elem_alternative_max: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_max only for lists and maps' ]] elif elem_schema_name: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_schema only for lists and maps' ]] elif elem_main_schema: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_main_schema only for lists and maps' ]] elif elem_alternative_schema_name: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_alternative_schema only for lists and maps' ]] elif elem_required: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_required only for lists and maps' ]] elif elem_non_empty: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_non_empty only for lists and maps' ]] elif elem_choices: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_choices only for lists and maps' ]] elif elem_regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_regex only for lists and maps' ]] elif elem_min: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_min only for lists and maps' ]] elif elem_max: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a definition should have elem_max only for lists and maps' ]] elem_type_default = elem_type or '' if ((elem_type_default == 'simple_dict') and (not elem_alternative_type) and (not elem_alternative_schema_name)): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('elem_type: ' + elem_type_default), 'msg: a simple_dict for elem_type must have an elem_alternative_type or ' + 'elem_alternative_schema' ]] elif elem_main_schema and (elem_type_default != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('elem_type: ' + elem_type_default), 'msg: elem_main_schema should be defined only when elem_type ' + 'is defined and is simple_dict' ]] elif elem_alternative_type and (elem_type_default != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('elem_type: ' + elem_type_default), 'msg: elem_alternative_type should be defined only when elem_type ' + 'is defined and is simple_dict' ]] elif elem_alternative_schema_name and (elem_type_default != 'simple_dict'): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('elem_type: ' + elem_type_default), 'msg: elem_alternative_schema should be defined only when elem_type ' + 'is defined and is simple_dict' ]] elif elem_alternative_type and elem_alternative_schema_name: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('elem_type: ' + elem_type_default), 'msg: define only one of elem_alternative_type or elem_alternative_schema, not both' ]] if not elem_alternative_type: if elem_alternative_choices: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have elem_alternative_choices only ' + 'when elem_alternative_type is defined' ]] elif elem_alternative_regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have elem_alternative_regex only ' + 'when elem_alternative_type is defined' ]] elif elem_alternative_min: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have elem_alternative_min only ' + 'when elem_alternative_type is defined' ]] elif elem_alternative_max: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: a definition should have elem_alternative_max only ' + 'when elem_alternative_type is defined' ]] if value_type in ['map', 'simple_map', 'list', 'simple_list']: if (not elem_type) and (not elem_schema_name): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a property definition with this type should have either a ' + 'elem_type or elem_schema property' ]] elif elem_type and elem_schema_name: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: a property definition with this type should not have ' + 'both elem_type and elem_schema properties' ]] if value_type == 'simple_dict': invalid_alternative_types = [ 'map', 'simple_map', 'dict', 'simple_dict' ] if alternative_type and (alternative_type in invalid_alternative_types): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('alternative_type: ' + alternative_type), str('msg: invalid alternative type for a ' + value_type), 'invalid alternative types: ', invalid_alternative_types, ]] if alternative_schema: schema_info_aux = schema_info_dict.get(alternative_schema) schema_info_aux_type = (schema_info_aux.get('type') if schema_info_aux else None) if schema_info_aux_type and (schema_info_aux_type in invalid_alternative_types): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('alternative schema type: ' + schema_info_aux_type), str('msg: invalid alternative schema type for a ' + value_type), 'invalid alternative types: ', invalid_alternative_types, ]] elif value_type in ['simple_map', 'simple_list']: invalid_elem_types = (['list', 'simple_list'] if (value_type == 'simple_list') else ['map', 'simple_map', 'dict', 'simple_dict']) if elem_type and (elem_type in invalid_elem_types): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('element type: ' + elem_type), str('msg: invalid element type for a ' + value_type), 'invalid element types: ', invalid_elem_types, ]] if elem_schema_name: schema_info_aux = schema_info_dict.get(elem_schema_name) schema_info_aux_type = schema_info_aux.get( 'type') if schema_info_aux else None if schema_info_aux_type and (schema_info_aux_type in invalid_elem_types): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('element schema type: ' + schema_info_aux_type), str('msg: invalid element schema type for a ' + value_type), 'invalid element types: ', invalid_elem_types, ]] props = schema_info.get('props') if ((props is None) and (not is_prop) and (not is_subelement) and (value_type in ['dict', 'simple_dict'])): if (value_type != 'simple_dict') or not main_schema: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: props not defined for schema' ]] if props and is_prop: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: props should not be defined inside a property (only in schemas)' ]] elif props and (value_type not in ['dict', 'simple_dict']): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: props should not be defined for a schema of this type' ]] elif props and (value_type == 'simple_dict') and main_schema: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: props should not be defined for a simple_dict with main_schema' ]] if choices and regex: return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: when choices is specified, regex cannot be specified', ]] elif choices and (minimum is not None): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: when choices is specified, min cannot be specified', ]] elif choices and (maximum is not None): return [[ str('context: dynamic schema'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: when choices is specified, max cannot be specified', ]] is_list = isinstance(value, list) is_dict = isinstance(value, dict) is_string = is_str(value) if non_empty: if is_list: if not value: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: list is empty' ]] elif is_dict: if not value: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: dict is empty' ]] else: if not str(value): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: value is empty' ]] if value is not None: if value_type in ['list']: if not is_list: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value expected to be a list', ]] elif value_type in ['dict', 'map']: if not is_dict: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value expected to be a dictionary' ]] elif value_type == 'simple_dict': if not is_dict: new_schema_info = dict( type=alternative_type, choices=alternative_choices, regex=alternative_regex, min=alternative_min, max=alternative_max, schema=alternative_schema, required=required, non_empty=non_empty, ) new_schema_data = dict( name=schema_name + ' (' + value_type + ' - alternative)', ctx=schema_ctx, info=new_schema_info, dict=schema_info_dict, prop=is_prop, subelement=is_subelement, required=required, simple=True, ) return validate_next_value(new_schema_data, value) elif value_type in ['simple_map', 'simple_list']: if (((value_type == 'simple_list') and not is_list) or ((value_type == 'simple_map') and not is_dict)): new_schema_info = dict( type=elem_type, alternative_type=elem_alternative_type, alternative_choices=elem_alternative_choices, alternative_regex=elem_alternative_regex, alternative_min=elem_alternative_min, alternative_max=elem_alternative_max, schema=elem_schema_name, required=elem_required, non_empty=elem_non_empty, choices=elem_choices, regex=elem_regex, min=elem_min, max=elem_max, ) new_schema_data = dict( name=schema_name + ' (' + value_type + ' - single)', ctx=schema_ctx, info=new_schema_info, dict=schema_info_dict, prop=is_prop, subelement=is_subelement, required=required, simple=True, ) return validate_next_value(new_schema_data, value) elif value_type == 'str': if not is_string: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value expected to be a string' ]] elif value_type != 'unknown': if is_list or is_dict: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value expected to be a primitive' ]] if value_type in primitive_types and (str(value) != ''): if (value_type == 'bool') and (not isinstance(value, bool)): if not is_bool(value): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value should be a boolean', ]] elif ((value_type == 'int') and (isinstance(value, bool) or not isinstance(value, int))): if (not is_str(value)) or (not is_int(value)): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value should be an integer', ]] elif (value_type == 'float') and (not isinstance(value, float)): if (not is_str(value)) or (not is_float(value)): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('value type: ' + str(type(value))), 'msg: value should be a float', ]] if choices: values_for_choices = (value or []) if is_list else [value] values_item_type = (elem_type or '') if is_list else (value_type or '') for value_item in values_for_choices: if isinstance(value_item, dict): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: value is a dictionary, but has choices defined for it', ]] elif isinstance(value_item, list): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), 'msg: value is a list, but has choices defined for it', ]] else: value_to_compare = (value_item if (values_item_type != 'bool') else to_bool(value_item)) if (value_to_compare not in choices): non_empty = (value_to_compare is not None) non_empty = non_empty and ( (values_item_type not in primitive_types) or (str(value_to_compare) != '')) if non_empty: type_info = ((value_type + ' (elem: ' + values_item_type + ')') if is_list else values_item_type) return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + type_info), str('value: ' + str(value_item)), 'msg: value is invalid', 'valid choices:', choices ]] if regex: pattern = re.compile(regex) if not pattern.search(value): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('regex: ' + regex), 'msg: value is invalid (not compatible with the regex specified)', ]] if minimum is not None: if value_type == 'str': if len(value) < minimum: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('min: ' + str(minimum)), 'msg: value is invalid (the length of the string is less than the minimum specified)', ]] elif value_type in ['int', 'float']: numeric_value = (to_int(value) if (value_type == 'int') else to_float(value)) if numeric_value < minimum: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('min: ' + str(minimum)), 'msg: value is invalid (numeric value is less than the minimum specified)', ]] else: return [[ str('context: dynamic schema (unexpected)'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: min property is not allowed with this value type', 'allowed types:', ['str', 'int', 'float'], ]] if maximum is not None: if value_type == 'str': if len(value) > maximum: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('max: ' + str(maximum)), 'msg: value is invalid (the length of the string is more than the maximum specified)', ]] elif value_type in ['int', 'float']: numeric_value = (to_int(value) if (value_type == 'int') else to_float(value)) if numeric_value > maximum: return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('max: ' + str(maximum)), 'msg: value is invalid (numeric value is more than the maximum specified)', ]] else: return [[ str('context: dynamic schema (unexpected)'), str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), 'msg: max property is not allowed with this value type', 'allowed types:', ['str', 'int', 'float'], ]] if main_schema: new_schema_info = dict( schema=main_schema, required=required, non_empty=non_empty, prop=is_prop, subelement=is_subelement, ) new_schema_data = dict( name=schema_name + ' (' + value_type + ' - main)', ctx=schema_ctx, info=new_schema_info, dict=schema_info_dict, required=required, simple=True, ) return validate_next_value(new_schema_data, value) if value_type != 'unknown': error_msgs = [] if is_list: for idx, elem_value in enumerate(value): new_schema_info = dict( type=elem_type, alternative_type=elem_alternative_type, alternative_choices=elem_alternative_choices, alternative_regex=elem_alternative_regex, alternative_min=elem_alternative_min, alternative_max=elem_alternative_max, schema=elem_schema_name, main_schema=elem_main_schema, alternative_schema=elem_alternative_schema_name, required=elem_required, non_empty=elem_non_empty, choices=elem_choices, regex=elem_regex, min=elem_min, max=elem_max, ) new_schema_data = dict(name=schema_name, ctx=schema_ctx + '[' + str(idx) + ']', info=new_schema_info, dict=schema_info_dict, prop=False, subelement=True, required=elem_required) error_msgs += validate_next_value(new_schema_data, elem_value) elif is_dict: keys = list(value.keys()) if elem_key_regex: for key in sorted(keys): pattern = re.compile(elem_key_regex) if not pattern.search(key): return [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('type: ' + value_type), str('key: ' + key), str('elem_key_regex: ' + elem_key_regex), 'msg: dictionary key is invalid (not compatible with the regex specified)', ]] # required or non-empty properties if props and (value_type in ['dict', 'simple_dict']): for key in list(props.keys()): prop_value = props.get(key) if prop_value and (prop_value.get('required') or prop_value.get('non_empty')): keys += [key] keys = list(set(keys)) for key in sorted(keys): elem_value = value.get(key) if value_type in ['dict', 'simple_dict']: if props: if key not in props: if not schema_info.get('lax'): error_msgs += [[ str('schema_name: ' + schema_name + schema_suffix), str('at: ' + (schema_ctx or '<root>')), str('property: ' + str(key)), 'msg: property not defined in schema', 'allowed: ', sorted(props.keys()) ]] else: new_schema_info = props.get(key) new_schema_data = dict( name=schema_name, ctx=schema_ctx + (schema_ctx and '.') + key, info=new_schema_info, dict=schema_info_dict, prop=True, subelement=False, required=new_schema_info.get('required')) error_msgs += validate_next_value( new_schema_data, elem_value) else: new_schema_info = dict( type=elem_type, alternative_type=elem_alternative_type, alternative_choices=elem_alternative_choices, alternative_regex=elem_alternative_regex, alternative_min=elem_alternative_min, alternative_max=elem_alternative_max, schema=elem_schema_name, main_schema=elem_main_schema, alternative_schema=elem_alternative_schema_name, required=elem_required, non_empty=elem_non_empty, choices=elem_choices, regex=elem_regex, min=elem_min, max=elem_max, ) new_schema_data = dict(name=schema_name, ctx=schema_ctx + '[' + key + ']', info=new_schema_info, dict=schema_info_dict, prop=False, subelement=True, required=elem_required) error_msgs += validate_next_value( new_schema_data, elem_value) return error_msgs return []
def prepare_node_host_dependencies( node_dependencies, hosts_data, instance_index, ignore_unknown_nodes=None, ): error_msgs = list() result = dict() dependency_result = dict() try: for dependency_name in sorted(list((node_dependencies or {}).keys())): error_msgs_dependency = list() try: node_dependency = node_dependencies.get( dependency_name ) dependency_type = node_dependency.get('type') dependency_hosts = node_dependency.get('hosts') dependency_limit = node_dependency.get('limit') required_amount = int(node_dependency.get('required_amount')) protocol = node_dependency.get('protocol') port = node_dependency.get('port') port = str(port) if (port is not None) else None result_host = None result_hosts = list() real_hosts_amount = 0 if dependency_limit != 0: if dependency_type == 'node': node_ip_type = node_dependency.get( 'node_ip_type') or 'private' new_dependency_hosts = [] for dependency_host in dependency_hosts: dependency_host_data = hosts_data.get(dependency_host) if dependency_host_data: node_ip_type_prop = ( 'private_ip' if (node_ip_type == 'private') else node_ip_type ) new_dependency_host = dependency_host_data.get( node_ip_type_prop ) local_target = to_bool( dependency_host_data.get('local') ) if not new_dependency_host: if local_target: new_dependency_host = dependency_host else: error_msgs_dependency += [[ str('node_ip_type: ' + str(node_ip_type)), 'msg: dependency host has no property for the node ip type', ]] new_dependency_hosts += [new_dependency_host or ''] elif ignore_unknown_nodes: new_dependency_host = dependency_host new_dependency_hosts += [new_dependency_host] elif required_amount == -1: if ignore_unknown_nodes: new_dependency_host = dependency_host new_dependency_hosts += [new_dependency_host] error_msgs_dependency += [[ str('dependency_host: ' + str(dependency_host)), 'msg: dependency host not defined', ]] else: error_msgs_dependency += [[ str('dependency_host: ' + str(dependency_host)), 'msg: dependency host not defined', ]] dependency_hosts = new_dependency_hosts hosts_amount = len(dependency_hosts) result_host_idx = 0 initial_idx = 0 final_idx_next = 0 single_host_included = True if hosts_amount > 0: result_host_idx = (instance_index - 1) % hosts_amount initial_idx = min( (result_host_idx * dependency_limit) % hosts_amount, hosts_amount ) final_idx_next = min( initial_idx + dependency_limit, hosts_amount ) final_idx_next_round = 0 if final_idx_next == hosts_amount: final_idx_next_round = min( dependency_limit - (final_idx_next - initial_idx), initial_idx ) if initial_idx < final_idx_next: result_hosts = dependency_hosts[initial_idx:final_idx_next] if final_idx_next_round: result_hosts += dependency_hosts[0:final_idx_next_round] if result_hosts: single_host_included = ( (initial_idx <= result_host_idx) and (result_host_idx < final_idx_next) ) or ( result_host_idx < final_idx_next_round ) real_hosts_amount = len(result_hosts) if required_amount == -1: if real_hosts_amount < dependency_limit: error_msgs_dependency += [[ str('required amount: ' + str(required_amount)), str('amount found: ' + str(real_hosts_amount)), str('amount expected: ' + str(dependency_limit)), 'msg: required amount is defined to require all hosts', str('initial_idx: ' + str(initial_idx)), str('final_idx_next: ' + str(final_idx_next)), result_hosts, ]] elif required_amount > 0: if real_hosts_amount < required_amount: error_msgs_dependency += [[ str('required amount: ' + str(required_amount)), str('amount found: ' + str(real_hosts_amount)), 'msg: the number of hosts is less than the required amount', ]] result_hosts_aux = result_hosts result_hosts = [] for host_aux in result_hosts_aux: host_aux = fill_host(host_aux, protocol, port) result_hosts += [host_aux] if result_hosts: result_host = dependency_hosts[result_host_idx] result_host = fill_host(result_host, protocol, port) if required_amount == -1: required_amount = real_hosts_amount dependency_result[dependency_name] = dict( original_type=dependency_type, required_amount=required_amount, single_host_included=single_host_included, host=result_host, host_list=result_hosts, ) except Exception as error: error_msgs_dependency += [[ 'msg: error when trying to prepare the host dependency data', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] for value in error_msgs_dependency: new_value = [ str('dependency name: ' + (dependency_name or '')) ] + value error_msgs += [new_value] result = dependency_result except Exception as error: error_msgs += [[ 'msg: error when trying to prepare the host dependencies data', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(result=result, error_msgs=error_msgs)
def get_validators(ctx_title, validator_files, task_data, env_data): result = list() error_msgs = list() try: task_data = task_data or dict() base_dir_prefix = task_data.get('base_dir_prefix') dict_to_validate = task_data.get('dict_to_validate') prop_names = task_data.get('prop_names') all_props = task_data.get('all_props') env = env_data.get('env') or dict() meta = env.get('meta') or dict() dev = env_data.get('dev') ignore_validators = to_bool(meta.get('ignore_validators')) ignore_validators = ( ignore_validators if (ignore_validators is not None) else dev ) dict_to_validate = dict_to_validate or dict() if validator_files: dict_to_validate = dict_to_validate or dict() if not isinstance(validator_files, list): validator_files = [validator_files] for validator_file in validator_files: validator_file = ( (base_dir_prefix + validator_file) if base_dir_prefix else validator_file ) if not validator_file: error_msgs += [[ str('context: ' + str(ctx_title or '')), str('msg: validator file not defined'), ]] elif os.path.exists(validator_file): if not ignore_validators: validator_data = dict() if all_props: validator_data = dict_to_validate else: for key in (prop_names or []): if dict_to_validate.get(key) is not None: validator_data[key] = dict_to_validate.get(key) result_item = dict( description=ctx_title, task=validator_file, data=validator_data, base_dir_prefix=base_dir_prefix or '', ) result += [result_item] else: error_msgs += [[ str('context: ' + str(ctx_title or '')), str('msg: validator file not found: ' + validator_file), ]] return dict(result=result, error_msgs=error_msgs) except Exception as error: error_msgs += [[ str('context: ' + str(ctx_title or '')), 'msg: error when trying to get the context validators', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs)
def prepare_node_dependencies(node_names, prepared_node_dict): result = dict() error_msgs = list() try: for node_name in (node_names or []): error_msgs_node = [] try: node_dependencies = dict() prepared_node = prepared_node_dict.get(node_name) dependencies = prepared_node.get('dependencies') local = to_bool(prepared_node.get('local')) origin_hosts_amount = prepared_node.get('amount') or 1 active_hosts_amount = len(prepared_node.get('active_hosts') or []) active_hosts_amount = active_hosts_amount if (not local) else ( active_hosts_amount if (active_hosts_amount > 0) else 1 ) if dependencies: for dependency_name in sorted(list(dependencies.keys())): dependency = dependencies.get(dependency_name) target_local = False error_msgs_dependency = [] if is_str(dependency) or isinstance(dependency, list): dependency = dict( type='url', host=dependency, ) when = to_bool(dependency.get('when'), True) if when: dependency_type = dependency.get('type') allowed_props_map = dict( node=[ 'type', 'required_amount', 'node_ip_type', 'limit', 'host', 'protocol', 'port', 'when', ], ip=[ 'type', 'required_amount', 'limit', 'host', 'protocol', 'port', 'when', ], url=[ 'type', 'required_amount', 'limit', 'host', 'protocol', 'port', 'when', ], ) allowed_props = allowed_props_map.get(dependency_type) for key in sorted(list(dependency.keys())): if key not in allowed_props: error_msgs_dependency += [[ str('property: ' + key), 'msg: invalid property for this node dependency type', 'allowed properties: ', allowed_props, ]] required_props = ['type', 'host'] for key in sorted(required_props): if dependency.get(key) is None: error_msgs_dependency += [[ str('property: ' + key), 'msg: required property not specified for this node dependency', ]] dependency_hosts = dependency.get('host') if is_str(dependency_hosts): dependency_hosts = [dependency_hosts] if dependency_hosts and (dependency_type == 'node'): target_node_names = dependency_hosts dependency_hosts = [] for target_node_name in (target_node_names or []): target_prepared_node = prepared_node_dict.get( target_node_name) if not target_prepared_node: error_msgs_dependency += [[ str('target node: ' + str(target_node_name or '')), 'msg: invalid target node', ]] else: dependency_hosts += target_prepared_node.get( 'active_hosts' ) or [] target_local = target_prepared_node.get('local') if not error_msgs_dependency: dependency_limit = dependency.get('limit') dependency_limit = int( dependency_limit if (dependency_limit is not None) else 1 ) dependency_required_amount = to_default_int( dependency.get('required_amount'), 0 ) dependency_real_limit = ( len(dependency_hosts) if (dependency_limit == -1) else min(dependency_limit, len(dependency_hosts)) ) if dependency_limit < -1: error_msgs_dependency += [[ str('dependency limit: ' + str(dependency_limit)), 'msg: dependency limit should be -1, 0, or a positive integer', ]] elif (dependency_required_amount != 0) and dependency_limit == 0: error_msgs_dependency += [[ str( 'dependency required amount: ' + str(dependency_required_amount) ), 'msg: dependency limit is defined as 0, but is required', ]] elif (dependency_required_amount != 0) and dependency_real_limit == 0: error_msgs_dependency += [[ str( 'dependency required amount: ' + str(dependency_required_amount) ), 'msg: dependency is required, but number of hosts defined is 0', ]] if dependency_type == 'node': error_msgs_dependency += [[ 'tip: take a look at the "amount" property of the target node(s)', 'target node(s): ' + str(dependency.get('host') or ''), ]] elif ( (dependency_required_amount != -1) and (dependency_real_limit < dependency_required_amount) ): error_msgs_dependency += [[ str( 'dependency required amount: ' + str(dependency_required_amount) ), str( 'target host amount: ' + str(dependency_real_limit) ), 'msg: dependency real limit is less than the required amount', ]] result_item = dict( type=dependency_type, origin_amount=origin_hosts_amount, required_amount=dependency_required_amount, limit=dependency_real_limit, node_ip_type=dependency.get('node_ip_type'), hosts=dependency_hosts, protocol=dependency.get('protocol'), port=dependency.get('port'), local=target_local, ) result_item_keys = list(result_item.keys()) for key in result_item_keys: if result_item.get(key) is None: result_item.pop(key, None) node_dependencies[dependency_name] = result_item for value in error_msgs_dependency: new_value = [ str('dependency name: ' + dependency_name), str('dependency type: ' + dependency_type), ] + value error_msgs_node += [new_value] node_result = dict( active_hosts_amount=active_hosts_amount, local=local, dependencies=node_dependencies, ) result[node_name] = node_result except Exception as error: error_msgs_node += [[ 'msg: error when trying to prepare the node dependency', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] for value in error_msgs_node: new_value = [str('node name: ' + (node_name or ''))] + value error_msgs += [new_value] return dict(result=result, error_msgs=error_msgs) except Exception as error: error_msgs += [[ 'msg: error when trying to prepare the nodes dependencies', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs)
def load_next_vars(file_relpath, ctx_params, data_info): error_msgs = list() try: directories = list() files = list() templates = list() plugin = data_info.get('plugin') ansible_vars = data_info.get('ansible_vars') pod = data_info.get('pod') validate = data_info.get('validate') pod_local_dir = pod.get('local_dir') try: file = pod_local_dir + '/' + file_relpath if validate and not os.path.exists(file): error_msgs += [[ str('file: ' + file_relpath), str('pod_local_dir: ' + pod_local_dir), 'msg: pod ctx file not found in the pod repository', ]] return dict(error_msgs=error_msgs) res_str = None try: res_str = lookup(plugin, ansible_vars, file, ctx_params) except Exception as error: error_msgs += [[ str('file: ' + file_relpath), 'msg: error when trying to load the pod ctx file', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs) res = None try: res = load_yaml(str(res_str)) except Exception as error: error_msgs += [[ str('file: ' + (file_relpath or '')), 'file content type: ' + str(type(res_str)), 'msg: error when trying to process the pod ctx file', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs) if res: if validate: task_data = dict( base_dir_prefix=None, dict_to_validate=res, all_props=True, ) info = validate_ctx_schema( ctx_title='validate pod ctx vars schema', schema_files=['schemas/pod_ctx.schema.yml'], task_data=task_data, ) error_msgs += (info.get('error_msgs') or []) if error_msgs: return dict(error_msgs=error_msgs) res_files = res.get('files') res_templates = res.get('templates') res_env_files = res.get('env_files') res_env_templates = res.get('env_templates') for res_file in (res_files or []): info = load_ctx_file(res_file, is_env=False, data_info=data_info) result_aux = info.get('result') error_msgs_aux = info.get('error_msgs') if error_msgs_aux: for value in error_msgs_aux: new_value = ['context: pod ctx file'] + value error_msgs += [new_value] else: directories += result_aux.get('directories') or [] files += result_aux.get('files') or [] for res_template in (res_templates or []): info = load_ctx_template( res_template, is_env=False, data_info=data_info) result_aux = info.get('result') error_msgs_aux = info.get('error_msgs') if error_msgs_aux: for value in error_msgs_aux: new_value = ['context: pod ctx template'] + value error_msgs += [new_value] else: directories += result_aux.get('directories') or [] templates += result_aux.get('templates') or [] for res_env_file in (res_env_files or []): info = load_ctx_file(res_env_file, is_env=True, data_info=data_info) result_aux = info.get('result') error_msgs_aux = info.get('error_msgs') if error_msgs_aux: for value in error_msgs_aux: new_value = ['context: pod ctx env file'] + value error_msgs += [new_value] else: directories += result_aux.get('directories') or [] files += result_aux.get('files') or [] for res_env_template in (res_env_templates or []): info = load_ctx_template( res_env_template, is_env=True, data_info=data_info) result_aux = info.get('result') error_msgs_aux = info.get('error_msgs') if error_msgs_aux: for value in error_msgs_aux: new_value = ['context: pod ctx env template'] + value error_msgs += [new_value] else: directories += result_aux.get('directories') or [] templates += result_aux.get('templates') or [] if not error_msgs: children = res.get('children') if children: for child in children: when = child.get('when') if to_bool(when if (when is not None) else True): child_error_msgs = list() child_name = child.get('name') child_params = child.get('params') task_data = dict( base_dir_prefix=pod_local_dir, dict_to_validate=child_params, all_props=True, ) info = validate_ctx_schema( ctx_title='validate pod ctx child vars schema', schema_files=child.get('schema'), task_data=task_data, ) child_error_msgs += (info.get('error_msgs') or []) res_child = None if not child_error_msgs: res_child = load_next_vars( file_relpath=child_name, ctx_params=child_params, data_info=data_info, ) child_error_msgs += res_child.get('error_msgs') or list() if child_error_msgs: for value in child_error_msgs: new_value = [str('ctx child: ' + child_name)] + value error_msgs += [new_value] else: child_result = res_child.get('result') child_directories = child_result.get('directories') child_files = child_result.get('files') child_templates = child_result.get('templates') if child_directories: directories += child_directories if child_files: files += child_files if child_templates: templates += child_templates except Exception as error: error_msgs += [[ str('file: ' + (file_relpath or '')), 'msg: error when trying to define the pod ctx vars', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs) result = dict( directories=directories, files=files, templates=templates, ) return dict(result=result, error_msgs=error_msgs) except Exception as error: error_msgs += [[ str('file: ' + (file_relpath or '')), 'msg: unknown error when trying to define the pod ctx vars', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs)
def load_ctx_template(current_template, is_env, data_info): error_msgs = list() try: directories = list() templates = list() pod = data_info.get('pod') env_data = data_info.get('env_data') previous = data_info.get('previous') validate = data_info.get('validate') pod_dir = pod.get('pod_dir') pod_local_dir = pod.get('local_dir') pod_tmp_dir = pod.get('tmp_dir') dir_names = previous.get('dir_names') file_names = previous.get('file_names') env_dir = env_data.get('env_dir') env_lax = env_data.get('lax') env = env_data.get('env') env_template_no_empty_lines = (env.get('meta') or dict()).get( 'template_no_empty_lines' ) default_dir_mode = 777 if env_lax else 755 default_file_mode = 666 if env_lax else 640 default_file_executable_mode = 777 if env_lax else 751 when = current_template.get('when') if to_bool(when if (when is not None) else True): src_relpath = current_template.get('src') src = (env_dir if is_env else pod_local_dir) + '/' + src_relpath local_src = src dest_relpath = current_template.get('dest') dest = pod_dir + '/' + dest_relpath dest_tmp = pod_tmp_dir + '/tpl/' + dest_relpath dest_dir = os.path.dirname(dest) dest_tmp_dir = os.path.dirname(dest_tmp) no_empty_lines = to_bool( current_template.get('no_empty_lines'), to_bool(env_template_no_empty_lines), ) if validate and not os.path.exists(local_src): error_msgs += [[ str('src: ' + (src_relpath or '')), 'msg: pod ctx template file not found', ]] template_params = current_template.get('params') or None if validate and not error_msgs: error_msgs_aux = list() base_dir_prefix = (env_dir if is_env else pod_local_dir) + '/' task_data = dict( base_dir_prefix=base_dir_prefix, dict_to_validate=template_params, all_props=True, ) info = validate_ctx_schema( ctx_title='validate pod ctx vars template schema', schema_files=current_template.get('schema'), task_data=task_data, ) error_msgs_aux += (info.get('error_msgs') or []) for value in (error_msgs_aux or []): new_value = [ str('src: ' + (src_relpath or '')), str('dest: ' + (dest_relpath or '')), ] + value error_msgs += [new_value] if error_msgs: return dict(error_msgs=error_msgs) if dest_dir not in dir_names: dir_names.add(dest_dir) dir_to_add = dict( path=dest_dir, mode=current_template.get('dir_mode') or default_dir_mode, ) directories += [dir_to_add] if dest_tmp_dir not in dir_names: dir_names.add(dest_tmp_dir) dir_to_add = dict( path=dest_tmp_dir, mode=current_template.get('dir_mode') or default_dir_mode, ) directories += [dir_to_add] if dest not in file_names: file_names.add(dest) template_to_add = dict( src=src, dest=dest, dest_tmp=dest_tmp, no_empty_lines=no_empty_lines, mode=( current_template.get('mode') or ( default_file_executable_mode if to_bool(current_template.get('executable')) else default_file_mode ) ), params=template_params, ) templates += [template_to_add] else: error_msgs += [[ str('src: ' + (src or '')), str('dest: ' + (dest or '')), 'msg: duplicate destination for the pod ctx template', ]] result = dict(directories=directories, templates=templates) return dict(result=result, error_msgs=error_msgs) except Exception as error: error_msgs += [[ 'msg: error when trying to load the pod ctx template', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs)
def load_ctx_file(current_file, is_env, data_info): error_msgs = list() try: directories = list() files = list() pod = data_info.get('pod') env_data = data_info.get('env_data') previous = data_info.get('previous') validate = data_info.get('validate') pod_dir = pod.get('pod_dir') pod_local_dir = pod.get('local_dir') dir_names = previous.get('dir_names') file_names = previous.get('file_names') env_dir = env_data.get('env_dir') env_lax = env_data.get('lax') default_dir_mode = 777 if env_lax else 755 default_file_mode = 666 if env_lax else 640 default_file_executable_mode = 777 if env_lax else 751 when = current_file.get('when') if to_bool(when if (when is not None) else True): src_relpath = current_file.get('src') src = (env_dir if is_env else pod_dir) + '/' + src_relpath local_src = src if is_env else (pod_local_dir + '/' + src_relpath) dest = pod_dir + '/' + current_file.get('dest') dest_dir = os.path.dirname(dest) if validate and not os.path.exists(local_src): error_msgs += [[ str('src: ' + (src_relpath or '')), 'msg: pod ctx file not found', ]] if dest_dir not in dir_names: dir_names.add(dest_dir) dir_to_add = dict( path=dest_dir, mode=current_file.get('dir_mode') or default_dir_mode, ) directories += [dir_to_add] if dest not in file_names: file_names.add(dest) file_to_add = dict( remote_src=not is_env, src=src, dest=dest, mode=( current_file.get('mode') or ( default_file_executable_mode if to_bool(current_file.get('executable')) else default_file_mode ) ), ) files += [file_to_add] else: error_msgs += [[ str('src: ' + (src or '')), str('dest: ' + (dest or '')), 'msg: duplicate destination for the pod ctx file', ]] result = dict(directories=directories, files=files) return dict(result=result, error_msgs=error_msgs) except Exception as error: error_msgs += [[ 'msg: error when trying to load the pod ctx file', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] return dict(error_msgs=error_msgs)
def prepare_ctx(ctx_name, env, env_init, env_original, env_info): error_msgs_ctx = list() result = dict() try: has_original_env = env_info.get('has_original_env') env_dir = env_info.get('env_dir') env_lax = env_info.get('env_lax') if not ctx_name: error_msgs_ctx += [['msg: context name not specified']] else: result['name'] = ctx_name main_dict = env.get('main') cloud_dict = env.get('cloud') or dict() if not main_dict: error_msgs_ctx += [[ 'msg: main environment dictionary not specified' ]] elif ctx_name not in main_dict: error_msgs_ctx += [[ 'msg: context not in the main environment dictionary' ]] else: ctx = main_dict.get(ctx_name) extend_cloud = to_bool(ctx.get('extend_cloud')) repo = ctx.get('repo') repo = repo if (repo is not None) else ( cloud_dict.get('repo') if extend_cloud else None) result['repo'] = repo ext_repos = ctx.get('ext_repos') ext_repos = ext_repos if (ext_repos is not None) else ( cloud_dict.get('ext_repos') if extend_cloud else None) result['ext_repos'] = ext_repos run_file = ctx.get('run_file') run_file = run_file if (run_file is not None) else ( cloud_dict.get('run_file') if extend_cloud else None) run_file = run_file or 'run' result['run_file'] = run_file repos = env.get('repos') if not repo: error_msgs_ctx += [[ 'msg: cloud repository not defined for the context ' + '(under the ' + ('main or cloud' if extend_cloud else 'main') + ' section) in the environment dictionary' ]] elif not repos: error_msgs_ctx += [[ 'msg: no repositories in the environment dictionary' ]] elif not repos.get(repo): error_msgs_ctx += [[ 'context: validate ctx repo', 'msg: repository not found: ' + repo, ]] for ext_repo in (ext_repos or []): ext_repo_name = ext_repo.get('repo') if not repos.get(ext_repo_name): error_msgs_ctx += [[ 'context: validate ctx env repo', str('msg: repository not found: ' + ext_repo_name), ]] collections = ctx.get('collections') collections = collections if (collections is not None) else ( cloud_dict.get('collections') if extend_cloud else None) if collections: prepared_collections = list() collection_namespaces = set() collection_only_namespaces = set() collection_dests = set() for collection_idx, collection in enumerate(collections): error_msgs_collection = list() collection_namespace = collection.get('namespace') collection_name = collection.get('collection') collection_path = collection.get('path') collection_dest = collection_namespace if collection_name: collection_dest = collection_namespace + '/' + collection_name if ((collection_namespace == 'lrd') and (not collection_name)): error_msgs_collection += [[ 'collection namespace: ' + str(collection_namespace), 'msg: reserved collection namespace (namespace only)', ]] elif ((collection_namespace == 'lrd') and (collection_name == 'cloud')): error_msgs_collection += [[ 'collection namespace: ' + str(collection_namespace), 'collection name: ' + str(collection_name), 'msg: reserved collection', ]] elif collection_namespace in collection_only_namespaces: error_msgs_collection += [[ 'collection namespace: ' + str(collection_namespace), 'msg: duplicate collection namespace ' + '(there is another collection with namespace only)', ]] elif ( not collection_name ) and collection_namespace in collection_namespaces: error_msgs_collection += [[ 'collection namespace: ' + str(collection_namespace), 'msg: duplicate collection namespace (namespace only)', ]] elif collection_dest in collection_dests: error_msgs_collection += [[ 'collection: ' + str(collection_dest), 'msg: duplicate collection', ]] if not collection_name: collection_only_namespaces.add( collection_namespace) if error_msgs_collection: for value in error_msgs_collection: new_value = [ str('collection item: #' + str(collection_idx + 1)) ] + value error_msgs_ctx += [new_value] else: collection_namespaces.add(collection_namespace) collection_dests.add(collection_dest) default_mode = 777 if env_lax else 755 prepared_collection = dict( src=collection_path, dest='ansible/collections/ansible_collections/' + collection_dest, mode=collection.get('mode') or default_mode, dir_mode=collection.get('dir_mode') or default_mode, absent=to_bool(collection.get('absent')) or False, ) prepared_collections += [prepared_collection] result['prepared_collections'] = prepared_collections ctl_env_schema = ctx.get('ctl_env_schema') original_env_schema = ctx.get('original_env_schema') nodes = ctx.get('nodes') if nodes: node_names = list() for node_idx, node in enumerate(nodes): error_msgs_node = list() node_name = node if is_str(node) else node.get('name') if not node_name: error_msgs_node += [[ 'msg: node name not defined', ]] elif node_name in node_names: error_msgs_node += [[ 'node name: ' + str(node_name), 'msg: duplicate node name', ]] if error_msgs_node: for value in error_msgs_node: new_value = [ str('node item: #' + str(node_idx + 1)) ] + value error_msgs_ctx += [new_value] else: node_names += [node_name] result['node_names'] = node_names if has_original_env and ctl_env_schema: error_msgs_ctx += [[ 'ctl_env_schema: ' + str(ctl_env_schema), 'msg: ctl_env_schema should not be defined the base environment file ' + '(only in the original environment file)' ]] elif (not has_original_env) and original_env_schema: error_msgs_ctx += [[ 'original_env_schema: ' + str(original_env_schema), 'msg: original_env_schema should not be defined the original environment file ' + '(only in the base environment file)' ]] if (not has_original_env) and ctl_env_schema: task_data = dict( base_dir_prefix=env_dir + '/', dict_to_validate=env_init.get('env_params'), all_props=True, ) info = validate_ctx_schema( ctx_title= 'validate the controller project environment params schema', schema_files=ctl_env_schema, task_data=task_data, ) error_msgs_ctx += (info.get('error_msgs') or []) if has_original_env and original_env_schema: task_data = dict( base_dir_prefix=env_dir + '/', dict_to_validate=env_original, all_props=True, ) info = validate_ctx_schema( ctx_title='validate the original environment schema', schema_files=original_env_schema, task_data=task_data, ) error_msgs_ctx += (info.get('error_msgs') or []) except Exception as error: error_msgs_ctx += [[ 'ctx_name: ' + str(ctx_name or ''), 'msg: error when trying to prepare the minimal context', 'error type: ' + str(type(error)), 'error details: ', traceback.format_exc().split('\n'), ]] error_msgs = list() for value in (error_msgs_ctx or []): new_value = [str('ctx: ' + str(ctx_name))] + value error_msgs += [new_value] return dict(result=result, error_msgs=error_msgs)