def parse_structured_value( name: str, field: Dict, context: ParseContext ) -> StructuredValue: """Parse something that might look like: {'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9} or {'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9} """ top_level = removeline_numbers(field).items() if not top_level: raise exc.DataGenSyntaxError( f"Strange datastructure ({field})", **context.line_num(field) ) elif len(top_level) > 1: raise exc.DataGenSyntaxError( f"Extra keys for field {name} : {top_level}", **context.line_num(field) ) [[function_name, args]] = top_level args = parse_structured_value_args(args, context) if function_name == "reference": return ReferenceValue(function_name, args, **context.line_num(field)) else: return StructuredValue(function_name, args, **context.line_num(field))
def parse_structured_value(name: str, field: Dict, context: ParseContext) -> Definition: """Parse something that might look like: {'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9} or {'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9} """ top_level = removeline_numbers(field).items() if not top_level: raise exc.DataGenSyntaxError(f"Strange datastructure ({field})", **context.line_num(field)) elif len(top_level) > 1: raise exc.DataGenSyntaxError( f"Extra keys for field {name} : {top_level}", **context.line_num(field)) [[function_name, args]] = top_level plugin = None if "." in function_name: namespace, name = function_name.split(".") plugin = context.parser_macros_plugins.get(namespace) if plugin: func = getattr(plugin, name) rc = func(context, args) return rc else: args = parse_structured_value_args(args, context) return StructuredValue(function_name, args, **context.line_num(field))
def parse_file(stream: IO[str], context: ParseContext) -> List[Dict]: stream_name = getattr(stream, "name", None) if stream_name: path = Path(fsdecode(stream.name)).absolute() else: path = Path("<stream>") try: data, line_numbers = yaml_safe_load_with_line_numbers(stream, str(path)) except YAMLError as y: raise exc.DataGenYamlSyntaxError( str(y), str(path), y.problem_mark.line + 1, ) context.line_numbers.update(line_numbers) if not isinstance(data, list): raise exc.DataGenSyntaxError( "Generator file should be a list (use '-' on top-level lines)", stream_name, 1, ) templates = parse_top_level_elements(path, data, context) return templates
def categorize_top_level_objects(data: List, context: ParseContext): """Look at all of the top-level declarations and categorize them""" top_level_collections: Dict = { "option": [], "include_file": [], "macro": [], "plugin": [], "object": [], } assert isinstance(data, list) for obj in data: if not isinstance(obj, dict): raise exc.DataGenSyntaxError( f"Top level elements in a data generation template should all be dictionaries, not {obj}", **context.line_num(data), ) parent_collection = None for collection in top_level_collections: if obj.get(collection): if parent_collection: raise exc.DataGenError( f"Top level element seems to match two name patterns: {collection, parent_collection}", **context.line_num(obj), ) parent_collection = collection if parent_collection: top_level_collections[parent_collection].append(obj) else: raise exc.DataGenError( f"Unknown object type {obj}", **context.line_num(obj) ) return top_level_collections
def categorize_top_level_objects(data: List, context: ParseContext): """Look at all of the top-level declarations and categorize them""" top_level_collections: Dict = { "option": [], "include_file": [], "macro": [], "plugin": [], "statement": [], # objects with side-effects } assert isinstance(data, list) for obj in data: if not isinstance(obj, dict): raise exc.DataGenSyntaxError( f"Top level elements in a data generation template should all be dictionaries, not {obj}", **context.line_num(data), ) obj_category = None for declaration, category in collection_rules.items(): typ = obj.get(declaration) if typ: if obj_category: raise exc.DataGenError( f"Top level element seems to match two name patterns: {declaration, obj_category}", **context.line_num(obj), ) obj_category = category if obj_category: top_level_collections[obj_category].append(obj) else: raise exc.DataGenError(f"Unknown object type {obj}", **context.line_num(obj)) return top_level_collections
def change_current_parent_object(self, obj: Dict): _old_parsed_template = self.current_parent_object self.current_parent_object = obj try: yield except exc.DataGenError: raise except Exception as e: raise exc.DataGenSyntaxError(str(e), **self.line_num()) from e finally: self.current_parent_object = _old_parsed_template
def parse_fields(fields: Dict, context: ParseContext) -> List[FieldFactory]: with context.change_current_parent_object(fields): if not isinstance(fields, dict): raise exc.DataGenSyntaxError( "Fields should be a dictionary (should not start with -) ", **context.line_num(), ) return [ parse_field(name, definition, context) for name, definition in fields.items() if name != "__line__" ]
def _coerce_to_string(val, context): if isinstance(val, str): return val elif isinstance(val, (int, bool, date)): # a few functions accept keyword arguments that YAML interprets # as types other than string return str(val) else: if val is None: val = "null (None)" raise exc.DataGenSyntaxError( f"Cannot interpret key `{val}`` as string", **context.line_num())
def parse_structured_value(name: str, field: Dict, context: ParseContext) -> Definition: """Parse something that might look like: {'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9} or {'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9} """ top_level = removeline_numbers(field).items() if not top_level: raise exc.DataGenSyntaxError(f"Strange datastructure ({field})", **context.line_num(field)) elif len(top_level) > 1: raise exc.DataGenSyntaxError( f"Extra keys for field {name} : {top_level}", **context.line_num(field)) [[function_name, args]] = top_level plugin = None if "." in function_name: namespace, name = function_name.split(".") plugin = context.parser_macros_plugins.get(namespace) if plugin: try: func = getattr(plugin, name) rc = func(context, args) return rc except AttributeError as e: # check if this is a regular runtime function. If so, # fall-through and handle it as if we had never # tried to parse it as a macro plugin if not hasattr(plugin, "Functions") and not hasattr( plugin.Functions, name): raise e args = parse_structured_value_args(args, context) return StructuredValue(function_name, args, **context.line_num(field))
def parse_object_template(yaml_sobj: Dict, context: ParseContext) -> ObjectTemplate: parsed_template: Any = parse_element( dct=yaml_sobj, element_type="object", mandatory_keys={}, optional_keys={ "fields": Dict, "friends": List, "include": str, "nickname": str, "just_once": bool, "count": (str, int, dict), }, context=context, ) if not context.top_level and parsed_template.just_once: raise exc.DataGenSyntaxError( "just_once can only be used at the top level") assert yaml_sobj with context.change_current_parent_object(yaml_sobj): sobj_def = {} sobj_def["tablename"] = parsed_template.object check_identifier(parsed_template.object, yaml_sobj, "Object names") fields: List friends: List sobj_def["fields"] = fields = [] sobj_def["friends"] = friends = [] parse_inclusions(yaml_sobj, fields, friends, context) fields.extend(parse_fields(parsed_template.fields or {}, context)) friends.extend(parse_friends(parsed_template.friends or [], context)) sobj_def["nickname"] = nickname = parsed_template.nickname check_identifier(nickname, yaml_sobj, "Nicknames") sobj_def["just_once"] = parsed_template.just_once or False sobj_def["line_num"] = parsed_template.line_num.line_num sobj_def["filename"] = parsed_template.line_num.filename count_expr = yaml_sobj.get("count") if count_expr is not None: parse_count_expression(yaml_sobj, sobj_def, context) new_template = ObjectTemplate(**sobj_def) context.register_template(new_template) return new_template
def parse_field_value( name: str, field, context: ParseContext, allow_structured_values=True ) -> Union[SimpleValue, StructuredValue, ObjectTemplate]: if isinstance(field, (str, Number, date, type(None))): return SimpleValue(field, **(context.line_num(field) or context.line_num())) elif isinstance(field, dict) and field.get("object"): with context.change_current_parent_object(field): return parse_object_template(field, context) elif isinstance(field, dict): return parse_structured_value(name, field, context) elif isinstance(field, list) and len(field) == 1 and isinstance(field[0], dict): # unwrap a list of a single item in this context because it is # a mistake and we can infer their real meaning return parse_field_value(name, field[0], context) else: raise exc.DataGenSyntaxError( f"Unknown field {type(field)} type for {name}. Should be a string or 'object': \n {field} ", **(context.line_num(field) or context.line_num()), )