def parse_structured_value(
name: str, field: Dict, context: ParseContext
) -> StructuredValue:
"""Parse something that might look like:
{'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9}
or
{'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9}
"""
top_level = removeline_numbers(field).items()
if not top_level:
raise exc.DataGenSyntaxError(
f"Strange datastructure ({field})", **context.line_num(field)
)
elif len(top_level) > 1:
raise exc.DataGenSyntaxError(
f"Extra keys for field {name} : {top_level}", **context.line_num(field)
)
[[function_name, args]] = top_level
args = parse_structured_value_args(args, context)
if function_name == "reference":
return ReferenceValue(function_name, args, **context.line_num(field))
else:
return StructuredValue(function_name, args, **context.line_num(field))
def parse_structured_value(name: str, field: Dict,
context: ParseContext) -> Definition:
"""Parse something that might look like:
{'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9}
or
{'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9}
"""
top_level = removeline_numbers(field).items()
if not top_level:
raise exc.DataGenSyntaxError(f"Strange datastructure ({field})",
**context.line_num(field))
elif len(top_level) > 1:
raise exc.DataGenSyntaxError(
f"Extra keys for field {name} : {top_level}",
**context.line_num(field))
[[function_name, args]] = top_level
plugin = None
if "." in function_name:
namespace, name = function_name.split(".")
plugin = context.parser_macros_plugins.get(namespace)
if plugin:
func = getattr(plugin, name)
rc = func(context, args)
return rc
else:
args = parse_structured_value_args(args, context)
return StructuredValue(function_name, args, **context.line_num(field))
def parse_file(stream: IO[str], context: ParseContext) -> List[Dict]:
stream_name = getattr(stream, "name", None)
if stream_name:
path = Path(fsdecode(stream.name)).absolute()
else:
path = Path("<stream>")
try:
data, line_numbers = yaml_safe_load_with_line_numbers(stream, str(path))
except YAMLError as y:
raise exc.DataGenYamlSyntaxError(
str(y),
str(path),
y.problem_mark.line + 1,
)
context.line_numbers.update(line_numbers)
if not isinstance(data, list):
raise exc.DataGenSyntaxError(
"Generator file should be a list (use '-' on top-level lines)",
stream_name,
1,
)
templates = parse_top_level_elements(path, data, context)
return templates
def categorize_top_level_objects(data: List, context: ParseContext):
"""Look at all of the top-level declarations and categorize them"""
top_level_collections: Dict = {
"option": [],
"include_file": [],
"macro": [],
"plugin": [],
"object": [],
}
assert isinstance(data, list)
for obj in data:
if not isinstance(obj, dict):
raise exc.DataGenSyntaxError(
f"Top level elements in a data generation template should all be dictionaries, not {obj}",
**context.line_num(data),
)
parent_collection = None
for collection in top_level_collections:
if obj.get(collection):
if parent_collection:
raise exc.DataGenError(
f"Top level element seems to match two name patterns: {collection, parent_collection}",
**context.line_num(obj),
)
parent_collection = collection
if parent_collection:
top_level_collections[parent_collection].append(obj)
else:
raise exc.DataGenError(
f"Unknown object type {obj}", **context.line_num(obj)
)
return top_level_collections
def categorize_top_level_objects(data: List, context: ParseContext):
"""Look at all of the top-level declarations and categorize them"""
top_level_collections: Dict = {
"option": [],
"include_file": [],
"macro": [],
"plugin": [],
"statement": [], # objects with side-effects
}
assert isinstance(data, list)
for obj in data:
if not isinstance(obj, dict):
raise exc.DataGenSyntaxError(
f"Top level elements in a data generation template should all be dictionaries, not {obj}",
**context.line_num(data),
)
obj_category = None
for declaration, category in collection_rules.items():
typ = obj.get(declaration)
if typ:
if obj_category:
raise exc.DataGenError(
f"Top level element seems to match two name patterns: {declaration, obj_category}",
**context.line_num(obj),
)
obj_category = category
if obj_category:
top_level_collections[obj_category].append(obj)
else:
raise exc.DataGenError(f"Unknown object type {obj}",
**context.line_num(obj))
return top_level_collections
def change_current_parent_object(self, obj: Dict):
_old_parsed_template = self.current_parent_object
self.current_parent_object = obj
try:
yield
except exc.DataGenError:
raise
except Exception as e:
raise exc.DataGenSyntaxError(str(e), **self.line_num()) from e
finally:
self.current_parent_object = _old_parsed_template
def parse_fields(fields: Dict, context: ParseContext) -> List[FieldFactory]:
with context.change_current_parent_object(fields):
if not isinstance(fields, dict):
raise exc.DataGenSyntaxError(
"Fields should be a dictionary (should not start with -) ",
**context.line_num(),
)
return [
parse_field(name, definition, context)
for name, definition in fields.items() if name != "__line__"
]
def _coerce_to_string(val, context):
if isinstance(val, str):
return val
elif isinstance(val, (int, bool, date)):
# a few functions accept keyword arguments that YAML interprets
# as types other than string
return str(val)
else:
if val is None:
val = "null (None)"
raise exc.DataGenSyntaxError(
f"Cannot interpret key `{val}`` as string", **context.line_num())
def parse_structured_value(name: str, field: Dict,
context: ParseContext) -> Definition:
"""Parse something that might look like:
{'choose': ['Option1', 'Option2', 'Option3', 'Option4'], '__line__': 9}
or
{'random_number': {'min': 10, 'max': 20, '__line__': 10} , '__line__': 9}
"""
top_level = removeline_numbers(field).items()
if not top_level:
raise exc.DataGenSyntaxError(f"Strange datastructure ({field})",
**context.line_num(field))
elif len(top_level) > 1:
raise exc.DataGenSyntaxError(
f"Extra keys for field {name} : {top_level}",
**context.line_num(field))
[[function_name, args]] = top_level
plugin = None
if "." in function_name:
namespace, name = function_name.split(".")
plugin = context.parser_macros_plugins.get(namespace)
if plugin:
try:
func = getattr(plugin, name)
rc = func(context, args)
return rc
except AttributeError as e:
# check if this is a regular runtime function. If so,
# fall-through and handle it as if we had never
# tried to parse it as a macro plugin
if not hasattr(plugin, "Functions") and not hasattr(
plugin.Functions, name):
raise e
args = parse_structured_value_args(args, context)
return StructuredValue(function_name, args, **context.line_num(field))
def parse_object_template(yaml_sobj: Dict,
context: ParseContext) -> ObjectTemplate:
parsed_template: Any = parse_element(
dct=yaml_sobj,
element_type="object",
mandatory_keys={},
optional_keys={
"fields": Dict,
"friends": List,
"include": str,
"nickname": str,
"just_once": bool,
"count": (str, int, dict),
},
context=context,
)
if not context.top_level and parsed_template.just_once:
raise exc.DataGenSyntaxError(
"just_once can only be used at the top level")
assert yaml_sobj
with context.change_current_parent_object(yaml_sobj):
sobj_def = {}
sobj_def["tablename"] = parsed_template.object
check_identifier(parsed_template.object, yaml_sobj, "Object names")
fields: List
friends: List
sobj_def["fields"] = fields = []
sobj_def["friends"] = friends = []
parse_inclusions(yaml_sobj, fields, friends, context)
fields.extend(parse_fields(parsed_template.fields or {}, context))
friends.extend(parse_friends(parsed_template.friends or [], context))
sobj_def["nickname"] = nickname = parsed_template.nickname
check_identifier(nickname, yaml_sobj, "Nicknames")
sobj_def["just_once"] = parsed_template.just_once or False
sobj_def["line_num"] = parsed_template.line_num.line_num
sobj_def["filename"] = parsed_template.line_num.filename
count_expr = yaml_sobj.get("count")
if count_expr is not None:
parse_count_expression(yaml_sobj, sobj_def, context)
new_template = ObjectTemplate(**sobj_def)
context.register_template(new_template)
return new_template
def parse_field_value(
name: str, field, context: ParseContext, allow_structured_values=True
) -> Union[SimpleValue, StructuredValue, ObjectTemplate]:
if isinstance(field, (str, Number, date, type(None))):
return SimpleValue(field, **(context.line_num(field) or context.line_num()))
elif isinstance(field, dict) and field.get("object"):
with context.change_current_parent_object(field):
return parse_object_template(field, context)
elif isinstance(field, dict):
return parse_structured_value(name, field, context)
elif isinstance(field, list) and len(field) == 1 and isinstance(field[0], dict):
# unwrap a list of a single item in this context because it is
# a mistake and we can infer their real meaning
return parse_field_value(name, field[0], context)
else:
raise exc.DataGenSyntaxError(
f"Unknown field {type(field)} type for {name}. Should be a string or 'object': \n {field} ",
**(context.line_num(field) or context.line_num()),
)
