def _parse_filename(filename: JsonType, params: Dict[str, Any], load_type: str) -> str: if not isinstance(filename, str): raise YATLSyntaxError( f"{load_type} must be given a string or list of strings: {filename}" ) filename = render_interpolation(filename, params) if not isinstance(filename, str): raise YATLSyntaxError( f"{load_type} directive must be given a string or list of strings: {filename}" ) return filename
def _render_for( key: str, value: JsonType, params: Dict[str, Any], defs: Dict[str, Def], rendered_obj: JsonType, ) -> JsonType: for_match = re.match( r"for\s*\(([a-zA-Z_][a-zA-Z0-9_]*)\s+in\s+([a-zA-Z_][a-zA-Z0-9_]*)\)\s*$", key[1:], ) if not for_match: raise YATLSyntaxError(f"Invalid for statement: {key}") var = for_match[1].strip() param = for_match[2].strip() try: iterable = params[param] except KeyError: raise YATLEnvironmentError(f"Missing parameter {param}") rendered_list = [] for elem in iterable: rendered_list.append(render_from_obj(value, {**params, var: elem}, defs)) return _shallow_merge(key, rendered_list, params, defs, rendered_obj)
def _create_use_args_from_list(value: list, df: Def) -> Dict[str, JsonType]: if len(df.args) != len(value): expected = ", ".join(df.args) received = len(value) raise YATLSyntaxError( f"def {df.name} expected args ({expected}) but received {received} args instead" ) return dict(zip(df.args, value))
def _parse_use_args(value: JsonType, df: Def) -> Dict[str, JsonType]: if not df.args: if value: raise YATLSyntaxError( f"def {df.name} takes no args but use directive passed a non-empty object" ) return {} if isinstance(value, dict): return _create_use_args_from_dict(value, df) elif isinstance(value, list): return _create_use_args_from_list(value, df) else: if len(df.args) != 1: raise YATLSyntaxError( f"def {df.name} takes {len(df.args)} args but is given a value; pass an object or list instead" ) return {df.args[0]: value}
def _update_obj( obj: JsonType, key: str, value: JsonType, params: Dict[str, Any], defs: Dict[str, Def], ) -> None: if not isinstance(obj, dict): raise YATLSyntaxError(f"Cannot add field {key} to non-object") interpolated_key = render_interpolation(key, params) obj[interpolated_key] = render_from_obj(value, params, defs)
def _create_use_args_from_dict(value: dict, df: Def) -> Dict[str, JsonType]: expected_args = {*df.args} received_args = {*value.keys()} if expected_args != received_args: expected = ", ".join(df.args) received = ", ".join(value.keys()) raise YATLSyntaxError( f"def {df.name} expected args ({expected}) but received ({received})" ) return value
def _parse_use_name(key: str) -> str: name_match = re.match( r""" \.use \s+ ([a-zA-Z_][a-zA-Z0-9_]*) \s* # Capture the name """, key, re.VERBOSE, ) if not name_match: raise YATLSyntaxError(f"Malformed use directive: {key}") return name_match[1]
def _parse_def_parts(key: str) -> Tuple[str, List[str]]: name_match = re.match( r""" \.def \s+ ([a-zA-Z_][a-zA-Z0-9_]*) \s* # Capture the name """, key, re.VERBOSE, ) if not name_match: raise YATLSyntaxError(f"Malformed def directive: {key}") name = name_match[1] if name_match.end() == len(key): # Handle ".def foo:" return name, [] args = key[name_match.end() :] args_match = re.match( r""" \( \s* ( # Capture the arg list (?:[a-zA-Z_][a-zA-Z0-9_]*) # First arg (?: \s* , \s* [a-zA-Z_][a-zA-Z0-9_]* # Subsequent args )* )? \s* \) \s* $ """, args, re.VERBOSE, ) if not args_match: raise YATLSyntaxError(f"Malformed def arguments: {key}") if not args_match[1]: # Handle ".def foo():" return name, [] return name, [a.strip() for a in args_match[1].split(",")]
def parse_expression(s: str) -> Tuple[str, str]: paren_nesting = 0 i = 0 while i < len(s): if s[i] == "(": paren_nesting += 1 elif s[i] == ")": paren_nesting -= 1 # TODO: Handle embedded strings if paren_nesting < 0: return s[:i], s[i + 1:] i += 1 raise YATLSyntaxError("Could not find end of expression")
def _render_if( if_key: str, if_value: JsonType, params: Dict[str, Any], defs: Dict[str, Def], rendered_obj: JsonType, ) -> Tuple[JsonType, bool]: # This regular expression captures both if and elif. if_match = re.match(r"\.(?:el)?if\s*\((.*)\)\s*$", if_key) if not if_match: raise YATLSyntaxError(f"Invalid if statement: {if_key}") condition = if_match[1].strip() if params[condition]: return _shallow_merge(if_key, if_value, params, defs, rendered_obj), True return rendered_obj, False
def _load_defaults( value: JsonType, params: Dict[str, Any], defs: Dict[str, Def] ) -> dict: if not isinstance(value, list): value = [value] accumulated_defaults: dict = {} for filename in value: filename = _parse_filename(filename, params, "load_defaults_from") defaults = _load_yaml(filename) if not isinstance(defaults, dict): raise YATLSyntaxError(f"{filename} must be an object at the top-level") rendered_defaults = render_from_obj(defaults, params, defs) accumulated_defaults = _deep_merge_dicts( accumulated_defaults, rendered_defaults ) return accumulated_defaults
def _shallow_merge( key: str, value: JsonType, params: Dict[str, Any], defs: Dict[str, Def], rendered_obj: JsonType, ) -> JsonType: rendered_value = render_from_obj(value, params, defs) if not _can_merge_values(rendered_obj, rendered_value): raise YATLSyntaxError( f"Cannot merge {_type_name(rendered_value)} with {_type_name(rendered_obj)} in {key}" ) if isinstance(rendered_value, dict): # Don't deep-merge, just shallow-update return {**rendered_obj, **rendered_value} # type: ignore elif isinstance(rendered_value, list): rendered_obj = rendered_obj or [] # Convert {} to [] return rendered_obj + rendered_value # type: ignore else: return rendered_value
def _store_def(key: str, value: JsonType, defs: Dict[str, Def]) -> None: name, args = _parse_def_parts(key) if len(args) != len({*args}): raise YATLSyntaxError(f"Duplicate name in def arguments: {key}") defs[name] = Def(name, args, value)
def render_from_obj( # noqa: C901 obj: JsonType, params: Dict[str, Any], defs: Dict[str, Def] ) -> JsonType: if isinstance(obj, dict): defaults_obj: JsonType = None rendered_obj: JsonType = {} last_if = None for key, value in obj.items(): if isinstance(key, str) and key.startswith("."): # Note, elif and else require Python 3.7+ or a custom YAML loader to preserve key order. if _is_if(key): rendered_obj, last_if = _render_if( key, value, params, defs, rendered_obj ) elif _is_elif(key): if last_if is None: raise YATLSyntaxError(f"elif does not follow if: {key}") if last_if is False: rendered_obj, last_if = _render_if( key, value, params, defs, rendered_obj ) elif key == ".else": if last_if is None: raise YATLSyntaxError(f"else does not follow if: {key}") if last_if is False: rendered_obj = _render_else( key, value, params, defs, rendered_obj ) else: last_if = None if key == ".load": rendered_obj = _render_load(value, params, defs, rendered_obj) elif key == ".load_defaults_from": defaults_obj = _load_defaults(value, params, defs) elif _is_for(key): rendered_obj = _render_for( key, value, params, defs, rendered_obj ) elif _is_def(key): _store_def(key, value, defs) elif _is_use(key): rendered_obj = _render_use( key, value, params, defs, rendered_obj ) else: _update_obj(rendered_obj, key, value, params, defs) else: last_if = None _update_obj(rendered_obj, key, value, params, defs) if defaults_obj: rendered_obj = _deep_merge_dicts(defaults_obj, rendered_obj) # type: ignore return rendered_obj elif isinstance(obj, list): rendered_obj = [] for elem in obj: rendered_elem = render_from_obj(elem, params, defs) if _can_extend_list(elem, rendered_elem): # Convert rendered_elem to [] if it's {} rendered_obj.extend(rendered_elem or []) # type: ignore else: rendered_obj.append(rendered_elem) return rendered_obj elif isinstance(obj, str): return render_interpolation(obj, params) else: return obj