def test_dir(): d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}}) assert "a" in dir(d) assert "b" in dir(d) assert "r" in dir(d) assert 4 in d.keys() assert "t" in d.r
def test_from_tuples(): args = [("a", 1), ("b", 2), ("c", {"d": 3})] d = MinyDict(*args) assert d.a == 1 assert d.b == 2 assert d.c.d == 3 d = MinyDict(args) assert d.a == 1 assert d.b == 2 assert d.c.d == 3
def test_copy(): d1 = MinyDict({"a": 1, "b": 2, "r": {"t": 5}}) d2 = d1.copy() d3 = d1.deepcopy() assert d1 == d2 assert d1 == d3 d2.a = 2 assert d1.a == 1 d3.b = 4 assert d1.b == 2
def load_defaults(default: Union[str, dict, MinyDict]): """ Set the default keys from `defaults`: * str/path -> must point to a yaml/json/pickle file then MinyDict.from_X will be used * dict -> Convert that dictionnary to a resolved MinyDict * list -> Recursively calls load_defaults on each element, then sequentially update an (initially empty) MinyDict to allow for hierarchical defaults. Args: allow (Union[str, dict, MinyDict]): The set of allowed keys as a (Miny)dict or a path to a file that `minydra.MinyDict` will be able to load (as `json`, `pickle` or `yaml`) """ # `defaults` is a path: load it with MinyDict.from_X if isinstance(default, (str, pathlib.Path)): # resolve path to file default = resolve_path(default) # ensure it exists assert default.exists() assert default.is_file() # check for known file formats if default.suffix not in {".json", ".yaml", ".yml", ".pickle", ".pkl"}: raise ValueError(f"{str(default)} is not a valid file extension.") # Load from YAML if default.suffix in {".yaml", ".yml"}: default = MinyDict.from_yaml(default) # Load from Pickle elif default.suffix in {".pickle", ".pkl"}: default = MinyDict.from_pickle(default) # Load from JSON else: default = MinyDict.from_json(default) # `defaults` is a dictionnary: convert it to a resolved MinyDict elif isinstance(default, dict): default = MinyDict(default).resolve() # `defaults` is a list: recursively call load_defaults on each element # then sequentially merge all dictionaries to enable hierarchical defaults elif isinstance(default, list): defaults = [Parser.load_defaults(d) for d in default] default = MinyDict() for d in defaults: default.update(d, strict=False) assert isinstance(default, MinyDict) return default
def test_update(): d1 = MinyDict({"a": 1, "b": 2, "r": {"t": 5}}) d2 = d1.copy() u1 = MinyDict({"a": 3, "r": {"t": 4, "z": 7}}) u2 = MinyDict({"a": 3, "r": {"t": 4}}) t1 = MinyDict({"a": 3, "b": 2, "r": {"t": 4, "z": 7}}) t2 = MinyDict({"a": 3, "b": 2, "r": {"t": 4}}) assert d1.update(u1) == t1 with pytest.raises(KeyError): d2.update(u1, strict=True) assert d2.update(u2, strict=False) == t2 with pytest.raises(TypeError): d1.update({1: 2}, {3: 4})
def test_from_dict(): d = MinyDict({1: 2, "a": 4, "b": {"c": 5}}) assert d[1] == 2 assert d["a"] == 4 assert d["b"]["c"] == 5 assert d.a == 4 assert d.b.c == 5
def test_basic(): assert run(["a=1", "b=1e-3", "c", "-d"]) == capture( MinyDict({ "a": 1, "b": 0.001, "c": True, "d": False }))
def test_in(): d = MinyDict({"a": 1, "b": 2, 4: "x"}) assert "a" in d assert "b" in d assert "x" not in d assert d.z is None assert 4 in d assert d.a == 1 assert d.b == 2 assert d[4] == "x"
def test_force_types(): assert (run([ "a___str=01", "b___bool=d", "c___int=1.3", "d___float=2", ]) == capture(MinyDict({ "a": "01", "b": True, "c": 1, "d": 2.0 })))
def test_freeze(): d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}}) d.freeze() with pytest.raises(AttributeError): d.r.t = 4 d.unfreeze() d.r.t = 4
def test_attr(): d = MinyDict({"a": 1, "b": 2, 4: "x"}) d.x = 3 assert d.x == 3 d.y = MinyDict({"w": 4}) assert d.y.w == 4 d.z = {"m": 0} assert d.z.m == 0
def _parse_args(self): # check arguments syntax Parser.check_args(self._argv) # edit configuration to use the command-line special (@) arguments self._parse_conf() # create a dictionary from the command-line string arguments args = Parser.map_argv( self._argv, self.conf.allow_overwrites, self.conf.warn_overwrites ) # parse the dictionary's values into known types args = Parser.parse_arg_types(args, self.conf.parse_env, self.conf.warn_env) # store the parsed & typed arguments in a MinyDict self.args = MinyDict(**args)
def test_empty(): assert run([]) == capture(MinyDict({}))
def __init__( self, verbose=0, allow_overwrites=False, warn_overwrites=True, parse_env=True, warn_env=True, defaults=None, strict=True, keep_special_kwargs=True, ) -> None: """ Create a Minydra Parser to parse arbitrary commandline argument as: * `key=value` * `positiveArg` (set to `True`) * `-negativeArg` (set to `False`) Args: verbose (int, optional): Wether to print received system arguments. Defaults to 0. allow_overwrites (bool, optional): Wether to allow for repeating arguments in the command line. Defaults to True. warn_overwrites (bool, optional): Wether to print a waring in case of a repeated argument (if that is allowed). Defaults to True. parse_env (bool, optional): Wether to parse environment variables specified as key or value in the command line. Defaults to True. warn_env (bool, optional): Wether to print a warning in case an environment variable is parsed but no value is found. Defaults to True. defaults (Union[str, dict, MinyDict], optional): Default arguments a (Miny)dict or a path to a file that `minydra.MinyDict` will be able to load (as `json`, `pickle` or `yaml`). Can also be provided with the special `@defaults=<value>` command-line argument. Defaults to None. strict (bool, optional): Wether to raise an error if an unknown argument is passed and `defaults` was provided. Can also be provided with the special `@strict=<true|false>` command-line argument. Defaults to True. keep_special_kwargs (bool, optional): Wether to keep special keywords like `@defaults` and `@strict` in the parsed arguments. Defaults to True. """ super().__init__() self.conf = MinyDict() # store initial arguments in a conf object to be able to # retrieve them later and most importantly to be able to # parse them in the `_parse_conf()` method self.conf.verbose = verbose self.conf.allow_overwrites = allow_overwrites self.conf.warn_overwrites = warn_overwrites self.conf.parse_env = parse_env self.conf.warn_env = warn_env self.conf.defaults = defaults self.conf.strict = strict self.conf.keep_special_kwargs = keep_special_kwargs self._argv = sys.argv[1:] self._parse_args() self._print("sys.argv:", self._argv) # defaults are provided if self.conf.defaults is not None or self.args["@defaults"]: # load defaults as a MinyDict default = self.load_defaults(self.args["@defaults"] or self.conf.defaults) # resolve existing args to properly override nested defaults args = self.args.deepcopy().resolve() args_defaults = args["@defaults"] args_strict = args["@strict"] # clean args if args_defaults is not None: del args["@defaults"] if args_strict is not None: self.conf.strict = args["@strict"] del args["@strict"] # update defaults from command-line args self.args = default.update(args, strict=self.conf.strict) # bring back special (@) kwargs if need be if self.conf.keep_special_kwargs: if args_defaults is not None: self.args["@defaults"] = args_defaults if args_strict is not None: self.args["@strict"] = args_strict # clean up special (@) kwargs if need be if not self.conf.keep_special_kwargs: for k in self.conf: self.args.pop(f"@{k}", None)
def test_to_dict(): d = MinyDict({"a": 1, "b": [{"c": 2}, {"d": {"e": 3}}]}) assert d.b[0].c == 2 assert d.to_dict() == {"a": 1, "b": [{"c": 2}, {"d": {"e": 3}}]}
def test_env(monkeypatch): monkeypatch.setenv("USER", "TestingUser") assert run(["a=$USER"]) == capture(MinyDict({"a": "TestingUser"}))
def test_dump_yaml(): d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}}) p = Path(d.to_yaml("d.yaml", verbose=1)) assert MinyDict.from_yaml(p) == d p.unlink()
def test_list(): assert run(["a=[2, 4, 'x']"]) == capture(MinyDict({"a": [2, 4, "x"]}))
def test_dump_pickle(): d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}}) p = Path(d.to_pickle("d.pkl", verbose=1)) assert MinyDict.from_pickle(p) == d p.unlink()
def test_dump_json(): d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}}) p = Path(d.to_json("d.json", verbose=1)) assert MinyDict.from_json(p) == d p.unlink()
def test_dict(): assert run(["a={}".format('{"b": 3}') ]) == capture(MinyDict({"a": { "b": 3 }}))
def test_pretty_print(): d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}}) d.pretty_print() d = MinyDict({"a": 1, "r": {"t": 5, "key": ("superlongvalue" * 10)}}) d.pretty_print() d = MinyDict({ "a": 1, "r": { "t": 5, ("superlongkey" * 10): 2 } }) # to be dealt with, not well handled d.pretty_print()
def test_protected(): d = MinyDict() with pytest.raises(AttributeError): d.update = 4 d["update"] = 4
def test_resolve(): d = MinyDict({"a.b.c": 2}) d.resolve() assert d == MinyDict({"a": {"b": {"c": 2}}}) d = MinyDict({"a.b.c": 2, "a.b.d": 3}) d.resolve() assert d == MinyDict({"a": {"b": {"c": 2, "d": 3}}}) d = MinyDict({"a.b.c": 2, "a.b.d": {"r.v": 4}, "x": {"o.p": 3}}) d.resolve() assert d == MinyDict({ "x": { "o": { "p": 3 } }, "a": { "b": { "c": 2, "d": { "r": { "v": 4 } } } } })
def test_dump_no_overwrite(): d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}}) p = Path(d.to_json("d.json")) with pytest.raises(FileExistsError): d.to_json(p, allow_overwrite=False) p.unlink() p = Path(d.to_pickle("d.pkl")) with pytest.raises(FileExistsError): d.to_pickle(p, allow_overwrite=False) assert MinyDict.from_pickle(p) == d p.unlink() p = Path(d.to_yaml("d.yaml")) with pytest.raises(FileExistsError): d.to_yaml(p, allow_overwrite=False) assert MinyDict.from_yaml(p) == d p.unlink()
def test_init(): _ = MinyDict()
def test_dotted(): assert run(["a.b.x=2"]) == capture(MinyDict({"a": {"b": {"x": 2}}}))
def test_defaults(): examples = Path(__file__).resolve().parent.parent / "examples" d1 = examples / "demo.json" d2 = examples / "demo2.json" y1 = examples / "demo.yaml" p = MinyDict({"a": "2", "c": 3, "d": {"e": {"f": 4, "g": 5}}}) pkl = p.to_pickle(Path(__file__).resolve().parent / "test.pkl") with patch.object(sys, "argv", [""]): args = minydra.resolved_args(defaults=p) assert args == p with patch.object(sys, "argv", ["", "d.e.f=2"]): args = minydra.resolved_args(defaults=p) assert args.d.e.f == 2 with patch.object(sys, "argv", ["", f"@defaults={str(d1)}"]): args = minydra.resolved_args() del args["@defaults"] assert args.to_dict() == json.loads(d1.read_text()) with patch.object(sys, "argv", ["", f"@defaults={str(y1)}"]): args = minydra.resolved_args() del args["@defaults"] assert args.to_dict() == MinyDict.from_yaml(y1) with patch.object(sys, "argv", ["", f"@defaults={str(pkl)}"]): args = minydra.resolved_args() del args["@defaults"] assert args.to_dict() == MinyDict.from_pickle(pkl) Path(pkl).unlink() with pytest.raises(ValueError): with patch.object( sys, "argv", ["", f"@defaults={str(d1).replace('.json', '.py')}"] ): args = minydra.resolved_args() with pytest.raises(KeyError): with patch.object(sys, "argv", ["", f"@defaults={str(d1)}", "new_key=3"]): args = minydra.resolved_args() del args["@defaults"] assert args.to_dict() == json.loads(d1.read_text()) with patch.object( sys, "argv", ["", f"@defaults={str(d1)}", "@strict=false", "new_key=3"] ): args = minydra.resolved_args(keep_special_kwargs=False) target = json.loads(d1.read_text()) target["new_key"] = 3 assert args.to_dict() == target with patch.object( sys, "argv", ["", f"@defaults={str(d1)}", "@strict=false", "new_key=3"] ): args = minydra.resolved_args() del args["@defaults"] del args["@strict"] target = json.loads(d1.read_text()) target["new_key"] = 3 assert args.to_dict() == target double_defaults = f"['{str(d1)}', '{str(d2)}']" with patch.object(sys, "argv", ["", f"@defaults={double_defaults}", "new_key=3"]): args = minydra.resolved_args() d1d = MinyDict.from_json(d1) d2d = MinyDict.from_json(d2) d1d = d1d.update(d2d) del args["@defaults"] assert args == d1d
class Parser: """ Minydra Parser: reads `sys.argv[1:]` into a `minydra.MinyDict`. Immediately runs `self._parse_args()`: storing a dictionnary of arguments in `self.dict_args` and a `minydra.MinyDict` of arguments in `self.args`. """ known_types = { "bool", "int", "float", "str", } type_separator = "___" def __init__( self, verbose=0, allow_overwrites=False, warn_overwrites=True, parse_env=True, warn_env=True, defaults=None, strict=True, keep_special_kwargs=True, ) -> None: """ Create a Minydra Parser to parse arbitrary commandline argument as: * `key=value` * `positiveArg` (set to `True`) * `-negativeArg` (set to `False`) Args: verbose (int, optional): Wether to print received system arguments. Defaults to 0. allow_overwrites (bool, optional): Wether to allow for repeating arguments in the command line. Defaults to True. warn_overwrites (bool, optional): Wether to print a waring in case of a repeated argument (if that is allowed). Defaults to True. parse_env (bool, optional): Wether to parse environment variables specified as key or value in the command line. Defaults to True. warn_env (bool, optional): Wether to print a warning in case an environment variable is parsed but no value is found. Defaults to True. defaults (Union[str, dict, MinyDict], optional): Default arguments a (Miny)dict or a path to a file that `minydra.MinyDict` will be able to load (as `json`, `pickle` or `yaml`). Can also be provided with the special `@defaults=<value>` command-line argument. Defaults to None. strict (bool, optional): Wether to raise an error if an unknown argument is passed and `defaults` was provided. Can also be provided with the special `@strict=<true|false>` command-line argument. Defaults to True. keep_special_kwargs (bool, optional): Wether to keep special keywords like `@defaults` and `@strict` in the parsed arguments. Defaults to True. """ super().__init__() self.conf = MinyDict() # store initial arguments in a conf object to be able to # retrieve them later and most importantly to be able to # parse them in the `_parse_conf()` method self.conf.verbose = verbose self.conf.allow_overwrites = allow_overwrites self.conf.warn_overwrites = warn_overwrites self.conf.parse_env = parse_env self.conf.warn_env = warn_env self.conf.defaults = defaults self.conf.strict = strict self.conf.keep_special_kwargs = keep_special_kwargs self._argv = sys.argv[1:] self._parse_args() self._print("sys.argv:", self._argv) # defaults are provided if self.conf.defaults is not None or self.args["@defaults"]: # load defaults as a MinyDict default = self.load_defaults(self.args["@defaults"] or self.conf.defaults) # resolve existing args to properly override nested defaults args = self.args.deepcopy().resolve() args_defaults = args["@defaults"] args_strict = args["@strict"] # clean args if args_defaults is not None: del args["@defaults"] if args_strict is not None: self.conf.strict = args["@strict"] del args["@strict"] # update defaults from command-line args self.args = default.update(args, strict=self.conf.strict) # bring back special (@) kwargs if need be if self.conf.keep_special_kwargs: if args_defaults is not None: self.args["@defaults"] = args_defaults if args_strict is not None: self.args["@strict"] = args_strict # clean up special (@) kwargs if need be if not self.conf.keep_special_kwargs: for k in self.conf: self.args.pop(f"@{k}", None) @staticmethod def load_defaults(default: Union[str, dict, MinyDict]): """ Set the default keys from `defaults`: * str/path -> must point to a yaml/json/pickle file then MinyDict.from_X will be used * dict -> Convert that dictionnary to a resolved MinyDict * list -> Recursively calls load_defaults on each element, then sequentially update an (initially empty) MinyDict to allow for hierarchical defaults. Args: allow (Union[str, dict, MinyDict]): The set of allowed keys as a (Miny)dict or a path to a file that `minydra.MinyDict` will be able to load (as `json`, `pickle` or `yaml`) """ # `defaults` is a path: load it with MinyDict.from_X if isinstance(default, (str, pathlib.Path)): # resolve path to file default = resolve_path(default) # ensure it exists assert default.exists() assert default.is_file() # check for known file formats if default.suffix not in {".json", ".yaml", ".yml", ".pickle", ".pkl"}: raise ValueError(f"{str(default)} is not a valid file extension.") # Load from YAML if default.suffix in {".yaml", ".yml"}: default = MinyDict.from_yaml(default) # Load from Pickle elif default.suffix in {".pickle", ".pkl"}: default = MinyDict.from_pickle(default) # Load from JSON else: default = MinyDict.from_json(default) # `defaults` is a dictionnary: convert it to a resolved MinyDict elif isinstance(default, dict): default = MinyDict(default).resolve() # `defaults` is a list: recursively call load_defaults on each element # then sequentially merge all dictionaries to enable hierarchical defaults elif isinstance(default, list): defaults = [Parser.load_defaults(d) for d in default] default = MinyDict() for d in defaults: default.update(d, strict=False) assert isinstance(default, MinyDict) return default def _print(self, *args, **kwargs): if self.conf.verbose > 0: print(*args, **kwargs) def _parse_args(self): # check arguments syntax Parser.check_args(self._argv) # edit configuration to use the command-line special (@) arguments self._parse_conf() # create a dictionary from the command-line string arguments args = Parser.map_argv( self._argv, self.conf.allow_overwrites, self.conf.warn_overwrites ) # parse the dictionary's values into known types args = Parser.parse_arg_types(args, self.conf.parse_env, self.conf.warn_env) # store the parsed & typed arguments in a MinyDict self.args = MinyDict(**args) def _parse_conf(self): # find all potentially overridable arguments special_keys = [f"@{k}" for k in self.conf.keys()] # find all minydra-specific command-line args: # they must start with an "@" AND be in special_keys # so that @unknown=4 will be kept as an argument, not a configuration # argument. command_line_special_keys = [ arg for arg in self._argv if arg.split("=")[0] in special_keys ] # No special (@) arguments: nothing to do if len(command_line_special_keys) == 0: return # create a dictionary of special (@) arguments ({key:value}) mapped_specials = Parser.map_argv(command_line_special_keys, False, False) # parse the dictionary's values into known types parsed_specials = Parser.parse_arg_types(mapped_specials, True, True) # override conf from special (@) command-line arguments self.conf.update({k[1:]: v for k, v in parsed_specials.items()}) @staticmethod def check_args(args: List[str]) -> None: """ Checks the syntax of received arguments. Only `key=value`, `positiveArg`, `-negativeArg` patters are allowed in `sys.argv[1:]`. Args: args (list): List of arguments from `sys.argv[1:]` Raises: MinydraWrongArgumentException: ` =` in arguments MinydraWrongArgumentException: `= ` in arguments MinydraWrongArgumentException: Command-line ends with `=` MinydraWrongArgumentException: Argument key starts with a `-` """ args_str = " ".join(args) if " =" in args_str: raise MinydraWrongArgumentException( "Found space around `=` sign." + " Named arguments should be used as key=value" ) if "= " in args_str: raise MinydraWrongArgumentException( "Found space around `=` sign." + " Named arguments should be used as key=value" ) if args_str.endswith("="): raise MinydraWrongArgumentException( "Missing argument after `=` sign." + " Named arguments should be used as key=value" ) for arg in args: if "=" in arg: if arg.startswith("-"): raise MinydraWrongArgumentException( "Cannot start a named argument with a -" ) if arg.startswith("."): raise MinydraWrongArgumentException("Cannot start an argument with a .") return @staticmethod def _force_type(value: str, type_str: str) -> Any: """ Applies a type constructor to a value, e.g. `float(value)`, if `type_str` is in `Parser.known_types`. Args: value (str): The value to transform. type_str (str): The string for the type constructor, as per `Parser.known_args`. Returns: Any: typed value if the `type_str` is known, `value` otherwise. """ if type_str == "bool": return bool(value) if type_str == "int": return int(float(value)) if type_str == "float": return float(value) if type_str == "str": return str(value) @staticmethod def _infer_arg_type(arg: Any, type_str: Optional[str] = None) -> Any: """ Parses an argument: returns it if it is not a string, else: * parses to int * parses to float * parses to bool * parses to a list * parses to a dict Type can be forced by appending `___type` to the key Recursive calls are applied to list and dict items Args: arg (Any): Argument to parse Returns: Any: parsed argument """ if not isinstance(arg, str): return arg if type_str is not None: return Parser._force_type(arg, type_str) if arg.isdigit(): return int(arg) try: arg = float(arg) return arg except ValueError: pass if arg.lower() == "true": return True if arg.lower() == "false": return False if arg.startswith("[") and arg.endswith("]"): return [Parser._infer_arg_type(v) for v in ast.literal_eval(arg)] if arg.startswith("{") and arg.endswith("}"): return { Parser._infer_arg_type(k): Parser._infer_arg_type(v) for k, v in ast.literal_eval(arg).items() } return arg @staticmethod def parse_arg_types( args: dict, parse_env: bool = True, warn_env: bool = True ) -> dict: """ Automatically infer individual arguments' types Args: args (dict): dictionnary of arguments parse_env (bool, optional): Whether to parse environment variables. Defaults to True. warn_env (bool, optional): Whether to print a warning in case an env variable cannot be found. Defaults to True. Returns: dict: Args dict with inferred type values """ typed = {} for k, v in args.items(): if parse_env: v = Parser.set_env(v, warn_env) type_str = None if Parser.type_separator in k: candidate_k, candidate_type_str = k.split(Parser.type_separator) if candidate_type_str in Parser.known_types: type_str = candidate_type_str k = candidate_k typed[k] = Parser._infer_arg_type(v, type_str) return typed @staticmethod def set_env(value: str, warn_env: bool = True) -> str: """ Replaces environment variables with their values Args: value (str): Environment variable identifier $VAR warn_env (bool, optional): Whether to print a warning if an env variable cannot be found. Defaults to True. Returns: str: Argument with resolved env variables """ if not isinstance(value, str): return value arg_vars = re.findall(r"\$([\w_]+)", value) for arg_var in arg_vars: env_var = os.getenv(arg_var) if env_var is None and warn_env: Parser.warn( f"Detected variable ${arg_var}" + ", but could not find it in the environment. " + f"Keeping raw value ${arg_var}.", ) else: value = value.replace("$" + arg_var, env_var) return value @staticmethod def warn(text: str): """ Print a warning as `"[Minydra Warning] " + text` Args: text (str): Warning text """ text = "[Minydra Warning] " + text print(text) @staticmethod def map_argv(args, allow_overwrites, warn_overwrites): """ Create a dictionnary from the list of arguments: * `key=value` -> `{key: value}` * `key` -> `{key: True}` * `-key` -> `{key: False}` Args: args (list): List of arguments to map into a dictionnary allow_overwrites (bool): Whether repeating arguments are allowed. If True, the last value is kept warn_overwrites (bool): Whether to print a warning in case an argument is repeated (and it is allowed) Raises: MinydraWrongArgumentException: An argument is repeated and allow_overwrites is False Returns: dict: Mapped arguments """ args_dict = {} for arg in args: if "=" in arg: values = arg.split("=") key = values[0] value = "=".join(values[1:]) key = key.strip() value = value.strip() elif arg.startswith("-"): key = arg[1:].strip() value = False else: key = arg.strip() value = True if key in args_dict: if not allow_overwrites: raise MinydraWrongArgumentException( f"Repeated argument {key} with `allow_overwrites=False`" ) if warn_overwrites: text = f"Repeated argument {key}," + " overwriting previous value." Parser.warn(text) args_dict[key] = value return args_dict