def test_attr():
d = MinyDict({"a": 1, "b": 2, 4: "x"})
d.x = 3
assert d.x == 3
d.y = MinyDict({"w": 4})
assert d.y.w == 4
d.z = {"m": 0}
assert d.z.m == 0
def test_from_tuples():
args = [("a", 1), ("b", 2), ("c", {"d": 3})]
d = MinyDict(*args)
assert d.a == 1
assert d.b == 2
assert d.c.d == 3
d = MinyDict(args)
assert d.a == 1
assert d.b == 2
assert d.c.d == 3
def test_pretty_print():
d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}})
d.pretty_print()
d = MinyDict({"a": 1, "r": {"t": 5, "key": ("superlongvalue" * 10)}})
d.pretty_print()
d = MinyDict({
"a": 1,
"r": {
"t": 5,
("superlongkey" * 10): 2
}
}) # to be dealt with, not well handled
d.pretty_print()
def load_defaults(default: Union[str, dict, MinyDict]):
"""
Set the default keys from `defaults`:
* str/path -> must point to a yaml/json/pickle file then
MinyDict.from_X will be used
* dict -> Convert that dictionnary to a resolved MinyDict
* list -> Recursively calls load_defaults on each element, then
sequentially update an (initially empty) MinyDict to allow
for hierarchical defaults.
Args:
allow (Union[str, dict, MinyDict]): The set of allowed keys as a
(Miny)dict or a path to a file that `minydra.MinyDict` will be able to
load (as `json`, `pickle` or `yaml`)
"""
# `defaults` is a path: load it with MinyDict.from_X
if isinstance(default, (str, pathlib.Path)):
# resolve path to file
default = resolve_path(default)
# ensure it exists
assert default.exists()
assert default.is_file()
# check for known file formats
if default.suffix not in {".json", ".yaml", ".yml", ".pickle", ".pkl"}:
raise ValueError(f"{str(default)} is not a valid file extension.")
# Load from YAML
if default.suffix in {".yaml", ".yml"}:
default = MinyDict.from_yaml(default)
# Load from Pickle
elif default.suffix in {".pickle", ".pkl"}:
default = MinyDict.from_pickle(default)
# Load from JSON
else:
default = MinyDict.from_json(default)
# `defaults` is a dictionnary: convert it to a resolved MinyDict
elif isinstance(default, dict):
default = MinyDict(default).resolve()
# `defaults` is a list: recursively call load_defaults on each element
# then sequentially merge all dictionaries to enable hierarchical defaults
elif isinstance(default, list):
defaults = [Parser.load_defaults(d) for d in default]
default = MinyDict()
for d in defaults:
default.update(d, strict=False)
assert isinstance(default, MinyDict)
return default
def test_freeze():
d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}})
d.freeze()
with pytest.raises(AttributeError):
d.r.t = 4
d.unfreeze()
d.r.t = 4
def test_from_dict():
d = MinyDict({1: 2, "a": 4, "b": {"c": 5}})
assert d[1] == 2
assert d["a"] == 4
assert d["b"]["c"] == 5
assert d.a == 4
assert d.b.c == 5
def test_dir():
d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}})
assert "a" in dir(d)
assert "b" in dir(d)
assert "r" in dir(d)
assert 4 in d.keys()
assert "t" in d.r
def test_update():
d1 = MinyDict({"a": 1, "b": 2, "r": {"t": 5}})
d2 = d1.copy()
u1 = MinyDict({"a": 3, "r": {"t": 4, "z": 7}})
u2 = MinyDict({"a": 3, "r": {"t": 4}})
t1 = MinyDict({"a": 3, "b": 2, "r": {"t": 4, "z": 7}})
t2 = MinyDict({"a": 3, "b": 2, "r": {"t": 4}})
assert d1.update(u1) == t1
with pytest.raises(KeyError):
d2.update(u1, strict=True)
assert d2.update(u2, strict=False) == t2
with pytest.raises(TypeError):
d1.update({1: 2}, {3: 4})
def test_basic():
assert run(["a=1", "b=1e-3", "c", "-d"]) == capture(
MinyDict({
"a": 1,
"b": 0.001,
"c": True,
"d": False
}))
def test_in():
d = MinyDict({"a": 1, "b": 2, 4: "x"})
assert "a" in d
assert "b" in d
assert "x" not in d
assert d.z is None
assert 4 in d
assert d.a == 1
assert d.b == 2
assert d[4] == "x"
def test_force_types():
assert (run([
"a___str=01",
"b___bool=d",
"c___int=1.3",
"d___float=2",
]) == capture(MinyDict({
"a": "01",
"b": True,
"c": 1,
"d": 2.0
})))
def test_copy():
d1 = MinyDict({"a": 1, "b": 2, "r": {"t": 5}})
d2 = d1.copy()
d3 = d1.deepcopy()
assert d1 == d2
assert d1 == d3
d2.a = 2
assert d1.a == 1
d3.b = 4
assert d1.b == 2
def test_resolve():
d = MinyDict({"a.b.c": 2})
d.resolve()
assert d == MinyDict({"a": {"b": {"c": 2}}})
d = MinyDict({"a.b.c": 2, "a.b.d": 3})
d.resolve()
assert d == MinyDict({"a": {"b": {"c": 2, "d": 3}}})
d = MinyDict({"a.b.c": 2, "a.b.d": {"r.v": 4}, "x": {"o.p": 3}})
d.resolve()
assert d == MinyDict({
"x": {
"o": {
"p": 3
}
},
"a": {
"b": {
"c": 2,
"d": {
"r": {
"v": 4
}
}
}
}
})
def _parse_args(self):
# check arguments syntax
Parser.check_args(self._argv)
# edit configuration to use the command-line special (@) arguments
self._parse_conf()
# create a dictionary from the command-line string arguments
args = Parser.map_argv(
self._argv, self.conf.allow_overwrites, self.conf.warn_overwrites
)
# parse the dictionary's values into known types
args = Parser.parse_arg_types(args, self.conf.parse_env, self.conf.warn_env)
# store the parsed & typed arguments in a MinyDict
self.args = MinyDict(**args)
def test_dump_no_overwrite():
d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}})
p = Path(d.to_json("d.json"))
with pytest.raises(FileExistsError):
d.to_json(p, allow_overwrite=False)
p.unlink()
p = Path(d.to_pickle("d.pkl"))
with pytest.raises(FileExistsError):
d.to_pickle(p, allow_overwrite=False)
assert MinyDict.from_pickle(p) == d
p.unlink()
p = Path(d.to_yaml("d.yaml"))
with pytest.raises(FileExistsError):
d.to_yaml(p, allow_overwrite=False)
assert MinyDict.from_yaml(p) == d
p.unlink()
def test_empty():
assert run([]) == capture(MinyDict({}))
def test_list():
assert run(["a=[2, 4, 'x']"]) == capture(MinyDict({"a": [2, 4, "x"]}))
def test_dotted():
assert run(["a.b.x=2"]) == capture(MinyDict({"a": {"b": {"x": 2}}}))
def test_to_dict():
d = MinyDict({"a": 1, "b": [{"c": 2}, {"d": {"e": 3}}]})
assert d.b[0].c == 2
assert d.to_dict() == {"a": 1, "b": [{"c": 2}, {"d": {"e": 3}}]}
def test_dump_yaml():
d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}})
p = Path(d.to_yaml("d.yaml", verbose=1))
assert MinyDict.from_yaml(p) == d
p.unlink()
def test_dict():
assert run(["a={}".format('{"b": 3}')
]) == capture(MinyDict({"a": {
"b": 3
}}))
def test_protected():
d = MinyDict()
with pytest.raises(AttributeError):
d.update = 4
d["update"] = 4
def __init__(
self,
verbose=0,
allow_overwrites=False,
warn_overwrites=True,
parse_env=True,
warn_env=True,
defaults=None,
strict=True,
keep_special_kwargs=True,
) -> None:
"""
Create a Minydra Parser to parse arbitrary commandline argument as:
* `key=value`
* `positiveArg` (set to `True`)
* `-negativeArg` (set to `False`)
Args:
verbose (int, optional): Wether to print received system arguments.
Defaults to 0.
allow_overwrites (bool, optional): Wether to allow for repeating arguments
in the command line. Defaults to True.
warn_overwrites (bool, optional): Wether to print a waring in case of a
repeated argument (if that is allowed). Defaults to True.
parse_env (bool, optional): Wether to parse environment variables specified
as key or value in the command line. Defaults to True.
warn_env (bool, optional): Wether to print a warning in case an environment
variable is parsed but no value is found. Defaults to True.
defaults (Union[str, dict, MinyDict], optional): Default arguments
a (Miny)dict or a path to a file that `minydra.MinyDict` will be able to
load (as `json`, `pickle` or `yaml`). Can also be provided with the
special `@defaults=<value>` command-line argument. Defaults to None.
strict (bool, optional): Wether to raise an error if an unknown argument is
passed and `defaults` was provided. Can also be provided with the
special `@strict=<true|false>` command-line argument. Defaults to True.
keep_special_kwargs (bool, optional): Wether to keep special keywords like
`@defaults` and `@strict` in the parsed arguments. Defaults to True.
"""
super().__init__()
self.conf = MinyDict()
# store initial arguments in a conf object to be able to
# retrieve them later and most importantly to be able to
# parse them in the `_parse_conf()` method
self.conf.verbose = verbose
self.conf.allow_overwrites = allow_overwrites
self.conf.warn_overwrites = warn_overwrites
self.conf.parse_env = parse_env
self.conf.warn_env = warn_env
self.conf.defaults = defaults
self.conf.strict = strict
self.conf.keep_special_kwargs = keep_special_kwargs
self._argv = sys.argv[1:]
self._parse_args()
self._print("sys.argv:", self._argv)
# defaults are provided
if self.conf.defaults is not None or self.args["@defaults"]:
# load defaults as a MinyDict
default = self.load_defaults(self.args["@defaults"] or self.conf.defaults)
# resolve existing args to properly override nested defaults
args = self.args.deepcopy().resolve()
args_defaults = args["@defaults"]
args_strict = args["@strict"]
# clean args
if args_defaults is not None:
del args["@defaults"]
if args_strict is not None:
self.conf.strict = args["@strict"]
del args["@strict"]
# update defaults from command-line args
self.args = default.update(args, strict=self.conf.strict)
# bring back special (@) kwargs if need be
if self.conf.keep_special_kwargs:
if args_defaults is not None:
self.args["@defaults"] = args_defaults
if args_strict is not None:
self.args["@strict"] = args_strict
# clean up special (@) kwargs if need be
if not self.conf.keep_special_kwargs:
for k in self.conf:
self.args.pop(f"@{k}", None)
def test_dump_json():
d = MinyDict({"a": 1, "b": 2, "u": "x", "r": {"t": 5}})
p = Path(d.to_json("d.json", verbose=1))
assert MinyDict.from_json(p) == d
p.unlink()
def test_init():
_ = MinyDict()
def test_env(monkeypatch):
monkeypatch.setenv("USER", "TestingUser")
assert run(["a=$USER"]) == capture(MinyDict({"a": "TestingUser"}))
def test_defaults():
examples = Path(__file__).resolve().parent.parent / "examples"
d1 = examples / "demo.json"
d2 = examples / "demo2.json"
y1 = examples / "demo.yaml"
p = MinyDict({"a": "2", "c": 3, "d": {"e": {"f": 4, "g": 5}}})
pkl = p.to_pickle(Path(__file__).resolve().parent / "test.pkl")
with patch.object(sys, "argv", [""]):
args = minydra.resolved_args(defaults=p)
assert args == p
with patch.object(sys, "argv", ["", "d.e.f=2"]):
args = minydra.resolved_args(defaults=p)
assert args.d.e.f == 2
with patch.object(sys, "argv", ["", f"@defaults={str(d1)}"]):
args = minydra.resolved_args()
del args["@defaults"]
assert args.to_dict() == json.loads(d1.read_text())
with patch.object(sys, "argv", ["", f"@defaults={str(y1)}"]):
args = minydra.resolved_args()
del args["@defaults"]
assert args.to_dict() == MinyDict.from_yaml(y1)
with patch.object(sys, "argv", ["", f"@defaults={str(pkl)}"]):
args = minydra.resolved_args()
del args["@defaults"]
assert args.to_dict() == MinyDict.from_pickle(pkl)
Path(pkl).unlink()
with pytest.raises(ValueError):
with patch.object(
sys, "argv", ["", f"@defaults={str(d1).replace('.json', '.py')}"]
):
args = minydra.resolved_args()
with pytest.raises(KeyError):
with patch.object(sys, "argv", ["", f"@defaults={str(d1)}", "new_key=3"]):
args = minydra.resolved_args()
del args["@defaults"]
assert args.to_dict() == json.loads(d1.read_text())
with patch.object(
sys, "argv", ["", f"@defaults={str(d1)}", "@strict=false", "new_key=3"]
):
args = minydra.resolved_args(keep_special_kwargs=False)
target = json.loads(d1.read_text())
target["new_key"] = 3
assert args.to_dict() == target
with patch.object(
sys, "argv", ["", f"@defaults={str(d1)}", "@strict=false", "new_key=3"]
):
args = minydra.resolved_args()
del args["@defaults"]
del args["@strict"]
target = json.loads(d1.read_text())
target["new_key"] = 3
assert args.to_dict() == target
double_defaults = f"['{str(d1)}', '{str(d2)}']"
with patch.object(sys, "argv", ["", f"@defaults={double_defaults}", "new_key=3"]):
args = minydra.resolved_args()
d1d = MinyDict.from_json(d1)
d2d = MinyDict.from_json(d2)
d1d = d1d.update(d2d)
del args["@defaults"]
assert args == d1d
def test_dump_pickle():
d = MinyDict({"a": 1, "b": 2, 4: "x", "r": {"t": 5}})
p = Path(d.to_pickle("d.pkl", verbose=1))
assert MinyDict.from_pickle(p) == d
p.unlink()