def test_load_from_raises_if_file_is_directory(tmp_dir, dvc):
(tmp_dir / "data").mkdir()
with pytest.raises(ParamsLoadError) as exc_info:
Context.load_from(dvc.fs, "data")
assert str(exc_info.value) == "'data' is a directory"
def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
tree = LocalTree(None, config={})
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
dump_yaml(path1, d1, tree)
dump_yaml(path2, d2, tree)
context = Context.load_from(tree, path1)
c = Context.load_from(tree, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
fs = LocalFileSystem()
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
path1.dump(d1, fs=fs)
path2.dump(d2, fs=fs)
context = Context.load_from(fs, path1)
c = Context.load_from(fs, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_track(tmp_dir):
d = {
"lst": [
{
"foo0": "foo0",
"bar0": "bar0"
},
{
"foo1": "foo1",
"bar1": "bar1"
},
],
"dct": {
"foo": "foo",
"bar": "bar",
"baz": "baz"
},
}
tree = LocalTree(None, config={})
path = tmp_dir / "params.yaml"
dump_yaml(path, d, tree)
context = Context.load_from(tree, path)
def key_tracked(d, key):
assert len(d) == 1
return key in d[relpath(path)]
with context.track() as tracked:
context.select("lst")
assert key_tracked(tracked, "lst")
context.select("dct")
assert not key_tracked(tracked, "dct")
context.select("dct.foo")
assert key_tracked(tracked, "dct.foo")
# Currently, it's unable to track dictionaries, as it can be merged
# from multiple sources.
context.select("lst.0")
assert not key_tracked(tracked, "lst.0")
# FIXME: either support tracking list values in ParamsDependency
# or, prevent this from being tracked.
context.select("lst.0.foo0")
assert key_tracked(tracked, "lst.0.foo0")
def test_track(tmp_dir):
d = {
"lst": [
{
"foo0": "foo0",
"bar0": "bar0"
},
{
"foo1": "foo1",
"bar1": "bar1"
},
],
"dct": {
"foo": "foo",
"bar": "bar",
"baz": "baz"
},
}
fs = LocalFileSystem()
(tmp_dir / "params.yaml").dump(d, fs=fs)
context = Context.load_from(fs, "params.yaml")
def key_tracked(d, key):
assert len(d) == 1
return key in d["params.yaml"]
with context.track() as tracked:
context.select("lst")
assert key_tracked(tracked, "lst")
context.select("dct")
assert not key_tracked(tracked, "dct")
context.select("dct.foo")
assert key_tracked(tracked, "dct.foo")
# Currently, it's unable to track dictionaries, as it can be merged
# from multiple sources.
context.select("lst.0")
assert not key_tracked(tracked, "lst.0")
# FIXME: either support tracking list values in ParamsDependency
# or, prevent this from being tracked.
context.select("lst.0.foo0")
assert key_tracked(tracked, "lst.0.foo0")
def test_load_from(mocker):
def _yaml_load(*args, **kwargs):
return {"x": {"y": {"z": 5}, "lst": [1, 2, 3]}, "foo": "foo"}
mocker.patch("dvc.parsing.context.LOADERS", {".yaml": _yaml_load})
class tree:
def exists(self, _):
return True
file = "params.yaml"
c = Context.load_from(tree(), file)
assert asdict(c["x"].meta) == {
"source": file,
"dpaths": ["x"],
"local": False,
}
assert asdict(c["foo"].meta) == {
"source": file,
"local": False,
"dpaths": ["foo"],
}
assert asdict(c["x"]["y"].meta) == {
"source": file,
"dpaths": ["x", "y"],
"local": False,
}
assert asdict(c["x"]["y"]["z"].meta) == {
"source": file,
"dpaths": ["x", "y", "z"],
"local": False,
}
assert asdict(c["x"]["lst"].meta) == {
"source": file,
"dpaths": ["x", "lst"],
"local": False,
}
assert asdict(c["x"]["lst"][0].meta) == {
"source": file,
"dpaths": ["x", "lst", "0"],
"local": False,
}
def test_load_from(mocker):
d = {"x": {"y": {"z": 5}, "lst": [1, 2, 3]}, "foo": "foo"}
fs = mocker.Mock(
open=mock_open(read_data=dumps_yaml(d)),
**{
"exists.return_value": True,
"isdir.return_value": False
},
)
file = "params.yaml"
c = Context.load_from(fs, file)
assert asdict(c["x"].meta) == {
"source": file,
"dpaths": ["x"],
"local": False,
}
assert asdict(c["foo"].meta) == {
"source": file,
"local": False,
"dpaths": ["foo"],
}
assert asdict(c["x"]["y"].meta) == {
"source": file,
"dpaths": ["x", "y"],
"local": False,
}
assert asdict(c["x"]["y"]["z"].meta) == {
"source": file,
"dpaths": ["x", "y", "z"],
"local": False,
}
assert asdict(c["x"]["lst"].meta) == {
"source": file,
"dpaths": ["x", "lst"],
"local": False,
}
assert asdict(c["x"]["lst"][0].meta) == {
"source": file,
"dpaths": ["x", "lst", "0"],
"local": False,
}
def test_load_from_raises_if_file_not_exist(tmp_dir, dvc):
with pytest.raises(ParamsLoadError) as exc_info:
Context.load_from(dvc.fs, tmp_dir / DEFAULT_PARAMS_FILE)
assert str(exc_info.value) == "'params.yaml' does not exist"
