def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
tree = LocalTree(None, config={})
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
dump_yaml(path1, d1, tree)
dump_yaml(path2, d2, tree)
context = Context.load_from(tree, path1)
c = Context.load_from(tree, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
fs = LocalFileSystem()
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
path1.dump(d1, fs=fs)
path2.dump(d2, fs=fs)
context = Context.load_from(fs, path1)
c = Context.load_from(fs, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_clone():
d = {
"dct": {
"foo0": "foo0",
"bar0": "bar0",
"foo1": "foo1",
"bar1": "bar1",
},
"lst": [1, 2, 3],
}
c1 = Context(d)
c2 = Context.clone(c1)
c2["dct"]["foo0"] = "foo"
del c2["dct"]["foo1"]
assert c1 != c2
assert c1 == Context(d)
assert c2.select("lst.0") == Value(1)
with pytest.raises(KeyNotInContext):
c2.select("lst.1.not_existing_key")
def test_clone():
d = {
"lst": [
{
"foo0": "foo0",
"bar0": "bar0"
},
{
"foo1": "foo1",
"bar1": "bar1"
},
]
}
c1 = Context(d)
c2 = Context.clone(c1)
c2["lst"][0]["foo0"] = "foo"
del c2["lst"][1]["foo1"]
assert c1 != c2
assert c1 == Context(d)
assert c2.select("lst.0.foo0") == Value("foo")
with pytest.raises(ValueError):
c2.select("lst.1.foo1")
