def test_resolve_resolves_dict_keys():
d = {"dct": {"foo": "foobar", "persist": True}}
context = Context(d)
assert context.resolve({"${dct.foo}": {"persist": "${dct.persist}"}}) == {
"foobar": {"persist": True}
}
def test_load_from_raises_if_file_is_directory(tmp_dir, dvc):
(tmp_dir / "data").mkdir()
with pytest.raises(ParamsLoadError) as exc_info:
Context.load_from(dvc.fs, "data")
assert str(exc_info.value) == "'data' is a directory"
def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
fs = LocalFileSystem()
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
path1.dump(d1, fs=fs)
path2.dump(d2, fs=fs)
context = Context.load_from(fs, path1)
c = Context.load_from(fs, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_track_from_multiple_files(tmp_dir):
d1 = {"Train": {"us": {"lr": 10}}}
d2 = {"Train": {"us": {"layers": 100}}}
tree = LocalTree(None, config={})
path1 = tmp_dir / "params.yaml"
path2 = tmp_dir / "params2.yaml"
dump_yaml(path1, d1, tree)
dump_yaml(path2, d2, tree)
context = Context.load_from(tree, path1)
c = Context.load_from(tree, path2)
context.merge_update(c)
def key_tracked(d, path, key):
return key in d[relpath(path)]
with context.track() as tracked:
context.select("Train")
assert not (
key_tracked(tracked, path1, "Train")
or key_tracked(tracked, path2, "Train")
)
context.select("Train.us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("Train.us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
context = Context.clone(context)
assert not context._tracked_data
# let's see with an alias
context["us"] = context["Train"]["us"]
with context.track() as tracked:
context.select("us")
assert not (
key_tracked(tracked, path1, "Train.us")
or key_tracked(tracked, path2, "Train.us")
)
context.select("us.lr")
assert key_tracked(tracked, path1, "Train.us.lr") and not key_tracked(
tracked, path2, "Train.us.lr"
)
context.select("Train.us.layers")
assert not key_tracked(
tracked, path1, "Train.us.layers"
) and key_tracked(tracked, path2, "Train.us.layers")
def test_resolve_collection():
from tests.func.parsing import (
CONTEXT_DATA,
RESOLVED_DVC_YAML_DATA,
TEMPLATED_DVC_YAML_DATA,
)
context = Context(CONTEXT_DATA)
resolved = context.resolve(TEMPLATED_DVC_YAML_DATA)
assert resolved == RESOLVED_DVC_YAML_DATA
assert recurse_not_a_node(resolved)
def test_loop_context():
context = Context({"foo": "foo", "bar": "bar", "lst": [1, 2, 3]})
assert list(context) == ["foo", "bar", "lst"]
assert len(context) == 3
assert list(context["lst"]) == [Value(i) for i in [1, 2, 3]]
assert len(context["lst"]) == 3
assert list(context.items()) == [
("foo", Value("foo")),
("bar", Value("bar")),
("lst", CtxList([1, 2, 3])),
]
def test_set_multiple_interpolations(value):
context = Context(CONTEXT_DATA)
with pytest.raises(
ValueError,
match=r"Cannot set 'item', joining string with interpolated string",
):
DataResolver.set_context_from(context, {"thresh": 10, "item": value})
def test_context():
context = Context({"foo": "bar"})
assert context["foo"] == Value("bar")
context = Context(foo="bar")
assert context["foo"] == Value("bar")
context["foobar"] = "foobar"
assert context["foobar"] == Value("foobar")
del context["foobar"]
assert "foobar" not in context
assert "foo" in context
with pytest.raises(KeyError):
_ = context["foobar"]
def test_item_key_in_generated_stage_vars(tmp_dir, dvc, redefine, from_file):
context = Context(foo="bar")
vars_ = [redefine]
if from_file:
(tmp_dir / "test_params.yaml").dump(redefine)
vars_ = ["test_params.yaml"]
definition = make_foreach_def(
tmp_dir,
"build",
{"model1": {"thresh": "10"}, "model2": {"thresh": 5}},
{"vars": vars_, "cmd": "${item}"},
context,
)
with pytest.raises(ResolveError) as exc_info:
definition.resolve_all()
message = str(exc_info.value)
assert (
"failed to parse stage 'build@model1' in 'dvc.yaml': "
"attempted to modify reserved"
) in message
key_or_keys = "keys" if len(redefine) > 1 else "key"
assert f"{key_or_keys} {join(redefine)}" in message
if from_file:
assert "in 'test_params.yaml'" in message
assert context == {"foo": "bar"}
def test_set_already_exists():
context = Context({"item": "foo"})
with pytest.raises(ValueError,
match="Cannot set 'item', key already exists"):
DataResolver.set_context_from(context, {"item": "bar"})
assert context["item"] == Value("foo")
def test_node_value():
d = {"dct": {"foo": "bar"}, "lst": [1, 2, 3], "foo": "foo"}
context = Context(d)
assert isinstance(context, (Context, CtxDict))
assert isinstance(context["dct"], CtxDict)
assert isinstance(context["lst"], CtxList)
assert isinstance(context["foo"], Value)
assert isinstance(context["dct"]["foo"], Value)
assert isinstance(context["lst"][0], Value)
assert context.value == d
assert recurse_not_a_node(context.value)
assert isinstance(context.value["dct"], dict)
assert isinstance(context.value["lst"], list)
assert isinstance(context.value["foo"], str)
assert isinstance(context.value["dct"]["foo"], str)
assert isinstance(context.value["lst"][0], int)
assert isinstance(context["dct"].value, dict)
assert context["dct"]["foo"].value == "bar"
assert isinstance(context["lst"].value, list)
assert context["lst"][1].value == 2
assert context["foo"].value == "foo"
def make_entry_definition(wdir, name, data, context=None) -> EntryDefinition:
return EntryDefinition(
DataResolver(wdir.dvc, wdir.fs_path, {}),
context or Context(),
name,
data,
)
def test_context_dict_ignores_keys_except_str():
c = Context({"one": 1, 3: 3})
assert "one" in c
assert 3 not in c
c[3] = 3
assert 3 not in c
def make_foreach_def(
wdir, name, foreach_data, do_data=None, context=None
) -> ForeachDefinition:
return ForeachDefinition(
DataResolver(wdir.dvc, wdir, {}),
context or Context(),
name,
{"foreach": foreach_data, "do": do_data or {}},
)
def test_set_multiple_interpolations(value):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError,) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": value})
assert str(exc_info.value) == (
"Failed to set 'item': Cannot set 'item', "
"joining string with interpolated string is not supported"
)
def test_set():
context = Context(CONTEXT_DATA)
to_set = {
"foo": "foo",
"bar": "bar",
"pi": pi,
"true": True,
"false": False,
"none": "None",
"int": 1,
"lst2": [1, 2, 3],
"dct2": {"foo": "bar", "foobar": "foobar"},
}
DataResolver.set_context_from(context, to_set)
for key, value in to_set.items():
# FIXME: using for convenience, figure out better way to do it
assert context[key] == context._convert(key, value)
def test_set_nested_coll(coll):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": coll})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
"has nested dict/list"
)
def test_foreach_do_syntax_is_checked_once(tmp_dir, dvc, mocker):
do_def = {"cmd": "python script.py --epochs ${item}"}
data = {"foreach": [0, 1, 2, 3, 4], "do": do_def}
definition = ForeachDefinition(DataResolver(dvc, tmp_dir.fs_path, {}),
Context(), "build", data)
mock = mocker.patch("dvc.parsing.check_syntax_errors", return_value=True)
definition.resolve_all()
mock.assert_called_once_with(do_def, "build", "dvc.yaml")
def test_set_collection_interpolation(coll):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": coll})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
f"having interpolation inside '{type(coll).__name__}' "
"is not supported."
)
def test_foreach_data_is_only_resolved_once(tmp_dir, dvc, mocker):
context = Context(models=["foo", "bar", "baz"])
data = {"foreach": "${models}", "do": {}}
definition = ForeachDefinition(DataResolver(dvc, tmp_dir.fs_path, {}),
context, "build", data)
mock = mocker.spy(definition, "_resolve_foreach_data")
definition.resolve_all()
mock.assert_called_once_with()
def test_set_already_exists():
context = Context({"item": "foo"})
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"item": "bar"})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
"key already exists"
)
assert context["item"] == Value("foo")
def test_merge_list():
c1 = Context(lst=[1, 2, 3])
with pytest.raises(ValueError):
# cannot overwrite by default
c1.merge_update({"lst": [10, 11, 12]})
# lists are never merged
c1.merge_update({"lst": [10, 11, 12]}, overwrite=True)
assert c1.select("lst") == [10, 11, 12]
def test_foreach_data_expects_list_or_dict(tmp_dir, dvc, foreach_data):
context = Context(
{"foo": "bar", "dct": {"model1": "a-out"}, "lst": ["foo", "bar"]}
)
definition = make_foreach_def(tmp_dir, "build", foreach_data, {}, context)
with pytest.raises(ResolveError) as exc_info:
definition.resolve_all()
assert str(exc_info.value) == (
"failed to resolve 'stages.build.foreach' in 'dvc.yaml': "
"expected list/dictionary, got str"
)
def test_interpolate_non_string(tmp_dir, dvc):
definition = make_entry_definition(tmp_dir, "build",
{"cmd": "echo ${models}"},
Context(models={}))
with pytest.raises(ResolveError) as exc_info:
definition.resolve()
assert str(exc_info.value) == (
"failed to parse 'stages.build.cmd' in 'dvc.yaml':\n"
"Cannot interpolate data of type 'dict'")
assert definition.context == {"models": {}}
def test_clone():
d = {
"dct": {
"foo0": "foo0",
"bar0": "bar0",
"foo1": "foo1",
"bar1": "bar1",
},
"lst": [1, 2, 3],
}
c1 = Context(d)
c2 = Context.clone(c1)
c2["dct"]["foo0"] = "foo"
del c2["dct"]["foo1"]
assert c1 != c2
assert c1 == Context(d)
assert c2.select("lst.0") == Value(1)
with pytest.raises(KeyNotInContext):
c2.select("lst.1.not_existing_key")
def test_merge_list():
c1 = Context(lst=[1, 2, 3])
with pytest.raises(MergeError):
# cannot overwrite by default
c1.merge_update({"lst": [10, 11, 12]})
# lists are never merged
c1.merge_update({"lst": [10, 11, 12]}, overwrite=True)
node = c1.select("lst")
assert node == [10, 11, 12]
assert isinstance(node, CtxList)
assert node[0] == Value(10)
def test_foreach_overwriting_item_in_list(
tmp_dir, dvc, caplog, global_data, where
):
context = Context(global_data)
definition = make_foreach_def(
tmp_dir, "build", {"model1": 10, "model2": 5}, {}, context
)
with caplog.at_level(logging.WARNING, logger="dvc.parsing"):
definition.resolve_all()
assert caplog.messages == [
f"{where} already specified, "
"will be overwritten for stages generated from 'build'"
]
def test_with_simple_list_data(tmp_dir, dvc):
"""Testing a simple non-nested list as a foreach data"""
resolver = DataResolver(dvc, tmp_dir.fs_path, {})
context = Context()
data = {"foreach": ["foo", "bar", "baz"], "do": {"cmd": "echo ${item}"}}
definition = ForeachDefinition(resolver, context, "build", data)
assert definition.resolve_one("foo") == {"build@foo": {"cmd": "echo foo"}}
assert definition.resolve_one("bar") == {"build@bar": {"cmd": "echo bar"}}
# check that `foreach` item-key replacement didnot leave any leftovers.
assert not context
assert not resolver.tracked_vars["build@foo"]
assert not resolver.tracked_vars["build@bar"]
def test_wdir_failed_to_interpolate(tmp_dir, dvc, wdir, expected_msg):
definition = make_entry_definition(
tmp_dir,
"build",
{"wdir": wdir, "cmd": "echo ${models.bar}"},
Context(models={"bar": "bar"}),
)
with pytest.raises(ResolveError) as exc_info:
definition.resolve()
assert escape_ansi(str(exc_info.value)) == (
"failed to parse 'stages.build.wdir' in 'dvc.yaml':" + expected_msg
)
assert definition.context == {"models": {"bar": "bar"}}
def test_specified_key_does_not_exist(tmp_dir, dvc):
definition = make_entry_definition(
tmp_dir,
"build",
{"cmd": "echo ${models.foobar}"},
Context(models={"foo": "foo"}),
)
with pytest.raises(ResolveError) as exc_info:
definition.resolve()
assert str(exc_info.value) == (
"failed to parse 'stages.build.cmd' in 'dvc.yaml': "
"Could not find 'models.foobar'")
assert definition.context == {"models": {"foo": "foo"}}
