def test_set_with_foreach_and_on_stage_definition(tmp_dir, dvc):
iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}}
dump_json(tmp_dir / "params.json", iterable)
d = {
"vars": ["params.json"],
"stages": {
"build": {
"set": {"data": "${models}"},
"foreach": "${data}",
"do": {
"set": {"thresh": "${item.thresh}"},
"cmd": "command --value ${thresh}",
},
}
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build@us": {
"cmd": "command --value 10",
"params": [{"params.json": ["models.us.thresh"]}],
},
"build@gb": {
"cmd": "command --value 15",
"params": [{"params.json": ["models.gb.thresh"]}],
},
}
},
)
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
iterable = {"bar": "bar", "foo": "foo"}
dump_json(tmp_dir / "params.json", iterable)
d = {
"vars": ["params.json"],
"stages": {
"build": {
"cmd": "command --value ${bar}",
"params": [{"params.json": ["foo"]}],
"vars": ["params.json"],
},
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": "command --value bar",
"params": [{"params.json": ["bar", "foo"]}],
},
}
},
)
def test_with_params_section(tmp_dir, dvc):
"""Test that params section is also loaded for interpolation"""
d = {
"vars": [DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}],
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar} ${dict.foobar}",
"params": [{"params.json": ["value1"]}],
"vars": ["params.json"],
},
},
}
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}})
dump_json(tmp_dir / "params.json", {"dict": {"foobar": "foobar"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd": "echo foo bar foobar",
"params": [
"dict.bar",
{"params.json": ["dict.foobar", "value1"]},
],
}
}
},
)
def test_load_vars_from_file(tmp_dir, dvc):
dump_yaml(DEFAULT_PARAMS_FILE, DATA)
datasets = {"datasets": ["foo", "bar"]}
dump_json("params.json", datasets)
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
resolver = DataResolver(dvc, tmp_dir, d)
expected = deepcopy(DATA)
expected.update(datasets)
assert resolver.context == expected
def test_global_overwrite_error_on_imports(tmp_dir, dvc):
dump_yaml(DEFAULT_PARAMS_FILE, DATA)
dump_json("params.json", DATA)
d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]}
with pytest.raises(ResolveError) as exc_info:
DataResolver(dvc, tmp_dir, d)
assert str(
exc_info.value) == ("failed to parse 'vars' in 'dvc.yaml':\n"
"cannot redefine 'models.bar' from 'params.json' "
"as it already exists in 'params.yaml'")
def test_local_overwrite_error(tmp_dir, dvc, vars_, loc):
dump_yaml(DEFAULT_PARAMS_FILE, DATA)
dump_json("params.json", DATA)
d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
resolver = DataResolver(dvc, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(
exc_info.value) == ("failed to parse stage 'build' in 'dvc.yaml':\n"
f"cannot redefine 'models.bar' from '{loc}' "
"as it already exists in 'params.yaml'")
def test_with_templated_wdir(tmp_dir, dvc):
"""
Test that params from the resolved wdir are still loaded
and is used in the interpolation.
"""
d = {
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar}",
"params": ["value1"],
"wdir": "${dict.ws}",
"vars": [DEFAULT_PARAMS_FILE],
},
},
}
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE,
{"dict": {
"bar": "bar",
"ws": "data"
}})
data_dir = tmp_dir / "data"
data_dir.mkdir()
dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd": "echo foo bar",
"wdir": "data",
"params": ["value1"],
}
}
},
)
assert resolver.tracked_vars == {
"stage1": {
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {
"dict.foo": "foo"
},
DEFAULT_PARAMS_FILE: {
"dict.bar": "bar",
"dict.ws": "data"
},
}
}
assert resolver.context.imports == {str(tmp_dir / "params.yaml"): None}
assert resolver.context == {"dict": {"bar": "bar", "ws": "data"}}
def test_with_templated_wdir(tmp_dir, dvc):
"""
Test that params from the resolved wdir are still loaded
and is used in the interpolation.
"""
d = {
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar}",
"params": ["value1"],
"wdir": "${dict.ws}",
},
},
}
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE,
{"dict": {
"bar": "bar",
"ws": "data"
}})
data_dir = tmp_dir / "data"
data_dir.mkdir()
dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
root_params_file = os.path.relpath(tmp_dir / "params.yaml", data_dir)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd":
"echo foo bar",
"wdir":
"data",
"params": [
"dict.foo",
"value1",
{
root_params_file: ["dict.bar", "dict.ws"]
},
],
}
}
},
)
def test_stage_with_wdir(tmp_dir, dvc):
"""
Test that params file from wdir are also loaded
"""
d = {
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar}",
"params": ["value1"],
"wdir": "data",
"vars": [DEFAULT_PARAMS_FILE],
},
},
}
data_dir = tmp_dir / "data"
data_dir.mkdir()
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}})
dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd": "echo foo bar",
"wdir": "data",
"params": ["value1"],
}
}
},
)
assert resolver.tracked_vars == {
"stage1": {
str(PathInfo("data") / DEFAULT_PARAMS_FILE): {
"dict.foo": "foo"
},
DEFAULT_PARAMS_FILE: {
"dict.bar": "bar"
},
}
}
def test_stage_with_wdir(tmp_dir, dvc):
"""
Test that params file from wdir are also loaded
"""
d = {
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar}",
"params": ["value1"],
"wdir": "data",
},
},
}
data_dir = tmp_dir / "data"
data_dir.mkdir()
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}})
dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
root_params_file = os.path.relpath(tmp_dir / "params.yaml", data_dir)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd":
"echo foo bar",
"wdir":
"data",
"params": [
"dict.foo",
"value1",
{
root_params_file: ["dict.bar"]
},
],
}
}
},
)
def repo(tmp_dir, dvc):
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, DATA)
dump_json(tmp_dir / "params.json", DATA)
yield dvc
