def test_set_with_foreach(tmp_dir, dvc):
items = ["foo", "bar", "baz"]
d = {
"stages": {
"build": {
"set": {
"items": items
},
"foreach": "${items}",
"do": {
"cmd": "command --value ${item}"
},
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
f"build@{item}": {
"cmd": f"command --value {item}"
}
for item in items
}
},
)
def test_with_params_section(tmp_dir, dvc):
"""Test that params section is also loaded for interpolation"""
d = {
"vars": [DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}],
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar} ${dict.foobar}",
"params": [{"params.json": ["value1"]}],
"vars": ["params.json"],
},
},
}
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}})
dump_json(tmp_dir / "params.json", {"dict": {"foobar": "foobar"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"stage1": {
"cmd": "echo foo bar foobar",
"params": [
"dict.bar",
{"params.json": ["dict.foobar", "value1"]},
],
}
}
},
)
def test_set_already_exists():
context = Context({"item": "foo"})
with pytest.raises(ValueError,
match="Cannot set 'item', key already exists"):
DataResolver.set_context_from(context, {"item": "bar"})
assert context["item"] == Value("foo")
def test_wdir_failed_to_interpolate(tmp_dir, repo, wdir, expected_msg):
d = {"stages": {"build": {"wdir": wdir, "cmd": "echo ${models.bar}"}}}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert escape_ansi(str(exc_info.value)) == (
"failed to parse 'stages.build.wdir' in 'dvc.yaml':" + expected_msg)
def test_set_with_foreach_and_on_stage_definition(tmp_dir, dvc):
iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}}
dump_json(tmp_dir / "params.json", iterable)
d = {
"vars": ["params.json"],
"stages": {
"build": {
"set": {"data": "${models}"},
"foreach": "${data}",
"do": {
"set": {"thresh": "${item.thresh}"},
"cmd": "command --value ${thresh}",
},
}
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build@us": {
"cmd": "command --value 10",
"params": [{"params.json": ["models.us.thresh"]}],
},
"build@gb": {
"cmd": "command --value 15",
"params": [{"params.json": ["models.gb.thresh"]}],
},
}
},
)
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
iterable = {"bar": "bar", "foo": "foo"}
dump_json(tmp_dir / "params.json", iterable)
d = {
"vars": ["params.json"],
"stages": {
"build": {
"cmd": "command --value ${bar}",
"params": [{"params.json": ["foo"]}],
"vars": ["params.json"],
},
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": "command --value bar",
"params": [{"params.json": ["bar", "foo"]}],
},
}
},
)
def test_set_multiple_interpolations(value):
context = Context(CONTEXT_DATA)
with pytest.raises(
ValueError,
match=r"Cannot set 'item', joining string with interpolated string",
):
DataResolver.set_context_from(context, {"thresh": 10, "item": value})
def test_foreach_loop_templatized(tmp_dir, dvc):
params = {"models": {"us": {"thresh": 10}}}
vars_ = [{"models": {"gb": {"thresh": 15}}}]
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, params)
d = {
"vars": vars_,
"stages": {
"build": {
"foreach": "${models}",
"do": {"cmd": "python script.py --thresh ${item.thresh}"},
}
},
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build@gb": {"cmd": "python script.py --thresh 15"},
"build@us": {
"cmd": "python script.py --thresh 10",
"params": ["models.us.thresh"],
},
}
},
)
def test_vars_and_params_import(tmp_dir, dvc):
"""
Test that vars and params are both merged together for interpolation,
whilst tracking the "used" variables from params.
"""
d = {
"vars": [DEFAULT_PARAMS_FILE, {
"dict": {
"foo": "foobar"
}
}],
"stages": {
"stage1": {
"cmd": "echo ${dict.foo} ${dict.bar}"
}
},
}
dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}})
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(resolver.resolve(),
{"stages": {
"stage1": {
"cmd": "echo foobar bar"
}
}})
assert resolver.tracked_vars == {
"stage1": {
DEFAULT_PARAMS_FILE: {
"dict.bar": "bar"
}
}
}
def test_set(tmp_dir, dvc, value):
d = {
"stages": {
"build": {
"set": {
"item": value
},
"cmd": "python script.py --thresh ${item}",
"always_changed": "${item}",
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
if isinstance(value, bool):
stringified_value = "true" if value else "false"
else:
stringified_value = str(value)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": f"python script.py --thresh {stringified_value}",
"always_changed": value,
}
}
},
)
def test_coll(tmp_dir, dvc, coll):
d = {
"stages": {
"build": {
"set": {
"item": coll,
"thresh": 10
},
"cmd": "python script.py --thresh ${thresh}",
"outs": "${item}",
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": "python script.py --thresh 10",
"outs": coll
}
}
},
)
def test_foreach_loop_dict(tmp_dir, dvc):
iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}}
d = {
"stages": {
"build": {
"foreach": iterable["models"],
"do": {
"cmd": "python script.py ${item.thresh}"
},
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
f"build@{key}": {
"cmd": f"python script.py {item['thresh']}"
}
for key, item in iterable["models"].items()
}
},
)
def test_simple_foreach_loop(tmp_dir, dvc):
iterable = ["foo", "bar", "baz"]
d = {
"stages": {
"build": {
"foreach": iterable,
"do": {
"cmd": "python script.py ${item}"
},
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
f"build@{item}": {
"cmd": f"python script.py {item}"
}
for item in iterable
}
},
)
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc):
iterable = {"bar": "bar", "foo": "foo"}
(tmp_dir / "params.json").dump(iterable)
d = {
"vars": ["params.json"],
"stages": {
"build": {
"cmd": "command --value ${bar}",
"params": [{"params.json": ["foo"]}],
"vars": ["params.json"],
}
},
}
resolver = DataResolver(dvc, tmp_dir.fs_path, d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": "command --value bar",
"params": [{"params.json": ["foo"]}],
}
}
},
)
assert resolver.tracked_vars == {"build": {"params.json": {"bar": "bar"}}}
def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc):
iterable = {"bar": "bar", "foo": "foo"}
(tmp_dir / "params.yaml").dump(iterable)
d = {
"stages": {
"build": {
"cmd": "command --value ${bar}",
"params": [{"params.yaml": ["foo"]}],
"vars": ["params.yaml"],
}
}
}
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
assert_stage_equal(
resolver.resolve(),
{
"stages": {
"build": {
"cmd": "command --value bar",
"params": [{"params.yaml": ["foo"]}],
}
}
},
)
assert resolver.tracked_vars == {"build": {"params.yaml": {"bar": "bar"}}}
def resolved_data(self):
data = self.data
if self._enable_parametrization:
wdir = PathInfo(self.dvcfile.path).parent
with log_durations(logger.debug, "resolving values"):
resolver = DataResolver(self.repo, wdir, data)
data = resolver.resolve()
return data.get("stages", {})
def test_vars(tmp_dir, dvc):
d = deepcopy(TEMPLATED_DVC_YAML_DATA)
d["vars"] = [CONTEXT_DATA]
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA)
assert_stage_equal(resolver.resolve(), resolved_data)
assert not any(resolver.tracked_vars.values())
def test_interpolate_non_string(tmp_dir, repo):
d = {"stages": {"build": {"cmd": "echo ${models}"}}}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(exc_info.value) == (
"failed to parse 'stages.build.cmd' in 'dvc.yaml':\n"
"Cannot interpolate data of type 'dict'")
def test_specified_key_does_not_exist(tmp_dir, repo):
d = {"stages": {"build": {"cmd": "echo ${models.foobar}"}}}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(exc_info.value) == (
"failed to parse 'stages.build.cmd' in 'dvc.yaml': "
"Could not find 'models.foobar'")
def test_local_overwrite_error(tmp_dir, repo, vars_, loc):
d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(
exc_info.value) == ("failed to parse stage 'build' in 'dvc.yaml':\n"
f"cannot redefine 'models.bar' from '{loc}' "
"as it already exists in 'params.yaml'")
def test_set_multiple_interpolations(value):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError,) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": value})
assert str(exc_info.value) == (
"Failed to set 'item': Cannot set 'item', "
"joining string with interpolated string is not supported"
)
def test_set_nested_coll(coll):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": coll})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
"has nested dict/list"
)
def test_set_collection_interpolation(coll):
context = Context(CONTEXT_DATA)
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"thresh": 10, "item": coll})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
f"having interpolation inside '{type(coll).__name__}' "
"is not supported."
)
def stages(self):
data, _ = self._load()
if self.repo.config["feature"]["parametrization"]:
with log_durations(logger.debug, "resolving values"):
resolver = DataResolver(data)
data = resolver.resolve()
lockfile_data = self._lockfile.load()
return StageLoader(self, data.get("stages", {}), lockfile_data)
def test_simple(tmp_dir, dvc):
(tmp_dir / DEFAULT_PARAMS_FILE).dump(CONTEXT_DATA)
resolver = DataResolver(
dvc, PathInfo(str(tmp_dir)), deepcopy(TEMPLATED_DVC_YAML_DATA)
)
assert_stage_equal(resolver.resolve(), deepcopy(RESOLVED_DVC_YAML_DATA))
assert resolver.tracked_vars == {
"stage1": {DEFAULT_PARAMS_FILE: USED_VARS["stage1"]},
"stage2": {DEFAULT_PARAMS_FILE: USED_VARS["stage2"]},
}
def test_vars_load_partial(tmp_dir, dvc, local, vars_):
iterable = {"bar": "bar", "foo": "foo"}
(tmp_dir / "test_params.yaml").dump(iterable)
d = {"stages": {"build": {"cmd": "echo ${bar}"}}}
if local:
d["stages"]["build"]["vars"] = vars_
else:
d["vars"] = vars_
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolver.resolve()
def test_vars(tmp_dir, dvc):
d = deepcopy(TEMPLATED_DVC_YAML_DATA)
d["vars"] = [CONTEXT_DATA]
resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d)
resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA)
# `vars` section is not auto-tracked
del resolved_data["stages"]["stage1"]["params"]
del resolved_data["stages"]["stage2"]["params"]
assert_stage_equal(resolver.resolve(), resolved_data)
def test_set_already_exists():
context = Context({"item": "foo"})
with pytest.raises(ResolveError) as exc_info:
DataResolver.set_context_from(context, {"item": "bar"})
assert (
str(exc_info.value) == "Failed to set 'item': Cannot set 'item', "
"key already exists"
)
assert context["item"] == Value("foo")
def test_foreach_interpolation_key_does_not_exist(tmp_dir, repo):
d = {
"stages": {"build": {"foreach": "${modelss}", "do": {}}},
}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(exc_info.value) == (
"failed to parse 'stages.build.foreach' in 'dvc.yaml': "
"Could not find 'modelss'"
)
def test_foreach_expects_dict_or_list(tmp_dir, repo, foreach):
d = {
"stages": {"build": {"foreach": foreach, "do": {}}},
}
resolver = DataResolver(repo, tmp_dir, d)
with pytest.raises(ResolveError) as exc_info:
resolver.resolve()
assert str(exc_info.value) == (
"failed to resolve 'stages.build.foreach' in 'dvc.yaml': "
"expected list/dictionary, got str"
)