def test_set_with_foreach(tmp_dir, dvc): items = ["foo", "bar", "baz"] d = { "stages": { "build": { "set": { "items": items }, "foreach": "${items}", "do": { "cmd": "command --value ${item}" }, } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { f"build@{item}": { "cmd": f"command --value {item}" } for item in items } }, )
def test_with_params_section(tmp_dir, dvc): """Test that params section is also loaded for interpolation""" d = { "vars": [DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}], "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar} ${dict.foobar}", "params": [{"params.json": ["value1"]}], "vars": ["params.json"], }, }, } dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}}) dump_json(tmp_dir / "params.json", {"dict": {"foobar": "foobar"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar foobar", "params": [ "dict.bar", {"params.json": ["dict.foobar", "value1"]}, ], } } }, )
def test_set_already_exists(): context = Context({"item": "foo"}) with pytest.raises(ValueError, match="Cannot set 'item', key already exists"): DataResolver.set_context_from(context, {"item": "bar"}) assert context["item"] == Value("foo")
def test_wdir_failed_to_interpolate(tmp_dir, repo, wdir, expected_msg): d = {"stages": {"build": {"wdir": wdir, "cmd": "echo ${models.bar}"}}} resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert escape_ansi(str(exc_info.value)) == ( "failed to parse 'stages.build.wdir' in 'dvc.yaml':" + expected_msg)
def test_set_with_foreach_and_on_stage_definition(tmp_dir, dvc): iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}} dump_json(tmp_dir / "params.json", iterable) d = { "vars": ["params.json"], "stages": { "build": { "set": {"data": "${models}"}, "foreach": "${data}", "do": { "set": {"thresh": "${item.thresh}"}, "cmd": "command --value ${thresh}", }, } }, } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build@us": { "cmd": "command --value 10", "params": [{"params.json": ["models.us.thresh"]}], }, "build@gb": { "cmd": "command --value 15", "params": [{"params.json": ["models.gb.thresh"]}], }, } }, )
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc): iterable = {"bar": "bar", "foo": "foo"} dump_json(tmp_dir / "params.json", iterable) d = { "vars": ["params.json"], "stages": { "build": { "cmd": "command --value ${bar}", "params": [{"params.json": ["foo"]}], "vars": ["params.json"], }, }, } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": "command --value bar", "params": [{"params.json": ["bar", "foo"]}], }, } }, )
def test_set_multiple_interpolations(value): context = Context(CONTEXT_DATA) with pytest.raises( ValueError, match=r"Cannot set 'item', joining string with interpolated string", ): DataResolver.set_context_from(context, {"thresh": 10, "item": value})
def test_foreach_loop_templatized(tmp_dir, dvc): params = {"models": {"us": {"thresh": 10}}} vars_ = [{"models": {"gb": {"thresh": 15}}}] dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, params) d = { "vars": vars_, "stages": { "build": { "foreach": "${models}", "do": {"cmd": "python script.py --thresh ${item.thresh}"}, } }, } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build@gb": {"cmd": "python script.py --thresh 15"}, "build@us": { "cmd": "python script.py --thresh 10", "params": ["models.us.thresh"], }, } }, )
def test_vars_and_params_import(tmp_dir, dvc): """ Test that vars and params are both merged together for interpolation, whilst tracking the "used" variables from params. """ d = { "vars": [DEFAULT_PARAMS_FILE, { "dict": { "foo": "foobar" } }], "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar}" } }, } dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal(resolver.resolve(), {"stages": { "stage1": { "cmd": "echo foobar bar" } }}) assert resolver.tracked_vars == { "stage1": { DEFAULT_PARAMS_FILE: { "dict.bar": "bar" } } }
def test_set(tmp_dir, dvc, value): d = { "stages": { "build": { "set": { "item": value }, "cmd": "python script.py --thresh ${item}", "always_changed": "${item}", } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) if isinstance(value, bool): stringified_value = "true" if value else "false" else: stringified_value = str(value) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": f"python script.py --thresh {stringified_value}", "always_changed": value, } } }, )
def test_coll(tmp_dir, dvc, coll): d = { "stages": { "build": { "set": { "item": coll, "thresh": 10 }, "cmd": "python script.py --thresh ${thresh}", "outs": "${item}", } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": "python script.py --thresh 10", "outs": coll } } }, )
def test_foreach_loop_dict(tmp_dir, dvc): iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}} d = { "stages": { "build": { "foreach": iterable["models"], "do": { "cmd": "python script.py ${item.thresh}" }, } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { f"build@{key}": { "cmd": f"python script.py {item['thresh']}" } for key, item in iterable["models"].items() } }, )
def test_simple_foreach_loop(tmp_dir, dvc): iterable = ["foo", "bar", "baz"] d = { "stages": { "build": { "foreach": iterable, "do": { "cmd": "python script.py ${item}" }, } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { f"build@{item}": { "cmd": f"python script.py {item}" } for item in iterable } }, )
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc): iterable = {"bar": "bar", "foo": "foo"} (tmp_dir / "params.json").dump(iterable) d = { "vars": ["params.json"], "stages": { "build": { "cmd": "command --value ${bar}", "params": [{"params.json": ["foo"]}], "vars": ["params.json"], } }, } resolver = DataResolver(dvc, tmp_dir.fs_path, d) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": "command --value bar", "params": [{"params.json": ["foo"]}], } } }, ) assert resolver.tracked_vars == {"build": {"params.json": {"bar": "bar"}}}
def test_resolve_local_tries_to_load_globally_used_params_yaml(tmp_dir, dvc): iterable = {"bar": "bar", "foo": "foo"} (tmp_dir / "params.yaml").dump(iterable) d = { "stages": { "build": { "cmd": "command --value ${bar}", "params": [{"params.yaml": ["foo"]}], "vars": ["params.yaml"], } } } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": "command --value bar", "params": [{"params.yaml": ["foo"]}], } } }, ) assert resolver.tracked_vars == {"build": {"params.yaml": {"bar": "bar"}}}
def resolved_data(self): data = self.data if self._enable_parametrization: wdir = PathInfo(self.dvcfile.path).parent with log_durations(logger.debug, "resolving values"): resolver = DataResolver(self.repo, wdir, data) data = resolver.resolve() return data.get("stages", {})
def test_vars(tmp_dir, dvc): d = deepcopy(TEMPLATED_DVC_YAML_DATA) d["vars"] = [CONTEXT_DATA] resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA) assert_stage_equal(resolver.resolve(), resolved_data) assert not any(resolver.tracked_vars.values())
def test_interpolate_non_string(tmp_dir, repo): d = {"stages": {"build": {"cmd": "echo ${models}"}}} resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str(exc_info.value) == ( "failed to parse 'stages.build.cmd' in 'dvc.yaml':\n" "Cannot interpolate data of type 'dict'")
def test_specified_key_does_not_exist(tmp_dir, repo): d = {"stages": {"build": {"cmd": "echo ${models.foobar}"}}} resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str(exc_info.value) == ( "failed to parse 'stages.build.cmd' in 'dvc.yaml': " "Could not find 'models.foobar'")
def test_local_overwrite_error(tmp_dir, repo, vars_, loc): d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}} resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str( exc_info.value) == ("failed to parse stage 'build' in 'dvc.yaml':\n" f"cannot redefine 'models.bar' from '{loc}' " "as it already exists in 'params.yaml'")
def test_set_multiple_interpolations(value): context = Context(CONTEXT_DATA) with pytest.raises(ResolveError,) as exc_info: DataResolver.set_context_from(context, {"thresh": 10, "item": value}) assert str(exc_info.value) == ( "Failed to set 'item': Cannot set 'item', " "joining string with interpolated string is not supported" )
def test_set_nested_coll(coll): context = Context(CONTEXT_DATA) with pytest.raises(ResolveError) as exc_info: DataResolver.set_context_from(context, {"thresh": 10, "item": coll}) assert ( str(exc_info.value) == "Failed to set 'item': Cannot set 'item', " "has nested dict/list" )
def test_set_collection_interpolation(coll): context = Context(CONTEXT_DATA) with pytest.raises(ResolveError) as exc_info: DataResolver.set_context_from(context, {"thresh": 10, "item": coll}) assert ( str(exc_info.value) == "Failed to set 'item': Cannot set 'item', " f"having interpolation inside '{type(coll).__name__}' " "is not supported." )
def stages(self): data, _ = self._load() if self.repo.config["feature"]["parametrization"]: with log_durations(logger.debug, "resolving values"): resolver = DataResolver(data) data = resolver.resolve() lockfile_data = self._lockfile.load() return StageLoader(self, data.get("stages", {}), lockfile_data)
def test_simple(tmp_dir, dvc): (tmp_dir / DEFAULT_PARAMS_FILE).dump(CONTEXT_DATA) resolver = DataResolver( dvc, PathInfo(str(tmp_dir)), deepcopy(TEMPLATED_DVC_YAML_DATA) ) assert_stage_equal(resolver.resolve(), deepcopy(RESOLVED_DVC_YAML_DATA)) assert resolver.tracked_vars == { "stage1": {DEFAULT_PARAMS_FILE: USED_VARS["stage1"]}, "stage2": {DEFAULT_PARAMS_FILE: USED_VARS["stage2"]}, }
def test_vars_load_partial(tmp_dir, dvc, local, vars_): iterable = {"bar": "bar", "foo": "foo"} (tmp_dir / "test_params.yaml").dump(iterable) d = {"stages": {"build": {"cmd": "echo ${bar}"}}} if local: d["stages"]["build"]["vars"] = vars_ else: d["vars"] = vars_ resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) resolver.resolve()
def test_vars(tmp_dir, dvc): d = deepcopy(TEMPLATED_DVC_YAML_DATA) d["vars"] = [CONTEXT_DATA] resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) resolved_data = deepcopy(RESOLVED_DVC_YAML_DATA) # `vars` section is not auto-tracked del resolved_data["stages"]["stage1"]["params"] del resolved_data["stages"]["stage2"]["params"] assert_stage_equal(resolver.resolve(), resolved_data)
def test_set_already_exists(): context = Context({"item": "foo"}) with pytest.raises(ResolveError) as exc_info: DataResolver.set_context_from(context, {"item": "bar"}) assert ( str(exc_info.value) == "Failed to set 'item': Cannot set 'item', " "key already exists" ) assert context["item"] == Value("foo")
def test_foreach_interpolation_key_does_not_exist(tmp_dir, repo): d = { "stages": {"build": {"foreach": "${modelss}", "do": {}}}, } resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str(exc_info.value) == ( "failed to parse 'stages.build.foreach' in 'dvc.yaml': " "Could not find 'modelss'" )
def test_foreach_expects_dict_or_list(tmp_dir, repo, foreach): d = { "stages": {"build": {"foreach": foreach, "do": {}}}, } resolver = DataResolver(repo, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str(exc_info.value) == ( "failed to resolve 'stages.build.foreach' in 'dvc.yaml': " "expected list/dictionary, got str" )