def test_set_with_foreach_and_on_stage_definition(tmp_dir, dvc): iterable = {"models": {"us": {"thresh": 10}, "gb": {"thresh": 15}}} dump_json(tmp_dir / "params.json", iterable) d = { "vars": ["params.json"], "stages": { "build": { "set": {"data": "${models}"}, "foreach": "${data}", "do": { "set": {"thresh": "${item.thresh}"}, "cmd": "command --value ${thresh}", }, } }, } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build@us": { "cmd": "command --value 10", "params": [{"params.json": ["models.us.thresh"]}], }, "build@gb": { "cmd": "command --value 15", "params": [{"params.json": ["models.gb.thresh"]}], }, } }, )
def test_resolve_local_tries_to_load_globally_used_files(tmp_dir, dvc): iterable = {"bar": "bar", "foo": "foo"} dump_json(tmp_dir / "params.json", iterable) d = { "vars": ["params.json"], "stages": { "build": { "cmd": "command --value ${bar}", "params": [{"params.json": ["foo"]}], "vars": ["params.json"], }, }, } resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "build": { "cmd": "command --value bar", "params": [{"params.json": ["bar", "foo"]}], }, } }, )
def test_with_params_section(tmp_dir, dvc): """Test that params section is also loaded for interpolation""" d = { "vars": [DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}], "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar} ${dict.foobar}", "params": [{"params.json": ["value1"]}], "vars": ["params.json"], }, }, } dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}}) dump_json(tmp_dir / "params.json", {"dict": {"foobar": "foobar"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar foobar", "params": [ "dict.bar", {"params.json": ["dict.foobar", "value1"]}, ], } } }, )
def test_load_vars_from_file(tmp_dir, dvc): dump_yaml(DEFAULT_PARAMS_FILE, DATA) datasets = {"datasets": ["foo", "bar"]} dump_json("params.json", datasets) d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]} resolver = DataResolver(dvc, tmp_dir, d) expected = deepcopy(DATA) expected.update(datasets) assert resolver.context == expected
def test_global_overwrite_error_on_imports(tmp_dir, dvc): dump_yaml(DEFAULT_PARAMS_FILE, DATA) dump_json("params.json", DATA) d = {"vars": [DEFAULT_PARAMS_FILE, "params.json"]} with pytest.raises(ResolveError) as exc_info: DataResolver(dvc, tmp_dir, d) assert str( exc_info.value) == ("failed to parse 'vars' in 'dvc.yaml':\n" "cannot redefine 'models.bar' from 'params.json' " "as it already exists in 'params.yaml'")
def test_local_overwrite_error(tmp_dir, dvc, vars_, loc): dump_yaml(DEFAULT_PARAMS_FILE, DATA) dump_json("params.json", DATA) d = {"stages": {"build": {"cmd": "echo ${models.foo}", "vars": [vars_]}}} resolver = DataResolver(dvc, tmp_dir, d) with pytest.raises(ResolveError) as exc_info: resolver.resolve() assert str( exc_info.value) == ("failed to parse stage 'build' in 'dvc.yaml':\n" f"cannot redefine 'models.bar' from '{loc}' " "as it already exists in 'params.yaml'")
def test_with_templated_wdir(tmp_dir, dvc): """ Test that params from the resolved wdir are still loaded and is used in the interpolation. """ d = { "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar}", "params": ["value1"], "wdir": "${dict.ws}", "vars": [DEFAULT_PARAMS_FILE], }, }, } dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": { "bar": "bar", "ws": "data" }}) data_dir = tmp_dir / "data" data_dir.mkdir() dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar", "wdir": "data", "params": ["value1"], } } }, ) assert resolver.tracked_vars == { "stage1": { str(PathInfo("data") / DEFAULT_PARAMS_FILE): { "dict.foo": "foo" }, DEFAULT_PARAMS_FILE: { "dict.bar": "bar", "dict.ws": "data" }, } } assert resolver.context.imports == {str(tmp_dir / "params.yaml"): None} assert resolver.context == {"dict": {"bar": "bar", "ws": "data"}}
def test_with_templated_wdir(tmp_dir, dvc): """ Test that params from the resolved wdir are still loaded and is used in the interpolation. """ d = { "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar}", "params": ["value1"], "wdir": "${dict.ws}", }, }, } dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": { "bar": "bar", "ws": "data" }}) data_dir = tmp_dir / "data" data_dir.mkdir() dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) root_params_file = os.path.relpath(tmp_dir / "params.yaml", data_dir) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar", "wdir": "data", "params": [ "dict.foo", "value1", { root_params_file: ["dict.bar", "dict.ws"] }, ], } } }, )
def test_stage_with_wdir(tmp_dir, dvc): """ Test that params file from wdir are also loaded """ d = { "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar}", "params": ["value1"], "wdir": "data", "vars": [DEFAULT_PARAMS_FILE], }, }, } data_dir = tmp_dir / "data" data_dir.mkdir() dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}}) dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar", "wdir": "data", "params": ["value1"], } } }, ) assert resolver.tracked_vars == { "stage1": { str(PathInfo("data") / DEFAULT_PARAMS_FILE): { "dict.foo": "foo" }, DEFAULT_PARAMS_FILE: { "dict.bar": "bar" }, } }
def test_stage_with_wdir(tmp_dir, dvc): """ Test that params file from wdir are also loaded """ d = { "stages": { "stage1": { "cmd": "echo ${dict.foo} ${dict.bar}", "params": ["value1"], "wdir": "data", }, }, } data_dir = tmp_dir / "data" data_dir.mkdir() dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, {"dict": {"bar": "bar"}}) dump_json(data_dir / DEFAULT_PARAMS_FILE, {"dict": {"foo": "foo"}}) resolver = DataResolver(dvc, PathInfo(str(tmp_dir)), d) root_params_file = os.path.relpath(tmp_dir / "params.yaml", data_dir) assert_stage_equal( resolver.resolve(), { "stages": { "stage1": { "cmd": "echo foo bar", "wdir": "data", "params": [ "dict.foo", "value1", { root_params_file: ["dict.bar"] }, ], } } }, )
def repo(tmp_dir, dvc): dump_yaml(tmp_dir / DEFAULT_PARAMS_FILE, DATA) dump_json(tmp_dir / "params.json", DATA) yield dvc