def test_order(dvc): stage = create_stage(PipelineStage, dvc, deps=["input"], outs=["output"], params=["foo-param"], **kwargs) params, deps = split_params_deps(stage) deps[0].info = {"md5": "md-five"} params[0].info = {"foo-param": "value"} stage.outs[0].info = {"md5": "md5-output"} assert to_single_stage_lockfile(stage) == OrderedDict([ ("cmd", "command"), ("deps", [{ "path": "input", "md5": "md-five" }]), ("params", { "params.yaml": { "foo-param": "value" } }), ("outs", [{ "path": "output", "md5": "md5-output" }]), ])
def test_repro_multiple_params(tmp_dir, dvc): from tests.func.test_run_multistage import supported_params from dvc.stage.utils import split_params_deps with (tmp_dir / "params2.yaml").open("w+") as f: yaml.dump(supported_params, f) with (tmp_dir / "params.yaml").open("w+") as f: yaml.dump(supported_params, f) (tmp_dir / "foo").write_text("foo") stage = dvc.run( name="read_params", deps=["foo"], outs=["bar"], params=[ "params2.yaml:lists,floats,name", "answer,floats,nested.nested1", ], cmd="cat params2.yaml params.yaml > bar", ) params, deps = split_params_deps(stage) assert len(params) == 2 assert len(deps) == 1 assert len(stage.outs) == 1 lockfile = stage.dvcfile._lockfile assert lockfile.load()["read_params"]["params"] == { "params2.yaml": { "lists": [42, 42.0, "42"], "floats": 42.0, "name": "Answer", }, "params.yaml": { "answer": 42, "floats": 42.0, "nested.nested1": { "nested2": "42", "nested2-2": 41.99999 }, }, } data, _ = stage.dvcfile._load() params = data["stages"]["read_params"]["params"] custom, defaults = lsplit(lambda v: isinstance(v, dict), params) assert set(custom[0]["params2.yaml"]) == {"name", "lists", "floats"} assert set(defaults) == {"answer", "floats", "nested.nested1"} assert not dvc.reproduce(stage.addressing) with (tmp_dir / "params.yaml").open("w+") as f: params = deepcopy(supported_params) params["answer"] = 43 yaml.dump(params, f) assert dvc.reproduce(stage.addressing) == [stage]
def test_params_dump(tmp_dir, dvc, run_head): tmp_dir.gen(FS_STRUCTURE) stage = run_head( "foo", "bar", "foobar", name="copy-first-line", params=[ "params2.yaml:answer,lists,name", "params.yaml:lists,floats,nested.nested1,nested.nested1.nested2", ], ) initial_content = read_lock_file() lock = initial_content["stages"]["copy-first-line"] # lock stage key order: assert list(lock.keys()) == ["cmd", "deps", "params", "outs"] assert list(lock["params"].keys()) == ["params.yaml", "params2.yaml"] # # params keys are always sorted by the name assert list(lock["params"]["params.yaml"].keys()) == [ "floats", "lists", "nested.nested1", "nested.nested1.nested2", ] assert list(lock["params"]["params2.yaml"]) == ["answer", "lists", "name"] assert not dvc.reproduce(stage.addressing) # let's change the order of params and dump them in pipeline file params, _ = split_params_deps(stage) for param in params: param.params.reverse() stage.dvcfile._dump_pipeline_file(stage) assert not dvc.reproduce(stage.addressing) (tmp_dir / PIPELINE_LOCK).unlink() assert dvc.reproduce(stage.addressing) == [stage] assert_eq_lockfile(initial_content, read_lock_file()) # remove build-cache and check if the same structure is built for item in [dvc.stage_cache.cache_dir, PIPELINE_LOCK]: remove(item) assert dvc.reproduce(stage.addressing) == [stage] assert_eq_lockfile(initial_content, read_lock_file())