def test_order(dvc):
stage = create_stage(PipelineStage,
dvc,
deps=["input"],
outs=["output"],
params=["foo-param"],
**kwargs)
params, deps = split_params_deps(stage)
deps[0].info = {"md5": "md-five"}
params[0].info = {"foo-param": "value"}
stage.outs[0].info = {"md5": "md5-output"}
assert to_single_stage_lockfile(stage) == OrderedDict([
("cmd", "command"),
("deps", [{
"path": "input",
"md5": "md-five"
}]),
("params", {
"params.yaml": {
"foo-param": "value"
}
}),
("outs", [{
"path": "output",
"md5": "md5-output"
}]),
])
def test_repro_multiple_params(tmp_dir, dvc):
from tests.func.test_run_multistage import supported_params
from dvc.stage.utils import split_params_deps
with (tmp_dir / "params2.yaml").open("w+") as f:
yaml.dump(supported_params, f)
with (tmp_dir / "params.yaml").open("w+") as f:
yaml.dump(supported_params, f)
(tmp_dir / "foo").write_text("foo")
stage = dvc.run(
name="read_params",
deps=["foo"],
outs=["bar"],
params=[
"params2.yaml:lists,floats,name",
"answer,floats,nested.nested1",
],
cmd="cat params2.yaml params.yaml > bar",
)
params, deps = split_params_deps(stage)
assert len(params) == 2
assert len(deps) == 1
assert len(stage.outs) == 1
lockfile = stage.dvcfile._lockfile
assert lockfile.load()["read_params"]["params"] == {
"params2.yaml": {
"lists": [42, 42.0, "42"],
"floats": 42.0,
"name": "Answer",
},
"params.yaml": {
"answer": 42,
"floats": 42.0,
"nested.nested1": {
"nested2": "42",
"nested2-2": 41.99999
},
},
}
data, _ = stage.dvcfile._load()
params = data["stages"]["read_params"]["params"]
custom, defaults = lsplit(lambda v: isinstance(v, dict), params)
assert set(custom[0]["params2.yaml"]) == {"name", "lists", "floats"}
assert set(defaults) == {"answer", "floats", "nested.nested1"}
assert not dvc.reproduce(stage.addressing)
with (tmp_dir / "params.yaml").open("w+") as f:
params = deepcopy(supported_params)
params["answer"] = 43
yaml.dump(params, f)
assert dvc.reproduce(stage.addressing) == [stage]
def test_params_dump(tmp_dir, dvc, run_head):
tmp_dir.gen(FS_STRUCTURE)
stage = run_head(
"foo",
"bar",
"foobar",
name="copy-first-line",
params=[
"params2.yaml:answer,lists,name",
"params.yaml:lists,floats,nested.nested1,nested.nested1.nested2",
],
)
initial_content = read_lock_file()
lock = initial_content["stages"]["copy-first-line"]
# lock stage key order:
assert list(lock.keys()) == ["cmd", "deps", "params", "outs"]
assert list(lock["params"].keys()) == ["params.yaml", "params2.yaml"]
# # params keys are always sorted by the name
assert list(lock["params"]["params.yaml"].keys()) == [
"floats",
"lists",
"nested.nested1",
"nested.nested1.nested2",
]
assert list(lock["params"]["params2.yaml"]) == ["answer", "lists", "name"]
assert not dvc.reproduce(stage.addressing)
# let's change the order of params and dump them in pipeline file
params, _ = split_params_deps(stage)
for param in params:
param.params.reverse()
stage.dvcfile._dump_pipeline_file(stage)
assert not dvc.reproduce(stage.addressing)
(tmp_dir / PIPELINE_LOCK).unlink()
assert dvc.reproduce(stage.addressing) == [stage]
assert_eq_lockfile(initial_content, read_lock_file())
# remove build-cache and check if the same structure is built
for item in [dvc.stage_cache.cache_dir, PIPELINE_LOCK]:
remove(item)
assert dvc.reproduce(stage.addressing) == [stage]
assert_eq_lockfile(initial_content, read_lock_file())
