def test_list():
lst = [
{LocalOutput.PARAM_PATH: "foo", LocalTree.PARAM_CHECKSUM: "123"},
{LocalOutput.PARAM_PATH: "bar", LocalTree.PARAM_CHECKSUM: None},
{LocalOutput.PARAM_PATH: "baz"},
]
d = {Stage.PARAM_DEPS: lst}
SingleStageFile.validate(d)
lst[0][LocalOutput.PARAM_CACHE] = True
lst[1][LocalOutput.PARAM_CACHE] = False
d = {Stage.PARAM_OUTS: lst}
SingleStageFile.validate(d)
def test_meta_is_preserved(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
# Add meta to DVC-file
data = (tmp_dir / stage.path).parse()
data["meta"] = {"custom_key": 42}
(tmp_dir / stage.path).dump(data)
# Loading and dumping to test that it works and meta is retained
dvcfile = SingleStageFile(dvc, stage.path)
new_stage = dvcfile.stage
dvcfile.dump(new_stage)
new_data = (tmp_dir / stage.path).parse()
assert new_data["meta"] == data["meta"]
def test_ignored_in_checksum(self):
stage = self.dvc.run(
cmd="echo test > {}".format(self.FOO),
deps=[self.BAR],
outs=[self.FOO],
)
d = stage.dumpd()
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
with self.dvc.lock, self.dvc.state:
stage = SingleStageFile(self.dvc, stage.relpath).stage
self.assertFalse(stage.changed())
def test_ignored_in_checksum(self):
stage = self.dvc.run(
cmd=f"echo test > {self.FOO}",
deps=[self.BAR],
outs=[self.FOO],
single_stage=True,
)
d = stage.dumpd()
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
d = load_yaml(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
with self.dvc.lock:
stage = SingleStageFile(self.dvc, stage.relpath).stage
self.assertFalse(stage.changed())
def test_desc_is_preserved(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen("foo", "foo content")
data = (tmp_dir / stage.path).parse()
stage_desc = "test stage description"
out_desc = "test out description"
data["desc"] = stage_desc
data["outs"][0]["desc"] = out_desc
(tmp_dir / stage.path).dump(data)
dvcfile = SingleStageFile(dvc, stage.path)
new_stage = dvcfile.stage
dvcfile.dump(new_stage)
new_data = (tmp_dir / stage.path).parse()
assert new_data["desc"] == stage_desc
assert new_data["outs"][0]["desc"] == out_desc
def test(self):
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
d = load_yaml(stage.relpath)
# NOTE: checking that reloaded stage didn't change its checksum
md5 = "11111111111111111111111111111111"
d[stage.PARAM_MD5] = md5
dump_yaml(stage.relpath, d)
dvcfile = SingleStageFile(self.dvc, stage.relpath)
stage = dvcfile.stage
self.assertTrue(stage is not None)
dvcfile.dump(stage)
d = load_yaml(stage.relpath)
self.assertEqual(d[stage.PARAM_MD5], md5)
def migrate(dvc, path, name):
from dvc.dvcfile import SingleStageFile, PipelineFile
dvcfile = SingleStageFile(dvc, path)
stage = dvcfile.stage
stage.name = name
# Change stage path to future stage path (dvc.yaml file location)
stage.path = os.path.join(os.getcwd(), "dvc.yaml")
p_file = PipelineFile(dvc, "dvc.yaml")
# using internal APIs, there are checks on `dump()`.
p_file._dump_pipeline_file(stage)
p_file._dump_lockfile(stage)
logger.info("'{}' has been added to 'dvc.yaml' and 'dvc.lock'.")
os.rename(dvcfile.path, dvcfile.path + ".bak")
logger.info("'{0}' has been renamed to '{0}.bak'.".format(dvcfile.path))
logger.info("Delete it after carefully reviewing"
" 'dvc.lock' and 'dvc.yaml' or use it to rollback.")
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker):
from dvc.dvcfile import SingleStageFile
tmp_dir.gen("foo", "foo")
stage = run_copy("foo", "bar", single_stage=True)
assert not dvc.reproduce(stage.addressing)
from dvc.stage.run import cmd_run
m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run)
data = SingleStageFile(dvc, stage.path)._load()[0]
data["cmd"] = " ".join(stage.cmd.split()) # change cmd spacing by two
dump_yaml(stage.path, data)
assert dvc.status([stage.addressing]) == {
stage.addressing: ["changed checksum"]
}
assert dvc.reproduce(stage.addressing)[0] == stage
m.assert_called_once_with(stage, checkpoint_func=None)
def test_empty_list():
d = {Stage.PARAM_DEPS: []}
SingleStageFile.validate(d)
d = {Stage.PARAM_OUTS: []}
SingleStageFile.validate(d)
def test_none():
SingleStageFile.validate({Stage.PARAM_DEPS: None})
SingleStageFile.validate({Stage.PARAM_OUTS: None})
def test_object():
with pytest.raises(YAMLValidationError):
SingleStageFile.validate({Stage.PARAM_DEPS: {}})
with pytest.raises(YAMLValidationError):
SingleStageFile.validate({Stage.PARAM_OUTS: {}})
def test_cmd_str():
SingleStageFile.validate({Stage.PARAM_CMD: "cmd"})
def test_no_cmd():
SingleStageFile.validate({})
def test_cmd_none():
SingleStageFile.validate({Stage.PARAM_CMD: None})
def test_cmd_obj():
with pytest.raises(YAMLValidationError):
SingleStageFile.validate({Stage.PARAM_CMD: {}})
def test_cmd_obj():
with pytest.raises(StageFileFormatError):
SingleStageFile.validate({Stage.PARAM_CMD: {}})
def test_object():
with pytest.raises(StageFileFormatError):
SingleStageFile.validate({Stage.PARAM_DEPS: {}})
with pytest.raises(StageFileFormatError):
SingleStageFile.validate({Stage.PARAM_OUTS: {}})
