def test_list(): lst = [ {LocalOutput.PARAM_PATH: "foo", LocalTree.PARAM_CHECKSUM: "123"}, {LocalOutput.PARAM_PATH: "bar", LocalTree.PARAM_CHECKSUM: None}, {LocalOutput.PARAM_PATH: "baz"}, ] d = {Stage.PARAM_DEPS: lst} SingleStageFile.validate(d) lst[0][LocalOutput.PARAM_CACHE] = True lst[1][LocalOutput.PARAM_CACHE] = False d = {Stage.PARAM_OUTS: lst} SingleStageFile.validate(d)
def test_meta_is_preserved(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen("foo", "foo content") # Add meta to DVC-file data = (tmp_dir / stage.path).parse() data["meta"] = {"custom_key": 42} (tmp_dir / stage.path).dump(data) # Loading and dumping to test that it works and meta is retained dvcfile = SingleStageFile(dvc, stage.path) new_stage = dvcfile.stage dvcfile.dump(new_stage) new_data = (tmp_dir / stage.path).parse() assert new_data["meta"] == data["meta"]
def test_ignored_in_checksum(self): stage = self.dvc.run( cmd="echo test > {}".format(self.FOO), deps=[self.BAR], outs=[self.FOO], ) d = stage.dumpd() self.assertNotIn(Stage.PARAM_WDIR, d.keys()) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) with self.dvc.lock, self.dvc.state: stage = SingleStageFile(self.dvc, stage.relpath).stage self.assertFalse(stage.changed())
def test_ignored_in_checksum(self): stage = self.dvc.run( cmd=f"echo test > {self.FOO}", deps=[self.BAR], outs=[self.FOO], single_stage=True, ) d = stage.dumpd() self.assertNotIn(Stage.PARAM_WDIR, d.keys()) d = load_yaml(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) with self.dvc.lock: stage = SingleStageFile(self.dvc, stage.relpath).stage self.assertFalse(stage.changed())
def test_desc_is_preserved(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen("foo", "foo content") data = (tmp_dir / stage.path).parse() stage_desc = "test stage description" out_desc = "test out description" data["desc"] = stage_desc data["outs"][0]["desc"] = out_desc (tmp_dir / stage.path).dump(data) dvcfile = SingleStageFile(dvc, stage.path) new_stage = dvcfile.stage dvcfile.dump(new_stage) new_data = (tmp_dir / stage.path).parse() assert new_data["desc"] == stage_desc assert new_data["outs"][0]["desc"] == out_desc
def test(self): stages = self.dvc.add(self.FOO) self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) d = load_yaml(stage.relpath) # NOTE: checking that reloaded stage didn't change its checksum md5 = "11111111111111111111111111111111" d[stage.PARAM_MD5] = md5 dump_yaml(stage.relpath, d) dvcfile = SingleStageFile(self.dvc, stage.relpath) stage = dvcfile.stage self.assertTrue(stage is not None) dvcfile.dump(stage) d = load_yaml(stage.relpath) self.assertEqual(d[stage.PARAM_MD5], md5)
def migrate(dvc, path, name): from dvc.dvcfile import SingleStageFile, PipelineFile dvcfile = SingleStageFile(dvc, path) stage = dvcfile.stage stage.name = name # Change stage path to future stage path (dvc.yaml file location) stage.path = os.path.join(os.getcwd(), "dvc.yaml") p_file = PipelineFile(dvc, "dvc.yaml") # using internal APIs, there are checks on `dump()`. p_file._dump_pipeline_file(stage) p_file._dump_lockfile(stage) logger.info("'{}' has been added to 'dvc.yaml' and 'dvc.lock'.") os.rename(dvcfile.path, dvcfile.path + ".bak") logger.info("'{0}' has been renamed to '{0}.bak'.".format(dvcfile.path)) logger.info("Delete it after carefully reviewing" " 'dvc.lock' and 'dvc.yaml' or use it to rollback.")
def test_repro_when_cmd_changes(tmp_dir, dvc, run_copy, mocker): from dvc.dvcfile import SingleStageFile tmp_dir.gen("foo", "foo") stage = run_copy("foo", "bar", single_stage=True) assert not dvc.reproduce(stage.addressing) from dvc.stage.run import cmd_run m = mocker.patch("dvc.stage.run.cmd_run", wraps=cmd_run) data = SingleStageFile(dvc, stage.path)._load()[0] data["cmd"] = " ".join(stage.cmd.split()) # change cmd spacing by two dump_yaml(stage.path, data) assert dvc.status([stage.addressing]) == { stage.addressing: ["changed checksum"] } assert dvc.reproduce(stage.addressing)[0] == stage m.assert_called_once_with(stage, checkpoint_func=None)
def test_empty_list(): d = {Stage.PARAM_DEPS: []} SingleStageFile.validate(d) d = {Stage.PARAM_OUTS: []} SingleStageFile.validate(d)
def test_none(): SingleStageFile.validate({Stage.PARAM_DEPS: None}) SingleStageFile.validate({Stage.PARAM_OUTS: None})
def test_object(): with pytest.raises(YAMLValidationError): SingleStageFile.validate({Stage.PARAM_DEPS: {}}) with pytest.raises(YAMLValidationError): SingleStageFile.validate({Stage.PARAM_OUTS: {}})
def test_cmd_str(): SingleStageFile.validate({Stage.PARAM_CMD: "cmd"})
def test_no_cmd(): SingleStageFile.validate({})
def test_cmd_none(): SingleStageFile.validate({Stage.PARAM_CMD: None})
def test_cmd_obj(): with pytest.raises(YAMLValidationError): SingleStageFile.validate({Stage.PARAM_CMD: {}})
def test_cmd_obj(): with pytest.raises(StageFileFormatError): SingleStageFile.validate({Stage.PARAM_CMD: {}})
def test_object(): with pytest.raises(StageFileFormatError): SingleStageFile.validate({Stage.PARAM_DEPS: {}}) with pytest.raises(StageFileFormatError): SingleStageFile.validate({Stage.PARAM_OUTS: {}})