def test(self): """ Making sure that 'remote' syntax is handled properly for local outs. """ cwd = os.getcwd() remote = "myremote" ret = main(["remote", "add", remote, cwd]) self.assertEqual(ret, 0) self.dvc = DvcRepo() foo = "remote://{}/{}".format(remote, self.FOO) ret = main(["add", foo]) self.assertEqual(ret, 0) d = load_stage_file("foo.dvc") self.assertEqual(d["outs"][0]["path"], foo) bar = os.path.join(cwd, self.BAR) ret = main(["add", bar]) self.assertEqual(ret, 0) d = load_stage_file("bar.dvc") self.assertEqual(d["outs"][0]["path"], bar)
def test_default_wdir_is_not_written(self): stage = self.dvc.run(cmd="echo test > {}".format(self.FOO), outs=[self.FOO], wdir=".") d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) stage = self.dvc.run(cmd="echo test > {}".format(self.BAR), outs=[self.BAR]) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys())
def test_default_wdir_is_written(self): stage = self.dvc.run(cmd="echo test > {}".format(self.FOO), outs=[self.FOO], wdir=".") d = load_stage_file(stage.relpath) self.assertEqual(d[Stage.PARAM_WDIR], ".") stage = self.dvc.run(cmd="echo test > {}".format(self.BAR), outs=[self.BAR]) d = load_stage_file(stage.relpath) self.assertEqual(d[Stage.PARAM_WDIR], ".")
def test_commit_changed_md5(tmp_dir, dvc): tmp_dir.gen({"file": "file content"}) (stage, ) = dvc.add("file", no_commit=True) stage_file_content = load_stage_file(stage.path) stage_file_content["md5"] = "1111111111" dump_stage_file(stage.path, stage_file_content) with pytest.raises(StageCommitError): dvc.commit(stage.path) dvc.commit(stage.path, force=True) assert "md5" not in load_stage_file(stage.path)
def test_meta_is_preserved(tmp_dir, dvc): (stage, ) = tmp_dir.dvc_gen("foo", "foo content") # Add meta to DVC-file data = load_stage_file(stage.path) data["meta"] = {"custom_key": 42} dump_stage_file(stage.path, data) # Loading and dumping to test that it works and meta is retained new_stage = Stage.load(dvc, stage.path) new_stage.dump() new_data = load_stage_file(stage.path) assert new_data["meta"] == data["meta"]
def test_meta_is_preserved(dvc_repo): stage, = dvc_repo.add("foo") # Add meta to stage file data = load_stage_file(stage.path) data["meta"] = {"custom_key": 42} dump_stage_file(stage.path, data) # Loading and dumping to test that it works and meta is retained new_stage = Stage.load(dvc_repo, stage.path) new_stage.dump() new_data = load_stage_file(stage.path) assert new_data["meta"] == data["meta"]
def test_default_wdir_is_not_written(self): stage = self.dvc.run( cmd=f"echo test > {self.FOO}", outs=[self.FOO], wdir=".", single_stage=True, ) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) stage = self.dvc.run( cmd=f"echo test > {self.BAR}", outs=[self.BAR], single_stage=True, ) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys())
def test(self): d = load_stage_file(self.file1_stage) del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM] del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM] dump_stage_file(self.file1_stage, d) self.dvc.checkout(force=True)
def test(self): data_stage_file = self.DATA + Stage.STAGE_FILE_SUFFIX ret = main(["add", self.DATA]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(data_stage_file)) new_data_dir = "data_dir2" os.makedirs(new_data_dir) ret = main(["move", self.DATA, new_data_dir]) self.assertEqual(ret, 0) new_data_path = os.path.join(new_data_dir, os.path.basename(self.DATA)) new_data_stage_file = new_data_path + Stage.STAGE_FILE_SUFFIX self.assertFalse(os.path.exists(self.DATA)) self.assertFalse(os.path.exists(data_stage_file)) self.assertTrue(os.path.exists(new_data_path)) self.assertTrue(os.path.exists(new_data_stage_file)) new_stage_file = load_stage_file(new_data_stage_file) self.assertEqual( os.path.basename(self.DATA), new_stage_file["outs"][0]["path"] )
def test(self): # Use copy to test for changes in the inodes ret = main(["config", "cache.type", "copy"]) self.assertEqual(ret, 0) ret = main(["add", self.DATA_DIR]) self.assertEqual(0, ret) stage_path = self.DATA_DIR + Stage.STAGE_FILE_SUFFIX stage = load_stage_file(stage_path) staged_files = self.outs_info(stage) # move instead of remove, to lock inode assigned to stage_files[0].path # if we were to use remove, we might end up with same inode assigned to # newly checked out file shutil.move(staged_files[0].path, "random_name") ret = main(["checkout", "--force", stage_path]) self.assertEqual(ret, 0) checkedout_files = self.outs_info(stage) self.assertEqual(len(staged_files), len(checkedout_files)) self.assertEqual(staged_files[0].path, checkedout_files[0].path) self.assertNotEqual(staged_files[0].inode, checkedout_files[0].inode) self.assertEqual(staged_files[1].inode, checkedout_files[1].inode)
def test(self): d = load_stage_file(self.file1_stage) del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM] del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM] dump_stage_file(self.file1_stage, d) stages = self.dvc.reproduce(self.file1_stage) self.assertEqual(len(stages), 1)
def test(self): d = load_stage_file(self.file1_stage) del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM] del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM] dump_stage_file(self.file1_stage, d) with pytest.raises(CheckoutError): self.dvc.checkout(force=True)
def test(self): stages = self.dvc.add(self.FOO) self.assertEqual(len(stages), 1) stage = stages[0] self.assertTrue(stage is not None) d = load_stage_file(stage.relpath) # NOTE: checking that reloaded stage didn't change its checksum md5 = "11111111111111111111111111111111" d[stage.PARAM_MD5] = md5 dump_stage_file(stage.relpath, d) stage = Stage.load(self.dvc, stage.relpath) self.assertTrue(stage is not None) stage.dump() d = load_stage_file(stage.relpath) self.assertEqual(d[stage.PARAM_MD5], md5)
def _test(self): self._prepare_external_data() ret = main(["add", os.path.join(self.link_name, self.data_file_name)]) self.assertEqual(0, ret) stage_file = self.data_file_name + Stage.STAGE_FILE_SUFFIX self.assertTrue(os.path.exists(stage_file)) d = load_stage_file(stage_file) relative_data_path = posixpath.join(self.link_name, self.data_file_name) self.assertEqual(relative_data_path, d["outs"][0]["path"])
def test_ignored_in_checksum(self): stage = self.dvc.run( cmd="echo test > {}".format(self.FOO), deps=[self.BAR], outs=[self.FOO], ) d = stage.dumpd() self.assertEqual(d[stage.PARAM_WDIR], ".") d = load_stage_file(stage.relpath) self.assertEqual(d[stage.PARAM_WDIR], ".") del d[stage.PARAM_WDIR] dump_stage_file(stage.relpath, d) d = load_stage_file(stage.relpath) self.assertIsNone(d.get(stage.PARAM_WDIR)) with self.dvc.state: stage = Stage.load(self.dvc, stage.relpath) self.assertFalse(stage.changed())
def test_commit_changed_md5(dvc_repo, repo_dir): stages = dvc_repo.add(repo_dir.FOO, no_commit=True) assert len(stages) == 1 stage = stages[0] stage_file_content = load_stage_file(stage.path) stage_file_content["md5"] = "1111111111" dump_stage_file(stage.path, stage_file_content) with pytest.raises(StageCommitError): dvc_repo.commit(stage.path) dvc_repo.commit(stage.path, force=True)
def test_fname_changes_path_and_wdir(self): dname = "dir" os.mkdir(os.path.join(self._root_dir, dname)) foo = os.path.join(dname, self.FOO) fname = os.path.join(dname, "stage" + DVC_FILE_SUFFIX) stage = self.dvc.run(cmd="echo test > {}".format(foo), outs=[foo], fname=fname) self.assertEqual(stage.wdir, os.path.realpath(self._root_dir)) self.assertEqual(stage.path, os.path.join(os.path.realpath(self._root_dir), fname)) # Check that it is dumped properly (relative to fname) d = load_stage_file(stage.relpath) self.assertEqual(d[Stage.PARAM_WDIR], "..")
def test_ignored_in_checksum(self): stage = self.dvc.run( cmd="echo test > {}".format(self.FOO), deps=[self.BAR], outs=[self.FOO], ) d = stage.dumpd() self.assertNotIn(Stage.PARAM_WDIR, d.keys()) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) with self.dvc.lock, self.dvc.state: stage = Stage.load(self.dvc, stage.relpath) self.assertFalse(stage.changed())
def test_ignored_in_checksum(self): stage = self.dvc.run( cmd=f"echo test > {self.FOO}", deps=[self.BAR], outs=[self.FOO], single_stage=True, ) d = stage.dumpd() self.assertNotIn(Stage.PARAM_WDIR, d.keys()) d = load_stage_file(stage.relpath) self.assertNotIn(Stage.PARAM_WDIR, d.keys()) with self.dvc.lock, self.dvc.state: stage = SingleStageFile(self.dvc, stage.relpath).stage self.assertFalse(stage.changed())
def test_add(tmp_dir, dvc): (stage,) = tmp_dir.dvc_gen({"foo": "foo"}) md5, _ = file_md5("foo") assert stage is not None assert isinstance(stage, Stage) assert os.path.isfile(stage.path) assert len(stage.outs) == 1 assert len(stage.deps) == 0 assert stage.cmd is None assert stage.outs[0].info["md5"] == md5 assert stage.md5 == "411854a9fc55b3cebda28704e41b23b3" assert load_stage_file("foo.dvc") == { "md5": "411854a9fc55b3cebda28704e41b23b3", "outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}], }
def test(self): foo_stage_file_path = self.FOO + DVC_FILE_SUFFIX ret = main(["add", self.FOO]) self.assertEqual(ret, 0) self.assertTrue(os.path.exists(foo_stage_file_path)) target_foo_path = os.path.join(self.DATA_DIR, self.FOO) target_foo_stage_file_path = target_foo_path + DVC_FILE_SUFFIX ret = main(["move", self.FOO, self.DATA_DIR]) self.assertEqual(ret, 0) self.assertFalse(os.path.exists(self.FOO)) self.assertFalse(os.path.exists(foo_stage_file_path)) self.assertTrue(os.path.exists(target_foo_path)) self.assertTrue(os.path.exists(target_foo_stage_file_path)) new_stage = load_stage_file(target_foo_stage_file_path) self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
def _test(self): url = Local.get_url() self.main(["remote", "add", "-d", TEST_REMOTE, url]) stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar") self.main(["push"]) stage_file_path = stage.relpath content = load_stage_file(stage_file_path) del content["outs"][0]["md5"] dump_stage_file(stage_file_path, content) with self._caplog.at_level(logging.WARNING, logger="dvc"): self._caplog.clear() self.main(["status", "-c"]) expected_warning = ( "Output 'bar'(Stage: 'bar.dvc') is missing version info." " Cache for it will not be collected." " Use dvc repro to get your pipeline up to date.") assert expected_warning in self._caplog.text
def stage_should_contain_persist_flag(self, stage_file): stage_file_content = load_stage_file(stage_file) self.assertEqual( True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST])