def test(self):
"""
Making sure that 'remote' syntax is handled properly for local outs.
"""
cwd = os.getcwd()
remote = "myremote"
ret = main(["remote", "add", remote, cwd])
self.assertEqual(ret, 0)
self.dvc = DvcRepo()
foo = "remote://{}/{}".format(remote, self.FOO)
ret = main(["add", foo])
self.assertEqual(ret, 0)
d = load_stage_file("foo.dvc")
self.assertEqual(d["outs"][0]["path"], foo)
bar = os.path.join(cwd, self.BAR)
ret = main(["add", bar])
self.assertEqual(ret, 0)
d = load_stage_file("bar.dvc")
self.assertEqual(d["outs"][0]["path"], bar)
def test_default_wdir_is_not_written(self):
stage = self.dvc.run(cmd="echo test > {}".format(self.FOO),
outs=[self.FOO],
wdir=".")
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
stage = self.dvc.run(cmd="echo test > {}".format(self.BAR),
outs=[self.BAR])
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
def test_default_wdir_is_written(self):
stage = self.dvc.run(cmd="echo test > {}".format(self.FOO),
outs=[self.FOO],
wdir=".")
d = load_stage_file(stage.relpath)
self.assertEqual(d[Stage.PARAM_WDIR], ".")
stage = self.dvc.run(cmd="echo test > {}".format(self.BAR),
outs=[self.BAR])
d = load_stage_file(stage.relpath)
self.assertEqual(d[Stage.PARAM_WDIR], ".")
def test_commit_changed_md5(tmp_dir, dvc):
tmp_dir.gen({"file": "file content"})
(stage, ) = dvc.add("file", no_commit=True)
stage_file_content = load_stage_file(stage.path)
stage_file_content["md5"] = "1111111111"
dump_stage_file(stage.path, stage_file_content)
with pytest.raises(StageCommitError):
dvc.commit(stage.path)
dvc.commit(stage.path, force=True)
assert "md5" not in load_stage_file(stage.path)
def test_meta_is_preserved(tmp_dir, dvc):
(stage, ) = tmp_dir.dvc_gen("foo", "foo content")
# Add meta to DVC-file
data = load_stage_file(stage.path)
data["meta"] = {"custom_key": 42}
dump_stage_file(stage.path, data)
# Loading and dumping to test that it works and meta is retained
new_stage = Stage.load(dvc, stage.path)
new_stage.dump()
new_data = load_stage_file(stage.path)
assert new_data["meta"] == data["meta"]
def test_meta_is_preserved(dvc_repo):
stage, = dvc_repo.add("foo")
# Add meta to stage file
data = load_stage_file(stage.path)
data["meta"] = {"custom_key": 42}
dump_stage_file(stage.path, data)
# Loading and dumping to test that it works and meta is retained
new_stage = Stage.load(dvc_repo, stage.path)
new_stage.dump()
new_data = load_stage_file(stage.path)
assert new_data["meta"] == data["meta"]
def test_default_wdir_is_not_written(self):
stage = self.dvc.run(
cmd=f"echo test > {self.FOO}",
outs=[self.FOO],
wdir=".",
single_stage=True,
)
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
stage = self.dvc.run(
cmd=f"echo test > {self.BAR}", outs=[self.BAR], single_stage=True,
)
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
def test(self):
d = load_stage_file(self.file1_stage)
del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
dump_stage_file(self.file1_stage, d)
self.dvc.checkout(force=True)
def test(self):
data_stage_file = self.DATA + Stage.STAGE_FILE_SUFFIX
ret = main(["add", self.DATA])
self.assertEqual(ret, 0)
self.assertTrue(os.path.exists(data_stage_file))
new_data_dir = "data_dir2"
os.makedirs(new_data_dir)
ret = main(["move", self.DATA, new_data_dir])
self.assertEqual(ret, 0)
new_data_path = os.path.join(new_data_dir, os.path.basename(self.DATA))
new_data_stage_file = new_data_path + Stage.STAGE_FILE_SUFFIX
self.assertFalse(os.path.exists(self.DATA))
self.assertFalse(os.path.exists(data_stage_file))
self.assertTrue(os.path.exists(new_data_path))
self.assertTrue(os.path.exists(new_data_stage_file))
new_stage_file = load_stage_file(new_data_stage_file)
self.assertEqual(
os.path.basename(self.DATA), new_stage_file["outs"][0]["path"]
)
def test(self):
# Use copy to test for changes in the inodes
ret = main(["config", "cache.type", "copy"])
self.assertEqual(ret, 0)
ret = main(["add", self.DATA_DIR])
self.assertEqual(0, ret)
stage_path = self.DATA_DIR + Stage.STAGE_FILE_SUFFIX
stage = load_stage_file(stage_path)
staged_files = self.outs_info(stage)
# move instead of remove, to lock inode assigned to stage_files[0].path
# if we were to use remove, we might end up with same inode assigned to
# newly checked out file
shutil.move(staged_files[0].path, "random_name")
ret = main(["checkout", "--force", stage_path])
self.assertEqual(ret, 0)
checkedout_files = self.outs_info(stage)
self.assertEqual(len(staged_files), len(checkedout_files))
self.assertEqual(staged_files[0].path, checkedout_files[0].path)
self.assertNotEqual(staged_files[0].inode, checkedout_files[0].inode)
self.assertEqual(staged_files[1].inode, checkedout_files[1].inode)
def test(self):
d = load_stage_file(self.file1_stage)
del d[Stage.PARAM_OUTS][0][RemoteLOCAL.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][RemoteLOCAL.PARAM_CHECKSUM]
dump_stage_file(self.file1_stage, d)
stages = self.dvc.reproduce(self.file1_stage)
self.assertEqual(len(stages), 1)
def test(self):
d = load_stage_file(self.file1_stage)
del d[Stage.PARAM_OUTS][0][LocalRemote.PARAM_CHECKSUM]
del d[Stage.PARAM_DEPS][0][LocalRemote.PARAM_CHECKSUM]
dump_stage_file(self.file1_stage, d)
with pytest.raises(CheckoutError):
self.dvc.checkout(force=True)
def test(self):
stages = self.dvc.add(self.FOO)
self.assertEqual(len(stages), 1)
stage = stages[0]
self.assertTrue(stage is not None)
d = load_stage_file(stage.relpath)
# NOTE: checking that reloaded stage didn't change its checksum
md5 = "11111111111111111111111111111111"
d[stage.PARAM_MD5] = md5
dump_stage_file(stage.relpath, d)
stage = Stage.load(self.dvc, stage.relpath)
self.assertTrue(stage is not None)
stage.dump()
d = load_stage_file(stage.relpath)
self.assertEqual(d[stage.PARAM_MD5], md5)
def _test(self):
self._prepare_external_data()
ret = main(["add", os.path.join(self.link_name, self.data_file_name)])
self.assertEqual(0, ret)
stage_file = self.data_file_name + Stage.STAGE_FILE_SUFFIX
self.assertTrue(os.path.exists(stage_file))
d = load_stage_file(stage_file)
relative_data_path = posixpath.join(self.link_name,
self.data_file_name)
self.assertEqual(relative_data_path, d["outs"][0]["path"])
def test_ignored_in_checksum(self):
stage = self.dvc.run(
cmd="echo test > {}".format(self.FOO),
deps=[self.BAR],
outs=[self.FOO],
)
d = stage.dumpd()
self.assertEqual(d[stage.PARAM_WDIR], ".")
d = load_stage_file(stage.relpath)
self.assertEqual(d[stage.PARAM_WDIR], ".")
del d[stage.PARAM_WDIR]
dump_stage_file(stage.relpath, d)
d = load_stage_file(stage.relpath)
self.assertIsNone(d.get(stage.PARAM_WDIR))
with self.dvc.state:
stage = Stage.load(self.dvc, stage.relpath)
self.assertFalse(stage.changed())
def test_commit_changed_md5(dvc_repo, repo_dir):
stages = dvc_repo.add(repo_dir.FOO, no_commit=True)
assert len(stages) == 1
stage = stages[0]
stage_file_content = load_stage_file(stage.path)
stage_file_content["md5"] = "1111111111"
dump_stage_file(stage.path, stage_file_content)
with pytest.raises(StageCommitError):
dvc_repo.commit(stage.path)
dvc_repo.commit(stage.path, force=True)
def test_fname_changes_path_and_wdir(self):
dname = "dir"
os.mkdir(os.path.join(self._root_dir, dname))
foo = os.path.join(dname, self.FOO)
fname = os.path.join(dname, "stage" + DVC_FILE_SUFFIX)
stage = self.dvc.run(cmd="echo test > {}".format(foo),
outs=[foo],
fname=fname)
self.assertEqual(stage.wdir, os.path.realpath(self._root_dir))
self.assertEqual(stage.path,
os.path.join(os.path.realpath(self._root_dir), fname))
# Check that it is dumped properly (relative to fname)
d = load_stage_file(stage.relpath)
self.assertEqual(d[Stage.PARAM_WDIR], "..")
def test_ignored_in_checksum(self):
stage = self.dvc.run(
cmd="echo test > {}".format(self.FOO),
deps=[self.BAR],
outs=[self.FOO],
)
d = stage.dumpd()
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
with self.dvc.lock, self.dvc.state:
stage = Stage.load(self.dvc, stage.relpath)
self.assertFalse(stage.changed())
def test_ignored_in_checksum(self):
stage = self.dvc.run(
cmd=f"echo test > {self.FOO}",
deps=[self.BAR],
outs=[self.FOO],
single_stage=True,
)
d = stage.dumpd()
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
d = load_stage_file(stage.relpath)
self.assertNotIn(Stage.PARAM_WDIR, d.keys())
with self.dvc.lock, self.dvc.state:
stage = SingleStageFile(self.dvc, stage.relpath).stage
self.assertFalse(stage.changed())
def test_add(tmp_dir, dvc):
(stage,) = tmp_dir.dvc_gen({"foo": "foo"})
md5, _ = file_md5("foo")
assert stage is not None
assert isinstance(stage, Stage)
assert os.path.isfile(stage.path)
assert len(stage.outs) == 1
assert len(stage.deps) == 0
assert stage.cmd is None
assert stage.outs[0].info["md5"] == md5
assert stage.md5 == "411854a9fc55b3cebda28704e41b23b3"
assert load_stage_file("foo.dvc") == {
"md5": "411854a9fc55b3cebda28704e41b23b3",
"outs": [{"md5": "acbd18db4cc2f85cedef654fccc4a4d8", "path": "foo"}],
}
def test(self):
foo_stage_file_path = self.FOO + DVC_FILE_SUFFIX
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
self.assertTrue(os.path.exists(foo_stage_file_path))
target_foo_path = os.path.join(self.DATA_DIR, self.FOO)
target_foo_stage_file_path = target_foo_path + DVC_FILE_SUFFIX
ret = main(["move", self.FOO, self.DATA_DIR])
self.assertEqual(ret, 0)
self.assertFalse(os.path.exists(self.FOO))
self.assertFalse(os.path.exists(foo_stage_file_path))
self.assertTrue(os.path.exists(target_foo_path))
self.assertTrue(os.path.exists(target_foo_stage_file_path))
new_stage = load_stage_file(target_foo_stage_file_path)
self.assertEqual(self.FOO, new_stage["outs"][0]["path"])
def _test(self):
url = Local.get_url()
self.main(["remote", "add", "-d", TEST_REMOTE, url])
stage = self.dvc.run(outs=["bar"], cmd="echo bar > bar")
self.main(["push"])
stage_file_path = stage.relpath
content = load_stage_file(stage_file_path)
del content["outs"][0]["md5"]
dump_stage_file(stage_file_path, content)
with self._caplog.at_level(logging.WARNING, logger="dvc"):
self._caplog.clear()
self.main(["status", "-c"])
expected_warning = (
"Output 'bar'(Stage: 'bar.dvc') is missing version info."
" Cache for it will not be collected."
" Use dvc repro to get your pipeline up to date.")
assert expected_warning in self._caplog.text
def stage_should_contain_persist_flag(self, stage_file):
stage_file_content = load_stage_file(stage_file)
self.assertEqual(
True, stage_file_content["outs"][0][BaseOutput.PARAM_PERSIST])
