def test(self):
with open(self.CODE, "w+", encoding="utf-8") as fobj:
fobj.write("import sys\n")
fobj.write("import os\n")
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
fobj.write(" fobj.write('foo')\n")
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
ret = main([
"run",
"-d",
self.CODE,
"-o",
self.FOO,
"--single-stage",
"python",
self.CODE,
self.FOO,
])
self.assertEqual(ret, 0)
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(system.is_symlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
ret = main([
"run",
"--force",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
"-o",
self.FOO,
"python",
self.CODE,
self.FOO,
])
self.assertEqual(ret, 0)
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(system.is_symlink(self.FOO))
with open(self.FOO, encoding="utf-8") as fd:
self.assertEqual(fd.read(), "foo")
def test_cache_type_is_properly_overridden(tmp_dir, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]], "set cache type to symlinks"
)
erepo_dir.dvc_gen("file", "contents", "create file")
assert system.is_symlink(erepo_dir / "file")
Repo.get(os.fspath(erepo_dir), "file", "file_imported")
assert not system.is_symlink("file_imported")
assert (tmp_dir / "file_imported").read_text() == "contents"
def test_cache_type_is_properly_overridden(tmp_dir, scm, dvc, erepo_dir):
with erepo_dir.chdir():
with erepo_dir.dvc.config.edit() as conf:
conf["cache"]["type"] = "symlink"
erepo_dir.dvc.odb = ODBManager(erepo_dir.dvc)
erepo_dir.scm_add(
[erepo_dir.dvc.config.files["repo"]],
"set source repo cache type to symlink",
)
erepo_dir.dvc_gen("foo", "foo content", "create foo")
assert system.is_symlink(erepo_dir / "foo")
dvc.imp(os.fspath(erepo_dir), "foo", "foo_imported")
assert not system.is_symlink("foo_imported")
assert (tmp_dir / "foo_imported").read_text() == "foo content"
assert scm.is_ignored("foo_imported")
def test(self):
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
ret = main(["add", self.DATA_DIR])
self.assertEqual(ret, 0)
self.assertTrue(system.is_symlink(self.FOO))
old_foo_link = os.path.realpath(self.FOO)
self.assertTrue(system.is_symlink(self.DATA))
old_data_link = os.path.realpath(self.DATA)
old_cache_dir = self.dvc.odb.local.cache_dir
new_cache_dir = old_cache_dir + "_new"
os.rename(old_cache_dir, new_cache_dir)
ret = main(["cache", "dir", new_cache_dir])
self.assertEqual(ret, 0)
ret = main(["checkout", "-f"])
self.assertEqual(ret, 0)
self.assertTrue(system.is_symlink(self.FOO))
new_foo_link = os.path.realpath(self.FOO)
self.assertTrue(system.is_symlink(self.DATA))
new_data_link = os.path.realpath(self.DATA)
self.assertEqual(
relpath(old_foo_link, old_cache_dir),
relpath(new_foo_link, new_cache_dir),
)
self.assertEqual(
relpath(old_data_link, old_cache_dir),
relpath(new_data_link, new_cache_dir),
)
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.odb.local.cache_types = ["symlink"]
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(dvc.odb.local, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
assert system.is_symlink(os.path.join("dir", "new_file"))
def test_destroy(tmp_dir, dvc, run_copy):
dvc.config["cache"]["type"] = ["symlink"]
dvc.odb = ODBManager(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
run_copy("file", "file2", single_stage=True)
run_copy("file2", "file3", name="copy-file2-file3")
run_copy("file3", "file4", name="copy-file3-file4")
dvc.destroy()
# Remove all the files related to DVC
for path in [
".dvc",
".dvcignore",
"file.dvc",
"file2.dvc",
"dir.dvc",
PIPELINE_FILE,
PIPELINE_LOCK,
]:
assert not (tmp_dir / path).exists()
# Leave the rest of the files
for path in [
"file",
"file2",
"file3",
"file4",
"dir/file",
"dir/subdir/file",
]:
assert (tmp_dir / path).is_file()
# Make sure that data was unprotected after `destroy`
for path in [
"file",
"file2",
"file3",
"file4",
"dir",
"dir/file",
"dir/subdir",
"dir/subdir/file",
]:
assert not system.is_symlink(tmp_dir / path)
def contains_symlink_up_to(path: "StrPath", base_path: "StrPath"):
from dvc.fs import system
base_path = os.path.normcase(os.fspath(base_path))
path = os.path.normcase(os.fspath(path))
if base_path not in path:
raise BasePathNotInCheckedPathException(path, base_path)
if path == base_path:
return False
if system.is_symlink(path):
return True
if os.path.dirname(path) == path:
return False
return contains_symlink_up_to(os.path.dirname(path), base_path)
def test_remove_broken_symlink(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.odb.local.cache_types = ["symlink"]
(stage,) = dvc.add("foo")
remove(dvc.odb.local.cache_dir)
assert system.is_symlink("foo")
with pytest.raises(ObjectDBError):
dvc.remove(stage.addressing)
assert os.path.lexists("foo")
assert (tmp_dir / stage.relpath).exists()
dvc.remove(stage.addressing, outs=True)
assert not os.path.lexists("foo")
assert not (tmp_dir / stage.relpath).exists()
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
shutil.copy("bar", "foo")
copy_spy = mocker.spy(dvc.odb.local.fs, "copy")
dvc.add("foo")
assert copy_spy.mock.call_count == 0
@pytest.mark.parametrize(
"link,new_link,link_test_func",
[
("hardlink", "copy", lambda path: not system.is_hardlink(path)),
("symlink", "copy", lambda path: not system.is_symlink(path)),
("copy", "hardlink", system.is_hardlink),
("copy", "symlink", system.is_symlink),
],
)
def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir,
dvc):
dvc.config["cache"]["type"] = link
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.odb.local.fs, link)((tmp_dir / "bar").fs_path,
(tmp_dir / "foo").fs_path)
dvc.odb.local.cache_types = [new_link]