def test(self):
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
ret = main(["add", self.DATA_DIR])
self.assertEqual(ret, 0)
self.assertTrue(System.is_symlink(self.FOO))
old_foo_link = os.path.realpath(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
old_data_link = os.path.realpath(self.DATA)
old_cache_dir = self.dvc.cache.local.cache_dir
new_cache_dir = old_cache_dir + "_new"
os.rename(old_cache_dir, new_cache_dir)
ret = main(["cache", "dir", new_cache_dir])
self.assertEqual(ret, 0)
ret = main(["checkout", "-f"])
self.assertEqual(ret, 0)
self.assertTrue(System.is_symlink(self.FOO))
new_foo_link = os.path.realpath(self.FOO)
self.assertTrue(System.is_symlink(self.DATA))
new_data_link = os.path.realpath(self.DATA)
self.assertEqual(
relpath(old_foo_link, old_cache_dir),
relpath(new_foo_link, new_cache_dir),
)
self.assertEqual(
relpath(old_data_link, old_cache_dir),
relpath(new_data_link, new_cache_dir),
)
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.cache.local.cache_types = ["symlink"]
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = mocker.spy(dvc.cache.local, "unprotect")
dvc.add("dir")
assert not unprotect_spy.mock.called
assert System.is_symlink(os.path.join("dir", "new_file"))
def _link_matches(self, path_info):
is_hardlink = System.is_hardlink(path_info)
is_symlink = System.is_symlink(path_info)
is_copy_or_reflink = not is_hardlink and not is_symlink
cache_type = self._get_cache_type(path_info)
if cache_type == "symlink":
return is_symlink
if cache_type == "hardlink":
return is_hardlink
return is_copy_or_reflink
def contains_symlink_up_to(path, base_path):
base_path = fspath(base_path)
path = fspath(path)
if base_path not in path:
raise BasePathNotInCheckedPathException(path, base_path)
if path == base_path:
return False
if System.is_symlink(path):
return True
if os.path.dirname(path) == path:
return False
return contains_symlink_up_to(os.path.dirname(path), base_path)
def test_cache_type_is_properly_overridden(erepo):
erepo.dvc.config.set(Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE,
"symlink")
erepo.dvc.scm.add([erepo.dvc.config.config_file])
erepo.dvc.scm.commit("set cache type to symlinks")
src = erepo.FOO
dst = erepo.FOO + "_imported"
Repo.get(erepo.root_dir, src, dst)
assert not System.is_symlink(dst)
assert os.path.exists(dst)
assert os.path.isfile(dst)
def test_destroy(repo_dir, dvc_repo):
# NOTE: using symlink to ensure that data was unprotected after `destroy`
dvc_repo.config.set("cache", "type", "symlink")
foo_stage, = dvc_repo.add(repo_dir.FOO)
data_dir_stage, = dvc_repo.add(repo_dir.DATA_DIR)
dvc_repo.destroy()
assert not os.path.exists(dvc_repo.dvc_dir)
assert not os.path.exists(foo_stage.path)
assert not os.path.exists(data_dir_stage.path)
assert os.path.isfile(repo_dir.FOO)
assert os.path.isdir(repo_dir.DATA_DIR)
assert os.path.isfile(repo_dir.DATA)
assert os.path.isdir(repo_dir.DATA_SUB_DIR)
assert os.path.isfile(repo_dir.DATA_SUB)
assert not System.is_symlink(repo_dir.FOO)
assert not System.is_symlink(repo_dir.DATA_DIR)
assert not System.is_symlink(repo_dir.DATA)
assert not System.is_symlink(repo_dir.DATA_SUB)
def test_destroy(tmp_dir, dvc, run_copy):
from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK
dvc.config["cache"]["type"] = ["symlink"]
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
run_copy("file", "file2", single_stage=True)
run_copy("file2", "file3", name="copy-file2-file3")
run_copy("file3", "file4", name="copy-file3-file4")
dvc.destroy()
# Remove all the files related to DVC
for path in [
".dvc",
"file.dvc",
"file2.dvc",
"dir.dvc",
PIPELINE_FILE,
PIPELINE_LOCK,
]:
assert not (tmp_dir / path).exists()
# Leave the rest of the files
for path in [
"file",
"file2",
"file3",
"file4",
"dir/file",
"dir/subdir/file",
]:
assert (tmp_dir / path).is_file()
# Make sure that data was unprotected after `destroy`
for path in [
"file",
"file2",
"file3",
"file4",
"dir",
"dir/file",
"dir/subdir",
"dir/subdir/file",
]:
assert not System.is_symlink(tmp_dir / path)
def test(self):
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
ret = main(["add", self.FOO])
self.assertEqual(ret, 0)
remove(self.dvc.cache.local.cache_dir)
self.assertTrue(System.is_symlink(self.FOO))
ret = main(["remove", self.FOO + ".dvc"])
self.assertEqual(ret, 0)
self.assertFalse(os.path.lexists(self.FOO))
def test_readding_dir_should_not_unprotect_all(dvc_repo, repo_dir):
dvc_repo.cache.local.cache_types = ["symlink"]
dvc_repo.cache.local.protected = True
dvc_repo.add(repo_dir.DATA_DIR)
new_file = os.path.join(repo_dir.DATA_DIR, "new_file")
repo_dir.create(new_file, "new_content")
unprotect_spy = spy(RemoteLOCAL.unprotect)
with patch.object(RemoteLOCAL, "unprotect", unprotect_spy):
dvc_repo.add(repo_dir.DATA_DIR)
assert not unprotect_spy.mock.called
assert System.is_symlink(new_file)
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker):
tmp_dir.gen("dir/data", "data")
dvc.cache.local.cache_types = ["symlink"]
dvc.cache.local.protected = True
dvc.add("dir")
tmp_dir.gen("dir/new_file", "new_file_content")
unprotect_spy = spy(RemoteLOCAL.unprotect)
mocker.patch.object(RemoteLOCAL, "unprotect", unprotect_spy)
dvc.add("dir")
assert not unprotect_spy.mock.called
assert System.is_symlink(os.path.join("dir", "new_file"))
def _unprotect_file(path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def test_remove_broken_symlink(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.cache.local.cache_types = ["symlink"]
(stage, ) = dvc.add("foo")
remove(dvc.cache.local.cache_dir)
assert System.is_symlink("foo")
with pytest.raises(DvcException):
dvc.remove(stage.addressing)
assert os.path.lexists("foo")
assert (tmp_dir / stage.relpath).exists()
dvc.remove(stage.addressing, outs=True)
assert not os.path.lexists("foo")
assert not (tmp_dir / stage.relpath).exists()
def test_cache_type_is_properly_overridden(repo_dir, git, dvc_repo, erepo):
erepo.dvc.config.set(Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE,
"symlink")
erepo.dvc.scm.add([erepo.dvc.config.config_file])
erepo.dvc.scm.commit("set source repo cache type to symlinks")
src = erepo.FOO
dst = erepo.FOO + "_imported"
dvc_repo.imp(erepo.root_dir, src, dst)
assert not System.is_symlink(dst)
assert os.path.exists(dst)
assert os.path.isfile(dst)
assert filecmp.cmp(repo_dir.FOO, dst, shallow=False)
assert git.git.check_ignore(dst)
def _unprotect_file(path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
# The operations order is important here - if some application would
# access the file during the process of copyfile then it would get
# only the part of file. So, at first, the file should be copied with
# the temporary name, and then original file should be replaced by new.
copyfile(path, tmp)
remove(path)
os.rename(tmp, path)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def _unprotect_file(self, path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug(f"Unprotecting '{path}'")
tmp = os.path.join(os.path.dirname(path), "." + uuid())
# The operations order is important here - if some application
# would access the file during the process of copyfile then it
# would get only the part of file. So, at first, the file should be
# copied with the temporary name, and then original file should be
# replaced by new.
copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
remove(path)
os.rename(tmp, path)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, self.file_mode)
def _unprotect_file(self, path):
import stat
import uuid
from dvc.system import System
from dvc.utils import copyfile, move, remove
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def test_destroy(tmp_dir, dvc):
dvc.config["cache"]["type"] = ["symlink"]
dvc.cache = Cache(dvc)
tmp_dir.dvc_gen("file", "text")
tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}})
dvc.destroy()
# Remove all the files related to DVC
for path in [".dvc", "file.dvc", "dir.dvc"]:
assert not (tmp_dir / path).exists()
# Leave the rest of the files
for path in ["file", "dir/file", "dir/subdir/file"]:
assert (tmp_dir / path).is_file()
# Make sure that data was unprotected after `destroy`
for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]:
assert not System.is_symlink(fspath(tmp_dir / path))
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
shutil.copy("bar", "foo")
copy_spy = mocker.spy(dvc.cache.local.tree, "copy")
dvc.add("foo")
assert copy_spy.mock.call_count == 0
@pytest.mark.parametrize(
"link,new_link,link_test_func",
[
("hardlink", "copy", lambda path: not System.is_hardlink(path)),
("symlink", "copy", lambda path: not System.is_symlink(path)),
("copy", "hardlink", System.is_hardlink),
("copy", "symlink", System.is_symlink),
],
)
def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir,
dvc):
from dvc.path_info import PathInfo
dvc.config["cache"]["type"] = link
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"})
os.remove("foo")
getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo"))
def test_symlink(tmp_dir, ssh):
tmp_dir.gen("foo", "foo content")
ssh.symlink("foo", "link")
assert System.is_symlink("link")
def test(self):
with open(self.CODE, "w+") as fobj:
fobj.write("import sys\n")
fobj.write("import os\n")
fobj.write("with open(sys.argv[1], 'a+') as fobj:\n")
fobj.write(" fobj.write('foo')\n")
ret = main(["config", "cache.type", "symlink"])
self.assertEqual(ret, 0)
self.assertEqual(ret, 0)
ret = main(
[
"run",
"-d",
self.CODE,
"-o",
self.FOO,
"--single-stage",
"python",
self.CODE,
self.FOO,
]
)
self.assertEqual(ret, 0)
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
with open(self.FOO) as fd:
self.assertEqual(fd.read(), "foo")
ret = main(
[
"run",
"--force",
"--no-run-cache",
"--single-stage",
"-d",
self.CODE,
"-o",
self.FOO,
"python",
self.CODE,
self.FOO,
]
)
self.assertEqual(ret, 0)
if os.name == "nt":
# NOTE: Windows symlink perms don't propagate to the target
self.assertTrue(os.access(self.FOO, os.W_OK))
else:
self.assertFalse(os.access(self.FOO, os.W_OK))
self.assertTrue(System.is_symlink(self.FOO))
with open(self.FOO) as fd:
self.assertEqual(fd.read(), "foo")
def is_symlink(path):
return System.is_symlink(path)
def test_reflink(tmp_dir, ssh):
tmp_dir.gen("foo", "foo content")
ssh.reflink("foo", "link")
assert filecmp.cmp("foo", "link")
assert not System.is_symlink("link")
assert not System.is_hardlink("link")
def test_symlink(repo_dir, ssh):
ssh.symlink("foo", "link")
assert System.is_symlink("link")
def is_symlink(path_info):
return System.is_symlink(path_info)
def iscopy(self, path_info):
return not (System.is_symlink(path_info)
or System.is_hardlink(path_info))
def test_reflink(repo_dir, ssh):
ssh.reflink("foo", "link")
assert filecmp.cmp("foo", "link")
assert not System.is_symlink("link")
assert not System.is_hardlink("link")
