def test(self): ret = main(["config", "cache.type", "symlink"]) self.assertEqual(ret, 0) ret = main(["add", self.FOO]) self.assertEqual(ret, 0) ret = main(["add", self.DATA_DIR]) self.assertEqual(ret, 0) self.assertTrue(System.is_symlink(self.FOO)) old_foo_link = os.path.realpath(self.FOO) self.assertTrue(System.is_symlink(self.DATA)) old_data_link = os.path.realpath(self.DATA) old_cache_dir = self.dvc.cache.local.cache_dir new_cache_dir = old_cache_dir + "_new" os.rename(old_cache_dir, new_cache_dir) ret = main(["cache", "dir", new_cache_dir]) self.assertEqual(ret, 0) ret = main(["checkout", "-f"]) self.assertEqual(ret, 0) self.assertTrue(System.is_symlink(self.FOO)) new_foo_link = os.path.realpath(self.FOO) self.assertTrue(System.is_symlink(self.DATA)) new_data_link = os.path.realpath(self.DATA) self.assertEqual( relpath(old_foo_link, old_cache_dir), relpath(new_foo_link, new_cache_dir), ) self.assertEqual( relpath(old_data_link, old_cache_dir), relpath(new_data_link, new_cache_dir), )
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker): tmp_dir.gen("dir/data", "data") dvc.cache.local.cache_types = ["symlink"] dvc.add("dir") tmp_dir.gen("dir/new_file", "new_file_content") unprotect_spy = mocker.spy(dvc.cache.local, "unprotect") dvc.add("dir") assert not unprotect_spy.mock.called assert System.is_symlink(os.path.join("dir", "new_file"))
def _link_matches(self, path_info): is_hardlink = System.is_hardlink(path_info) is_symlink = System.is_symlink(path_info) is_copy_or_reflink = not is_hardlink and not is_symlink cache_type = self._get_cache_type(path_info) if cache_type == "symlink": return is_symlink if cache_type == "hardlink": return is_hardlink return is_copy_or_reflink
def contains_symlink_up_to(path, base_path): base_path = fspath(base_path) path = fspath(path) if base_path not in path: raise BasePathNotInCheckedPathException(path, base_path) if path == base_path: return False if System.is_symlink(path): return True if os.path.dirname(path) == path: return False return contains_symlink_up_to(os.path.dirname(path), base_path)
def test_cache_type_is_properly_overridden(erepo): erepo.dvc.config.set(Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink") erepo.dvc.scm.add([erepo.dvc.config.config_file]) erepo.dvc.scm.commit("set cache type to symlinks") src = erepo.FOO dst = erepo.FOO + "_imported" Repo.get(erepo.root_dir, src, dst) assert not System.is_symlink(dst) assert os.path.exists(dst) assert os.path.isfile(dst)
def test_destroy(repo_dir, dvc_repo): # NOTE: using symlink to ensure that data was unprotected after `destroy` dvc_repo.config.set("cache", "type", "symlink") foo_stage, = dvc_repo.add(repo_dir.FOO) data_dir_stage, = dvc_repo.add(repo_dir.DATA_DIR) dvc_repo.destroy() assert not os.path.exists(dvc_repo.dvc_dir) assert not os.path.exists(foo_stage.path) assert not os.path.exists(data_dir_stage.path) assert os.path.isfile(repo_dir.FOO) assert os.path.isdir(repo_dir.DATA_DIR) assert os.path.isfile(repo_dir.DATA) assert os.path.isdir(repo_dir.DATA_SUB_DIR) assert os.path.isfile(repo_dir.DATA_SUB) assert not System.is_symlink(repo_dir.FOO) assert not System.is_symlink(repo_dir.DATA_DIR) assert not System.is_symlink(repo_dir.DATA) assert not System.is_symlink(repo_dir.DATA_SUB)
def test_destroy(tmp_dir, dvc, run_copy): from dvc.dvcfile import PIPELINE_FILE, PIPELINE_LOCK dvc.config["cache"]["type"] = ["symlink"] dvc.cache = Cache(dvc) tmp_dir.dvc_gen("file", "text") tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}}) run_copy("file", "file2", single_stage=True) run_copy("file2", "file3", name="copy-file2-file3") run_copy("file3", "file4", name="copy-file3-file4") dvc.destroy() # Remove all the files related to DVC for path in [ ".dvc", "file.dvc", "file2.dvc", "dir.dvc", PIPELINE_FILE, PIPELINE_LOCK, ]: assert not (tmp_dir / path).exists() # Leave the rest of the files for path in [ "file", "file2", "file3", "file4", "dir/file", "dir/subdir/file", ]: assert (tmp_dir / path).is_file() # Make sure that data was unprotected after `destroy` for path in [ "file", "file2", "file3", "file4", "dir", "dir/file", "dir/subdir", "dir/subdir/file", ]: assert not System.is_symlink(tmp_dir / path)
def test(self): ret = main(["config", "cache.type", "symlink"]) self.assertEqual(ret, 0) ret = main(["add", self.FOO]) self.assertEqual(ret, 0) remove(self.dvc.cache.local.cache_dir) self.assertTrue(System.is_symlink(self.FOO)) ret = main(["remove", self.FOO + ".dvc"]) self.assertEqual(ret, 0) self.assertFalse(os.path.lexists(self.FOO))
def test_readding_dir_should_not_unprotect_all(dvc_repo, repo_dir): dvc_repo.cache.local.cache_types = ["symlink"] dvc_repo.cache.local.protected = True dvc_repo.add(repo_dir.DATA_DIR) new_file = os.path.join(repo_dir.DATA_DIR, "new_file") repo_dir.create(new_file, "new_content") unprotect_spy = spy(RemoteLOCAL.unprotect) with patch.object(RemoteLOCAL, "unprotect", unprotect_spy): dvc_repo.add(repo_dir.DATA_DIR) assert not unprotect_spy.mock.called assert System.is_symlink(new_file)
def test_readding_dir_should_not_unprotect_all(tmp_dir, dvc, mocker): tmp_dir.gen("dir/data", "data") dvc.cache.local.cache_types = ["symlink"] dvc.cache.local.protected = True dvc.add("dir") tmp_dir.gen("dir/new_file", "new_file_content") unprotect_spy = spy(RemoteLOCAL.unprotect) mocker.patch.object(RemoteLOCAL, "unprotect", unprotect_spy) dvc.add("dir") assert not unprotect_spy.mock.called assert System.is_symlink(os.path.join("dir", "new_file"))
def _unprotect_file(path): if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def test_remove_broken_symlink(tmp_dir, dvc): tmp_dir.gen("foo", "foo") dvc.cache.local.cache_types = ["symlink"] (stage, ) = dvc.add("foo") remove(dvc.cache.local.cache_dir) assert System.is_symlink("foo") with pytest.raises(DvcException): dvc.remove(stage.addressing) assert os.path.lexists("foo") assert (tmp_dir / stage.relpath).exists() dvc.remove(stage.addressing, outs=True) assert not os.path.lexists("foo") assert not (tmp_dir / stage.relpath).exists()
def test_cache_type_is_properly_overridden(repo_dir, git, dvc_repo, erepo): erepo.dvc.config.set(Config.SECTION_CACHE, Config.SECTION_CACHE_TYPE, "symlink") erepo.dvc.scm.add([erepo.dvc.config.config_file]) erepo.dvc.scm.commit("set source repo cache type to symlinks") src = erepo.FOO dst = erepo.FOO + "_imported" dvc_repo.imp(erepo.root_dir, src, dst) assert not System.is_symlink(dst) assert os.path.exists(dst) assert os.path.isfile(dst) assert filecmp.cmp(repo_dir.FOO, dst, shallow=False) assert git.git.check_ignore(dst)
def _unprotect_file(path): if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4())) # The operations order is important here - if some application would # access the file during the process of copyfile then it would get # only the part of file. So, at first, the file should be copied with # the temporary name, and then original file should be replaced by new. copyfile(path, tmp) remove(path) os.rename(tmp, path) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def _unprotect_file(self, path): if System.is_symlink(path) or System.is_hardlink(path): logger.debug(f"Unprotecting '{path}'") tmp = os.path.join(os.path.dirname(path), "." + uuid()) # The operations order is important here - if some application # would access the file during the process of copyfile then it # would get only the part of file. So, at first, the file should be # copied with the temporary name, and then original file should be # replaced by new. copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path))) remove(path) os.rename(tmp, path) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, self.file_mode)
def _unprotect_file(self, path): import stat import uuid from dvc.system import System from dvc.utils import copyfile, move, remove if System.is_symlink(path) or System.is_hardlink(path): logger.debug("Unprotecting '{}'".format(path)) tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4())) move(path, tmp) copyfile(tmp, path) remove(tmp) else: logger.debug("Skipping copying for '{}', since it is not " "a symlink or a hardlink.".format(path)) os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def test_destroy(tmp_dir, dvc): dvc.config["cache"]["type"] = ["symlink"] dvc.cache = Cache(dvc) tmp_dir.dvc_gen("file", "text") tmp_dir.dvc_gen({"dir": {"file": "lorem", "subdir/file": "ipsum"}}) dvc.destroy() # Remove all the files related to DVC for path in [".dvc", "file.dvc", "dir.dvc"]: assert not (tmp_dir / path).exists() # Leave the rest of the files for path in ["file", "dir/file", "dir/subdir/file"]: assert (tmp_dir / path).is_file() # Make sure that data was unprotected after `destroy` for path in ["file", "dir", "dir/file", "dir/subdir", "dir/subdir/file"]: assert not System.is_symlink(fspath(tmp_dir / path))
tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) shutil.copy("bar", "foo") copy_spy = mocker.spy(dvc.cache.local.tree, "copy") dvc.add("foo") assert copy_spy.mock.call_count == 0 @pytest.mark.parametrize( "link,new_link,link_test_func", [ ("hardlink", "copy", lambda path: not System.is_hardlink(path)), ("symlink", "copy", lambda path: not System.is_symlink(path)), ("copy", "hardlink", System.is_hardlink), ("copy", "symlink", System.is_symlink), ], ) def test_should_relink_on_repeated_add(link, new_link, link_test_func, tmp_dir, dvc): from dvc.path_info import PathInfo dvc.config["cache"]["type"] = link tmp_dir.dvc_gen({"foo": "foo", "bar": "bar"}) os.remove("foo") getattr(dvc.cache.local.tree, link)(PathInfo("bar"), PathInfo("foo"))
def test_symlink(tmp_dir, ssh): tmp_dir.gen("foo", "foo content") ssh.symlink("foo", "link") assert System.is_symlink("link")
def test(self): with open(self.CODE, "w+") as fobj: fobj.write("import sys\n") fobj.write("import os\n") fobj.write("with open(sys.argv[1], 'a+') as fobj:\n") fobj.write(" fobj.write('foo')\n") ret = main(["config", "cache.type", "symlink"]) self.assertEqual(ret, 0) self.assertEqual(ret, 0) ret = main( [ "run", "-d", self.CODE, "-o", self.FOO, "--single-stage", "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) if os.name == "nt": # NOTE: Windows symlink perms don't propagate to the target self.assertTrue(os.access(self.FOO, os.W_OK)) else: self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) with open(self.FOO) as fd: self.assertEqual(fd.read(), "foo") ret = main( [ "run", "--force", "--no-run-cache", "--single-stage", "-d", self.CODE, "-o", self.FOO, "python", self.CODE, self.FOO, ] ) self.assertEqual(ret, 0) if os.name == "nt": # NOTE: Windows symlink perms don't propagate to the target self.assertTrue(os.access(self.FOO, os.W_OK)) else: self.assertFalse(os.access(self.FOO, os.W_OK)) self.assertTrue(System.is_symlink(self.FOO)) with open(self.FOO) as fd: self.assertEqual(fd.read(), "foo")
def is_symlink(path): return System.is_symlink(path)
def test_reflink(tmp_dir, ssh): tmp_dir.gen("foo", "foo content") ssh.reflink("foo", "link") assert filecmp.cmp("foo", "link") assert not System.is_symlink("link") assert not System.is_hardlink("link")
def test_symlink(repo_dir, ssh): ssh.symlink("foo", "link") assert System.is_symlink("link")
def is_symlink(path_info): return System.is_symlink(path_info)
def iscopy(self, path_info): return not (System.is_symlink(path_info) or System.is_hardlink(path_info))
def test_reflink(repo_dir, ssh): ssh.reflink("foo", "link") assert filecmp.cmp("foo", "link") assert not System.is_symlink("link") assert not System.is_hardlink("link")