def repo_fs(self): from dvc.fs.repo import RepoFileSystem return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf)
def test_isdir_isfile(tmp_dir, dvc): tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}) fs = RepoFileSystem(repo=dvc) assert fs.isdir("datadir") assert not fs.isfile("datadir") assert not fs.isdvc("datadir") assert not fs.isdir("datafile") assert fs.isfile("datafile") assert not fs.isdvc("datafile") dvc.add(["datadir", "datafile"]) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() assert fs.isdir("datadir") assert not fs.isfile("datadir") assert fs.isdvc("datadir") assert not fs.isdir("datafile") assert fs.isfile("datafile") assert fs.isdvc("datafile")
def test_exists_isdir_isfile_dirty(tmp_dir, dvc): tmp_dir.dvc_gen( {"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}} ) fs = RepoFileSystem(repo=dvc) shutil.rmtree(tmp_dir / "datadir") (tmp_dir / "datafile").unlink() root = PathInfo(tmp_dir) assert fs.exists(root / "datafile") assert fs.exists(root / "datadir") assert fs.exists(root / "datadir" / "foo") assert fs.isfile(root / "datafile") assert not fs.isfile(root / "datadir") assert fs.isfile(root / "datadir" / "foo") assert not fs.isdir(root / "datafile") assert fs.isdir(root / "datadir") assert not fs.isdir(root / "datadir" / "foo") # NOTE: creating file instead of dir and dir instead of file tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}}) assert fs.exists(root / "datafile") assert fs.exists(root / "datadir") assert not fs.exists(root / "datadir" / "foo") assert fs.exists(root / "datafile" / "foo") assert not fs.isfile(root / "datafile") assert fs.isfile(root / "datadir") assert not fs.isfile(root / "datadir" / "foo") assert fs.isfile(root / "datafile" / "foo") assert fs.isdir(root / "datafile") assert not fs.isdir(root / "datadir") assert not fs.isdir(root / "datadir" / "foo") assert not fs.isdir(root / "datafile" / "foo")
def test_subrepos(tmp_dir, scm, dvc): tmp_dir.scm_gen( {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, commit="dir/repo.txt", ) subrepo1 = tmp_dir / "dir" / "repo" subrepo2 = tmp_dir / "dir" / "repo2" for repo in [subrepo1, subrepo2]: make_subrepo(repo, scm) subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO") subrepo2.dvc_gen( {"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR" ) dvc._reset() fs = RepoFileSystem(repo=dvc, subrepos=True) def assert_fs_belongs_to_repo(ret_val): method = fs._get_repo def f(*args, **kwargs): r = method(*args, **kwargs) assert r.root_dir == ret_val.root_dir return r return f with mock.patch.object( fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc) ): assert fs.exists(subrepo1 / "foo") is True assert fs.exists(subrepo1 / "bar") is False assert fs.isfile(subrepo1 / "foo") is True assert fs.isfile(subrepo1 / "dir1" / "bar") is True assert fs.isfile(subrepo1 / "dir1") is False assert fs.isdir(subrepo1 / "dir1") is True assert fs.isdir(subrepo1 / "dir1" / "bar") is False assert fs.isdvc(subrepo1 / "foo") is True with mock.patch.object( fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc) ): assert fs.exists(subrepo2 / "lorem") is True assert fs.exists(subrepo2 / "ipsum") is False assert fs.isfile(subrepo2 / "lorem") is True assert fs.isfile(subrepo2 / "dir2" / "ipsum") is True assert fs.isfile(subrepo2 / "dir2") is False assert fs.isdir(subrepo2 / "dir2") is True assert fs.isdir(subrepo2 / "dir2" / "ipsum") is False assert fs.isdvc(subrepo2 / "lorem") is True
def test_repo_fs_no_subrepos(tmp_dir, dvc, scm): tmp_dir.scm_gen( {"dir": {"repo.txt": "file to confuse RepoFileSystem"}}, commit="dir/repo.txt", ) tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo") subrepo = tmp_dir / "dir" / "repo" make_subrepo(subrepo, scm) subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO") subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR") # using fs that does not have dvcignore dvc._reset() fs = RepoFileSystem(repo=dvc) expected = [ tmp_dir / ".dvcignore", tmp_dir / ".gitignore", tmp_dir / "lorem", tmp_dir / "lorem.dvc", tmp_dir / "dir", tmp_dir / "dir" / "repo.txt", ] actual = [] for root, dirs, files in fs.walk(tmp_dir, dvcfiles=True): for entry in dirs + files: actual.append(os.path.normpath(os.path.join(root, entry))) expected = [str(path) for path in expected] assert set(actual) == set(expected) assert len(actual) == len(expected) assert fs.isfile(tmp_dir / "lorem") is True assert fs.isfile(tmp_dir / "dir" / "repo" / "foo") is False assert fs.isdir(tmp_dir / "dir" / "repo") is False assert fs.isdir(tmp_dir / "dir") is True assert fs.isdvc(tmp_dir / "lorem") is True assert fs.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False assert fs.exists(tmp_dir / "dir" / "repo.txt") is True assert fs.exists(tmp_dir / "repo" / "ipsum") is False
def collect( self, targets: List[str] = None, revs: List[str] = None, recursive: bool = False, ) -> Dict[str, Dict]: """Collects all props and data for plots. Returns a structure like: {rev: {plots.csv: { props: {x: ..., "header": ..., ...}, data: "...data as a string...", }}} Data parsing is postponed, since it's affected by props. """ from dvc.fs.repo import RepoFileSystem from dvc.utils.collections import ensure_list targets = ensure_list(targets) data: Dict[str, Dict] = {} for rev in self.repo.brancher(revs=revs): # .brancher() adds unwanted workspace if revs is not None and rev not in revs: continue rev = rev or "workspace" fs = RepoFileSystem(self.repo) plots = _collect_plots(self.repo, targets, rev, recursive) for path_info, props in plots.items(): if rev not in data: data[rev] = {} if fs.isdir(path_info): plot_files = [] for pi in fs.walk_files(path_info): plot_files.append((pi, relpath(pi, self.repo.root_dir))) else: plot_files = [(path_info, relpath(path_info, self.repo.root_dir))] for path, repo_path in plot_files: data[rev].update({repo_path: {"props": props}}) # Load data from git or dvc cache try: with fs.open(path) as fd: data[rev][repo_path]["data"] = fd.read() except FileNotFoundError: # This might happen simply because cache is absent logger.debug( ("Could not find '%s' on '%s'. " "File will not be plotted"), path, rev, ) except UnicodeDecodeError: logger.debug( ("'%s' at '%s' is binary file. It will not be " "plotted."), path, rev, ) return data
def diff(self, a_rev="HEAD", b_rev=None, targets=None): """ By default, it compares the workspace with the last commit's fs. This implementation differs from `git diff` since DVC doesn't have the concept of `index`, but it keeps the same interface, thus, `dvc diff` would be the same as `dvc diff HEAD`. """ if self.scm.no_commits: return {} from dvc.fs.repo import RepoFileSystem repo_fs = RepoFileSystem(self) b_rev = b_rev if b_rev else "workspace" results = {} missing_targets = {} for rev in self.brancher(revs=[a_rev, b_rev]): if rev == "workspace" and rev != b_rev: # brancher always returns workspace, but we only need to compute # workspace paths/checksums if b_rev was None continue targets_path_infos = None if targets is not None: # convert targets to path_infos, and capture any missing targets targets_path_infos, missing_targets[rev] = _targets_to_path_infos( repo_fs, targets) results[rev] = _paths_checksums(self, repo_fs, targets_path_infos) if targets is not None: # check for overlapping missing targets between a_rev and b_rev for target in set(missing_targets[a_rev]) & set( missing_targets[b_rev]): raise PathMissingError(target, self) old = results[a_rev] new = results[b_rev] # Compare paths between the old and new fs. # set() efficiently converts dict keys to a set added = sorted(set(new) - set(old)) deleted_or_missing = set(old) - set(new) if b_rev == "workspace": # missing status is only applicable when diffing local workspace # against a commit missing = sorted(_filter_missing(repo_fs, deleted_or_missing)) else: missing = [] deleted = sorted(deleted_or_missing - set(missing)) modified = sorted(set(old) & set(new)) # Cases when file was changed and renamed are resulted # in having deleted and added record # To cover such cases we need to change hashing function # to produce rolling/chunking hash renamed = _calculate_renamed(new, old, added, deleted) for renamed_item in renamed: added.remove(renamed_item["path"]["new"]) deleted.remove(renamed_item["path"]["old"]) ret = { "added": [{ "path": path, "hash": new[path] } for path in added], "deleted": [{ "path": path, "hash": old[path] } for path in deleted], "modified": [{ "path": path, "hash": { "old": old[path], "new": new[path] } } for path in modified if old[path] != new[path]], "renamed": renamed, "not in cache": [{ "path": path, "hash": old[path] } for path in missing], } return ret if any(ret.values()) else {}
def test_isdir_isfile(tmp_dir, dvc): tmp_dir.gen( { "datafile": "data", "datadir": { "foo": "foo", "bar": "bar", }, "subdir": { "baz": "baz", "data": { "abc": "abc", "xyz": "xyz", }, }, }, ) fs = RepoFileSystem(repo=dvc) assert fs.isdir("datadir") assert not fs.isfile("datadir") assert not fs.isdvc("datadir") assert not fs.isdir("datafile") assert fs.isfile("datafile") assert not fs.isdvc("datafile") dvc.add([ "datadir", "datafile", os.path.join("subdir", "baz"), os.path.join("subdir", "data"), ]) shutil.rmtree(tmp_dir / "datadir") shutil.rmtree(tmp_dir / "subdir" / "data") (tmp_dir / "datafile").unlink() (tmp_dir / "subdir" / "baz").unlink() assert fs.isdir("datadir") assert not fs.isfile("datadir") assert fs.isdvc("datadir") assert not fs.isdir("datafile") assert fs.isfile("datafile") assert fs.isdvc("datafile") assert fs.isdir("subdir") assert not fs.isfile("subdir") assert not fs.isdvc("subdir") assert fs.isfile(os.path.join("subdir", "baz")) assert fs.isdir(os.path.join("subdir", "data"))
def test_walk_not_a_dir(tmp_dir, dvc): tmp_dir.dvc_gen("foo", "foo") fs = RepoFileSystem(repo=dvc) for _ in fs.walk("foo"): pass
def test_walk_missing(tmp_dir, dvc): fs = RepoFileSystem(repo=dvc) for _ in fs.walk("dir"): pass