def repo_fs(self):
from dvc.fs.repo import RepoFileSystem
return RepoFileSystem(self, subrepos=self.subrepos, **self._fs_conf)
def test_isdir_isfile(tmp_dir, dvc):
tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})
fs = RepoFileSystem(repo=dvc)
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert not fs.isdvc("datafile")
dvc.add(["datadir", "datafile"])
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert fs.isdvc("datafile")
def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
)
fs = RepoFileSystem(repo=dvc)
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
root = PathInfo(tmp_dir)
assert fs.exists(root / "datafile")
assert fs.exists(root / "datadir")
assert fs.exists(root / "datadir" / "foo")
assert fs.isfile(root / "datafile")
assert not fs.isfile(root / "datadir")
assert fs.isfile(root / "datadir" / "foo")
assert not fs.isdir(root / "datafile")
assert fs.isdir(root / "datadir")
assert not fs.isdir(root / "datadir" / "foo")
# NOTE: creating file instead of dir and dir instead of file
tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}})
assert fs.exists(root / "datafile")
assert fs.exists(root / "datadir")
assert not fs.exists(root / "datadir" / "foo")
assert fs.exists(root / "datafile" / "foo")
assert not fs.isfile(root / "datafile")
assert fs.isfile(root / "datadir")
assert not fs.isfile(root / "datadir" / "foo")
assert fs.isfile(root / "datafile" / "foo")
assert fs.isdir(root / "datafile")
assert not fs.isdir(root / "datadir")
assert not fs.isdir(root / "datadir" / "foo")
assert not fs.isdir(root / "datafile" / "foo")
def test_subrepos(tmp_dir, scm, dvc):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
commit="dir/repo.txt",
)
subrepo1 = tmp_dir / "dir" / "repo"
subrepo2 = tmp_dir / "dir" / "repo2"
for repo in [subrepo1, subrepo2]:
make_subrepo(repo, scm)
subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
subrepo2.dvc_gen(
{"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
)
dvc._reset()
fs = RepoFileSystem(repo=dvc, subrepos=True)
def assert_fs_belongs_to_repo(ret_val):
method = fs._get_repo
def f(*args, **kwargs):
r = method(*args, **kwargs)
assert r.root_dir == ret_val.root_dir
return r
return f
with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc)
):
assert fs.exists(subrepo1 / "foo") is True
assert fs.exists(subrepo1 / "bar") is False
assert fs.isfile(subrepo1 / "foo") is True
assert fs.isfile(subrepo1 / "dir1" / "bar") is True
assert fs.isfile(subrepo1 / "dir1") is False
assert fs.isdir(subrepo1 / "dir1") is True
assert fs.isdir(subrepo1 / "dir1" / "bar") is False
assert fs.isdvc(subrepo1 / "foo") is True
with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc)
):
assert fs.exists(subrepo2 / "lorem") is True
assert fs.exists(subrepo2 / "ipsum") is False
assert fs.isfile(subrepo2 / "lorem") is True
assert fs.isfile(subrepo2 / "dir2" / "ipsum") is True
assert fs.isfile(subrepo2 / "dir2") is False
assert fs.isdir(subrepo2 / "dir2") is True
assert fs.isdir(subrepo2 / "dir2" / "ipsum") is False
assert fs.isdvc(subrepo2 / "lorem") is True
def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
commit="dir/repo.txt",
)
tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
subrepo = tmp_dir / "dir" / "repo"
make_subrepo(subrepo, scm)
subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR")
# using fs that does not have dvcignore
dvc._reset()
fs = RepoFileSystem(repo=dvc)
expected = [
tmp_dir / ".dvcignore",
tmp_dir / ".gitignore",
tmp_dir / "lorem",
tmp_dir / "lorem.dvc",
tmp_dir / "dir",
tmp_dir / "dir" / "repo.txt",
]
actual = []
for root, dirs, files in fs.walk(tmp_dir, dvcfiles=True):
for entry in dirs + files:
actual.append(os.path.normpath(os.path.join(root, entry)))
expected = [str(path) for path in expected]
assert set(actual) == set(expected)
assert len(actual) == len(expected)
assert fs.isfile(tmp_dir / "lorem") is True
assert fs.isfile(tmp_dir / "dir" / "repo" / "foo") is False
assert fs.isdir(tmp_dir / "dir" / "repo") is False
assert fs.isdir(tmp_dir / "dir") is True
assert fs.isdvc(tmp_dir / "lorem") is True
assert fs.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False
assert fs.exists(tmp_dir / "dir" / "repo.txt") is True
assert fs.exists(tmp_dir / "repo" / "ipsum") is False
def collect(
self,
targets: List[str] = None,
revs: List[str] = None,
recursive: bool = False,
) -> Dict[str, Dict]:
"""Collects all props and data for plots.
Returns a structure like:
{rev: {plots.csv: {
props: {x: ..., "header": ..., ...},
data: "...data as a string...",
}}}
Data parsing is postponed, since it's affected by props.
"""
from dvc.fs.repo import RepoFileSystem
from dvc.utils.collections import ensure_list
targets = ensure_list(targets)
data: Dict[str, Dict] = {}
for rev in self.repo.brancher(revs=revs):
# .brancher() adds unwanted workspace
if revs is not None and rev not in revs:
continue
rev = rev or "workspace"
fs = RepoFileSystem(self.repo)
plots = _collect_plots(self.repo, targets, rev, recursive)
for path_info, props in plots.items():
if rev not in data:
data[rev] = {}
if fs.isdir(path_info):
plot_files = []
for pi in fs.walk_files(path_info):
plot_files.append((pi, relpath(pi,
self.repo.root_dir)))
else:
plot_files = [(path_info,
relpath(path_info, self.repo.root_dir))]
for path, repo_path in plot_files:
data[rev].update({repo_path: {"props": props}})
# Load data from git or dvc cache
try:
with fs.open(path) as fd:
data[rev][repo_path]["data"] = fd.read()
except FileNotFoundError:
# This might happen simply because cache is absent
logger.debug(
("Could not find '%s' on '%s'. "
"File will not be plotted"),
path,
rev,
)
except UnicodeDecodeError:
logger.debug(
("'%s' at '%s' is binary file. It will not be "
"plotted."),
path,
rev,
)
return data
def diff(self, a_rev="HEAD", b_rev=None, targets=None):
"""
By default, it compares the workspace with the last commit's fs.
This implementation differs from `git diff` since DVC doesn't have
the concept of `index`, but it keeps the same interface, thus,
`dvc diff` would be the same as `dvc diff HEAD`.
"""
if self.scm.no_commits:
return {}
from dvc.fs.repo import RepoFileSystem
repo_fs = RepoFileSystem(self)
b_rev = b_rev if b_rev else "workspace"
results = {}
missing_targets = {}
for rev in self.brancher(revs=[a_rev, b_rev]):
if rev == "workspace" and rev != b_rev:
# brancher always returns workspace, but we only need to compute
# workspace paths/checksums if b_rev was None
continue
targets_path_infos = None
if targets is not None:
# convert targets to path_infos, and capture any missing targets
targets_path_infos, missing_targets[rev] = _targets_to_path_infos(
repo_fs, targets)
results[rev] = _paths_checksums(self, repo_fs, targets_path_infos)
if targets is not None:
# check for overlapping missing targets between a_rev and b_rev
for target in set(missing_targets[a_rev]) & set(
missing_targets[b_rev]):
raise PathMissingError(target, self)
old = results[a_rev]
new = results[b_rev]
# Compare paths between the old and new fs.
# set() efficiently converts dict keys to a set
added = sorted(set(new) - set(old))
deleted_or_missing = set(old) - set(new)
if b_rev == "workspace":
# missing status is only applicable when diffing local workspace
# against a commit
missing = sorted(_filter_missing(repo_fs, deleted_or_missing))
else:
missing = []
deleted = sorted(deleted_or_missing - set(missing))
modified = sorted(set(old) & set(new))
# Cases when file was changed and renamed are resulted
# in having deleted and added record
# To cover such cases we need to change hashing function
# to produce rolling/chunking hash
renamed = _calculate_renamed(new, old, added, deleted)
for renamed_item in renamed:
added.remove(renamed_item["path"]["new"])
deleted.remove(renamed_item["path"]["old"])
ret = {
"added": [{
"path": path,
"hash": new[path]
} for path in added],
"deleted": [{
"path": path,
"hash": old[path]
} for path in deleted],
"modified": [{
"path": path,
"hash": {
"old": old[path],
"new": new[path]
}
} for path in modified if old[path] != new[path]],
"renamed":
renamed,
"not in cache": [{
"path": path,
"hash": old[path]
} for path in missing],
}
return ret if any(ret.values()) else {}
def test_isdir_isfile(tmp_dir, dvc):
tmp_dir.gen(
{
"datafile": "data",
"datadir": {
"foo": "foo",
"bar": "bar",
},
"subdir": {
"baz": "baz",
"data": {
"abc": "abc",
"xyz": "xyz",
},
},
}, )
fs = RepoFileSystem(repo=dvc)
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert not fs.isdvc("datafile")
dvc.add([
"datadir",
"datafile",
os.path.join("subdir", "baz"),
os.path.join("subdir", "data"),
])
shutil.rmtree(tmp_dir / "datadir")
shutil.rmtree(tmp_dir / "subdir" / "data")
(tmp_dir / "datafile").unlink()
(tmp_dir / "subdir" / "baz").unlink()
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert fs.isdvc("datafile")
assert fs.isdir("subdir")
assert not fs.isfile("subdir")
assert not fs.isdvc("subdir")
assert fs.isfile(os.path.join("subdir", "baz"))
assert fs.isdir(os.path.join("subdir", "data"))
def test_walk_not_a_dir(tmp_dir, dvc):
tmp_dir.dvc_gen("foo", "foo")
fs = RepoFileSystem(repo=dvc)
for _ in fs.walk("foo"):
pass
def test_walk_missing(tmp_dir, dvc):
fs = RepoFileSystem(repo=dvc)
for _ in fs.walk("dir"):
pass
