def test_subrepos(tmp_dir, scm, dvc, mocker):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse DvcFileSystem"}},
commit="dir/repo.txt",
)
subrepo1 = tmp_dir / "dir" / "repo"
subrepo2 = tmp_dir / "dir" / "repo2"
for repo in [subrepo1, subrepo2]:
make_subrepo(repo, scm)
with subrepo1.chdir():
subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
with subrepo2.chdir():
subrepo2.dvc_gen(
{"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
)
dvc._reset()
fs = DvcFileSystem(repo=dvc, subrepos=True)
def assert_fs_belongs_to_repo(ret_val):
method = fs.fs._get_repo
def f(*args, **kwargs):
r = method(*args, **kwargs)
assert r.root_dir == ret_val.root_dir
return r
return f
with mock.patch.object(
fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc)
):
assert fs.exists("dir/repo/foo") is True
assert fs.exists("dir/repo/bar") is False
assert fs.isfile("dir/repo/foo") is True
assert fs.isfile("dir/repo/dir1/bar") is True
assert fs.isfile("dir/repo/dir1") is False
assert fs.isdir("dir/repo/dir1") is True
assert fs.isdir("dir/repo/dir1/bar") is False
assert fs.isdvc("dir/repo/foo") is True
with mock.patch.object(
fs.fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc)
):
assert fs.exists("dir/repo2/lorem") is True
assert fs.exists("dir/repo2/ipsum") is False
assert fs.isfile("dir/repo2/lorem") is True
assert fs.isfile("dir/repo2/dir2/ipsum") is True
assert fs.isfile("dir/repo2/dir2") is False
assert fs.isdir("dir/repo2/dir2") is True
assert fs.isdir("dir/repo2/dir2/ipsum") is False
assert fs.isdvc("dir/repo2/lorem") is True
def _collect_paths(
repo: "Repo",
targets: Iterable[str],
recursive: bool = False,
rev: str = None,
) -> StrPaths:
from dvc.fs.dvc import DvcFileSystem
fs = DvcFileSystem(repo=repo)
fs_paths = [fs.from_os_path(target) for target in targets]
target_paths: StrPaths = []
for fs_path in fs_paths:
if recursive and fs.isdir(fs_path):
target_paths.extend(fs.find(fs_path))
rel = fs.path.relpath(fs_path)
if not fs.exists(fs_path):
if rev == "workspace" or rev == "":
logger.warning("'%s' was not found in current workspace.", rel)
else:
logger.warning("'%s' was not found at: '%s'.", rel, rev)
target_paths.append(fs_path)
return target_paths
def test_isdir_isfile(tmp_dir, dvc):
tmp_dir.gen({"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}})
fs = DvcFileSystem(repo=dvc)
assert not fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdir("datafile")
assert not fs.isfile("datafile")
dvc.add(["datadir", "datafile"])
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
def test_isdir_mixed(tmp_dir, dvc):
tmp_dir.gen({"dir": {"foo": "foo", "bar": "bar"}})
dvc.add(str(tmp_dir / "dir" / "foo"))
fs = DvcFileSystem(repo=dvc)
assert fs.isdir("dir")
assert not fs.isfile("dir")
def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
)
fs = DvcFileSystem(repo=dvc)
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
assert fs.exists("datafile")
assert fs.exists("datadir")
assert fs.exists("datadir/foo")
assert fs.isfile("datafile")
assert not fs.isfile("datadir")
assert fs.isfile("datadir/foo")
assert not fs.isdir("datafile")
assert fs.isdir("datadir")
assert not fs.isdir("datadir/foo")
# NOTE: creating file instead of dir and dir instead of file
tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}})
assert fs.exists("datafile")
assert fs.exists("datadir")
assert not fs.exists("datadir/foo")
assert fs.exists("datafile/foo")
assert not fs.isfile("datafile")
assert fs.isfile("datadir")
assert not fs.isfile("datadir/foo")
assert fs.isfile("datafile/foo")
assert fs.isdir("datafile")
assert not fs.isdir("datadir")
assert not fs.isdir("datadir/foo")
assert not fs.isdir("datafile/foo")
def test_dvcfs_no_subrepos(tmp_dir, dvc, scm):
tmp_dir.scm_gen(
{"dir": {
"repo.txt": "file to confuse DvcFileSystem"
}},
commit="dir/repo.txt",
)
tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
subrepo = tmp_dir / "dir" / "repo"
make_subrepo(subrepo, scm)
with subrepo.chdir():
subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR")
# using fs that does not have dvcignore
dvc._reset()
fs = DvcFileSystem(repo=dvc)
expected = [
"/.dvcignore",
"/.gitignore",
"/lorem",
"/lorem.dvc",
"/dir",
"/dir/repo.txt",
]
actual = []
for root, dirs, files in fs.walk("/", dvcfiles=True):
for entry in dirs + files:
actual.append(posixpath.join(root, entry))
assert set(actual) == set(expected)
assert len(actual) == len(expected)
assert fs.isfile("lorem") is True
assert fs.isfile("dir/repo/foo") is False
assert fs.isdir("dir/repo") is False
assert fs.isdir("dir") is True
assert fs.isdvc("lorem") is True
assert fs.isdvc("dir/repo/dir1") is False
assert fs.exists("dir/repo.txt") is True
assert fs.exists("repo/ipsum") is False
def _collect_from_revision(
self,
targets: Optional[List[str]] = None,
revision: Optional[str] = None,
recursive: bool = False,
onerror: Optional[Callable] = None,
props: Optional[Dict] = None,
):
from dvc.fs.dvc import DvcFileSystem
fs = DvcFileSystem(repo=self.repo)
plots = _collect_plots(self.repo, targets, revision, recursive)
res: Dict[str, Any] = {}
for fs_path, rev_props in plots.items():
base = os.path.join(*fs.path.relparts(fs_path, fs.fs.root_marker))
if fs.isdir(fs_path):
plot_files = []
unpacking_res = _unpack_dir_files(fs, fs_path, onerror=onerror)
if "data" in unpacking_res:
for pi in unpacking_res.get( # pylint: disable=E1101
"data"):
plot_files.append((
pi,
os.path.join(base, *fs.path.relparts(pi, fs_path)),
))
else:
res[base] = unpacking_res
else:
plot_files = [(fs_path, base)]
props = props or {}
for path, repo_path in plot_files:
joined_props = {**rev_props, **props}
res[repo_path] = {"props": joined_props}
res[repo_path].update({
"data_source":
partial(
parse,
fs,
path,
props=joined_props,
onerror=onerror,
)
})
return res
def test_isdir_isfile(tmp_dir, dvc):
tmp_dir.gen(
{
"datafile": "data",
"datadir": {
"foo": "foo",
"bar": "bar",
},
"subdir": {
"baz": "baz",
"data": {
"abc": "abc",
"xyz": "xyz",
},
},
},
)
fs = DvcFileSystem(repo=dvc)
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert not fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert not fs.isdvc("datafile")
dvc.add(
[
"datadir",
"datafile",
os.path.join("subdir", "baz"),
os.path.join("subdir", "data"),
]
)
shutil.rmtree(tmp_dir / "datadir")
shutil.rmtree(tmp_dir / "subdir" / "data")
(tmp_dir / "datafile").unlink()
(tmp_dir / "subdir" / "baz").unlink()
assert fs.isdir("datadir")
assert not fs.isfile("datadir")
assert fs.isdvc("datadir")
assert not fs.isdir("datafile")
assert fs.isfile("datafile")
assert fs.isdvc("datafile")
assert fs.isdir("subdir")
assert not fs.isfile("subdir")
assert not fs.isdvc("subdir")
assert fs.isfile("subdir/baz")
assert fs.isdir("subdir/data")
