def test_subrepos(tmp_dir, scm, dvc):
tmp_dir.scm_gen(
{"dir": {"repo.txt": "file to confuse RepoFileSystem"}},
commit="dir/repo.txt",
)
subrepo1 = tmp_dir / "dir" / "repo"
subrepo2 = tmp_dir / "dir" / "repo2"
for repo in [subrepo1, subrepo2]:
make_subrepo(repo, scm)
subrepo1.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
subrepo2.dvc_gen(
{"lorem": "lorem", "dir2": {"ipsum": "ipsum"}}, commit="BAR"
)
dvc._reset()
fs = RepoFileSystem(repo=dvc, subrepos=True)
def assert_fs_belongs_to_repo(ret_val):
method = fs._get_repo
def f(*args, **kwargs):
r = method(*args, **kwargs)
assert r.root_dir == ret_val.root_dir
return r
return f
with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo1.dvc)
):
assert fs.exists(subrepo1 / "foo") is True
assert fs.exists(subrepo1 / "bar") is False
assert fs.isfile(subrepo1 / "foo") is True
assert fs.isfile(subrepo1 / "dir1" / "bar") is True
assert fs.isfile(subrepo1 / "dir1") is False
assert fs.isdir(subrepo1 / "dir1") is True
assert fs.isdir(subrepo1 / "dir1" / "bar") is False
assert fs.isdvc(subrepo1 / "foo") is True
with mock.patch.object(
fs, "_get_repo", side_effect=assert_fs_belongs_to_repo(subrepo2.dvc)
):
assert fs.exists(subrepo2 / "lorem") is True
assert fs.exists(subrepo2 / "ipsum") is False
assert fs.isfile(subrepo2 / "lorem") is True
assert fs.isfile(subrepo2 / "dir2" / "ipsum") is True
assert fs.isfile(subrepo2 / "dir2") is False
assert fs.isdir(subrepo2 / "dir2") is True
assert fs.isdir(subrepo2 / "dir2" / "ipsum") is False
assert fs.isdvc(subrepo2 / "lorem") is True
def _collect_paths(
repo: "Repo",
targets: Iterable[str],
recursive: bool = False,
rev: str = None,
):
from dvc.fs.repo import RepoFileSystem
path_infos = [PathInfo(os.path.abspath(target)) for target in targets]
fs = RepoFileSystem(repo)
target_infos = []
for path_info in path_infos:
if recursive and fs.isdir(path_info):
target_infos.extend(repo.dvcignore.walk_files(fs, path_info))
if not fs.exists(path_info):
if not recursive:
if rev == "workspace" or rev == "":
logger.warning(
"'%s' was not found in current workspace.", path_info
)
else:
logger.warning(
"'%s' was not found at: '%s'.", path_info, rev
)
continue
target_infos.append(path_info)
return target_infos
def test_exists(tmp_dir, dvc):
tmp_dir.gen("foo", "foo")
dvc.add("foo")
(tmp_dir / "foo").unlink()
fs = RepoFileSystem(repo=dvc)
assert fs.exists("foo")
def _collect_paths(
repo: "Repo",
targets: Iterable[str],
recursive: bool = False,
rev: str = None,
):
from dvc.fs.repo import RepoFileSystem
from dvc.utils import relpath
fs_paths = [os.path.abspath(target) for target in targets]
fs = RepoFileSystem(repo)
target_paths = []
for fs_path in fs_paths:
if recursive and fs.isdir(fs_path):
target_paths.extend(repo.dvcignore.find(fs, fs_path))
if not fs.exists(fs_path):
rel = relpath(fs_path)
if rev == "workspace" or rev == "":
logger.warning("'%s' was not found in current workspace.", rel)
else:
logger.warning("'%s' was not found at: '%s'.", rel, rev)
target_paths.append(fs_path)
return target_paths
def test_repo_fs_no_subrepos(tmp_dir, dvc, scm):
tmp_dir.scm_gen(
{"dir": {
"repo.txt": "file to confuse RepoFileSystem"
}},
commit="dir/repo.txt",
)
tmp_dir.dvc_gen({"lorem": "lorem"}, commit="add foo")
subrepo = tmp_dir / "dir" / "repo"
make_subrepo(subrepo, scm)
with subrepo.chdir():
subrepo.dvc_gen({"foo": "foo", "dir1": {"bar": "bar"}}, commit="FOO")
subrepo.scm_gen({"ipsum": "ipsum"}, commit="BAR")
# using fs that does not have dvcignore
dvc._reset()
fs = RepoFileSystem(repo=dvc)
expected = [
tmp_dir / ".dvcignore",
tmp_dir / ".gitignore",
tmp_dir / "lorem",
tmp_dir / "lorem.dvc",
tmp_dir / "dir",
tmp_dir / "dir" / "repo.txt",
]
actual = []
for root, dirs, files in fs.walk(tmp_dir.fs_path, dvcfiles=True):
for entry in dirs + files:
actual.append(os.path.normpath(os.path.join(root, entry)))
expected = [str(path) for path in expected]
assert set(actual) == set(expected)
assert len(actual) == len(expected)
assert fs.isfile(tmp_dir / "lorem") is True
assert fs.isfile(tmp_dir / "dir" / "repo" / "foo") is False
assert fs.isdir(tmp_dir / "dir" / "repo") is False
assert fs.isdir(tmp_dir / "dir") is True
assert fs.isdvc(tmp_dir / "lorem") is True
assert fs.isdvc(tmp_dir / "dir" / "repo" / "dir1") is False
assert fs.exists(tmp_dir / "dir" / "repo.txt") is True
assert fs.exists(tmp_dir / "repo" / "ipsum") is False
def test_exists_isdir_isfile_dirty(tmp_dir, dvc):
tmp_dir.dvc_gen(
{"datafile": "data", "datadir": {"foo": "foo", "bar": "bar"}}
)
fs = RepoFileSystem(repo=dvc)
shutil.rmtree(tmp_dir / "datadir")
(tmp_dir / "datafile").unlink()
root = PathInfo(tmp_dir)
assert fs.exists(root / "datafile")
assert fs.exists(root / "datadir")
assert fs.exists(root / "datadir" / "foo")
assert fs.isfile(root / "datafile")
assert not fs.isfile(root / "datadir")
assert fs.isfile(root / "datadir" / "foo")
assert not fs.isdir(root / "datafile")
assert fs.isdir(root / "datadir")
assert not fs.isdir(root / "datadir" / "foo")
# NOTE: creating file instead of dir and dir instead of file
tmp_dir.gen({"datadir": "data", "datafile": {"foo": "foo", "bar": "bar"}})
assert fs.exists(root / "datafile")
assert fs.exists(root / "datadir")
assert not fs.exists(root / "datadir" / "foo")
assert fs.exists(root / "datafile" / "foo")
assert not fs.isfile(root / "datafile")
assert fs.isfile(root / "datadir")
assert not fs.isfile(root / "datadir" / "foo")
assert fs.isfile(root / "datafile" / "foo")
assert fs.isdir(root / "datafile")
assert not fs.isdir(root / "datadir")
assert not fs.isdir(root / "datadir" / "foo")
assert not fs.isdir(root / "datafile" / "foo")
