def list_cache_paths(self, prefix=None, progress_callback=None):
assert prefix is None
assert self.path_info is not None
if progress_callback:
for path in walk_files(self.path_info):
progress_callback()
yield path
else:
yield from walk_files(self.path_info)
def list_cache_paths(self, prefix=None, progress_callback=None):
assert self.path_info is not None
if prefix:
path_info = self.path_info / prefix[:2]
if not self.tree.exists(path_info):
return
else:
path_info = self.path_info
if progress_callback:
for path in walk_files(path_info):
progress_callback()
yield path
else:
yield from walk_files(path_info)
def _list_paths(self, prefix=None, progress_callback=None):
assert self.fs.path_info is not None
if prefix:
path_info = self.fs.path_info / prefix[:2]
if not self.fs.exists(path_info):
return
else:
path_info = self.fs.path_info
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
if progress_callback:
for path in walk_files(path_info):
progress_callback()
yield path
else:
yield from walk_files(path_info)
def add(self, paths: Iterable[str]):
from dvc.utils.fs import walk_files
if isinstance(paths, str):
paths = [paths]
files = []
for path in paths:
if not os.path.isabs(path) and self._submodules:
# NOTE: If path is inside a submodule, Dulwich expects the
# staged paths to be relative to the submodule root (not the
# parent git repo root). We append path to root_dir here so
# that the result of relpath(path, root_dir) is actually the
# path relative to the submodule root.
path_info = PathInfo(path).relative_to(self.root_dir)
for sm_path in self._submodules.values():
if path_info.isin(sm_path):
path = os.path.join(self.root_dir,
path_info.relative_to(sm_path))
break
if os.path.isdir(path):
files.extend(walk_files(path))
else:
files.append(path)
for fpath in files:
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
self.repo.stage(relpath(fpath, self.root_dir))
def _list_paths(self, prefix=None, progress_callback=None):
assert self.fs_path is not None
if prefix:
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
if not self.fs.exists(fs_path):
return
else:
fs_path = self.fs_path
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
if progress_callback:
for path in walk_files(fs_path):
progress_callback()
yield path
else:
yield from walk_files(fs_path)
def _list_paths(self, prefix=None):
assert self.fs_path is not None
if prefix:
fs_path = self.fs.path.join(self.fs_path, prefix[:2])
if not self.fs.exists(fs_path):
return
else:
fs_path = self.fs_path
# NOTE: use utils.fs walk_files since fs.walk_files will not follow
# symlinks
yield from walk_files(fs_path)
def add(self, paths: Union[str, Iterable[str]], update=False):
from dvc.utils.fs import walk_files
assert paths or update
if isinstance(paths, str):
paths = [paths]
if update and not paths:
self.repo.stage(list(self.repo.open_index()))
return
files: List[bytes] = []
for path in paths:
if not os.path.isabs(path) and self._submodules:
# NOTE: If path is inside a submodule, Dulwich expects the
# staged paths to be relative to the submodule root (not the
# parent git repo root). We append path to root_dir here so
# that the result of relpath(path, root_dir) is actually the
# path relative to the submodule root.
path_info = PathInfo(path).relative_to(self.root_dir)
for sm_path in self._submodules.values():
if path_info.isin(sm_path):
path = os.path.join(
self.root_dir, path_info.relative_to(sm_path)
)
break
if os.path.isdir(path):
files.extend(
os.fsencode(relpath(fpath, self.root_dir))
for fpath in walk_files(path)
)
else:
files.append(os.fsencode(relpath(path, self.root_dir)))
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
if update:
index = self.repo.open_index()
if os.name == "nt":
# NOTE: we need git/unix separator to compare against index
# paths but repo.stage() expects to be called with OS paths
self.repo.stage(
[
fname
for fname in files
if fname.replace(b"\\", b"/") in index
]
)
else:
self.repo.stage([fname for fname in files if fname in index])
else:
self.repo.stage(files)
def test_download_dir(remote, tmpdir):
path = str(tmpdir / "data")
to_info = PathInfo(path)
remote.download(remote.path_info / "data", to_info)
assert os.path.isdir(path)
data_dir = tmpdir / "data"
assert len(list(walk_files(path))) == 7
assert (data_dir / "alice").read_text(encoding="utf-8") == "alice"
assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha"
assert (data_dir /
"subdir-file.txt").read_text(encoding="utf-8") == "subdir"
assert (data_dir / "subdir" / "1").read_text(encoding="utf-8") == "1"
assert (data_dir / "subdir" / "2").read_text(encoding="utf-8") == "2"
assert (data_dir / "subdir" / "3").read_text(encoding="utf-8") == "3"
assert (data_dir / "subdir" /
"empty_file").read_text(encoding="utf-8") == ""
def add(self, paths: Iterable[str]):
from dvc.utils.fs import walk_files
if isinstance(paths, str):
paths = [paths]
files = []
for path in paths:
if not os.path.isabs(path):
path = os.path.join(self.root_dir, path)
if os.path.isdir(path):
files.extend(walk_files(path))
else:
files.append(path)
for fpath in files:
# NOTE: this doesn't check gitignore, same as GitPythonBackend.add
self.repo.stage(relpath(fpath, self.root_dir))
def test_partial_checkout(tmp_dir, dvc, target):
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
shutil.rmtree("dir")
stats = dvc.checkout([target])
assert stats["added"] == ["dir" + os.sep]
assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
def list_cache_paths(self):
assert self.path_info is not None
return walk_files(self.path_info)
def test_walk_files(tmp_dir):
assert list(walk_files(".")) == list(walk_files(tmp_dir))
