def list_cache_paths(self, prefix=None, progress_callback=None): assert prefix is None assert self.path_info is not None if progress_callback: for path in walk_files(self.path_info): progress_callback() yield path else: yield from walk_files(self.path_info)
def list_cache_paths(self, prefix=None, progress_callback=None): assert self.path_info is not None if prefix: path_info = self.path_info / prefix[:2] if not self.tree.exists(path_info): return else: path_info = self.path_info if progress_callback: for path in walk_files(path_info): progress_callback() yield path else: yield from walk_files(path_info)
def _list_paths(self, prefix=None, progress_callback=None): assert self.fs.path_info is not None if prefix: path_info = self.fs.path_info / prefix[:2] if not self.fs.exists(path_info): return else: path_info = self.fs.path_info # NOTE: use utils.fs walk_files since fs.walk_files will not follow # symlinks if progress_callback: for path in walk_files(path_info): progress_callback() yield path else: yield from walk_files(path_info)
def add(self, paths: Iterable[str]): from dvc.utils.fs import walk_files if isinstance(paths, str): paths = [paths] files = [] for path in paths: if not os.path.isabs(path) and self._submodules: # NOTE: If path is inside a submodule, Dulwich expects the # staged paths to be relative to the submodule root (not the # parent git repo root). We append path to root_dir here so # that the result of relpath(path, root_dir) is actually the # path relative to the submodule root. path_info = PathInfo(path).relative_to(self.root_dir) for sm_path in self._submodules.values(): if path_info.isin(sm_path): path = os.path.join(self.root_dir, path_info.relative_to(sm_path)) break if os.path.isdir(path): files.extend(walk_files(path)) else: files.append(path) for fpath in files: # NOTE: this doesn't check gitignore, same as GitPythonBackend.add self.repo.stage(relpath(fpath, self.root_dir))
def _list_paths(self, prefix=None, progress_callback=None): assert self.fs_path is not None if prefix: fs_path = self.fs.path.join(self.fs_path, prefix[:2]) if not self.fs.exists(fs_path): return else: fs_path = self.fs_path # NOTE: use utils.fs walk_files since fs.walk_files will not follow # symlinks if progress_callback: for path in walk_files(fs_path): progress_callback() yield path else: yield from walk_files(fs_path)
def _list_paths(self, prefix=None): assert self.fs_path is not None if prefix: fs_path = self.fs.path.join(self.fs_path, prefix[:2]) if not self.fs.exists(fs_path): return else: fs_path = self.fs_path # NOTE: use utils.fs walk_files since fs.walk_files will not follow # symlinks yield from walk_files(fs_path)
def add(self, paths: Union[str, Iterable[str]], update=False): from dvc.utils.fs import walk_files assert paths or update if isinstance(paths, str): paths = [paths] if update and not paths: self.repo.stage(list(self.repo.open_index())) return files: List[bytes] = [] for path in paths: if not os.path.isabs(path) and self._submodules: # NOTE: If path is inside a submodule, Dulwich expects the # staged paths to be relative to the submodule root (not the # parent git repo root). We append path to root_dir here so # that the result of relpath(path, root_dir) is actually the # path relative to the submodule root. path_info = PathInfo(path).relative_to(self.root_dir) for sm_path in self._submodules.values(): if path_info.isin(sm_path): path = os.path.join( self.root_dir, path_info.relative_to(sm_path) ) break if os.path.isdir(path): files.extend( os.fsencode(relpath(fpath, self.root_dir)) for fpath in walk_files(path) ) else: files.append(os.fsencode(relpath(path, self.root_dir))) # NOTE: this doesn't check gitignore, same as GitPythonBackend.add if update: index = self.repo.open_index() if os.name == "nt": # NOTE: we need git/unix separator to compare against index # paths but repo.stage() expects to be called with OS paths self.repo.stage( [ fname for fname in files if fname.replace(b"\\", b"/") in index ] ) else: self.repo.stage([fname for fname in files if fname in index]) else: self.repo.stage(files)
def test_download_dir(remote, tmpdir): path = str(tmpdir / "data") to_info = PathInfo(path) remote.download(remote.path_info / "data", to_info) assert os.path.isdir(path) data_dir = tmpdir / "data" assert len(list(walk_files(path))) == 7 assert (data_dir / "alice").read_text(encoding="utf-8") == "alice" assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha" assert (data_dir / "subdir-file.txt").read_text(encoding="utf-8") == "subdir" assert (data_dir / "subdir" / "1").read_text(encoding="utf-8") == "1" assert (data_dir / "subdir" / "2").read_text(encoding="utf-8") == "2" assert (data_dir / "subdir" / "3").read_text(encoding="utf-8") == "3" assert (data_dir / "subdir" / "empty_file").read_text(encoding="utf-8") == ""
def add(self, paths: Iterable[str]): from dvc.utils.fs import walk_files if isinstance(paths, str): paths = [paths] files = [] for path in paths: if not os.path.isabs(path): path = os.path.join(self.root_dir, path) if os.path.isdir(path): files.extend(walk_files(path)) else: files.append(path) for fpath in files: # NOTE: this doesn't check gitignore, same as GitPythonBackend.add self.repo.stage(relpath(fpath, self.root_dir))
def test_partial_checkout(tmp_dir, dvc, target): tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}}) shutil.rmtree("dir") stats = dvc.checkout([target]) assert stats["added"] == ["dir" + os.sep] assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
def list_cache_paths(self): assert self.path_info is not None return walk_files(self.path_info)
def test_walk_files(tmp_dir): assert list(walk_files(".")) == list(walk_files(tmp_dir))