def add(repo, target, recursive=False, no_commit=False, fname=None):
if recursive and fname:
raise RecursiveAddingWhileUsingFilename()
targets = [target]
if os.path.isdir(target) and recursive:
targets = [
file
for file in walk_files(target)
if not Stage.is_stage_file(file)
if os.path.basename(file) != repo.scm.ignore_file
if not repo.scm.is_tracked(file)
]
stages = _create_stages(repo, targets, fname, no_commit)
repo.check_dag(repo.stages() + stages)
for stage in stages:
stage.dump()
repo.remind_to_git_add()
return stages
def get_mtime_and_size(path, dvcignore):
if os.path.isdir(fspath_py35(path)):
size = 0
files_mtimes = {}
for file_path in walk_files(path, dvcignore):
try:
stat = os.stat(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stat.st_size
files_mtimes[file_path] = stat.st_mtime
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = os.stat(fspath_py35(path))
size = base_stat.st_size
mtime = base_stat.st_mtime
mtime = int(nanotime.timestamp(mtime))
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(mtime), str(size)
def _find_all_targets(repo, target, recursive):
if os.path.isdir(target) and recursive:
return [
file for file in walk_files(target)
if not repo.is_dvc_internal(file) if not Stage.is_stage_file(file)
if not repo.scm.belongs_to_scm(file)
if not repo.scm.is_tracked(file)
]
return [target]
def _discard_working_directory_changes(self, path, dir_info, force=False):
working_dir_files = set(path for path in walk_files(path))
cached_files = set(
os.path.join(path, file["relpath"]) for file in dir_info)
delta = working_dir_files - cached_files
for file in delta:
self.safe_remove({"scheme": "local", "path": file}, force=force)
def outs_info(self, stage):
FileInfo = collections.namedtuple("FileInfo", "path inode")
paths = [
path for output in stage.outs for path in walk_files(output.path)
]
return [
FileInfo(path=path, inode=System.inode(path)) for path in paths
]
def outs_info(self, stage):
FileInfo = collections.namedtuple("FileInfo", "path inode")
paths = [
path for output in stage["outs"]
for path in walk_files(output["path"], self.dvc.dvcignore)
]
return [
FileInfo(path=path, inode=System.inode(path)) for path in paths
]
def test_download_dir(remote, tmpdir):
path = str(tmpdir / "data")
to_info = PathInfo(path)
remote.download(remote.path_info / "data", to_info)
assert os.path.isdir(path)
data_dir = tmpdir / "data"
assert len(list(walk_files(path, None))) == 7
assert (data_dir / "alice").read_text(encoding="utf-8") == "alice"
assert (data_dir / "alpha").read_text(encoding="utf-8") == "alpha"
assert (data_dir / "subdir-file.txt").read_text(
encoding="utf-8"
) == "subdir"
assert (data_dir / "subdir" / "1").read_text(encoding="utf-8") == "1"
assert (data_dir / "subdir" / "2").read_text(encoding="utf-8") == "2"
assert (data_dir / "subdir" / "3").read_text(encoding="utf-8") == "3"
assert (data_dir / "subdir" / "empty_file").read_text(
encoding="utf-8"
) == ""
def _unprotect_dir(self, path, allow_copy=True):
for fname in walk_files(path, self.repo.dvcignore):
self._unprotect_file(fname, allow_copy)
def _unprotect_dir(path):
for path in walk_files(path):
RemoteLOCAL._unprotect_file(path)
def walk_files(self, path_info):
for fname in walk_files(path_info, self.repo.dvcignore):
yield PathInfo(fname)
def list_cache_paths(self):
assert self.path_info is not None
return walk_files(self.path_info)
def _unprotect_dir(self, path):
for fname in walk_files(path, self.repo.dvcignore):
RemoteLOCAL._unprotect_file(fname)
def test_walk_files(tmp_dir):
assert list(walk_files(".")) == list(walk_files(tmp_dir))
def _unprotect_dir(path):
for path in walk_files(path):
_unprotect_file(path)
def test_partial_checkout(tmp_dir, dvc, target):
tmp_dir.dvc_gen({"dir": {"subdir": {"file": "file"}, "other": "other"}})
shutil.rmtree("dir")
dvc.checkout([target])
assert list(walk_files("dir")) == [os.path.join("dir", "subdir", "file")]
def _files_set(root, dvcignore):
return {to_posixpath(f) for f in walk_files(root, dvcignore)}