def get_mtime_and_size(path, dvcignore): if os.path.isdir(fspath_py35(path)): size = 0 files_mtimes = {} for file_path in walk_files(path, dvcignore): try: stat = os.stat(file_path) except OSError as exc: # NOTE: broken symlink case. if exc.errno != errno.ENOENT: raise continue size += stat.st_size files_mtimes[file_path] = stat.st_mtime # We track file changes and moves, which cannot be detected with simply # max(mtime(f) for f in non_ignored_files) mtime = dict_md5(files_mtimes) else: base_stat = os.stat(fspath_py35(path)) size = base_stat.st_size mtime = base_stat.st_mtime mtime = int(nanotime.timestamp(mtime)) # State of files handled by dvc is stored in db as TEXT. # We cast results to string for later comparisons with stored values. return str(mtime), str(size)
def copyfile(src, dest, no_progress_bar=False, name=None): """Copy file with progress bar""" from dvc.exceptions import DvcException from dvc.progress import Tqdm from dvc.system import System src = fspath_py35(src) dest = fspath_py35(dest) name = name if name else os.path.basename(dest) total = os.stat(src).st_size if os.path.isdir(dest): dest = os.path.join(dest, os.path.basename(src)) try: System.reflink(src, dest) except DvcException: with open(src, "rb") as fsrc, open(dest, "wb+") as fdest: with Tqdm.wrapattr( fdest, "write", desc=name, disable=no_progress_bar, total=total, bytes=True, ) as fdest_wrapped: while True: buf = fsrc.read(LOCAL_CHUNK_SIZE) if not buf: break fdest_wrapped.write(buf)
def remove(path): logger.debug("Removing '%s'", path) path = fspath_py35(path) try: if os.path.isdir(path): shutil.rmtree(path, onerror=_chmod) else: _chmod(os.unlink, path, None) except OSError as exc: if exc.errno != errno.ENOENT: raise
def test_non_cached_output(tmp_path, erepo): os.chdir(erepo.root_dir) erepo.dvc.run(outs_no_cache=["non_cached_file"], cmd="echo hello > non_cached_file") erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"]) erepo.dvc.scm.commit("add non-cached output") os.chdir(fspath_py35(tmp_path)) Repo.get(erepo.root_dir, "non_cached_file") src = os.path.join(erepo.root_dir, "non_cached_file") assert os.path.isfile("non_cached_file") assert filecmp.cmp(src, "non_cached_file", shallow=False)
def move(src, dst, mode=None): """Atomically move src to dst and chmod it with mode. Moving is performed in two stages to make the whole operation atomic in case src and dst are on different filesystems and actual physical copying of data is happening. """ src = fspath_py35(src) dst = fspath_py35(dst) dst = os.path.abspath(dst) tmp = "{}.{}".format(dst, str(uuid())) if os.path.islink(src): shutil.copy(os.readlink(src), tmp) os.unlink(src) else: shutil.move(src, tmp) if mode is not None: os.chmod(tmp, mode) shutil.move(tmp, dst)
def makedirs(path, exist_ok=False, mode=None): path = fspath_py35(path) if mode is None: os.makedirs(path, exist_ok=exist_ok) return # utilize umask to set proper permissions since Python 3.7 the `mode` # `makedirs` argument no longer affects the file permission bits of # newly-created intermediate-level directories. umask = os.umask(0o777 - mode) try: os.makedirs(path, exist_ok=exist_ok) finally: os.umask(umask)
def normalize_path(path): return os.path.normpath(fspath_py35(path))