def get_mtime_and_size(path, dvcignore):
if os.path.isdir(fspath_py35(path)):
size = 0
files_mtimes = {}
for file_path in walk_files(path, dvcignore):
try:
stat = os.stat(file_path)
except OSError as exc:
# NOTE: broken symlink case.
if exc.errno != errno.ENOENT:
raise
continue
size += stat.st_size
files_mtimes[file_path] = stat.st_mtime
# We track file changes and moves, which cannot be detected with simply
# max(mtime(f) for f in non_ignored_files)
mtime = dict_md5(files_mtimes)
else:
base_stat = os.stat(fspath_py35(path))
size = base_stat.st_size
mtime = base_stat.st_mtime
mtime = int(nanotime.timestamp(mtime))
# State of files handled by dvc is stored in db as TEXT.
# We cast results to string for later comparisons with stored values.
return str(mtime), str(size)
def copyfile(src, dest, no_progress_bar=False, name=None):
"""Copy file with progress bar"""
from dvc.exceptions import DvcException
from dvc.progress import Tqdm
from dvc.system import System
src = fspath_py35(src)
dest = fspath_py35(dest)
name = name if name else os.path.basename(dest)
total = os.stat(src).st_size
if os.path.isdir(dest):
dest = os.path.join(dest, os.path.basename(src))
try:
System.reflink(src, dest)
except DvcException:
with open(src, "rb") as fsrc, open(dest, "wb+") as fdest:
with Tqdm.wrapattr(
fdest,
"write",
desc=name,
disable=no_progress_bar,
total=total,
bytes=True,
) as fdest_wrapped:
while True:
buf = fsrc.read(LOCAL_CHUNK_SIZE)
if not buf:
break
fdest_wrapped.write(buf)
def remove(path):
logger.debug("Removing '%s'", path)
path = fspath_py35(path)
try:
if os.path.isdir(path):
shutil.rmtree(path, onerror=_chmod)
else:
_chmod(os.unlink, path, None)
except OSError as exc:
if exc.errno != errno.ENOENT:
raise
def test_non_cached_output(tmp_path, erepo):
os.chdir(erepo.root_dir)
erepo.dvc.run(outs_no_cache=["non_cached_file"],
cmd="echo hello > non_cached_file")
erepo.dvc.scm.add(["non_cached_file", "non_cached_file.dvc"])
erepo.dvc.scm.commit("add non-cached output")
os.chdir(fspath_py35(tmp_path))
Repo.get(erepo.root_dir, "non_cached_file")
src = os.path.join(erepo.root_dir, "non_cached_file")
assert os.path.isfile("non_cached_file")
assert filecmp.cmp(src, "non_cached_file", shallow=False)
def move(src, dst, mode=None):
"""Atomically move src to dst and chmod it with mode.
Moving is performed in two stages to make the whole operation atomic in
case src and dst are on different filesystems and actual physical copying
of data is happening.
"""
src = fspath_py35(src)
dst = fspath_py35(dst)
dst = os.path.abspath(dst)
tmp = "{}.{}".format(dst, str(uuid()))
if os.path.islink(src):
shutil.copy(os.readlink(src), tmp)
os.unlink(src)
else:
shutil.move(src, tmp)
if mode is not None:
os.chmod(tmp, mode)
shutil.move(tmp, dst)
def makedirs(path, exist_ok=False, mode=None):
path = fspath_py35(path)
if mode is None:
os.makedirs(path, exist_ok=exist_ok)
return
# utilize umask to set proper permissions since Python 3.7 the `mode`
# `makedirs` argument no longer affects the file permission bits of
# newly-created intermediate-level directories.
umask = os.umask(0o777 - mode)
try:
os.makedirs(path, exist_ok=exist_ok)
finally:
os.umask(umask)
def normalize_path(path):
return os.path.normpath(fspath_py35(path))
