def test_copyfile(self):
src = 'file1'
dest = 'file2'
dest_dir = 'testdir'
with open(src, 'w+') as f:
f.write('file1contents')
os.mkdir(dest_dir)
utils.copyfile(src, dest)
self.assertTrue(filecmp.cmp(src, dest, shallow=False))
utils.copyfile(src, dest_dir)
self.assertTrue(
filecmp.cmp(src, '{}/{}'.format(dest_dir, src), shallow=False))
shutil.rmtree(dest_dir)
os.remove(src)
os.remove(dest)
def _unprotect_file(self, path):
import stat
import uuid
from dvc.system import System
from dvc.utils import copyfile, move, remove
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def upload(self, paths, path_infos, names=None):
names = self._verify_path_args(path_infos, paths, names)
for path, path_info, name in zip(paths, path_infos, names):
if path_info['scheme'] != 'local':
raise NotImplementedError
Logger.debug("Uploading '{}' to '{}'".format(
path, path_info['path']))
if not name:
name = os.path.basename(path)
self._makedirs(path_info['path'])
try:
copyfile(path, path_info['path'], name=name)
except Exception as exc:
Logger.error("Failed to upload '{}' tp '{}'".format(
path, path_info['path']))
def test_copyfile(self):
src = "file1"
dest = "file2"
dest_dir = "testdir"
with open(src, "w+") as f:
f.write("file1contents")
os.mkdir(dest_dir)
utils.copyfile(src, dest)
self.assertTrue(filecmp.cmp(src, dest, shallow=False))
utils.copyfile(src, dest_dir)
self.assertTrue(
filecmp.cmp(src, "{}/{}".format(dest_dir, src), shallow=False))
shutil.rmtree(dest_dir)
os.remove(src)
os.remove(dest)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != "local":
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
if not name:
name = to_info.name
makedirs(fspath_py35(to_info.parent), exist_ok=True)
tmp_file = tmp_fname(to_info)
try:
copyfile(
fspath_py35(from_info),
tmp_file,
no_progress_bar=no_progress_bar,
name=name,
)
move(tmp_file, fspath_py35(to_info))
except Exception:
logger.exception(
"failed to download '{}' to '{}'".format(
from_info, to_info
)
)
continue
def _unprotect_file(path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + str(uuid()))
# The operations order is important here - if some application
# would access the file during the process of copyfile then it
# would get only the part of file. So, at first, the file should be
# copied with the temporary name, and then original file should be
# replaced by new.
copyfile(path, tmp, name="Unprotecting '{}'".format(relpath(path)))
remove(path)
os.rename(tmp, path)
else:
logger.debug(
"Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path)
)
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "local":
raise NotImplementedError
if to_info["scheme"] != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(
from_info["path"], to_info["path"]))
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
tmp_file = tmp_fname(to_info["path"])
try:
copyfile(
from_info["path"],
tmp_file,
no_progress_bar=no_progress_bar,
name=name,
)
os.rename(tmp_file, to_info["path"])
except Exception:
logger.error("failed to download '{}' to '{}'".format(
from_info["path"], to_info["path"]))
continue
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info['scheme'] != 'local':
raise NotImplementedError
if from_info['scheme'] != 'local':
raise NotImplementedError
logger.debug("Uploading '{}' to '{}'".format(
from_info['path'], to_info['path']))
if not name:
name = os.path.basename(from_info['path'])
self._makedirs(to_info['path'])
try:
copyfile(from_info['path'], to_info['path'], name=name)
except Exception:
logger.error("failed to upload '{}' to '{}'".format(
from_info['path'], to_info['path']))
def upload(self, from_infos, to_infos, names=None):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info["scheme"] != "local":
raise NotImplementedError
if from_info["scheme"] != "local":
raise NotImplementedError
logger.debug("Uploading '{}' to '{}'".format(
from_info["path"], to_info["path"]))
if not name:
name = os.path.basename(from_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
copyfile(from_info["path"], to_info["path"], name=name)
except Exception:
logger.error("failed to upload '{}' to '{}'".format(
from_info["path"], to_info["path"]))
def test_copyfile(path, repo_dir):
src = repo_dir.FOO
dest = path
src_info = PathInfo(repo_dir.BAR)
dest_info = PathInfo(path)
copyfile(src, dest)
if os.path.isdir(dest):
assert filecmp.cmp(
src, os.path.join(dest, os.path.basename(src)), shallow=False
)
else:
assert filecmp.cmp(src, dest, shallow=False)
copyfile(src_info, dest_info)
if os.path.isdir(dest_info.fspath):
assert filecmp.cmp(
src_info.fspath,
os.path.join(dest_info.fspath, os.path.basename(src_info.fspath)),
shallow=False,
)
else:
assert filecmp.cmp(src_info.fspath, dest_info.fspath, shallow=False)
def push(self, item):
Logger.debug('sync to cloud ' + item.cache.dvc + " " +
self.storage_path)
copyfile(item.cache.dvc, self.storage_path)
return item
def sync_from_cloud(self, item):
Logger.debug('sync from cloud ' + self.storage_path + " " +
item.resolved_cache.dvc)
copyfile(self.storage_path, item.resolved_cache.dvc)
def _download(
self, from_info, to_file, name=None, no_progress_bar=False, **_kwargs
):
copyfile(
from_info, to_file, no_progress_bar=no_progress_bar, name=name
)
def push(self, path):
Logger.debug('sync to cloud ' + path + " " + self.storage_path)
copyfile(path, self.storage_path)
return path
def copy_file(input, output):
"""
Copy single file from local path.
"""
copyfile(input, output)
def _push_key(self, key, path):
self._makedirs(key.path)
copyfile(path, key.path)
return path
def sync_to_cloud(self, item):
Logger.debug('sync to cloud ' + item.resolved_cache.dvc + " " +
self.storage_path)
copyfile(item.resolved_cache.dvc, self.storage_path)
