def _save_file(self, path):
md5 = self.state.update(path)
cache = self.get(md5)
if self._changed(md5):
move(path, cache)
self.state.update(cache)
self._checkout(path, md5)
def download(
self,
from_infos,
to_infos,
names=None,
no_progress_bar=False,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
tmp_file = tmp_fname(to_info)
if not name:
name = to_info.name
cb = None if no_progress_bar else Callback(name)
makedirs(fspath_py35(to_info.parent), exist_ok=True)
try:
self.oss_service.get_object_to_file(from_info.path,
tmp_file,
progress_callback=cb)
except Exception:
logger.warning("failed to download '{}'".format(from_info))
else:
move(tmp_file, fspath_py35(to_info))
finally:
if not no_progress_bar:
progress.finish_target(name)
def _move(self, inp, outp):
# moving in two stages to make the whole operation atomic in
# case inp and outp are in different filesystems and actual
# physical copying of data is happening
tmp = "{}.{}".format(outp, str(uuid.uuid4()))
move(inp, tmp)
move(tmp, outp)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "s3":
raise NotImplementedError
if to_info["scheme"] == "s3":
self.copy(from_info, to_info, s3=s3)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["bucket"], from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
if no_progress_bar:
cb = None
else:
total = s3.head_object(
Bucket=from_info["bucket"], Key=from_info["path"]
)["ContentLength"]
cb = Callback(name, total)
s3.download_file(
from_info["bucket"],
from_info["path"],
tmp_file,
Callback=cb,
)
except Exception:
msg = "failed to download '{}/{}'".format(
from_info["bucket"], from_info["path"]
)
logger.error(msg)
continue
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
names = self._verify_path_args(from_infos, to_infos, names)
with self.transfer_context() as ctx:
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info, ctx=ctx)
continue
if to_info.scheme != "local":
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(from_info, to_info)
logger.debug(msg)
tmp_file = tmp_fname(to_info)
if not name:
name = to_info.name
if not no_progress_bar:
# real progress is not always available,
# lets at least show start and finish
progress.update_target(name, 0, None)
makedirs(fspath_py35(to_info.parent), exist_ok=True)
try:
self._download(
from_info,
tmp_file,
name=name,
ctx=ctx,
resume=resume,
no_progress_bar=no_progress_bar,
)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
continue
move(tmp_file, fspath_py35(to_info))
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
gs = self.gs
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "gs":
raise NotImplementedError
if to_info["scheme"] == "gs":
self.copy(from_info, to_info, gs=gs)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["bucket"], from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
if not no_progress_bar:
# percent_cb is not available for download_to_filename, so
# lets at least update progress at pathpoints(start, finish)
progress.update_target(name, 0, None)
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
bucket = gs.bucket(from_info["bucket"])
blob = bucket.get_blob(from_info["path"])
blob.download_to_filename(tmp_file)
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.exception(
msg.format(
from_info["bucket"], from_info["path"], to_info["path"]
)
)
continue
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def move(self, from_info, to_info):
if from_info.scheme != "local" or to_info.scheme != "local":
raise NotImplementedError
self.makedirs(to_info.parent)
if self.isfile(from_info):
mode = self._file_mode
else:
mode = self._dir_mode
move(from_info, to_info, mode=mode)
def _download_to(self, url, target_file, callback=None, resume=False):
request = self._request("GET", url, stream=True)
partial_file = target_file + ".part"
mode, transferred_bytes = self._determine_mode_get_transferred_bytes(
partial_file, resume)
self._validate_existing_file_size(transferred_bytes, partial_file)
self._write_request_content(mode, partial_file, request,
transferred_bytes, callback)
move(partial_file, target_file)
def move(self, from_info, to_info):
if from_info.scheme != "local" or to_info.scheme != "local":
raise NotImplementedError
inp = from_info.fspath
outp = to_info.fspath
# moving in two stages to make the whole operation atomic in
# case inp and outp are in different filesystems and actual
# physical copying of data is happening
tmp = "{}.{}".format(outp, str(uuid.uuid4()))
move(inp, tmp)
move(tmp, outp)
def download(
self,
from_info,
to_info,
name=None,
no_progress_bar=False,
file_mode=None,
dir_mode=None,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
if not no_progress_bar:
# real progress is not always available,
# lets at least show start and finish
progress.update_target(name, 0, None)
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
if not no_progress_bar:
progress.finish_target(name)
return 0
def _unprotect_file(self, path):
import stat
import uuid
from dvc.utils import copyfile, move, remove
self.logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def _unprotect_file(path):
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), "." + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def dump_dir_cache(self, md5, dir_info):
path = self.get(md5)
dname = os.path.dirname(path)
assert self.is_dir_cache(path)
assert isinstance(dir_info, list)
if not os.path.isdir(dname):
os.makedirs(dname)
# NOTE: Writing first and renaming after that
# to make sure that the operation is atomic.
tmp = '{}.{}'.format(path, str(uuid.uuid4()))
with open(tmp, 'w+') as fd:
json.dump(dir_info, fd, sort_keys=True)
move(tmp, path)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != self.scheme:
raise NotImplementedError
if to_info["scheme"] != "local":
raise NotImplementedError
bucket = from_info["bucket"]
path = from_info["path"]
logger.debug(
"Downloading '{}/{}' to '{}'".format(
bucket, path, to_info["path"]
)
)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
cb = None if no_progress_bar else Callback(name)
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
self.blob_service.get_blob_to_path(
bucket, path, tmp_file, progress_callback=cb
)
except Exception:
msg = "failed to download '{}/{}'".format(bucket, path)
logger.warning(msg)
else:
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "local":
raise NotImplementedError
if to_info["scheme"] != "local":
raise NotImplementedError
logger.debug(
"Downloading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
tmp_file = tmp_fname(to_info["path"])
try:
copyfile(
from_info["path"],
tmp_file,
no_progress_bar=no_progress_bar,
name=name,
)
move(tmp_file, to_info["path"])
except Exception:
logger.exception(
"failed to download '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
)
continue
def download(
self,
from_info,
to_info,
name=None,
no_progress_bar=False,
file_mode=None,
dir_mode=None,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
makedirs(to_info.parent, exist_ok=True, mode=dir_mode)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, to_info, mode=file_mode)
return 0
def _unprotect_file(self, path):
import stat
import uuid
from dvc.system import System
from dvc.utils import copyfile, move, remove
if System.is_symlink(path) or System.is_hardlink(path):
logger.debug("Unprotecting '{}'".format(path))
tmp = os.path.join(os.path.dirname(path), '.' + str(uuid.uuid4()))
move(path, tmp)
copyfile(tmp, path)
remove(tmp)
else:
logger.debug("Skipping copying for '{}', since it is not "
"a symlink or a hardlink.".format(path))
os.chmod(path, os.stat(path).st_mode | stat.S_IWRITE)
def move(self, from_info, to_info):
if from_info['scheme'] != 'local' or to_info['scheme'] != 'local':
raise NotImplementedError
move(from_info['path'], to_info['path'])
def _move(self, inp, outp):
# moving in two stages to make last the move atomic in
# case inp and outp are in different filesystems
tmp = '{}.{}'.format(outp, str(uuid.uuid4()))
move(inp, tmp)
move(tmp, outp)
def move(self, from_info, to_info):
if from_info["scheme"] != "local" or to_info["scheme"] != "local":
raise NotImplementedError
move(from_info["path"], to_info["path"])
