def download(
self,
from_infos,
to_infos,
names=None,
no_progress_bar=False,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
tmp_file = tmp_fname(to_info)
if not name:
name = to_info.name
cb = None if no_progress_bar else Callback(name)
makedirs(fspath_py35(to_info.parent), exist_ok=True)
try:
self.oss_service.get_object_to_file(from_info.path,
tmp_file,
progress_callback=cb)
except Exception:
logger.warning("failed to download '{}'".format(from_info))
else:
move(tmp_file, fspath_py35(to_info))
finally:
if not no_progress_bar:
progress.finish_target(name)
def upload(self, from_infos, to_infos, names=None, no_progress_bar=False):
names = self._verify_path_args(to_infos, from_infos, names)
for from_info, to_info, name in zip(from_infos, to_infos, names):
if to_info["scheme"] != "local":
raise NotImplementedError
if from_info["scheme"] != "local":
raise NotImplementedError
logger.debug("Uploading '{}' to '{}'".format(
from_info["path"], to_info["path"]))
if not name:
name = os.path.basename(from_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
tmp_file = tmp_fname(to_info["path"])
try:
copyfile(
from_info["path"],
tmp_file,
name=name,
no_progress_bar=no_progress_bar,
)
os.rename(tmp_file, to_info["path"])
except Exception:
logger.exception("failed to upload '{}' to '{}'".format(
from_info["path"], to_info["path"]))
def install(self, cache_dir=None, force=False):
if self.installed and not force:
logger.info(
"Skipping installing '{}'('{}') as it is already "
"installed.".format(self.name, self.url)
)
return
makedirs(self.repos_dir, exist_ok=True)
# installing package to a temporary directory until we are sure that
# it has been installed correctly.
#
# Note that we can't use tempfile.TemporaryDirectory is using symlinks
# to tmpfs, so we won't be able to use move properly.
tmp_dir = os.path.join(self.repos_dir, "." + str(shortuuid.uuid()))
try:
self._install_to(tmp_dir, cache_dir)
except ExternalRepoError:
if os.path.exists(tmp_dir):
remove(tmp_dir)
raise
if self.installed:
self.uninstall()
shutil.move(tmp_dir, self.path)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "s3":
raise NotImplementedError
if to_info["scheme"] == "s3":
self.copy(from_info, to_info, s3=s3)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["bucket"], from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
if no_progress_bar:
cb = None
else:
total = s3.head_object(
Bucket=from_info["bucket"], Key=from_info["path"]
)["ContentLength"]
cb = Callback(name, total)
s3.download_file(
from_info["bucket"],
from_info["path"],
tmp_file,
Callback=cb,
)
except Exception:
msg = "failed to download '{}/{}'".format(
from_info["bucket"], from_info["path"]
)
logger.error(msg)
continue
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
names = self._verify_path_args(from_infos, to_infos, names)
with self.transfer_context() as ctx:
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info, ctx=ctx)
continue
if to_info.scheme != "local":
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(from_info, to_info)
logger.debug(msg)
tmp_file = tmp_fname(to_info)
if not name:
name = to_info.name
if not no_progress_bar:
# real progress is not always available,
# lets at least show start and finish
progress.update_target(name, 0, None)
makedirs(fspath_py35(to_info.parent), exist_ok=True)
try:
self._download(
from_info,
tmp_file,
name=name,
ctx=ctx,
resume=resume,
no_progress_bar=no_progress_bar,
)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
continue
move(tmp_file, fspath_py35(to_info))
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
gs = self.gs
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != "gs":
raise NotImplementedError
if to_info["scheme"] == "gs":
self.copy(from_info, to_info, gs=gs)
continue
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(
from_info["bucket"], from_info["path"], to_info["path"]
)
logger.debug(msg)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
if not no_progress_bar:
# percent_cb is not available for download_to_filename, so
# lets at least update progress at pathpoints(start, finish)
progress.update_target(name, 0, None)
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
bucket = gs.bucket(from_info["bucket"])
blob = bucket.get_blob(from_info["path"])
blob.download_to_filename(tmp_file)
except Exception:
msg = "failed to download '{}/{}' to '{}'"
logger.exception(
msg.format(
from_info["bucket"], from_info["path"], to_info["path"]
)
)
continue
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def _upload(
self, from_file, to_info, name=None, no_progress_bar=False, **_kwargs
):
makedirs(fspath_py35(to_info.parent), exist_ok=True)
tmp_file = tmp_fname(to_info)
copyfile(
from_file, tmp_file, name=name, no_progress_bar=no_progress_bar
)
os.rename(tmp_file, fspath_py35(to_info))
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(to_infos, from_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] not in ["http", "https"]:
raise NotImplementedError
if to_info["scheme"] != "local":
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(
from_info["path"], to_info["path"]
)
logger.debug(msg)
if not name:
name = os.path.basename(to_info["path"])
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
total = self._content_length(from_info["path"])
if no_progress_bar or not total:
cb = None
else:
cb = ProgressBarCallback(name, total)
try:
self._download_to(
from_info["path"],
to_info["path"],
callback=cb,
resume=resume,
)
except Exception:
msg = "failed to download '{}'".format(from_info["path"])
logger.error(msg)
continue
if not no_progress_bar:
progress.finish_target(name)
def download(
self,
from_infos,
to_infos,
names=None,
no_progress_bar=False,
resume=False,
):
names = self._verify_path_args(to_infos, from_infos, names)
fails = 0
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
msg = "Downloading '{}' to '{}'".format(from_info, to_info)
logger.debug(msg)
if not name:
name = to_info.name
makedirs(fspath_py35(to_info.parent), exist_ok=True)
total = self._content_length(from_info.url)
if no_progress_bar or not total:
cb = None
else:
cb = ProgressBarCallback(name, total)
try:
self._download_to(from_info.url,
to_info.fspath,
callback=cb,
resume=resume)
except Exception:
fails += 1
msg = "failed to download '{}'".format(from_info)
logger.exception(msg)
continue
if not no_progress_bar:
progress.finish_target(name)
return fails
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info["scheme"] != self.scheme:
raise NotImplementedError
if to_info["scheme"] != "local":
raise NotImplementedError
bucket = from_info["bucket"]
path = from_info["path"]
logger.debug(
"Downloading '{}/{}' to '{}'".format(
bucket, path, to_info["path"]
)
)
tmp_file = tmp_fname(to_info["path"])
if not name:
name = os.path.basename(to_info["path"])
cb = None if no_progress_bar else Callback(name)
makedirs(os.path.dirname(to_info["path"]), exist_ok=True)
try:
self.blob_service.get_blob_to_path(
bucket, path, tmp_file, progress_callback=cb
)
except Exception:
msg = "failed to download '{}/{}'".format(bucket, path)
logger.warning(msg)
else:
move(tmp_file, to_info["path"])
if not no_progress_bar:
progress.finish_target(name)
def download(self, src, dest, no_progress_bar=False, progress_title=None):
from uuid import uuid4
self._sftp_connect()
makedirs(posixpath.dirname(dest), exist_ok=True)
tmp_file = dest + "." + str(uuid4())
if no_progress_bar:
self._sftp.get(src, tmp_file)
else:
if not progress_title:
progress_title = os.path.basename(dest)
self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
progress.finish_target(progress_title)
os.rename(tmp_file, dest)
def download(self, src, dest, no_progress_bar=False, progress_title=None):
self._sftp_connect()
makedirs(posixpath.dirname(dest), exist_ok=True)
tmp_file = tmp_fname(dest)
if no_progress_bar:
self._sftp.get(src, tmp_file)
else:
if not progress_title:
progress_title = os.path.basename(dest)
self._sftp.get(src, tmp_file, callback=create_cb(progress_title))
progress.finish_target(progress_title)
if os.path.exists(dest):
os.remove(dest)
os.rename(tmp_file, dest)
def download(self, from_info, to_info, name=None, no_progress_bar=False):
if not hasattr(self, "_download"):
raise RemoteActionNotImplemented("download", self.scheme)
if from_info.scheme != self.scheme:
raise NotImplementedError
if to_info.scheme == self.scheme != "local":
self.copy(from_info, to_info)
return 0
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
name = name or to_info.name
if not no_progress_bar:
# real progress is not always available,
# lets at least show start and finish
progress.update_target(name, 0, None)
makedirs(fspath_py35(to_info.parent), exist_ok=True)
tmp_file = tmp_fname(to_info)
try:
self._download(from_info,
tmp_file,
name=name,
no_progress_bar=no_progress_bar)
except Exception:
msg = "failed to download '{}' to '{}'"
logger.exception(msg.format(from_info, to_info))
return 1 # 1 fail
move(tmp_file, fspath_py35(to_info))
if not no_progress_bar:
progress.finish_target(name)
return 0
def download(
self,
from_infos,
to_infos,
no_progress_bar=False,
names=None,
resume=False,
):
names = self._verify_path_args(from_infos, to_infos, names)
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info.scheme != "local":
raise NotImplementedError
if to_info.scheme != "local":
raise NotImplementedError
logger.debug("Downloading '{}' to '{}'".format(from_info, to_info))
if not name:
name = to_info.name
makedirs(fspath_py35(to_info.parent), exist_ok=True)
tmp_file = tmp_fname(to_info)
try:
copyfile(
fspath_py35(from_info),
tmp_file,
no_progress_bar=no_progress_bar,
name=name,
)
move(tmp_file, fspath_py35(to_info))
except Exception:
logger.exception(
"failed to download '{}' to '{}'".format(
from_info, to_info
)
)
continue