def _download(self, make_dest_dirs=True):
with self.httpr(stream=True) as httpr:
if self.exists():
# Find out if the source file is gzip compressed in order to keep
# compression intact after the download.
# Per default requests decompresses .gz files.
# More detials can be found here: https://stackoverflow.com/questions/25749345/how-to-download-gz-files-with-requests-in-python-without-decoding-it?noredirect=1&lq=1
# Since data transferred with HTTP compression need to be decompressed automatically
# check the header and decode if the content is encoded.
if (not self.name.endswith(".gz")
and httpr.headers.get("Content-Encoding") == "gzip"):
# Decode non-gzipped sourcefiles automatically.
# This is needed to decompress uncompressed files that are compressed
# for the transfer by HTTP compression.
httpr.raw.decode_content = True
# if the destination path does not exist
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_path),
exist_ok=True)
with open(self.local_path, "wb") as f:
shutil.copyfileobj(httpr.raw, f)
os_sync() # ensure flush to disk
else:
raise HTTPFileException(
"The file does not seem to exist remotely: %s" %
self.remote_file())
def download(self, make_dest_dirs=True):
if self.exists():
# if the destination path does not exist, make it
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_file()), exist_ok=True)
self._dropboxc.files_download_to_file(self.local_file(),
self.dropbox_file())
os_sync() # ensure flush to disk
else:
raise DropboxFileException(
"The file does not seem to exist remotely: %s" %
self.dropbox_file())
def download(self, make_dest_dirs=True):
if self.exists():
# if the destination path does not exist, make it
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_file()), exist_ok=True)
with self.webdavc() as webdavc:
self.loop.run_until_complete(
self.conn.download(self.webdav_file, self.local_file()))
os_sync() # ensure flush to disk
else:
raise WorkflowError(
"The file does not seem to exist remotely: %s" %
self.webdav_file)
def download(self, make_dest_dirs=True):
with self.connection_pool.item() as sftpc:
if self.exists():
# if the destination path does not exist
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_path), exist_ok=True)
sftpc.get(
remotepath=self.remote_path,
localpath=self.local_path,
preserve_mtime=True,
)
os_sync() # ensure flush to disk
else:
raise SFTPFileException(
"The file does not seem to exist remotely: %s" % self.local_file()
)
def download(self):
if self.exists():
if self.size() == 0:
# Globus erroneously thinks that a transfer is incomplete if a
# file is empty. Hence we manually touch the local file.
self.local_touch_or_create()
return self.local_file()
# Download file. Wait for staging.
source = self.remote_file()
target = "file://" + os.path.abspath(self.local_file())
self._globus("-parallel", "4", "-create-dest", "-recurse", "-dp",
source, target)
os_sync()
return self.local_file()
return None
def download(self, make_dest_dirs=True):
with self.ftpc() as ftpc:
if self.exists():
# if the destination path does not exist
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_path),
exist_ok=True)
try:
# requires write access
ftpc.synchronize_times()
except:
pass
ftpc.download(source=self.remote_path, target=self.local_path)
os_sync() # ensure flush to disk
else:
raise FTPFileException(
"The file does not seem to exist remotely: %s" %
self.local_file())
def download(self):
if self.exists():
if self.size() == 0:
# Globus erroneously thinks that a transfer is incomplete if a
# file is empty. Hence we manually touch the local file.
self.local_touch_or_create()
return self.local_file()
# Download file. Wait for staging.
source = self.remote_file()
target = "file://" + os.path.abspath(self.local_file())
# disable all timeouts (file transfers can take a long time)
self._gfal("copy", "-p", "-f", "-n", "4", "-t", "0", "-T", "0",
source, target)
os_sync()
return self.local_file()
return None
def download(self, make_dest_dirs=True):
if self.exists():
if make_dest_dirs:
os.makedirs(os.path.dirname(self.local_path), exist_ok=True)
# force irods to overwrite existing file if this option is set
opt = {}
if self.kwargs.get("overwrite"):
opt[kw.FORCE_FLAG_KW] = ""
# get object and change timestamp
obj = self._irods_session.data_objects.get(self.remote_path,
self.local_path,
options=opt)
os.utime(self.local_path, (self.atime(), self.mtime()))
os_sync()
else:
raise WorkflowError(
"The file does not seem to exist remotely: %s" %
self.local_file())
def download(self):
if not self.exists():
return None
# Create the directory for the intended file
os.makedirs(os.path.dirname(self.local_file()), exist_ok=True)
# ideally we could calculate hash while streaming to file with provided function
# https://github.com/googleapis/python-storage/issues/29
with open(self.local_file(), "wb") as blob_file:
parser = Crc32cCalculator(blob_file)
self.blob.download_to_file(parser)
os_sync()
# Compute local hash and verify correct
if parser.hexdigest() != self.blob.crc32c:
os.remove(self.local_file())
raise CheckSumMismatchException(
"The checksum of %s does not match." % self.local_file())
return self.local_file()
def download(self):
self._s3c.download_from_s3(self.s3_bucket, self.s3_key,
self.local_file())
os_sync() # ensure flush to disk
