def open_read(self, path, mode="r"):
local_tmp_file = get_local_tempfile("download-%s" %
os.path.basename(path))
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
self.download(path, local_tmp_file)
return _DeleteOnCloseFile(local_tmp_file, mode)
def open_read(self, path, mode="r"):
account, container, blob = self._path_to_account_container_and_blob(path)
assert self.account == account
local_tmp_file = get_local_tempfile("download-%s" % os.path.basename(path))
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
self.download_as_file(container, blob, local_tmp_file)
return _DeleteOnCloseFile(local_tmp_file, mode)
def _open_read(
self,
remote_path,
local_path=None,
delete_file_on_close=True,
chunksize=None,
chunk_callback=lambda _: False,
):
"""Downloads the object contents to local file system.
Optionally stops after the first chunk for which chunk_callback returns True.
"""
chunksize = chunksize or self.chunksize
bucket, obj = self._path_to_bucket_and_key(remote_path)
tmp_file_path = local_path or get_local_tempfile(os.path.basename(remote_path))
with open(tmp_file_path, "wb") as fp:
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
if delete_file_on_close:
return_fp = _DeleteOnCloseFile(tmp_file_path, "r")
else:
return_fp = fp
# Special case empty files because chunk-based downloading doesn't work.
result = self.client.objects().get(bucket=bucket, object=obj).execute()
if int(result["size"]) == 0:
return return_fp
request = self.client.objects().get_media(bucket=bucket, object=obj)
downloader = http.MediaIoBaseDownload(fp, request, chunksize=chunksize)
attempts = 0
done = False
while not done:
error = None
try:
_, done = downloader.next_chunk()
if chunk_callback(fp):
done = True
except errors.HttpError as err:
error = err
if err.resp.status < 500:
raise
logger.warning("Error downloading file, retrying", exc_info=True)
except RETRYABLE_ERRORS as err:
logger.warning("Error downloading file, retrying", exc_info=True)
error = err
if error:
attempts += 1
if attempts >= NUM_RETRIES:
raise error
else:
attempts = 0
return return_fp