def open_read(self, path, mode="r"):
local_tmp_file = get_local_tempfile("download-%s" %
os.path.basename(path))
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
self.download(path, local_tmp_file)
return _DeleteOnCloseFile(local_tmp_file, mode)
def open_read(self, path, mode="r"):
account, container, blob = self._path_to_account_container_and_blob(path)
assert self.account == account
local_tmp_file = get_local_tempfile("download-%s" % os.path.basename(path))
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
self.download_as_file(container, blob, local_tmp_file)
return _DeleteOnCloseFile(local_tmp_file, mode)
def _open_read(
self,
remote_path,
local_path=None,
delete_file_on_close=True,
chunksize=None,
chunk_callback=lambda _: False,
):
"""Downloads the object contents to local file system.
Optionally stops after the first chunk for which chunk_callback returns True.
"""
chunksize = chunksize or self.chunksize
bucket, obj = self._path_to_bucket_and_key(remote_path)
tmp_file_path = local_path or get_local_tempfile(os.path.basename(remote_path))
with open(tmp_file_path, "wb") as fp:
# We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879
if delete_file_on_close:
return_fp = _DeleteOnCloseFile(tmp_file_path, "r")
else:
return_fp = fp
# Special case empty files because chunk-based downloading doesn't work.
result = self.client.objects().get(bucket=bucket, object=obj).execute()
if int(result["size"]) == 0:
return return_fp
request = self.client.objects().get_media(bucket=bucket, object=obj)
downloader = http.MediaIoBaseDownload(fp, request, chunksize=chunksize)
attempts = 0
done = False
while not done:
error = None
try:
_, done = downloader.next_chunk()
if chunk_callback(fp):
done = True
except errors.HttpError as err:
error = err
if err.resp.status < 500:
raise
logger.warning("Error downloading file, retrying", exc_info=True)
except RETRYABLE_ERRORS as err:
logger.warning("Error downloading file, retrying", exc_info=True)
error = err
if error:
attempts += 1
if attempts >= NUM_RETRIES:
raise error
else:
attempts = 0
return return_fp
def __init__(self, command, input_pipe=None):
"""
Initializes a InputPipeProcessWrapper instance.
:param command: a subprocess.Popen instance with stdin=input_pipe and
stdout=subprocess.PIPE.
Alternatively, just its args argument as a convenience.
"""
self._command = command
self._input_pipe = input_pipe
self._original_input = True
if input_pipe is not None:
try:
input_pipe.fileno()
except AttributeError:
# subprocess require a fileno to work, if not present we copy to disk first
self._original_input = False
self._tmp_file = get_local_tempfile("databand-process_tmp")
with open(self._tmp_file, "wb") as f:
self._tmp_file = f.name
while True:
chunk = input_pipe.read(io.DEFAULT_BUFFER_SIZE)
if not chunk:
break
f.write(chunk)
input_pipe.close()
self._input_pipe = FileWrapper(
io.BufferedReader(io.FileIO(self._tmp_file, "r")))
self._process = (command if isinstance(command, subprocess.Popen) else
self.create_subprocess(command))
# we want to keep a circular reference to avoid garbage collection
# when the object is used in, e.g., pipe.read()
self._process._selfref = self
def generate_tmp_path(self, path):
return get_local_tempfile(os.path.basename(path))