def open_read(self, path, mode="r"): local_tmp_file = get_local_tempfile("download-%s" % os.path.basename(path)) # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879 self.download(path, local_tmp_file) return _DeleteOnCloseFile(local_tmp_file, mode)
def open_read(self, path, mode="r"): account, container, blob = self._path_to_account_container_and_blob(path) assert self.account == account local_tmp_file = get_local_tempfile("download-%s" % os.path.basename(path)) # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879 self.download_as_file(container, blob, local_tmp_file) return _DeleteOnCloseFile(local_tmp_file, mode)
def _open_read( self, remote_path, local_path=None, delete_file_on_close=True, chunksize=None, chunk_callback=lambda _: False, ): """Downloads the object contents to local file system. Optionally stops after the first chunk for which chunk_callback returns True. """ chunksize = chunksize or self.chunksize bucket, obj = self._path_to_bucket_and_key(remote_path) tmp_file_path = local_path or get_local_tempfile(os.path.basename(remote_path)) with open(tmp_file_path, "wb") as fp: # We can't return the tempfile reference because of a bug in python: http://bugs.python.org/issue18879 if delete_file_on_close: return_fp = _DeleteOnCloseFile(tmp_file_path, "r") else: return_fp = fp # Special case empty files because chunk-based downloading doesn't work. result = self.client.objects().get(bucket=bucket, object=obj).execute() if int(result["size"]) == 0: return return_fp request = self.client.objects().get_media(bucket=bucket, object=obj) downloader = http.MediaIoBaseDownload(fp, request, chunksize=chunksize) attempts = 0 done = False while not done: error = None try: _, done = downloader.next_chunk() if chunk_callback(fp): done = True except errors.HttpError as err: error = err if err.resp.status < 500: raise logger.warning("Error downloading file, retrying", exc_info=True) except RETRYABLE_ERRORS as err: logger.warning("Error downloading file, retrying", exc_info=True) error = err if error: attempts += 1 if attempts >= NUM_RETRIES: raise error else: attempts = 0 return return_fp
def __init__(self, command, input_pipe=None): """ Initializes a InputPipeProcessWrapper instance. :param command: a subprocess.Popen instance with stdin=input_pipe and stdout=subprocess.PIPE. Alternatively, just its args argument as a convenience. """ self._command = command self._input_pipe = input_pipe self._original_input = True if input_pipe is not None: try: input_pipe.fileno() except AttributeError: # subprocess require a fileno to work, if not present we copy to disk first self._original_input = False self._tmp_file = get_local_tempfile("databand-process_tmp") with open(self._tmp_file, "wb") as f: self._tmp_file = f.name while True: chunk = input_pipe.read(io.DEFAULT_BUFFER_SIZE) if not chunk: break f.write(chunk) input_pipe.close() self._input_pipe = FileWrapper( io.BufferedReader(io.FileIO(self._tmp_file, "r"))) self._process = (command if isinstance(command, subprocess.Popen) else self.create_subprocess(command)) # we want to keep a circular reference to avoid garbage collection # when the object is used in, e.g., pipe.read() self._process._selfref = self
def generate_tmp_path(self, path): return get_local_tempfile(os.path.basename(path))