def _main(self, client, bucket, key, fileobj, extra_args, callbacks, max_attempts, download_output_manager, io_chunksize, start_index=0, bandwidth_limiter=None): """Downloads an object and places content into io queue :param client: The client to use when calling GetObject :param bucket: The bucket to download from :param key: The key to download from :param fileobj: The file handle to write content to :param exta_args: Any extra arguements to include in GetObject request :param callbacks: List of progress callbacks to invoke on download :param max_attempts: The number of retries to do when downloading :param download_output_manager: The download output manager associated with the current download. :param io_chunksize: The size of each io chunk to read from the download stream and queue in the io queue. :param start_index: The location in the file to start writing the content of the key to. :param bandwidth_limiter: The bandwidth limiter to use when throttling the downloading of data in streams. """ last_exception = None for i in range(max_attempts): try: current_index = start_index response = client.get_object( Bucket=bucket, Key=key, **extra_args) streaming_body = StreamReaderProgress( response['Body'], callbacks) if bandwidth_limiter: streaming_body = \ bandwidth_limiter.get_bandwith_limited_stream( streaming_body, self._transfer_coordinator) chunks = DownloadChunkIterator(streaming_body, io_chunksize) for chunk in chunks: # If the transfer is done because of a cancellation # or error somewhere else, stop trying to submit more # data to be written and break out of the download. if not self._transfer_coordinator.done(): self._handle_io( download_output_manager, fileobj, chunk, current_index ) current_index += len(chunk) else: return return except S3_RETRYABLE_DOWNLOAD_ERRORS as e: logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e # Also invoke the progress callbacks to indicate that we # are trying to download the stream again and all progress # for this GetObject has been lost. invoke_progress_callbacks( callbacks, start_index - current_index) continue raise RetriesExceededError(last_exception)
def _download_range(self, bucket, key, filename, part_size, num_parts, callback, part_index): try: range_param = self._calculate_range_param( part_size, part_index, num_parts) max_attempts = self._config.num_download_attempts last_exception = None for i in range(max_attempts): try: logger.debug("Making get_object call.") response = self._client.get_object( Bucket=bucket, Key=key, Range=range_param) streaming_body = StreamReaderProgress( response['Body'], callback) buffer_size = 1024 * 16 current_index = part_size * part_index for chunk in iter(lambda: streaming_body.read(buffer_size), b''): self._ioqueue.put((current_index, chunk)) current_index += len(chunk) return except (socket.timeout, socket.error, ReadTimeoutError, IncompleteReadError) as e: logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e continue raise RetriesExceededError(last_exception) finally: logger.debug("EXITING _download_range for part: %s", part_index)
def _do_get_object(self, bucket, key, extra_args, temp_filename, offset): last_exception = None for i in range(self._MAX_ATTEMPTS): try: response = self._client.get_object( Bucket=bucket, Key=key, **extra_args) self._write_to_file(temp_filename, offset, response['Body']) return except S3_RETRYABLE_DOWNLOAD_ERRORS as e: logger.debug('Retrying exception caught (%s), ' 'retrying request, (attempt %s / %s)', e, i+1, self._MAX_ATTEMPTS, exc_info=True) last_exception = e raise RetriesExceededError(last_exception)
def _get_object(self, bucket, key, filename, extra_args, callback): # precondition: num_download_attempts > 0 max_attempts = self._config.num_download_attempts last_exception = None for i in range(max_attempts): try: return self._do_get_object(bucket, key, filename, extra_args, callback) except (socket.timeout, socket.error, ReadTimeoutError, IncompleteReadError) as e: # TODO: we need a way to reset the callback if the # download failed. logger.debug("Retrying exception caught (%s), " "retrying request, (attempt %s / %s)", e, i, max_attempts, exc_info=True) last_exception = e continue raise RetriesExceededError(last_exception)