Пример #1
0
    def _main(self, client, bucket, key, fileobj, extra_args, callbacks,
              max_attempts, download_output_manager, io_chunksize,
              start_index=0, bandwidth_limiter=None):
        """Downloads an object and places content into io queue

        :param client: The client to use when calling GetObject
        :param bucket: The bucket to download from
        :param key: The key to download from
        :param fileobj: The file handle to write content to
        :param exta_args: Any extra arguements to include in GetObject request
        :param callbacks: List of progress callbacks to invoke on download
        :param max_attempts: The number of retries to do when downloading
        :param download_output_manager: The download output manager associated
            with the current download.
        :param io_chunksize: The size of each io chunk to read from the
            download stream and queue in the io queue.
        :param start_index: The location in the file to start writing the
            content of the key to.
        :param bandwidth_limiter: The bandwidth limiter to use when throttling
            the downloading of data in streams.
        """
        last_exception = None
        for i in range(max_attempts):
            try:
                current_index = start_index
                response = client.get_object(
                    Bucket=bucket, Key=key, **extra_args)
                streaming_body = StreamReaderProgress(
                    response['Body'], callbacks)
                if bandwidth_limiter:
                    streaming_body = \
                        bandwidth_limiter.get_bandwith_limited_stream(
                            streaming_body, self._transfer_coordinator)

                chunks = DownloadChunkIterator(streaming_body, io_chunksize)
                for chunk in chunks:
                    # If the transfer is done because of a cancellation
                    # or error somewhere else, stop trying to submit more
                    # data to be written and break out of the download.
                    if not self._transfer_coordinator.done():
                        self._handle_io(
                            download_output_manager, fileobj, chunk,
                            current_index
                        )
                        current_index += len(chunk)
                    else:
                        return
                return
            except S3_RETRYABLE_DOWNLOAD_ERRORS as e:
                logger.debug("Retrying exception caught (%s), "
                             "retrying request, (attempt %s / %s)", e, i,
                             max_attempts, exc_info=True)
                last_exception = e
                # Also invoke the progress callbacks to indicate that we
                # are trying to download the stream again and all progress
                # for this GetObject has been lost.
                invoke_progress_callbacks(
                    callbacks, start_index - current_index)
                continue
        raise RetriesExceededError(last_exception)
Пример #2
0
    def _download_range(self, bucket, key, filename,
                        part_size, num_parts, callback, part_index):
        try:
            range_param = self._calculate_range_param(
                part_size, part_index, num_parts)

            max_attempts = self._config.num_download_attempts
            last_exception = None
            for i in range(max_attempts):
                try:
                    logger.debug("Making get_object call.")
                    response = self._client.get_object(
                        Bucket=bucket, Key=key, Range=range_param)
                    streaming_body = StreamReaderProgress(
                        response['Body'], callback)
                    buffer_size = 1024 * 16
                    current_index = part_size * part_index
                    for chunk in iter(lambda: streaming_body.read(buffer_size),
                                      b''):
                        self._ioqueue.put((current_index, chunk))
                        current_index += len(chunk)
                    return
                except (socket.timeout, socket.error,
                        ReadTimeoutError, IncompleteReadError) as e:
                    logger.debug("Retrying exception caught (%s), "
                                 "retrying request, (attempt %s / %s)", e, i,
                                 max_attempts, exc_info=True)
                    last_exception = e
                    continue
            raise RetriesExceededError(last_exception)
        finally:
            logger.debug("EXITING _download_range for part: %s", part_index)
Пример #3
0
 def _do_get_object(self, bucket, key, extra_args, temp_filename, offset):
     last_exception = None
     for i in range(self._MAX_ATTEMPTS):
         try:
             response = self._client.get_object(
                 Bucket=bucket, Key=key, **extra_args)
             self._write_to_file(temp_filename, offset, response['Body'])
             return
         except S3_RETRYABLE_DOWNLOAD_ERRORS as e:
             logger.debug('Retrying exception caught (%s), '
                          'retrying request, (attempt %s / %s)', e, i+1,
                          self._MAX_ATTEMPTS, exc_info=True)
             last_exception = e
     raise RetriesExceededError(last_exception)
Пример #4
0
 def _get_object(self, bucket, key, filename, extra_args, callback):
     # precondition: num_download_attempts > 0
     max_attempts = self._config.num_download_attempts
     last_exception = None
     for i in range(max_attempts):
         try:
             return self._do_get_object(bucket, key, filename,
                                        extra_args, callback)
         except (socket.timeout, socket.error,
                 ReadTimeoutError, IncompleteReadError) as e:
             # TODO: we need a way to reset the callback if the
             # download failed.
             logger.debug("Retrying exception caught (%s), "
                          "retrying request, (attempt %s / %s)", e, i,
                          max_attempts, exc_info=True)
             last_exception = e
             continue
     raise RetriesExceededError(last_exception)