def test_invoke_progress_callbacks(self):
recording_subscriber = RecordingSubscriber()
invoke_progress_callbacks([recording_subscriber.on_progress], 2)
self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)
def test_invoke_progress_callbacks_with_no_progress(self):
recording_subscriber = RecordingSubscriber()
invoke_progress_callbacks([recording_subscriber.on_progress], 0)
self.assertEqual(len(recording_subscriber.on_progress_calls), 0)
def _main(self,
client,
bucket,
key,
fileobj,
extra_args,
callbacks,
max_attempts,
download_output_manager,
io_chunksize,
start_index=0):
"""Downloads an object and places content into io queue
:param client: The client to use when calling GetObject
:param bucket: The bucket to download from
:param key: The key to download from
:param fileobj: The file handle to write content to
:param exta_args: Any extra arguements to include in GetObject request
:param callbacks: List of progress callbacks to invoke on download
:param max_attempts: The number of retries to do when downloading
:param download_output_manager: The download output manager associated
with the current download.
:param io_chunksize: The size of each io chunk to read from the
download stream and queue in the io queue.
:param start_index: The location in the file to start writing the
content of the key to.
"""
last_exception = None
for i in range(max_attempts):
try:
response = client.get_object(Bucket=bucket,
Key=key,
**extra_args)
streaming_body = StreamReaderProgress(response['Body'],
callbacks)
current_index = start_index
chunks = DownloadChunkIterator(streaming_body, io_chunksize)
for chunk in chunks:
# If the transfer is done because of a cancellation
# or error somewhere else, stop trying to submit more
# data to be written and break out of the download.
if not self._transfer_coordinator.done():
self._handle_io(download_output_manager, fileobj,
chunk, current_index)
current_index += len(chunk)
else:
return
return
except S3_RETRYABLE_ERRORS as e:
logger.debug(
"Retrying exception caught (%s), "
"retrying request, (attempt %s / %s)",
e,
i,
max_attempts,
exc_info=True)
last_exception = e
# Also invoke the progress callbacks to indicate that we
# are trying to download the stream again and all progress
# for this GetObject has been lost.
invoke_progress_callbacks(callbacks,
start_index - current_index)
continue
raise RetriesExceededError(last_exception)