def test_multipart_download_with_multiple_parts(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
# For testing purposes, we're testing with a multipart threshold
# of 4 bytes and a chunksize of 4 bytes. Given b'foobarbaz',
# this should result in 3 calls. In python slices this would be:
# r[0:4], r[4:8], r[8:9]. But the Range param will be slightly
# different because they use inclusive ranges.
config = TransferConfig(multipart_threshold=4, multipart_chunksize=4)
downloader = MultipartDownloader(client, config, InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
# We're storing these in **extra because the assertEqual
# below is really about verifying we have the correct value
# for the Range param.
extra = {'Bucket': 'bucket', 'Key': 'key'}
self.assertEqual(
client.get_object.call_args_list,
# Note these are inclusive ranges.
[
mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=4-7', **extra),
mock.call(Range='bytes=8-', **extra)
])
def test_retry_on_failures_from_stream_reads(self):
# If we get an exception during a call to the response body's .read()
# method, we should retry the request.
client = mock.Mock()
response_body = b'foobarbaz'
stream_with_errors = mock.Mock()
stream_with_errors.read.side_effect = [
socket.error("fake error"), response_body
]
client.get_object.return_value = {'Body': stream_with_errors}
config = TransferConfig(multipart_threshold=4, multipart_chunksize=4)
downloader = MultipartDownloader(client, config, InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
# We're storing these in **extra because the assertEqual
# below is really about verifying we have the correct value
# for the Range param.
extra = {'Bucket': 'bucket', 'Key': 'key'}
self.assertEqual(
client.get_object.call_args_list,
# The first call to range=0-3 fails because of the
# side_effect above where we make the .read() raise a
# socket.error.
# The second call to range=0-3 then succeeds.
[
mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=4-7', **extra),
mock.call(Range='bytes=8-', **extra)
])
def test_retry_on_failures_from_stream_reads(self):
# If we get an exception during a call to the response body's .read()
# method, we should retry the request.
client = mock.Mock()
response_body = b'foobarbaz'
stream_with_errors = mock.Mock()
stream_with_errors.read.side_effect = [
socket.error("fake error"),
response_body
]
client.get_object.return_value = {'Body': stream_with_errors}
config = TransferConfig(multipart_threshold=4,
multipart_chunksize=4)
downloader = MultipartDownloader(client, config,
InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
# We're storing these in **extra because the assertEqual
# below is really about verifying we have the correct value
# for the Range param.
extra = {'Bucket': 'bucket', 'Key': 'key'}
self.assertEqual(client.get_object.call_args_list,
# The first call to range=0-3 fails because of the
# side_effect above where we make the .read() raise a
# socket.error.
# The second call to range=0-3 then succeeds.
[mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=4-7', **extra),
mock.call(Range='bytes=8-', **extra)])
def test_multipart_download_with_multiple_parts(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
# For testing purposes, we're testing with a multipart threshold
# of 4 bytes and a chunksize of 4 bytes. Given b'foobarbaz',
# this should result in 3 calls. In python slices this would be:
# r[0:4], r[4:8], r[8:9]. But the Range param will be slightly
# different because they use inclusive ranges.
config = TransferConfig(multipart_threshold=4,
multipart_chunksize=4)
downloader = MultipartDownloader(client, config,
InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
# We're storing these in **extra because the assertEqual
# below is really about verifying we have the correct value
# for the Range param.
extra = {'Bucket': 'bucket', 'Key': 'key'}
self.assertEqual(client.get_object.call_args_list,
# Note these are inclusive ranges.
[mock.call(Range='bytes=0-3', **extra),
mock.call(Range='bytes=4-7', **extra),
mock.call(Range='bytes=8-', **extra)])
def test_exception_raised_on_exceeded_retries(self):
client = mock.Mock()
response_body = b'foobarbaz'
stream_with_errors = mock.Mock()
stream_with_errors.read.side_effect = socket.error("fake error")
client.get_object.return_value = {'Body': stream_with_errors}
config = TransferConfig(multipart_threshold=4, multipart_chunksize=4)
downloader = MultipartDownloader(client, config, InMemoryOSLayer({}),
SequentialExecutor)
with self.assertRaises(RetriesExceededError):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
def test_multipart_download_uses_correct_client_calls(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
downloader = MultipartDownloader(client, TransferConfig(),
InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
client.get_object.assert_called_with(Range='bytes=0-',
Bucket='bucket',
Key='key')
def test_exception_raised_on_exceeded_retries(self):
client = mock.Mock()
response_body = b'foobarbaz'
stream_with_errors = mock.Mock()
stream_with_errors.read.side_effect = socket.error("fake error")
client.get_object.return_value = {'Body': stream_with_errors}
config = TransferConfig(multipart_threshold=4,
multipart_chunksize=4)
downloader = MultipartDownloader(client, config,
InMemoryOSLayer({}),
SequentialExecutor)
with self.assertRaises(RetriesExceededError):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
def test_multipart_download_uses_correct_client_calls(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
downloader = MultipartDownloader(client, TransferConfig(),
InMemoryOSLayer({}),
SequentialExecutor)
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
client.get_object.assert_called_with(
Range='bytes=0-',
Bucket='bucket',
Key='key'
)
def test_io_thread_failure_triggers_shutdown(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
os_layer = mock.Mock()
mock_fileobj = mock.MagicMock()
mock_fileobj.__enter__.return_value = mock_fileobj
mock_fileobj.write.side_effect = Exception("fake IO error")
os_layer.open.return_value = mock_fileobj
downloader = MultipartDownloader(client, TransferConfig(), os_layer,
SequentialExecutor)
# We're verifying that the exception raised from the IO future
# propogates back up via download_file().
with self.assertRaisesRegexp(Exception, "fake IO error"):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
def test_io_thread_failure_triggers_shutdown(self):
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
os_layer = mock.Mock()
mock_fileobj = mock.MagicMock()
mock_fileobj.__enter__.return_value = mock_fileobj
mock_fileobj.write.side_effect = Exception("fake IO error")
os_layer.open.return_value = mock_fileobj
downloader = MultipartDownloader(client, TransferConfig(),
os_layer, SequentialExecutor)
# We're verifying that the exception raised from the IO future
# propogates back up via download_file().
with self.assertRaisesRegexp(Exception, "fake IO error"):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
def test_download_futures_fail_triggers_shutdown(self):
class FailedDownloadParts(SequentialExecutor):
def __init__(self, max_workers):
self.is_first = True
def submit(self, function):
future = futures.Future()
if self.is_first:
# This is the download_parts_thread.
future.set_exception(
Exception("fake download parts error"))
self.is_first = False
return future
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
downloader = MultipartDownloader(client, TransferConfig(),
InMemoryOSLayer({}),
FailedDownloadParts)
with self.assertRaisesRegexp(Exception, "fake download parts error"):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})
def test_download_futures_fail_triggers_shutdown(self):
class FailedDownloadParts(SequentialExecutor):
def __init__(self, max_workers):
self.is_first = True
def submit(self, function):
future = super(FailedDownloadParts, self).submit(function)
if self.is_first:
# This is the download_parts_thread.
future.set_exception(
Exception("fake download parts error"))
self.is_first = False
return future
client = mock.Mock()
response_body = b'foobarbaz'
client.get_object.return_value = {'Body': six.BytesIO(response_body)}
downloader = MultipartDownloader(client, TransferConfig(),
InMemoryOSLayer({}),
FailedDownloadParts)
with self.assertRaisesRegexp(Exception, "fake download parts error"):
downloader.download_file('bucket', 'key', 'filename',
len(response_body), {})