def test_retry_on_failures_from_stream_reads(self): # If we get an exception during a call to the response body's .read() # method, we should retry the request. client = mock.Mock() response_body = b'foobarbaz' stream_with_errors = mock.Mock() stream_with_errors.read.side_effect = [ socket.error("fake error"), response_body ] client.get_object.return_value = {'Body': stream_with_errors} config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) downloader = MultipartDownloader(client, config, InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) # We're storing these in **extra because the assertEqual # below is really about verifying we have the correct value # for the Range param. extra = {'Bucket': 'bucket', 'Key': 'key'} self.assertEqual( client.get_object.call_args_list, # The first call to range=0-3 fails because of the # side_effect above where we make the .read() raise a # socket.error. # The second call to range=0-3 then succeeds. [ mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=4-7', **extra), mock.call(Range='bytes=8-', **extra) ])
def test_retry_on_failures_from_stream_reads(self): # If we get an exception during a call to the response body's .read() # method, we should retry the request. client = mock.Mock() response_body = b'foobarbaz' stream_with_errors = mock.Mock() stream_with_errors.read.side_effect = [ socket.error("fake error"), response_body ] client.get_object.return_value = {'Body': stream_with_errors} config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) downloader = MultipartDownloader(client, config, InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) # We're storing these in **extra because the assertEqual # below is really about verifying we have the correct value # for the Range param. extra = {'Bucket': 'bucket', 'Key': 'key'} self.assertEqual(client.get_object.call_args_list, # The first call to range=0-3 fails because of the # side_effect above where we make the .read() raise a # socket.error. # The second call to range=0-3 then succeeds. [mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=4-7', **extra), mock.call(Range='bytes=8-', **extra)])
def test_multipart_download_with_multiple_parts(self): client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} # For testing purposes, we're testing with a multipart threshold # of 4 bytes and a chunksize of 4 bytes. Given b'foobarbaz', # this should result in 3 calls. In python slices this would be: # r[0:4], r[4:8], r[8:9]. But the Range param will be slightly # different because they use inclusive ranges. config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) downloader = MultipartDownloader(client, config, InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) # We're storing these in **extra because the assertEqual # below is really about verifying we have the correct value # for the Range param. extra = {'Bucket': 'bucket', 'Key': 'key'} self.assertEqual( client.get_object.call_args_list, # Note these are inclusive ranges. [ mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=4-7', **extra), mock.call(Range='bytes=8-', **extra) ])
def test_multipart_download_with_multiple_parts(self): client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} # For testing purposes, we're testing with a multipart threshold # of 4 bytes and a chunksize of 4 bytes. Given b'foobarbaz', # this should result in 3 calls. In python slices this would be: # r[0:4], r[4:8], r[8:9]. But the Range param will be slightly # different because they use inclusive ranges. config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) downloader = MultipartDownloader(client, config, InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) # We're storing these in **extra because the assertEqual # below is really about verifying we have the correct value # for the Range param. extra = {'Bucket': 'bucket', 'Key': 'key'} self.assertEqual(client.get_object.call_args_list, # Note these are inclusive ranges. [mock.call(Range='bytes=0-3', **extra), mock.call(Range='bytes=4-7', **extra), mock.call(Range='bytes=8-', **extra)])
def test_io_thread_fails_to_open_triggers_shutdown_error(self): client = mock.Mock() client.get_object.return_value = {'Body': six.BytesIO(b'asdf')} os_layer = mock.Mock(spec=OSUtils) os_layer.open.side_effect = IOError("Can't open file") downloader = MultipartDownloader(client, TransferConfig(), os_layer, SequentialExecutor) # We're verifying that the exception raised from the IO future # propogates back up via download_file(). with self.assertRaisesRegexp(IOError, "Can't open file"): downloader.download_file('bucket', 'key', 'filename', len(b'asdf'), {})
def test_exception_raised_on_exceeded_retries(self): client = mock.Mock() response_body = b'foobarbaz' stream_with_errors = mock.Mock() stream_with_errors.read.side_effect = socket.error("fake error") client.get_object.return_value = {'Body': stream_with_errors} config = TransferConfig(multipart_threshold=4, multipart_chunksize=4) downloader = MultipartDownloader(client, config, InMemoryOSLayer({}), SequentialExecutor) with self.assertRaises(RetriesExceededError): downloader.download_file('bucket', 'key', 'filename', len(response_body), {})
def test_multipart_download_uses_correct_client_calls(self): client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} downloader = MultipartDownloader(client, TransferConfig(), InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) client.get_object.assert_called_with(Range='bytes=0-', Bucket='bucket', Key='key')
def test_io_thread_fails_to_open_triggers_shutdown_error(self): client = mock.Mock() client.get_object.return_value = { 'Body': six.BytesIO(b'asdf') } os_layer = mock.Mock(spec=OSUtils) os_layer.open.side_effect = IOError("Can't open file") downloader = MultipartDownloader( client, TransferConfig(), os_layer, SequentialExecutor) # We're verifying that the exception raised from the IO future # propogates back up via download_file(). with self.assertRaisesRegexp(IOError, "Can't open file"): downloader.download_file('bucket', 'key', 'filename', len(b'asdf'), {})
def test_multipart_download_uses_correct_client_calls(self): client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} downloader = MultipartDownloader(client, TransferConfig(), InMemoryOSLayer({}), SequentialExecutor) downloader.download_file('bucket', 'key', 'filename', len(response_body), {}) client.get_object.assert_called_with( Range='bytes=0-', Bucket='bucket', Key='key' )
def test_multipart_download_with_multiple_parts_and_extra_args(self): client = Session().create_client('s3') stubber = Stubber(client) response_body = b'foobarbaz' response = {'Body': six.BytesIO(response_body)} expected_params = { 'Range': mock.ANY, 'Bucket': mock.ANY, 'Key': mock.ANY, 'RequestPayer': 'requester'} stubber.add_response('get_object', response, expected_params) stubber.activate() downloader = MultipartDownloader( client, TransferConfig(), InMemoryOSLayer({}), SequentialExecutor) downloader.download_file( 'bucket', 'key', 'filename', len(response_body), {'RequestPayer': 'requester'}) stubber.assert_no_pending_responses()
def test_io_thread_failure_triggers_shutdown(self): client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} os_layer = mock.Mock() mock_fileobj = mock.MagicMock() mock_fileobj.__enter__.return_value = mock_fileobj mock_fileobj.write.side_effect = Exception("fake IO error") os_layer.open.return_value = mock_fileobj downloader = MultipartDownloader(client, TransferConfig(), os_layer, SequentialExecutor) # We're verifying that the exception raised from the IO future # propogates back up via download_file(). with self.assertRaisesRegexp(Exception, "fake IO error"): downloader.download_file('bucket', 'key', 'filename', len(response_body), {})
def test_download_futures_fail_triggers_shutdown(self): class FailedDownloadParts(SequentialExecutor): def __init__(self, max_workers): self.is_first = True def submit(self, function): future = super(FailedDownloadParts, self).submit(function) if self.is_first: # This is the download_parts_thread. future.set_exception( Exception("fake download parts error")) self.is_first = False return future client = mock.Mock() response_body = b'foobarbaz' client.get_object.return_value = {'Body': six.BytesIO(response_body)} downloader = MultipartDownloader(client, TransferConfig(), InMemoryOSLayer({}), FailedDownloadParts) with self.assertRaisesRegexp(Exception, "fake download parts error"): downloader.download_file('bucket', 'key', 'filename', len(response_body), {})