def test_cntrl_c_in_context_manager_cancels_incomplete_transfers(self):
# The purpose of this test is to make sure if an error is raised
# in the body of the context manager, incomplete transfers will
# be cancelled with value of the exception wrapped by a CancelledError
# NOTE: The fact that delete() was chosen to test this is arbitrary
# other than it is the easiet to set up for the stubber.
# The specific operation is not important to the purpose of this test.
num_transfers = 100
futures = []
for _ in range(num_transfers):
self.stubber.add_response('delete_object', {})
manager = TransferManager(
self.client,
TransferConfig(max_request_concurrency=1,
max_submission_concurrency=1))
try:
with manager:
for i in range(num_transfers):
futures.append(manager.delete('mybucket', 'mykey'))
raise KeyboardInterrupt()
except KeyboardInterrupt:
# At least one of the submitted futures should have been
# cancelled.
with self.assertRaisesRegexp(CancelledError,
'KeyboardInterrupt()'):
for future in futures:
future.result()
def test_enable_disable_callbacks_only_ever_registered_once(self):
body = SignalTransferringBody()
request = create_request_object({
'method': 'PUT',
'url': 'https://s3.amazonaws.com',
'body': body,
'headers': {},
'context': {}
})
# Create two TransferManager's using the same client
TransferManager(self.client)
TransferManager(self.client)
self.client.meta.events.emit('request-created.s3',
request=request,
operation_name='PutObject')
# The client should have only have the enable/disable callback
# handlers registered once depite being used for two different
# TransferManagers.
self.assertEqual(
body.signal_transferring_call_count, 1,
'The enable_callback() should have only ever been registered once')
self.assertEqual(
body.signal_not_transferring_call_count, 1,
'The disable_callback() should have only ever been registered '
'once')
def test_uses_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
self.stream = six.BytesIO(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_head_object_response()
self.add_successful_get_object_responses()
start = time.time()
future = self.manager.download(self.bucket, self.key, self.filename,
self.extra_args)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
# Ensure that the contents are correct
with open(self.filename, 'rb') as f:
self.assertEqual(self.content, f.read())
def test_limits_in_memory_chunks_for_fileobj(self):
# Limit the maximum in memory chunks to one but make number of
# threads more than one. This means that the upload will have to
# happen sequentially despite having many threads available because
# data is sequentially partitioned into chunks in memory and since
# there can only every be one in memory chunk, each upload part will
# have to happen one at a time.
self.config.max_request_concurrency = 10
self.config.max_in_memory_upload_chunks = 1
self._manager = TransferManager(self.client, self.config)
# Add some default stubbed responses.
# These responses are added in order of part number so if the
# multipart upload is not done sequentially, which it should because
# we limit the in memory upload chunks to one, the stubber will
# raise exceptions for mismatching parameters for partNumber when
# once the upload() method is called on the transfer manager.
# If there is a mismatch, the stubber error will propogate on
# the future.result()
self.add_create_multipart_response_with_default_expected_params()
self.add_upload_part_responses_with_default_expected_params()
self.add_complete_multipart_response_with_default_expected_params()
with open(self.filename, 'rb') as f:
future = self.manager.upload(f, self.bucket, self.key,
self.extra_args)
future.result()
# Make sure that the stubber had all of its stubbed responses consumed.
self.assert_expected_client_calls_were_correct()
# Ensure the contents were uploaded in sequentially order by checking
# the sent contents were in order.
self.assert_upload_part_bodies_were_correct()
def setUp(self):
super(TestMultipartUpload, self).setUp()
self.chunksize = 4
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=self.chunksize)
self._manager = TransferManager(self.client, self.config)
self.multipart_id = 'my-upload-id'
def test_uses_provided_osutil(self):
osutil = RecordingOSUtils()
# Use the recording os utility for the transfer manager
self._manager = TransferManager(self.client, self.config, osutil)
self.add_put_object_response_with_default_expected_params()
future = self.manager.upload(self.filename, self.bucket, self.key)
future.result()
# The upload should have used the os utility. We check this by making
# sure that the recorded opens are as expected.
expected_opens = [(self.filename, 'rb')]
self.assertEqual(osutil.open_records, expected_opens)
def test_uses_provided_osutil(self):
osutil = RecordingOSUtils()
# Use the recording os utility for the transfer manager
self._manager = TransferManager(self.client, self.config, osutil)
self.add_head_object_response()
self.add_successful_get_object_responses()
future = self.manager.download(**self.create_call_kwargs())
future.result()
# The osutil should have had its open() method invoked when opening
# a temporary file and its rename_file() method invoked when the
# the temporary file was moved to its final location.
self.assertEqual(len(osutil.open_records), 1)
self.assertEqual(len(osutil.rename_records), 1)
def test_retry_failure(self):
self.add_head_object_response()
max_retries = 3
self.config.num_download_attempts = max_retries
self._manager = TransferManager(self.client, self.config)
# Add responses that fill up the maximum number of retries.
self.add_n_retryable_get_object_responses(max_retries)
future = self.manager.download(**self.create_call_kwargs())
# A retry exceeded error should have happened.
with self.assertRaises(RetriesExceededError):
future.result()
# All of the retries should have been used up.
self.stubber.assert_no_pending_responses()
def setUp(self):
super(BaseCopyTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_chunksize=MIN_UPLOAD_CHUNKSIZE,
multipart_threshold=MIN_UPLOAD_CHUNKSIZE *
4)
self._manager = TransferManager(self.client, self.config)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.copy_source = {'Bucket': 'mysourcebucket', 'Key': 'mysourcekey'}
self.extra_args = {}
self.subscribers = []
self.half_chunksize = int(MIN_UPLOAD_CHUNKSIZE / 2)
self.content = b'0' * (2 * MIN_UPLOAD_CHUNKSIZE + self.half_chunksize)
def setUp(self):
super(BaseDownloadTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory to write to
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# Create a stream to read from
self.content = b'my content'
self.stream = six.BytesIO(self.content)
def create_transfer_manager(client, config, osutil=None):
"""Creates a transfer manager based on configuration
:type client: ibm_boto3.client
:param client: The S3 client to use
:type config: ibm_boto3.s3.transfer.TransferConfig
:param config: The transfer config to use
:type osutil: ibm_s3transfer.utils.OSUtils
:param osutil: The os utility to use
:rtype: ibm_s3transfer.manager.TransferManager
:returns: A transfer manager based on parameters provided
"""
executor_cls = None
if not config.use_threads:
executor_cls = NonThreadedExecutor
return TransferManager(client, config, osutil, executor_cls)
def test_sigv4_progress_callbacks_invoked_once(self):
# Reset the client and manager to use sigv4
self.reset_stubber_with_new_client(
{'config': Config(signature_version='s3v4')})
self.client.meta.events.register('before-parameter-build.s3.*',
self.collect_body)
self._manager = TransferManager(self.client, self.config)
# Add the stubbed response.
self.add_put_object_response_with_default_expected_params()
subscriber = RecordingSubscriber()
future = self.manager.upload(self.filename,
self.bucket,
self.key,
subscribers=[subscriber])
future.result()
self.assert_expected_client_calls_were_correct()
# The amount of bytes seen should be the same as the file size
self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def setUp(self):
super(BaseUploadTest, self).setUp()
# TODO: We do not want to use the real MIN_UPLOAD_CHUNKSIZE
# when we're adjusting parts.
# This is really wasteful and fails CI builds because self.contents
# would normally use 10MB+ of memory.
# Until there's an API to configure this, we're patching this with
# a min size of 1. We can't patch MIN_UPLOAD_CHUNKSIZE directly
# because it's already bound to a default value in the
# chunksize adjuster. Instead we need to patch out the
# chunksize adjuster class.
self.adjuster_patch = mock.patch(
'ibm_s3transfer.upload.ChunksizeAdjuster',
lambda: ChunksizeAdjuster(min_size=1))
self.adjuster_patch.start()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory with files to read from
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.content = b'my content'
with open(self.filename, 'wb') as f:
f.write(self.content)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.sent_bodies = []
self.client.meta.events.register('before-parameter-build.s3.*',
self.collect_body)
def test_upload_with_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
with open(self.filename, 'wb') as f:
f.write(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_put_object_response_with_default_expected_params()
start = time.time()
future = self.manager.upload(self.filename, self.bucket, self.key)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
self.assert_expected_client_calls_were_correct()
self.assert_put_object_body_was_correct()
def test_use_custom_executor_implementation(self):
mocked_executor_cls = mock.Mock(BaseExecutor)
transfer_manager = TransferManager(self.client,
executor_cls=mocked_executor_cls)
transfer_manager.delete('bucket', 'key')
self.assertTrue(mocked_executor_cls.return_value.submit.called)
def setUp(self):
super(TestDeleteObject, self).setUp()
self.bucket = 'mybucket'
self.key = 'mykey'
self.manager = TransferManager(self.client)
def setUp(self):
self.client = mock.Mock()
self.manager = mock.Mock(TransferManager(self.client))
self.transfer = S3Transfer(manager=self.manager)
self.callback = mock.Mock()
def create_transfer_manager(self, config=None):
return TransferManager(self.client, config=config)
def setUp(self):
super(TestRangedDownload, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=4)
self._manager = TransferManager(self.client, self.config)
def test_unicode_exception_in_context_manager(self):
with self.assertRaises(ArbitraryException):
with TransferManager(self.client):
raise ArbitraryException(u'\u2713')
def create_transfer_manager(credentials):
cos_client = create_client(credentials)
return TransferManager(cos_client)
