def test_enable_disable_callbacks_only_ever_registered_once(self):
body = CallbackEnablingBody()
request = create_request_object({
'method': 'PUT',
'url': 'https://s3.amazonaws.com',
'body': body,
'headers': {},
'context': {}
})
# Create two TransferManager's using the same client
TransferManager(self.client)
TransferManager(self.client)
self.client.meta.events.emit('request-created.s3',
request=request,
operation_name='PutObject')
# The client should have only have the enable/disable callback
# handlers registered once depite being used for two different
# TransferManagers.
self.assertEqual(
body.enable_callback_call_count, 1,
'The enable_callback() should have only ever been registered once')
self.assertEqual(
body.disable_callback_call_count, 1,
'The disable_callback() should have only ever been registered '
'once')
def test_limits_in_memory_chunks_for_fileobj(self):
# Limit the maximum in memory chunks to one but make number of
# threads more than one. This means that the upload will have to
# happen sequentially despite having many threads available because
# data is sequentially partitioned into chunks in memory and since
# there can only every be one in memory chunk, each upload part will
# have to happen one at a time.
self.config.max_request_concurrency = 10
self.config.max_in_memory_upload_chunks = 1
self._manager = TransferManager(self.client, self.config)
# Add some default stubbed responses.
# These responses are added in order of part number so if the
# multipart upload is not done sequentially, which it should because
# we limit the in memory upload chunks to one, the stubber will
# raise exceptions for mismatching parameters for partNumber when
# once the upload() method is called on the transfer manager.
# If there is a mismatch, the stubber error will propogate on
# the future.result()
self.add_create_multipart_response_with_default_expected_params()
self.add_upload_part_responses_with_default_expected_params()
self.add_complete_multipart_response_with_default_expected_params()
with open(self.filename, 'rb') as f:
future = self.manager.upload(f, self.bucket, self.key,
self.extra_args)
future.result()
# Make sure that the stubber had all of its stubbed responses consumed.
self.assert_expected_client_calls_were_correct()
# Ensure the contents were uploaded in sequentially order by checking
# the sent contents were in order.
self.assert_upload_part_bodies_were_correct()
def test_uses_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
self.stream = six.BytesIO(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_head_object_response()
self.add_successful_get_object_responses()
start = time.time()
future = self.manager.download(self.bucket, self.key, self.filename,
self.extra_args)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
# Ensure that the contents are correct
with open(self.filename, 'rb') as f:
self.assertEqual(self.content, f.read())
def __call__(self, client, result_queue):
"""Creates a S3TransferHandler instance
:type client: botocore.client.Client
:param client: The client to power the S3TransferHandler
:type result_queue: queue.Queue
:param result_queue: The result queue to be used to process results
for the S3TransferHandler
:returns: A S3TransferHandler instance
"""
transfer_config = create_transfer_config_from_runtime_config(
self._runtime_config)
transfer_config.max_in_memory_upload_chunks = self.MAX_IN_MEMORY_CHUNKS
transfer_config.max_in_memory_download_chunks = \
self.MAX_IN_MEMORY_CHUNKS
transfer_manager = TransferManager(client, transfer_config)
LOGGER.debug(
"Using a multipart threshold of %s and a part size of %s",
transfer_config.multipart_threshold,
transfer_config.multipart_chunksize
)
result_recorder = ResultRecorder()
result_processor_handlers = [result_recorder]
self._add_result_printer(result_recorder, result_processor_handlers)
result_processor = ResultProcessor(
result_queue, result_processor_handlers)
command_result_recorder = CommandResultRecorder(
result_queue, result_recorder, result_processor)
return S3TransferHandler(
transfer_manager, self._cli_params, command_result_recorder)
def test_cntrl_c_in_context_manager_cancels_incomplete_transfers(self):
# The purpose of this test is to make sure if an error is raised
# in the body of the context manager, incomplete transfers will
# be cancelled with value of the exception wrapped by a CancelledError
# NOTE: The fact that delete() was chosen to test this is arbitrary
# other than it is the easiet to set up for the stubber.
# The specific operation is not important to the purpose of this test.
num_transfers = 100
futures = []
for _ in range(num_transfers):
self.stubber.add_response('delete_object', {})
manager = TransferManager(
self.client,
TransferConfig(max_request_concurrency=1,
max_submission_concurrency=1))
try:
with manager:
for i in range(num_transfers):
futures.append(manager.delete('mybucket', 'mykey'))
raise KeyboardInterrupt()
except KeyboardInterrupt:
# At least one of the submitted futures should have been
# cancelled.
with self.assertRaisesRegexp(CancelledError,
'KeyboardInterrupt()'):
for future in futures:
future.result()
def setUp(self):
super(TestMultipartUpload, self).setUp()
self.chunksize = 4
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=self.chunksize)
self._manager = TransferManager(self.client, self.config)
self.multipart_id = 'my-upload-id'
def test_can_disable_bucket_validation(self):
s3_object_lambda_arn = (
'arn:aws:s3-object-lambda:us-west-2:123456789012:'
'accesspoint:my-accesspoint')
config = TransferConfig()
manager = TransferManager(self.client, config)
manager.VALIDATE_SUPPORTED_BUCKET_VALUES = False
manager.delete(s3_object_lambda_arn, 'my-key')
def __init__(self, bucket, endpoint, id, key, region):
self.bucket = bucket
self.service_endpoint = endpoint
self.aws_access_key_id = id
self.aws_secret_access_key = key
self.region_name = region
self.client = boto3.client(
's3',
endpoint_url=self.service_endpoint,
aws_access_key_id=self.aws_access_key_id,
aws_secret_access_key=self.aws_secret_access_key,
region_name=self.region_name)
self.transfer = TransferManager(self.client, None, None, None)
def test_uses_provided_osutil(self):
osutil = RecordingOSUtils()
# Use the recording os utility for the transfer manager
self._manager = TransferManager(self.client, self.config, osutil)
self.add_put_object_response_with_default_expected_params()
future = self.manager.upload(self.filename, self.bucket, self.key)
future.result()
# The upload should have used the os utility. We check this by making
# sure that the recorded opens are as expected.
self.assertEqual(osutil.open_records, [(self.filename, 'rb')])
def call(self, files):
# There is only ever one file in a stream transfer.
file = files[0]
if self._manager is not None:
manager = self._manager
else:
manager = TransferManager(file.client, self.config)
if file.operation_name == 'upload':
bucket, key = find_bucket_key(file.dest)
return self._upload(manager, bucket, key)
elif file.operation_name == 'download':
bucket, key = find_bucket_key(file.src)
return self._download(manager, bucket, key)
def test_uses_provided_osutil(self):
osutil = RecordingOSUtils()
# Use the recording os utility for the transfer manager
self._manager = TransferManager(self.client, self.config, osutil)
self.add_head_object_response()
self.add_successful_get_object_responses()
future = self.manager.download(**self.create_call_kwargs())
future.result()
# The osutil should have had its open() method invoked when opening
# a temporary file and its rename_file() method invoked when the
# the temporary file was moved to its final location.
self.assertEqual(len(osutil.open_records), 1)
self.assertEqual(len(osutil.rename_records), 1)
def setUp(self):
super(BaseCopyTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.copy_source = {
'Bucket': 'mysourcebucket',
'Key': 'mysourcekey'
}
self.extra_args = {}
self.subscribers = []
self.content = b'my content'
def __init__(self,
s3_client,
bucket_name,
prefix=None,
kms_key_id=None,
force_upload=False,
transfer_manager=None):
self.bucket_name = bucket_name
self.prefix = prefix
self.kms_key_id = kms_key_id or None
self.force_upload = force_upload
self.s3 = s3_client
self.transfer_manager = transfer_manager
if not transfer_manager:
self.transfer_manager = TransferManager(self.s3)
def test_retry_failure(self):
self.add_head_object_response()
max_retries = 3
self.config.num_download_attempts = max_retries
self._manager = TransferManager(self.client, self.config)
# Add responses that fill up the maximum number of retries.
self.add_n_retryable_get_object_responses(max_retries)
future = self.manager.download(**self.create_call_kwargs())
# A retry exceeded error should have happened.
with self.assertRaises(RetriesExceededError):
future.result()
# All of the retries should have been used up.
self.stubber.assert_no_pending_responses()
def setUp(self):
super(BaseCopyTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_chunksize=MIN_UPLOAD_CHUNKSIZE,
multipart_threshold=MIN_UPLOAD_CHUNKSIZE *
4)
self._manager = TransferManager(self.client, self.config)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.copy_source = {'Bucket': 'mysourcebucket', 'Key': 'mysourcekey'}
self.extra_args = {}
self.subscribers = []
self.half_chunksize = int(MIN_UPLOAD_CHUNKSIZE / 2)
self.content = b'0' * (2 * MIN_UPLOAD_CHUNKSIZE + self.half_chunksize)
def __init__(self, client=None, config=None, osutil=None, manager=None):
if not client and not manager:
raise ValueError(
'Either a boto3.Client or s3transfer.manager.TransferManager '
'must be provided')
if manager and any([client, config, osutil]):
raise ValueError(
'Manager cannot be provided with client, config, '
'nor osutil. These parameters are mutually exclusive.')
if config is None:
config = TransferConfig()
if osutil is None:
osutil = OSUtils()
if manager:
self._manager = manager
else:
self._manager = TransferManager(client, config, osutil)
def setUp(self):
super(BaseDownloadTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory to write to
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# Create a stream to read from
self.content = b'my content'
self.stream = six.BytesIO(self.content)
def create_transfer_manager(client, config, osutil=None):
"""Creates a transfer manager based on configuration
:type client: boto3.client
:param client: The S3 client to use
:type config: boto3.s3.transfer.TransferConfig
:param config: The transfer config to use
:type osutil: s3transfer.utils.OSUtils
:param osutil: The os utility to use
:rtype: s3transfer.manager.TransferManager
:returns: A transfer manager based on parameters provided
"""
executor_cls = None
if not config.use_threads:
executor_cls = NonThreadedExecutor
return TransferManager(client, config, osutil, executor_cls)
def test_sigv4_progress_callbacks_invoked_once(self):
# Reset the client and manager to use sigv4
self.reset_stubber_with_new_client(
{'config': Config(signature_version='s3v4')})
self.client.meta.events.register(
'before-parameter-build.s3.*', self.collect_body)
self._manager = TransferManager(self.client, self.config)
# Add the stubbed response.
self.add_put_object_response_with_default_expected_params()
subscriber = RecordingSubscriber()
future = self.manager.upload(
self.filename, self.bucket, self.key, subscribers=[subscriber])
future.result()
self.assert_expected_client_calls_were_correct()
# The amount of bytes seen should be the same as the file size
self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def __call__(
self, client: S3Client, result_queue: "Queue[Any]"
) -> S3TransferHandler:
"""Create a S3TransferHandler instance.
Args:
client: The client to power the S3TransferHandler.
result_queue: The result queue to be used to process results
for the S3TransferHandler.
"""
transfer_config = create_transfer_config_from_runtime_config(
self._runtime_config
)
transfer_config.max_in_memory_upload_chunks = self.MAX_IN_MEMORY_CHUNKS
transfer_config.max_in_memory_download_chunks = self.MAX_IN_MEMORY_CHUNKS
transfer_manager = TransferManager(client, transfer_config)
LOGGER.debug(
"Using a multipart threshold of %s and a part size of %s",
transfer_config.multipart_threshold,
transfer_config.multipart_chunksize,
)
result_recorder = ResultRecorder()
result_processor_handlers: List[Any] = [result_recorder]
self._add_result_printer(result_recorder, result_processor_handlers)
result_processor = ResultProcessor(
result_queue=result_queue, result_handlers=result_processor_handlers
)
command_result_recorder = CommandResultRecorder(
result_queue=result_queue,
result_recorder=result_recorder,
result_processor=result_processor,
)
return S3TransferHandler(
transfer_manager=transfer_manager,
config_params=self._config_params,
result_command_recorder=command_result_recorder,
)
def __init__(self, s3_bucket, s3_key, boto3_s3_client=None):
import boto3
from s3transfer.manager import TransferManager, TransferConfig
self.s3_bucket = s3_bucket
self.s3_key = s3_key
self.s3_client = boto3_s3_client
if self.s3_client is None:
self.s3_client = boto3.client('s3')
self._internal_queue = BlockingReaderWriterByteStream()
self._boto3_multipart_upload_workaround_buffer = b''
self.temp_s3_key = self.s3_key + '-{:0>10}-tmp'.format(
random.randrange(0, 1e10))
# don't start the upload until we've written at least
# boto3.TransferConfig.multipart_threshold bytes
self._transfer_manager = TransferManager(self.s3_client,
TransferConfig())
self._upload_future = None
def setUp(self):
super(BaseUploadTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory with files to read from
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.content = b'my content'
with open(self.filename, 'wb') as f:
f.write(self.content)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.sent_bodies = []
self.client.meta.events.register(
'before-parameter-build.s3.*', self.collect_body)
def test_upload_with_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
with open(self.filename, 'wb') as f:
f.write(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_put_object_response_with_default_expected_params()
start = time.time()
future = self.manager.upload(self.filename, self.bucket, self.key)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
self.assert_expected_client_calls_were_correct()
self.assert_put_object_body_was_correct()
def setUp(self):
super(BaseUploadTest, self).setUp()
# TODO: We do not want to use the real MIN_UPLOAD_CHUNKSIZE
# when we're adjusting parts.
# This is really wasteful and fails CI builds because self.contents
# would normally use 10MB+ of memory.
# Until there's an API to configure this, we're patching this with
# a min size of 1. We can't patch MIN_UPLOAD_CHUNKSIZE directly
# because it's already bound to a default value in the
# chunksize adjuster. Instead we need to patch out the
# chunksize adjuster class.
self.adjuster_patch = mock.patch('s3transfer.upload.ChunksizeAdjuster',
lambda: ChunksizeAdjuster(min_size=1))
self.adjuster_patch.start()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory with files to read from
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.content = b'my content'
with open(self.filename, 'wb') as f:
f.write(self.content)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.sent_bodies = []
self.client.meta.events.register('before-parameter-build.s3.*',
self.collect_body)
def test_config_property(self):
config = TransferConfig()
manager = TransferManager(self.client, config)
self.assertIs(manager.config, config)
def test_use_custom_executor_implementation(self):
mocked_executor_cls = mock.Mock(BaseExecutor)
transfer_manager = TransferManager(self.client,
executor_cls=mocked_executor_cls)
transfer_manager.delete('bucket', 'key')
self.assertTrue(mocked_executor_cls.return_value.submit.called)
def setUp(self):
super(TestRangedDownload, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=4)
self._manager = TransferManager(self.client, self.config)
def test_client_property(self):
manager = TransferManager(self.client)
self.assertIs(manager.client, self.client)
def setUp(self):
self.client = mock.Mock()
self.manager = mock.Mock(TransferManager(self.client))
self.transfer = S3Transfer(manager=self.manager)
self.callback = mock.Mock()
def setUp(self):
super(TestDeleteObject, self).setUp()
self.bucket = 'mybucket'
self.key = 'mykey'
self.manager = TransferManager(self.client)
