def test_uses_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
self.stream = six.BytesIO(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_head_object_response()
self.add_successful_get_object_responses()
start = time.time()
future = self.manager.download(self.bucket, self.key, self.filename,
self.extra_args)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
# Ensure that the contents are correct
with open(self.filename, 'rb') as f:
self.assertEqual(self.content, f.read())
def test_cntrl_c_in_context_manager_cancels_incomplete_transfers(self):
# The purpose of this test is to make sure if an error is raised
# in the body of the context manager, incomplete transfers will
# be cancelled with value of the exception wrapped by a CancelledError
# NOTE: The fact that delete() was chosen to test this is arbitrary
# other than it is the easiet to set up for the stubber.
# The specific operation is not important to the purpose of this test.
num_transfers = 100
futures = []
for _ in range(num_transfers):
self.stubber.add_response('delete_object', {})
manager = TransferManager(
self.client,
TransferConfig(max_request_concurrency=1,
max_submission_concurrency=1))
try:
with manager:
for i in range(num_transfers):
futures.append(manager.delete('mybucket', 'mykey'))
raise KeyboardInterrupt()
except KeyboardInterrupt:
# At least one of the submitted futures should have been
# cancelled.
with self.assertRaisesRegexp(CancelledError,
'KeyboardInterrupt()'):
for future in futures:
future.result()
def test_can_disable_bucket_validation(self):
s3_object_lambda_arn = (
'arn:aws:s3-object-lambda:us-west-2:123456789012:'
'accesspoint:my-accesspoint')
config = TransferConfig()
manager = TransferManager(self.client, config)
manager.VALIDATE_SUPPORTED_BUCKET_VALUES = False
manager.delete(s3_object_lambda_arn, 'my-key')
def setUp(self):
super(TestMultipartUpload, self).setUp()
self.chunksize = 4
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=self.chunksize)
self._manager = TransferManager(self.client, self.config)
self.multipart_id = 'my-upload-id'
def setUp(self):
self.client = mock.Mock()
self.transfer_config = TransferConfig()
self.cli_params = {}
self.source_bucket = 'source-bucket'
self.source_key = 'source-key'
self.bucket = 'bucket'
self.key = 'key'
self.future = self.get_transfer_future()
def setUp(self):
super(BaseSubmissionTaskTest, self).setUp()
self.config = TransferConfig()
self.osutil = OSUtils()
self.executor = BoundedExecutor(
1000, 1, {
IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10),
IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10)
})
def setUp(self):
super(BaseCopyTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.copy_source = {
'Bucket': 'mysourcebucket',
'Key': 'mysourcekey'
}
self.extra_args = {}
self.subscribers = []
self.content = b'my content'
def setUp(self):
super(BaseCopyTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_chunksize=MIN_UPLOAD_CHUNKSIZE,
multipart_threshold=MIN_UPLOAD_CHUNKSIZE *
4)
self._manager = TransferManager(self.client, self.config)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.copy_source = {'Bucket': 'mysourcebucket', 'Key': 'mysourcekey'}
self.extra_args = {}
self.subscribers = []
self.half_chunksize = int(MIN_UPLOAD_CHUNKSIZE / 2)
self.content = b'0' * (2 * MIN_UPLOAD_CHUNKSIZE + self.half_chunksize)
def setUp(self):
super(BaseDownloadTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory to write to
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# Create a stream to read from
self.content = b'my content'
self.stream = six.BytesIO(self.content)
def __init__(self, s3_bucket, s3_key, boto3_s3_client=None):
import boto3
from s3transfer.manager import TransferManager, TransferConfig
self.s3_bucket = s3_bucket
self.s3_key = s3_key
self.s3_client = boto3_s3_client
if self.s3_client is None:
self.s3_client = boto3.client('s3')
self._internal_queue = BlockingReaderWriterByteStream()
self._boto3_multipart_upload_workaround_buffer = b''
self.temp_s3_key = self.s3_key + '-{:0>10}-tmp'.format(
random.randrange(0, 1e10))
# don't start the upload until we've written at least
# boto3.TransferConfig.multipart_threshold bytes
self._transfer_manager = TransferManager(self.s3_client,
TransferConfig())
self._upload_future = None
def create_transfer_config_from_runtime_config(runtime_config):
"""
Creates an equivalent s3transfer TransferConfig
:type runtime_config: dict
:argument runtime_config: A valid RuntimeConfig-generated dict.
:returns: A TransferConfig with the same configuration as the runtime
config.
"""
translation_map = {
'max_concurrent_requests': 'max_request_concurrency',
'max_queue_size': 'max_request_queue_size'
}
kwargs = {}
for key, value in runtime_config.items():
new_key = translation_map.get(key, key)
kwargs[new_key] = value
return TransferConfig(**kwargs)
def create_transfer_config_from_runtime_config(runtime_config):
"""
Creates an equivalent s3transfer TransferConfig
:type runtime_config: dict
:argument runtime_config: A valid RuntimeConfig-generated dict.
:returns: A TransferConfig with the same configuration as the runtime
config.
"""
translation_map = {
'max_concurrent_requests': 'max_request_concurrency',
'max_queue_size': 'max_request_queue_size',
'multipart_threshold': 'multipart_threshold',
'multipart_chunksize': 'multipart_chunksize',
}
kwargs = {}
for key, value in runtime_config.items():
if key not in translation_map:
continue
kwargs[translation_map[key]] = value
return TransferConfig(**kwargs)
def setUp(self):
super(BaseUploadTest, self).setUp()
# TODO: We do not want to use the real MIN_UPLOAD_CHUNKSIZE
# when we're adjusting parts.
# This is really wasteful and fails CI builds because self.contents
# would normally use 10MB+ of memory.
# Until there's an API to configure this, we're patching this with
# a min size of 1. We can't patch MIN_UPLOAD_CHUNKSIZE directly
# because it's already bound to a default value in the
# chunksize adjuster. Instead we need to patch out the
# chunksize adjuster class.
self.adjuster_patch = mock.patch('s3transfer.upload.ChunksizeAdjuster',
lambda: ChunksizeAdjuster(min_size=1))
self.adjuster_patch.start()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory with files to read from
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.content = b'my content'
with open(self.filename, 'wb') as f:
f.write(self.content)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.sent_bodies = []
self.client.meta.events.register('before-parameter-build.s3.*',
self.collect_body)
def test_upload_with_bandwidth_limiter(self):
self.content = b'a' * 1024 * 1024
with open(self.filename, 'wb') as f:
f.write(self.content)
self.config = TransferConfig(max_request_concurrency=1,
max_bandwidth=len(self.content) / 2)
self._manager = TransferManager(self.client, self.config)
self.add_put_object_response_with_default_expected_params()
start = time.time()
future = self.manager.upload(self.filename, self.bucket, self.key)
future.result()
# This is just a smoke test to make sure that the limiter is
# being used and not necessary its exactness. So we set the maximum
# bandwidth to len(content)/2 per sec and make sure that it is
# noticeably slower. Ideally it will take more than two seconds, but
# given tracking at the beginning of transfers are not entirely
# accurate setting at the initial start of a transfer, we give us
# some flexibility by setting the expected time to half of the
# theoretical time to take.
self.assertGreaterEqual(time.time() - start, 1)
self.assert_expected_client_calls_were_correct()
self.assert_put_object_body_was_correct()
def setUp(self):
super(BaseUploadTest, self).setUp()
self.config = TransferConfig(max_request_concurrency=1)
self._manager = TransferManager(self.client, self.config)
# Create a temporary directory with files to read from
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.content = b'my content'
with open(self.filename, 'wb') as f:
f.write(self.content)
# Initialize some default arguments
self.bucket = 'mybucket'
self.key = 'mykey'
self.extra_args = {}
self.subscribers = []
# A list to keep track of all of the bodies sent over the wire
# and their order.
self.sent_bodies = []
self.client.meta.events.register(
'before-parameter-build.s3.*', self.collect_body)
def create_transfer_config_from_runtime_config(
runtime_config: TransferConfigDict, ) -> TransferConfig:
"""Create an equivalent s3transfer TransferConfig.
Args:
runtime_config: A valid RuntimeConfig-generated dict.
Returns:
A TransferConfig with the same configuration as the runtime config.
"""
translation_map = {
"max_bandwidth": "max_bandwidth",
"max_concurrent_requests": "max_request_concurrency",
"max_queue_size": "max_request_queue_size",
"multipart_chunksize": "multipart_chunksize",
"multipart_threshold": "multipart_threshold",
}
kwargs: Dict[str, Any] = {}
for key, value in runtime_config.items():
if key not in translation_map:
continue
kwargs[translation_map[key]] = value
return TransferConfig(**kwargs)
def copy(self,
CopySource,
Bucket,
Key,
ExtraArgs=None,
Callback=None,
SourceClient=None,
Config=None):
"""Copy an object from one S3 location to another.
This is a managed transfer which will perform a multipart copy in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.resource('s3')
copy_source = {
'Bucket': 'mybucket',
'Key': 'mykey'
}
s3.meta.client.copy(copy_source, 'otherbucket', 'otherkey')
:type CopySource: dict
:param CopySource: The name of the source bucket, key name of the
source object, and optional version ID of the source object. The
dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
that the ``VersionId`` key is optional and may be omitted.
:type Bucket: str
:param Bucket: The name of the bucket to copy to
:type Key: str
:param Key: The name of the key to copy to
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation
:type Callback: method
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the copy.
:type SourceClient: botocore or boto3 Client
:param SourceClient: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the current client is used as the client
for the source object.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
copy.
"""
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with TransferManager(self, config) as manager:
future = manager.copy(copy_source=CopySource,
bucket=Bucket,
key=Key,
extra_args=ExtraArgs,
subscribers=subscribers,
source_client=SourceClient)
return future.result()
def setUp(self):
super(TestRangedDownload, self).setUp()
self.config = TransferConfig(max_request_concurrency=1,
multipart_threshold=1,
multipart_chunksize=4)
self._manager = TransferManager(self.client, self.config)
def download_fileobj(self,
Bucket,
Key,
Fileobj,
ExtraArgs=None,
Callback=None,
Config=None):
"""Download an object from S3 to a file-like object.
The file-like object must be in binary mode.
This is a managed transfer which will perform a multipart download in
multiple threads if necessary.
Usage::
import boto3
s3 = boto3.client('s3')
with open('filename', 'wb') as data:
s3.download_fileobj('mybucket', 'mykey', data)
:type Fileobj: a file-like object
:param Fileobj: A file-like object to upload. At a minimum, it must
implement the `write` method and must accept bytes.
:type Bucket: str
:param Bucket: The name of the bucket to download from.
:type Key: str
:param Key: The name of the key to download from.
:type ExtraArgs: dict
:param ExtraArgs: Extra arguments that may be passed to the
client operation.
:type Callback: method
:param Callback: A method which takes a number of bytes transferred to
be periodically called during the download.
:type Config: boto3.s3.transfer.TransferConfig
:param Config: The transfer configuration to be used when performing the
download.
"""
if not hasattr(Fileobj, 'write'):
raise ValueError('Fileobj must implement write')
subscribers = None
if Callback is not None:
subscribers = [ProgressCallbackInvoker(Callback)]
config = Config
if config is None:
config = TransferConfig()
with TransferManager(self, config) as manager:
future = manager.download(bucket=Bucket,
key=Key,
fileobj=Fileobj,
extra_args=ExtraArgs,
subscribers=subscribers)
return future.result()
def test_config_property(self):
config = TransferConfig()
manager = TransferManager(self.client, config)
self.assertIs(manager.config, config)
def setUp(self):
super(TestDownload, self).setUp()
self.multipart_threshold = 5 * 1024 * 1024
self.config = TransferConfig(
multipart_threshold=self.multipart_threshold)
def test_exception_on_zero_attr_value(self):
with self.assertRaises(ValueError):
TransferConfig(max_request_queue_size=0)
def setUp(self):
super(BaseUploadInputManagerTest, self).setUp()
self.osutil = OSUtils()
self.config = TransferConfig()
self.recording_subscriber = RecordingSubscriber()
self.subscribers.append(self.recording_subscriber)
