def __call__(self, client, result_queue):
"""Creates a S3TransferHandler instance
:type client: botocore.client.Client
:param client: The client to power the S3TransferHandler
:type result_queue: queue.Queue
:param result_queue: The result queue to be used to process results
for the S3TransferHandler
:returns: A S3TransferHandler instance
"""
transfer_config = create_transfer_config_from_runtime_config(
self._runtime_config)
transfer_config.max_in_memory_upload_chunks = self.MAX_IN_MEMORY_CHUNKS
transfer_config.max_in_memory_download_chunks = \
self.MAX_IN_MEMORY_CHUNKS
transfer_manager = TransferManager(client, transfer_config)
LOGGER.debug(
"Using a multipart threshold of %s and a part size of %s",
transfer_config.multipart_threshold,
transfer_config.multipart_chunksize
)
result_recorder = ResultRecorder()
result_processor_handlers = [result_recorder]
self._add_result_printer(result_recorder, result_processor_handlers)
result_processor = ResultProcessor(
result_queue, result_processor_handlers)
command_result_recorder = CommandResultRecorder(
result_queue, result_recorder, result_processor)
return S3TransferHandler(
transfer_manager, self._cli_params, command_result_recorder)
def __call__(self, client, result_queue):
"""Creates a S3TransferHandler instance
:type client: botocore.client.Client
:param client: The client to power the S3TransferHandler
:type result_queue: queue.Queue
:param result_queue: The result queue to be used to process results
for the S3TransferHandler
:returns: A S3TransferHandler instance
"""
transfer_config = create_transfer_config_from_runtime_config(
self._runtime_config)
transfer_config.max_in_memory_upload_chunks = self.MAX_IN_MEMORY_CHUNKS
transfer_config.max_in_memory_download_chunks = \
self.MAX_IN_MEMORY_CHUNKS
transfer_manager = TransferManager(client, transfer_config)
LOGGER.debug(
"Using a multipart threshold of %s and a part size of %s",
transfer_config.multipart_threshold,
transfer_config.multipart_chunksize
)
result_recorder = ResultRecorder()
result_processor_handlers = [result_recorder]
self._add_result_printer(result_recorder, result_processor_handlers)
result_processor = ResultProcessor(
result_queue, result_processor_handlers)
command_result_recorder = CommandResultRecorder(
result_queue, result_recorder, result_processor)
return S3TransferHandler(
transfer_manager, self._cli_params, command_result_recorder)
def test_convert(self):
runtime_config = {
'multipart_threshold': 1,
'multipart_chunksize': 2,
'max_concurrent_requests': 3,
'max_queue_size': 4
}
result = transferconfig.create_transfer_config_from_runtime_config(
runtime_config)
self.assertEqual(result.multipart_threshold, 1)
self.assertEqual(result.multipart_chunksize, 2)
self.assertEqual(result.max_request_concurrency, 3)
self.assertEqual(result.max_request_queue_size, 4)
def __init__(self,
session,
params,
result_queue=None,
runtime_config=None,
manager=None):
super(S3TransferStreamHandler,
self).__init__(session, params, result_queue, runtime_config)
self.config = create_transfer_config_from_runtime_config(
self._runtime_config)
# Restrict the maximum chunks to 1 per thread.
self.config.max_in_memory_upload_chunks = \
self.MAX_IN_MEMORY_CHUNKS
self.config.max_in_memory_download_chunks = \
self.MAX_IN_MEMORY_CHUNKS
self._manager = manager
def test_convert(self):
runtime_config = {
'multipart_threshold': 1,
'multipart_chunksize': 2,
'max_concurrent_requests': 3,
'max_queue_size': 4,
'addressing_style': 'path',
'use_accelerate_endpoint': True,
# This is a TransferConfig only option, it should
# just be ignored if it's in the ~/.aws/config for now.
'max_in_memory_upload_chunks': 1000,
}
result = transferconfig.create_transfer_config_from_runtime_config(
runtime_config)
self.assertEqual(result.multipart_threshold, 1)
self.assertEqual(result.multipart_chunksize, 2)
self.assertEqual(result.max_request_concurrency, 3)
self.assertEqual(result.max_request_queue_size, 4)
self.assertNotEqual(result.max_in_memory_upload_chunks, 1000)
