def __call__(self, client, result_queue): """Creates a S3TransferHandler instance :type client: botocore.client.Client :param client: The client to power the S3TransferHandler :type result_queue: queue.Queue :param result_queue: The result queue to be used to process results for the S3TransferHandler :returns: A S3TransferHandler instance """ transfer_config = create_transfer_config_from_runtime_config( self._runtime_config) transfer_config.max_in_memory_upload_chunks = self.MAX_IN_MEMORY_CHUNKS transfer_config.max_in_memory_download_chunks = \ self.MAX_IN_MEMORY_CHUNKS transfer_manager = TransferManager(client, transfer_config) LOGGER.debug( "Using a multipart threshold of %s and a part size of %s", transfer_config.multipart_threshold, transfer_config.multipart_chunksize ) result_recorder = ResultRecorder() result_processor_handlers = [result_recorder] self._add_result_printer(result_recorder, result_processor_handlers) result_processor = ResultProcessor( result_queue, result_processor_handlers) command_result_recorder = CommandResultRecorder( result_queue, result_recorder, result_processor) return S3TransferHandler( transfer_manager, self._cli_params, command_result_recorder)
def test_convert(self): runtime_config = { 'multipart_threshold': 1, 'multipart_chunksize': 2, 'max_concurrent_requests': 3, 'max_queue_size': 4 } result = transferconfig.create_transfer_config_from_runtime_config( runtime_config) self.assertEqual(result.multipart_threshold, 1) self.assertEqual(result.multipart_chunksize, 2) self.assertEqual(result.max_request_concurrency, 3) self.assertEqual(result.max_request_queue_size, 4)
def __init__(self, session, params, result_queue=None, runtime_config=None, manager=None): super(S3TransferStreamHandler, self).__init__(session, params, result_queue, runtime_config) self.config = create_transfer_config_from_runtime_config( self._runtime_config) # Restrict the maximum chunks to 1 per thread. self.config.max_in_memory_upload_chunks = \ self.MAX_IN_MEMORY_CHUNKS self.config.max_in_memory_download_chunks = \ self.MAX_IN_MEMORY_CHUNKS self._manager = manager
def test_convert(self): runtime_config = { 'multipart_threshold': 1, 'multipart_chunksize': 2, 'max_concurrent_requests': 3, 'max_queue_size': 4, 'addressing_style': 'path', 'use_accelerate_endpoint': True, # This is a TransferConfig only option, it should # just be ignored if it's in the ~/.aws/config for now. 'max_in_memory_upload_chunks': 1000, } result = transferconfig.create_transfer_config_from_runtime_config( runtime_config) self.assertEqual(result.multipart_threshold, 1) self.assertEqual(result.multipart_chunksize, 2) self.assertEqual(result.max_request_concurrency, 3) self.assertEqual(result.max_request_queue_size, 4) self.assertNotEqual(result.max_in_memory_upload_chunks, 1000)