def setUp(self):
super(TestGetObjectSubmitter, self).setUp()
self.transfer_config = ProcessTransferConfig()
self.client_factory = mock.Mock(ClientFactory)
self.client_factory.create_client.return_value = self.client
self.transfer_monitor = TransferMonitor()
self.osutil = mock.Mock(OSUtils)
self.download_request_queue = queue.Queue()
self.worker_queue = queue.Queue()
self.submitter = GetObjectSubmitter(
transfer_config=self.transfer_config,
client_factory=self.client_factory,
transfer_monitor=self.transfer_monitor,
osutil=self.osutil,
download_request_queue=self.download_request_queue,
worker_queue=self.worker_queue,
)
self.transfer_id = self.transfer_monitor.notify_new_transfer()
self.bucket = 'bucket'
self.key = 'key'
self.filename = 'myfile'
self.temp_filename = 'myfile.temp'
self.osutil.get_temp_filename.return_value = self.temp_filename
self.extra_args = {}
self.expected_size = None
def setUp(self):
super(TestProcessPoolDownloader, self).setUp()
self.multipart_threshold = 5 * 1024 * 1024
self.config = ProcessTransferConfig(
multipart_threshold=self.multipart_threshold
)
self.client_kwargs = {'region_name': self.region}
def setUp(self):
# The stubbed client needs to run in a manager to be shared across
# processes and have it properly consume the stubbed response across
# processes.
self.manager = StubbedClientManager()
self.manager.start()
self.stubbed_client = self.manager.StubbedClient()
self.stubbed_client_factory = StubbedClientFactory(self.stubbed_client)
self.client_factory_patch = mock.patch(
'ibm_s3transfer.processpool.ClientFactory',
self.stubbed_client_factory
)
self.client_factory_patch.start()
self.files = FileCreator()
self.config = ProcessTransferConfig(
max_request_processes=1
)
self.downloader = ProcessPoolDownloader(config=self.config)
self.bucket = 'mybucket'
self.key = 'mykey'
self.filename = self.files.full_path('filename')
self.remote_contents = b'my content'
self.stream = six.BytesIO(self.remote_contents)
def test_download_file_ranged_download(self):
half_of_content_length = int(len(self.remote_contents) / 2)
self.stubbed_client.add_response(
'head_object', {'ContentLength': len(self.remote_contents)})
self.stubbed_client.add_response('get_object', {
'Body':
six.BytesIO(self.remote_contents[:half_of_content_length])
})
self.stubbed_client.add_response('get_object', {
'Body':
six.BytesIO(self.remote_contents[half_of_content_length:])
})
downloader = ProcessPoolDownloader(config=ProcessTransferConfig(
multipart_chunksize=half_of_content_length,
multipart_threshold=half_of_content_length,
max_request_processes=1))
with downloader:
downloader.download_file(self.bucket, self.key, self.filename)
self.assert_contents(self.filename, self.remote_contents)