def _submit_ranged_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager,
transfer_future):
call_args = transfer_future.meta.call_args
# Get the needed progress callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Determine the number of parts
part_size = config.multipart_chunksize
num_parts = int(math.ceil(transfer_future.meta.size /
float(part_size)))
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once all downloads
# are complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor))
for i in range(num_parts):
# Calculate the range parameter
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
# Inject the Range parameter to the parameters to be passed in
# as extra args
extra_args = {'Range': range_parameter}
extra_args.update(call_args.extra_args)
finalize_download_invoker.increment()
# Submit the ranged downloads
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'start_index': i * part_size,
'download_output_manager': download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement]),
tag=get_object_tag)
finalize_download_invoker.finalize()
def _submit_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager, transfer_future):
call_args = transfer_future.meta.call_args
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Get the needed callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once the download
# is complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor))
finalize_download_invoker.increment()
# Submit the task to download the object.
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': call_args.extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'download_output_manager':
download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement
]),
tag=get_object_tag)
finalize_download_invoker.finalize()
def setUp(self):
self.ref_results = []
self.invoker = CountCallbackInvoker(self.invoke_callback)
