def _submit_ranged_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager,
transfer_future):
call_args = transfer_future.meta.call_args
# Get the needed progress callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Determine the number of parts
part_size = config.multipart_chunksize
num_parts = int(
math.ceil(transfer_future.meta.size / float(part_size)))
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
ranged_downloads = []
for i in range(num_parts):
# Calculate the range parameter
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
# Inject the Range parameter to the parameters to be passed in
# as extra args
extra_args = {'Range': range_parameter}
extra_args.update(call_args.extra_args)
# Submit the ranged downloads
ranged_downloads.append(
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'start_index': i * part_size,
'download_output_manager': download_output_manager,
'io_chunksize': config.io_chunksize,
}
),
tag=get_object_tag
)
)
# Send the necessary tasks to complete the download.
self._complete_download(
request_executor, io_executor, download_output_manager,
ranged_downloads)
def _submit_ranged_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager,
transfer_future):
call_args = transfer_future.meta.call_args
# Get the needed progress callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Determine the number of parts
part_size = config.multipart_chunksize
num_parts = int(math.ceil(transfer_future.meta.size /
float(part_size)))
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once all downloads
# are complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor))
for i in range(num_parts):
# Calculate the range parameter
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
# Inject the Range parameter to the parameters to be passed in
# as extra args
extra_args = {'Range': range_parameter}
extra_args.update(call_args.extra_args)
finalize_download_invoker.increment()
# Submit the ranged downloads
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'start_index': i * part_size,
'download_output_manager': download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement]),
tag=get_object_tag)
finalize_download_invoker.finalize()
def _submit_ranged_get_object_jobs(self, download_file_request,
temp_filename, size):
part_size = self._transfer_config.multipart_chunksize
num_parts = calculate_num_parts(size, part_size)
self._notify_jobs_to_complete(download_file_request.transfer_id,
num_parts)
for i in range(num_parts):
offset = i * part_size
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
get_object_kwargs = {'Range': range_parameter}
get_object_kwargs.update(download_file_request.extra_args)
self._submit_get_object_job(
transfer_id=download_file_request.transfer_id,
bucket=download_file_request.bucket,
key=download_file_request.key,
temp_filename=temp_filename,
offset=offset,
extra_args=get_object_kwargs,
filename=download_file_request.filename,
)
def _submit_multipart_request(self, client, config, osutil,
request_executor, transfer_future):
call_args = transfer_future.meta.call_args
# Submit the request to create a multipart upload and make sure it
# does not include any of the arguments used for copy part.
create_multipart_extra_args = {}
for param, val in call_args.extra_args.items():
if param not in self.CREATE_MULTIPART_ARGS_BLACKLIST:
create_multipart_extra_args[param] = val
create_multipart_future = self._transfer_coordinator.submit(
request_executor,
CreateMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': create_multipart_extra_args,
}
)
)
# Determine how many parts are needed based on filesize and
# desired chunksize.
part_size = config.multipart_chunksize
adjuster = ChunksizeAdjuster()
part_size = adjuster.adjust_chunksize(
part_size, transfer_future.meta.size)
num_parts = int(
math.ceil(transfer_future.meta.size / float(part_size)))
# Submit requests to upload the parts of the file.
part_futures = []
progress_callbacks = get_callbacks(transfer_future, 'progress')
for part_number in range(1, num_parts + 1):
extra_part_args = self._extra_upload_part_args(
call_args.extra_args)
# The part number for upload part starts at 1 while the
# range parameter starts at zero, so just subtract 1 off of
# the part number
extra_part_args['CopySourceRange'] = calculate_range_parameter(
part_size, part_number-1, num_parts, transfer_future.meta.size)
# Get the size of the part copy as well for the progress
# callbacks.
size = self._get_transfer_size(
part_size, part_number-1, num_parts, transfer_future.meta.size
)
part_futures.append(
self._transfer_coordinator.submit(
request_executor,
CopyPartTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'copy_source': call_args.copy_source,
'bucket': call_args.bucket,
'key': call_args.key,
'part_number': part_number,
'extra_args': extra_part_args,
'callbacks': progress_callbacks,
'size': size
},
pending_main_kwargs={
'upload_id': create_multipart_future
}
)
)
)
complete_multipart_extra_args = self._extra_complete_multipart_args(
call_args.extra_args)
# Submit the request to complete the multipart upload.
self._transfer_coordinator.submit(
request_executor,
CompleteMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': complete_multipart_extra_args,
},
pending_main_kwargs={
'upload_id': create_multipart_future,
'parts': part_futures
},
is_final=True
)
)
def _submit_multipart_request(self, client, config, osutil,
request_executor, transfer_future):
call_args = transfer_future.meta.call_args
# Submit the request to create a multipart upload and make sure it
# does not include any of the arguments used for copy part.
create_multipart_extra_args = {}
for param, val in call_args.extra_args.items():
if param not in self.CREATE_MULTIPART_ARGS_BLACKLIST:
create_multipart_extra_args[param] = val
create_multipart_future = self._transfer_coordinator.submit(
request_executor,
CreateMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'extra_args': create_multipart_extra_args,
}
)
)
# Determine how many parts are needed based on filesize and
# desired chunksize.
part_size = config.multipart_chunksize
num_parts = int(
math.ceil(transfer_future.meta.size / float(part_size)))
# Submit requests to upload the parts of the file.
part_futures = []
progress_callbacks = get_callbacks(transfer_future, 'progress')
for part_number in range(1, num_parts + 1):
extra_part_args = self._extra_upload_part_args(
call_args.extra_args)
# The part number for upload part starts at 1 while the
# range parameter starts at zero, so just subtract 1 off of
# the part number
extra_part_args['CopySourceRange'] = calculate_range_parameter(
part_size, part_number-1, num_parts, transfer_future.meta.size)
# Get the size of the part copy as well for the progress
# callbacks.
size = self._get_transfer_size(
part_size, part_number-1, num_parts, transfer_future.meta.size
)
part_futures.append(
self._transfer_coordinator.submit(
request_executor,
CopyPartTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'copy_source': call_args.copy_source,
'bucket': call_args.bucket,
'key': call_args.key,
'part_number': part_number,
'extra_args': extra_part_args,
'callbacks': progress_callbacks,
'size': size
},
pending_main_kwargs={
'upload_id': create_multipart_future
}
)
)
)
# Submit the request to complete the multipart upload.
self._transfer_coordinator.submit(
request_executor,
CompleteMultipartUploadTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key
},
pending_main_kwargs={
'upload_id': create_multipart_future,
'parts': part_futures
},
is_final=True
)
)
def test_last_part_with_total_size(self):
range_val = calculate_range_parameter(self.part_size,
self.part_index,
num_parts=2,
total_size=8)
self.assertEqual(range_val, 'bytes=5-7')
def test_calculate_range_paramter(self):
range_val = calculate_range_parameter(self.part_size, self.part_index,
self.num_parts)
self.assertEqual(range_val, 'bytes=5-9')
def test_last_part_with_total_size(self):
range_val = calculate_range_parameter(
self.part_size, self.part_index, num_parts=2, total_size=8)
self.assertEqual(range_val, 'bytes=5-7')
def test_calculate_range_paramter(self):
range_val = calculate_range_parameter(
self.part_size, self.part_index, self.num_parts)
self.assertEqual(range_val, 'bytes=5-9')
