def _submit_ranged_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager,
transfer_future):
call_args = transfer_future.meta.call_args
# Get the needed progress callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Determine the number of parts
part_size = config.multipart_chunksize
num_parts = int(math.ceil(transfer_future.meta.size /
float(part_size)))
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once all downloads
# are complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor))
for i in range(num_parts):
# Calculate the range parameter
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
# Inject the Range parameter to the parameters to be passed in
# as extra args
extra_args = {'Range': range_parameter}
extra_args.update(call_args.extra_args)
finalize_download_invoker.increment()
# Submit the ranged downloads
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'start_index': i * part_size,
'download_output_manager': download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement]),
tag=get_object_tag)
finalize_download_invoker.finalize()
def _submit_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager, transfer_future):
call_args = transfer_future.meta.call_args
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Get the needed callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once the download
# is complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor))
finalize_download_invoker.increment()
# Submit the task to download the object.
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': call_args.extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'download_output_manager':
download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement
]),
tag=get_object_tag)
finalize_download_invoker.finalize()
def setUp(self):
self.ref_results = []
self.invoker = CountCallbackInvoker(self.invoke_callback)
class TestCountCallbackInvoker(unittest.TestCase):
def invoke_callback(self):
self.ref_results.append('callback invoked')
def assert_callback_invoked(self):
self.assertEqual(self.ref_results, ['callback invoked'])
def assert_callback_not_invoked(self):
self.assertEqual(self.ref_results, [])
def setUp(self):
self.ref_results = []
self.invoker = CountCallbackInvoker(self.invoke_callback)
def test_increment(self):
self.invoker.increment()
self.assertEqual(self.invoker.current_count, 1)
def test_decrement(self):
self.invoker.increment()
self.invoker.increment()
self.invoker.decrement()
self.assertEqual(self.invoker.current_count, 1)
def test_count_cannot_go_below_zero(self):
with self.assertRaises(RuntimeError):
self.invoker.decrement()
def test_callback_invoked_only_once_finalized(self):
self.invoker.increment()
self.invoker.decrement()
self.assert_callback_not_invoked()
self.invoker.finalize()
# Callback should only be invoked once finalized
self.assert_callback_invoked()
def test_callback_invoked_after_finalizing_and_count_reaching_zero(self):
self.invoker.increment()
self.invoker.finalize()
# Make sure that it does not get invoked immediately after
# finalizing as the count is currently one
self.assert_callback_not_invoked()
self.invoker.decrement()
self.assert_callback_invoked()
def test_cannot_increment_after_finalization(self):
self.invoker.finalize()
with self.assertRaises(RuntimeError):
self.invoker.increment()
def setUp(self):
self.ref_results = []
self.invoker = CountCallbackInvoker(self.invoke_callback)
class TestCountCallbackInvoker(unittest.TestCase):
def invoke_callback(self):
self.ref_results.append('callback invoked')
def assert_callback_invoked(self):
self.assertEqual(self.ref_results, ['callback invoked'])
def assert_callback_not_invoked(self):
self.assertEqual(self.ref_results, [])
def setUp(self):
self.ref_results = []
self.invoker = CountCallbackInvoker(self.invoke_callback)
def test_increment(self):
self.invoker.increment()
self.assertEqual(self.invoker.current_count, 1)
def test_decrement(self):
self.invoker.increment()
self.invoker.increment()
self.invoker.decrement()
self.assertEqual(self.invoker.current_count, 1)
def test_count_cannot_go_below_zero(self):
with self.assertRaises(RuntimeError):
self.invoker.decrement()
def test_callback_invoked_only_once_finalized(self):
self.invoker.increment()
self.invoker.decrement()
self.assert_callback_not_invoked()
self.invoker.finalize()
# Callback should only be invoked once finalized
self.assert_callback_invoked()
def test_callback_invoked_after_finalizing_and_count_reaching_zero(self):
self.invoker.increment()
self.invoker.finalize()
# Make sure that it does not get invoked immediately after
# finalizing as the count is currently one
self.assert_callback_not_invoked()
self.invoker.decrement()
self.assert_callback_invoked()
def test_cannot_increment_after_finalization(self):
self.invoker.finalize()
with self.assertRaises(RuntimeError):
self.invoker.increment()
def _submit_ranged_download_request(self, client, config, osutil,
request_executor, io_executor,
download_output_manager,
transfer_future):
call_args = transfer_future.meta.call_args
# Get the needed progress callbacks for the task
progress_callbacks = get_callbacks(transfer_future, 'progress')
# Get a handle to the file that will be used for writing downloaded
# contents
fileobj = download_output_manager.get_fileobj_for_io_writes(
transfer_future)
# Determine the number of parts
part_size = config.multipart_chunksize
num_parts = int(
math.ceil(transfer_future.meta.size / float(part_size)))
# Get any associated tags for the get object task.
get_object_tag = download_output_manager.get_download_task_tag()
# Callback invoker to submit the final io task once all downloads
# are complete.
finalize_download_invoker = CountCallbackInvoker(
self._get_final_io_task_submission_callback(
download_output_manager, io_executor
)
)
for i in range(num_parts):
# Calculate the range parameter
range_parameter = calculate_range_parameter(
part_size, i, num_parts)
# Inject the Range parameter to the parameters to be passed in
# as extra args
extra_args = {'Range': range_parameter}
extra_args.update(call_args.extra_args)
finalize_download_invoker.increment()
# Submit the ranged downloads
self._transfer_coordinator.submit(
request_executor,
GetObjectTask(
transfer_coordinator=self._transfer_coordinator,
main_kwargs={
'client': client,
'bucket': call_args.bucket,
'key': call_args.key,
'fileobj': fileobj,
'extra_args': extra_args,
'callbacks': progress_callbacks,
'max_attempts': config.num_download_attempts,
'start_index': i * part_size,
'download_output_manager': download_output_manager,
'io_chunksize': config.io_chunksize,
},
done_callbacks=[finalize_download_invoker.decrement]
),
tag=get_object_tag
)
finalize_download_invoker.finalize()
