def _submit_ranged_download_request(self, client, config, osutil, request_executor, io_executor, download_output_manager, transfer_future): call_args = transfer_future.meta.call_args # Get the needed progress callbacks for the task progress_callbacks = get_callbacks(transfer_future, 'progress') # Get a handle to the file that will be used for writing downloaded # contents fileobj = download_output_manager.get_fileobj_for_io_writes( transfer_future) # Determine the number of parts part_size = config.multipart_chunksize num_parts = int(math.ceil(transfer_future.meta.size / float(part_size))) # Get any associated tags for the get object task. get_object_tag = download_output_manager.get_download_task_tag() # Callback invoker to submit the final io task once all downloads # are complete. finalize_download_invoker = CountCallbackInvoker( self._get_final_io_task_submission_callback( download_output_manager, io_executor)) for i in range(num_parts): # Calculate the range parameter range_parameter = calculate_range_parameter( part_size, i, num_parts) # Inject the Range parameter to the parameters to be passed in # as extra args extra_args = {'Range': range_parameter} extra_args.update(call_args.extra_args) finalize_download_invoker.increment() # Submit the ranged downloads self._transfer_coordinator.submit( request_executor, GetObjectTask( transfer_coordinator=self._transfer_coordinator, main_kwargs={ 'client': client, 'bucket': call_args.bucket, 'key': call_args.key, 'fileobj': fileobj, 'extra_args': extra_args, 'callbacks': progress_callbacks, 'max_attempts': config.num_download_attempts, 'start_index': i * part_size, 'download_output_manager': download_output_manager, 'io_chunksize': config.io_chunksize, }, done_callbacks=[finalize_download_invoker.decrement]), tag=get_object_tag) finalize_download_invoker.finalize()
def setUp(self): self.ref_results = [] self.invoker = CountCallbackInvoker(self.invoke_callback)
class TestCountCallbackInvoker(unittest.TestCase): def invoke_callback(self): self.ref_results.append('callback invoked') def assert_callback_invoked(self): self.assertEqual(self.ref_results, ['callback invoked']) def assert_callback_not_invoked(self): self.assertEqual(self.ref_results, []) def setUp(self): self.ref_results = [] self.invoker = CountCallbackInvoker(self.invoke_callback) def test_increment(self): self.invoker.increment() self.assertEqual(self.invoker.current_count, 1) def test_decrement(self): self.invoker.increment() self.invoker.increment() self.invoker.decrement() self.assertEqual(self.invoker.current_count, 1) def test_count_cannot_go_below_zero(self): with self.assertRaises(RuntimeError): self.invoker.decrement() def test_callback_invoked_only_once_finalized(self): self.invoker.increment() self.invoker.decrement() self.assert_callback_not_invoked() self.invoker.finalize() # Callback should only be invoked once finalized self.assert_callback_invoked() def test_callback_invoked_after_finalizing_and_count_reaching_zero(self): self.invoker.increment() self.invoker.finalize() # Make sure that it does not get invoked immediately after # finalizing as the count is currently one self.assert_callback_not_invoked() self.invoker.decrement() self.assert_callback_invoked() def test_cannot_increment_after_finalization(self): self.invoker.finalize() with self.assertRaises(RuntimeError): self.invoker.increment()