Ejemplo n.º 1
0
    def _submit_ranged_download_request(self, client, config, osutil,
                                        request_executor, io_executor,
                                        download_output_manager,
                                        transfer_future):
        call_args = transfer_future.meta.call_args

        # Get the needed progress callbacks for the task
        progress_callbacks = get_callbacks(transfer_future, 'progress')

        # Get a handle to the file that will be used for writing downloaded
        # contents
        fileobj = download_output_manager.get_fileobj_for_io_writes(
            transfer_future)

        # Determine the number of parts
        part_size = config.multipart_chunksize
        num_parts = int(math.ceil(transfer_future.meta.size /
                                  float(part_size)))

        # Get any associated tags for the get object task.
        get_object_tag = download_output_manager.get_download_task_tag()

        # Callback invoker to submit the final io task once all downloads
        # are complete.
        finalize_download_invoker = CountCallbackInvoker(
            self._get_final_io_task_submission_callback(
                download_output_manager, io_executor))
        for i in range(num_parts):
            # Calculate the range parameter
            range_parameter = calculate_range_parameter(
                part_size, i, num_parts)

            # Inject the Range parameter to the parameters to be passed in
            # as extra args
            extra_args = {'Range': range_parameter}
            extra_args.update(call_args.extra_args)
            finalize_download_invoker.increment()
            # Submit the ranged downloads
            self._transfer_coordinator.submit(
                request_executor,
                GetObjectTask(
                    transfer_coordinator=self._transfer_coordinator,
                    main_kwargs={
                        'client': client,
                        'bucket': call_args.bucket,
                        'key': call_args.key,
                        'fileobj': fileobj,
                        'extra_args': extra_args,
                        'callbacks': progress_callbacks,
                        'max_attempts': config.num_download_attempts,
                        'start_index': i * part_size,
                        'download_output_manager': download_output_manager,
                        'io_chunksize': config.io_chunksize,
                    },
                    done_callbacks=[finalize_download_invoker.decrement]),
                tag=get_object_tag)
        finalize_download_invoker.finalize()
class TestCountCallbackInvoker(unittest.TestCase):
    def invoke_callback(self):
        self.ref_results.append('callback invoked')

    def assert_callback_invoked(self):
        self.assertEqual(self.ref_results, ['callback invoked'])

    def assert_callback_not_invoked(self):
        self.assertEqual(self.ref_results, [])

    def setUp(self):
        self.ref_results = []
        self.invoker = CountCallbackInvoker(self.invoke_callback)

    def test_increment(self):
        self.invoker.increment()
        self.assertEqual(self.invoker.current_count, 1)

    def test_decrement(self):
        self.invoker.increment()
        self.invoker.increment()
        self.invoker.decrement()
        self.assertEqual(self.invoker.current_count, 1)

    def test_count_cannot_go_below_zero(self):
        with self.assertRaises(RuntimeError):
            self.invoker.decrement()

    def test_callback_invoked_only_once_finalized(self):
        self.invoker.increment()
        self.invoker.decrement()
        self.assert_callback_not_invoked()
        self.invoker.finalize()
        # Callback should only be invoked once finalized
        self.assert_callback_invoked()

    def test_callback_invoked_after_finalizing_and_count_reaching_zero(self):
        self.invoker.increment()
        self.invoker.finalize()
        # Make sure that it does not get invoked immediately after
        # finalizing as the count is currently one
        self.assert_callback_not_invoked()
        self.invoker.decrement()
        self.assert_callback_invoked()

    def test_cannot_increment_after_finalization(self):
        self.invoker.finalize()
        with self.assertRaises(RuntimeError):
            self.invoker.increment()