def setUp(self):
     self.subscriber = RecordingSubscriber()
     self.second_subscriber = RecordingSubscriber()
     self.call_args = CallArgs(
         subscribers=[self.subscriber, self.second_subscriber])
     self.transfer_meta = TransferMeta(self.call_args)
     self.transfer_future = TransferFuture(self.transfer_meta)
Exemple #2
0
    def test_progress_subscribers_on_upload(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)
        file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt')
        future = transfer_manager.upload(
            file, self.bucket_name, '20mb.txt',
            subscribers=[subscriber])
        self.addCleanup(self.delete_object, '20mb.txt')

        future.result()
        # The callback should have been called enough times such that
        # the total amount of bytes we've seen (via the "amount"
        # arg to the callback function) should be the size
        # of the file we uploaded.
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
Exemple #3
0
    def test_progress_subscribers_on_upload(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)
        file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt')
        future = transfer_manager.upload(
            file, self.bucket_name, '20mb.txt',
            subscribers=[subscriber])
        self.addCleanup(self.delete_object, '20mb.txt')

        future.result()
        # The callback should have been called enough times such that
        # the total amount of bytes we've seen (via the "amount"
        # arg to the callback function) should be the size
        # of the file we uploaded.
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
Exemple #4
0
    def test_progress_subscribers_on_download(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)

        filename = self.files.create_file_with_size(
            'foo.txt', filesize=20 * 1024 * 1024)
        self.upload_file(filename, '20mb.txt')

        download_path = os.path.join(self.files.rootdir, '20mb.txt')

        future = transfer_manager.download(
            self.bucket_name, '20mb.txt', download_path,
            subscribers=[subscriber])
        future.result()
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
Exemple #5
0
    def test_callbacks_invoked(self):
        subscriber = RecordingSubscriber()
        self.callbacks.append(subscriber.on_progress)
        self.stubber.add_response(
            'copy_object', service_response={},
            expected_params={
                'Bucket': self.bucket, 'Key': self.key,
                'CopySource': self.copy_source
            }
        )
        task = self.get_copy_task()
        task()

        self.stubber.assert_no_pending_responses()
        self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
    def test_progress_subscribers_on_download(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)

        filename = self.files.create_file_with_size(
            'foo.txt', filesize=20 * 1024 * 1024)
        self.upload_file(filename, '20mb.txt')

        download_path = os.path.join(self.files.rootdir, '20mb.txt')

        future = transfer_manager.download(
            self.bucket_name, '20mb.txt', download_path,
            subscribers=[subscriber])
        future.result()
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
Exemple #7
0
    def test_cleanups_only_ran_once_on_exception(self):
        # We want to be able to handle the case where the final task completes
        # and anounces done but there is an error in the submission task
        # which will cause it to need to anounce done as well. In this case,
        # we do not want the done callbacks to be invoke more than once.

        final_task = self.get_task(FailureTask, is_final=True)
        self.main_kwargs['executor'] = self.executor
        self.main_kwargs['tasks_to_submit'] = [final_task]

        submission_task = self.get_task(ExceptionSubmissionTask,
                                        main_kwargs=self.main_kwargs)

        subscriber = RecordingSubscriber()
        self.call_args.subscribers.append(subscriber)

        # Add the done callback to the callbacks to be invoked when the
        # transfer is done.
        done_callbacks = get_callbacks(self.transfer_future, 'done')
        for done_callback in done_callbacks:
            self.transfer_coordinator.add_done_callback(done_callback)

        submission_task()

        # Make sure the task failed to start
        self.assertEqual(self.transfer_coordinator.status, 'failed')

        # Make sure the on_done callback of the subscriber is called only once.
        self.assertEqual(subscriber.on_done_calls,
                         [{
                             'future': self.transfer_future
                         }])
Exemple #8
0
    def test_on_queued_callbacks(self):
        submission_task = self.get_task(
            NOOPSubmissionTask, main_kwargs=self.main_kwargs)

        subscriber = RecordingSubscriber()
        self.call_args.subscribers.append(subscriber)
        submission_task()
        # Make sure the on_queued callback of the subscriber is called.
        self.assertEqual(
            subscriber.on_queued_calls, [{'future': self.transfer_future}])
Exemple #9
0
    def test_sigv4_progress_callbacks_invoked_once(self):
        # Reset the client and manager to use sigv4
        self.reset_stubber_with_new_client(
            {'config': Config(signature_version='s3v4')})
        self.client.meta.events.register(
            'before-parameter-build.s3.*', self.collect_body)
        self._manager = TransferManager(self.client, self.config)

        # Add the stubbed response.
        self.add_put_object_response_with_default_expected_params()

        subscriber = RecordingSubscriber()
        future = self.manager.upload(
            self.filename, self.bucket, self.key, subscribers=[subscriber])
        future.result()
        self.assert_expected_client_calls_were_correct()

        # The amount of bytes seen should be the same as the file size
        self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
Exemple #10
0
    def test_sigv4_progress_callbacks_invoked_once(self):
        # Reset the client and manager to use sigv4
        self.reset_stubber_with_new_client(
            {'config': Config(signature_version='s3v4')})
        self.client.meta.events.register(
            'before-parameter-build.s3.*', self.collect_body)
        self._manager = TransferManager(self.client, self.config)

        # Add the stubbed response.
        self.add_put_object_response_with_default_expected_params()

        subscriber = RecordingSubscriber()
        future = self.manager.upload(
            self.filename, self.bucket, self.key, subscribers=[subscriber])
        future.result()
        self.assert_expected_client_calls_were_correct()

        # The amount of bytes seen should be the same as the file size
        self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
Exemple #11
0
 def test_callbacks_invoked(self):
     subscriber = RecordingSubscriber()
     self.callbacks.append(subscriber.on_progress)
     self.stubber.add_response(
         'upload_part_copy', service_response={
             'CopyPartResult': {
                 'ETag': self.result_etag
             }
         },
         expected_params={
             'Bucket': self.bucket, 'Key': self.key,
             'CopySource': self.copy_source, 'UploadId': self.upload_id,
             'PartNumber': self.part_number,
             'CopySourceRange': self.copy_source_range
         }
     )
     task = self.get_copy_task()
     self.assertEqual(
         task(), {'PartNumber': self.part_number, 'ETag': self.result_etag})
     self.stubber.assert_no_pending_responses()
     self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
Exemple #12
0
    def test_progress_subscribers_on_copy(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)
        key = '20mb.txt'
        new_key = '20mb-copy.txt'

        filename = self.files.create_file_with_size(
            key, filesize=20 * 1024 * 1024)
        self.upload_file(filename, key)

        future = transfer_manager.copy(
            copy_source={'Bucket': self.bucket_name, 'Key': key},
            bucket=self.bucket_name,
            key=new_key,
            subscribers=[subscriber]
        )

        future.result()
        # The callback should have been called enough times such that
        # the total amount of bytes we've seen (via the "amount"
        # arg to the callback function) should be the size
        # of the file we uploaded.
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
    def test_progress_subscribers_on_copy(self):
        subscriber = RecordingSubscriber()
        transfer_manager = self.create_transfer_manager(self.config)
        key = '20mb.txt'
        new_key = '20mb-copy.txt'

        filename = self.files.create_file_with_size(key,
                                                    filesize=20 * 1024 * 1024)
        self.upload_file(filename, key)

        future = transfer_manager.copy(copy_source={
            'Bucket': self.bucket_name,
            'Key': key
        },
                                       bucket=self.bucket_name,
                                       key=new_key,
                                       subscribers=[subscriber])

        future.result()
        # The callback should have been called enough times such that
        # the total amount of bytes we've seen (via the "amount"
        # arg to the callback function) should be the size
        # of the file we uploaded.
        self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
Exemple #14
0
    def test_retry_rewinds_callbacks(self):
        self.add_head_object_response()
        # Insert a response that will trigger a retry after one read of the
        # stream has been made.
        self.add_n_retryable_get_object_responses(1, num_reads=1)
        # Add the normal responses to simulate the download proceeding
        # as normal after the retry.
        self.add_successful_get_object_responses()

        recorder_subscriber = RecordingSubscriber()
        # Set the streaming to a size that is smaller than the data we
        # currently provide to it to simulate rewinds of callbacks.
        self.config.io_chunksize = 3
        future = self.manager.download(subscribers=[recorder_subscriber],
                                       **self.create_call_kwargs())
        future.result()

        # Ensure that there is no more remaining responses and that contents
        # are correct.
        self.stubber.assert_no_pending_responses()
        with open(self.filename, 'rb') as f:
            self.assertEqual(self.content, f.read())

        # Assert that the number of bytes seen is equal to the length of
        # downloaded content.
        self.assertEqual(recorder_subscriber.calculate_bytes_seen(),
                         len(self.content))

        # Also ensure that the second progress invocation was negative three
        # becasue a retry happened on the second read of the stream and we
        # know that the chunk size for each read is 3.
        progress_byte_amts = [
            call['bytes_transferred']
            for call in recorder_subscriber.on_progress_calls
        ]
        self.assertEqual(-3, progress_byte_amts[1])
Exemple #15
0
    def test_retry_rewinds_callbacks(self):
        self.add_head_object_response()
        # Insert a response that will trigger a retry after one read of the
        # stream has been made.
        self.add_n_retryable_get_object_responses(1, num_reads=1)
        # Add the normal responses to simulate the download proceeding
        # as normal after the retry.
        self.add_successful_get_object_responses()

        recorder_subscriber = RecordingSubscriber()
        # Set the streaming to a size that is smaller than the data we
        # currently provide to it to simulate rewinds of callbacks.
        self.config.io_chunksize = 3
        future = self.manager.download(
            subscribers=[recorder_subscriber], **self.create_call_kwargs())
        future.result()

        # Ensure that there is no more remaining responses and that contents
        # are correct.
        self.stubber.assert_no_pending_responses()
        with open(self.filename, 'rb') as f:
            self.assertEqual(self.content, f.read())

        # Assert that the number of bytes seen is equal to the length of
        # downloaded content.
        self.assertEqual(
            recorder_subscriber.calculate_bytes_seen(), len(self.content))

        # Also ensure that the second progress invocation was negative three
        # becasue a retry happened on the second read of the stream and we
        # know that the chunk size for each read is 3.
        progress_byte_amts = [
            call['bytes_transferred'] for call in
            recorder_subscriber.on_progress_calls
        ]
        self.assertEqual(-3, progress_byte_amts[1])
Exemple #16
0
    def test_calls_done_callbacks_on_exception(self):
        submission_task = self.get_task(
            ExceptionSubmissionTask, main_kwargs=self.main_kwargs)

        subscriber = RecordingSubscriber()
        self.call_args.subscribers.append(subscriber)

        # Add the done callback to the callbacks to be invoked when the
        # transfer is done.
        done_callbacks = get_callbacks(self.transfer_future, 'done')
        for done_callback in done_callbacks:
            self.transfer_coordinator.add_done_callback(done_callback)
        submission_task()

        # Make sure the task failed to start
        self.assertEqual(self.transfer_coordinator.status, 'failed')

        # Make sure the on_done callback of the subscriber is called.
        self.assertEqual(
            subscriber.on_done_calls, [{'future': self.transfer_future}])
 def test_invoke_progress_callbacks_with_no_progress(self):
     recording_subscriber = RecordingSubscriber()
     invoke_progress_callbacks([recording_subscriber.on_progress], 0)
     self.assertEqual(len(recording_subscriber.on_progress_calls), 0)
 def test_invoke_progress_callbacks(self):
     recording_subscriber = RecordingSubscriber()
     invoke_progress_callbacks([recording_subscriber.on_progress], 2)
     self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)
Exemple #19
0
 def setUp(self):
     super(BaseUploadInputManagerTest, self).setUp()
     self.osutil = OSUtils()
     self.config = TransferConfig()
     self.recording_subscriber = RecordingSubscriber()
     self.subscribers.append(self.recording_subscriber)
Exemple #20
0
 def test_invoke_progress_callbacks(self):
     recording_subscriber = RecordingSubscriber()
     invoke_progress_callbacks([recording_subscriber.on_progress], 2)
     self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)