def test_progress_subscribers_on_upload(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt') future = transfer_manager.upload( file, self.bucket_name, '20mb.txt', subscribers=[subscriber]) self.addCleanup(self.delete_object, '20mb.txt') future.result() # The callback should have been called enough times such that # the total amount of bytes we've seen (via the "amount" # arg to the callback function) should be the size # of the file we uploaded. self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_upload(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt') future = transfer_manager.upload( file, self.bucket_name, '20mb.txt', subscribers=[subscriber]) self.addCleanup(self.delete_object, '20mb.txt') future.result() # The callback should have been called enough times such that # the total amount of bytes we've seen (via the "amount" # arg to the callback function) should be the size # of the file we uploaded. self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_download(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) filename = self.files.create_file_with_size( 'foo.txt', filesize=20 * 1024 * 1024) self.upload_file(filename, '20mb.txt') download_path = os.path.join(self.files.rootdir, '20mb.txt') future = transfer_manager.download( self.bucket_name, '20mb.txt', download_path, subscribers=[subscriber]) future.result() self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_callbacks_invoked(self): subscriber = RecordingSubscriber() self.callbacks.append(subscriber.on_progress) self.stubber.add_response( 'copy_object', service_response={}, expected_params={ 'Bucket': self.bucket, 'Key': self.key, 'CopySource': self.copy_source } ) task = self.get_copy_task() task() self.stubber.assert_no_pending_responses() self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
def test_progress_subscribers_on_download(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) filename = self.files.create_file_with_size( 'foo.txt', filesize=20 * 1024 * 1024) self.upload_file(filename, '20mb.txt') download_path = os.path.join(self.files.rootdir, '20mb.txt') future = transfer_manager.download( self.bucket_name, '20mb.txt', download_path, subscribers=[subscriber]) future.result() self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_sigv4_progress_callbacks_invoked_once(self): # Reset the client and manager to use sigv4 self.reset_stubber_with_new_client( {'config': Config(signature_version='s3v4')}) self.client.meta.events.register( 'before-parameter-build.s3.*', self.collect_body) self._manager = TransferManager(self.client, self.config) # Add the stubbed response. self.add_put_object_response_with_default_expected_params() subscriber = RecordingSubscriber() future = self.manager.upload( self.filename, self.bucket, self.key, subscribers=[subscriber]) future.result() self.assert_expected_client_calls_were_correct() # The amount of bytes seen should be the same as the file size self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def test_sigv4_progress_callbacks_invoked_once(self): # Reset the client and manager to use sigv4 self.reset_stubber_with_new_client( {'config': Config(signature_version='s3v4')}) self.client.meta.events.register( 'before-parameter-build.s3.*', self.collect_body) self._manager = TransferManager(self.client, self.config) # Add the stubbed response. self.add_put_object_response_with_default_expected_params() subscriber = RecordingSubscriber() future = self.manager.upload( self.filename, self.bucket, self.key, subscribers=[subscriber]) future.result() self.assert_expected_client_calls_were_correct() # The amount of bytes seen should be the same as the file size self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def test_callbacks_invoked(self): subscriber = RecordingSubscriber() self.callbacks.append(subscriber.on_progress) self.stubber.add_response( 'upload_part_copy', service_response={ 'CopyPartResult': { 'ETag': self.result_etag } }, expected_params={ 'Bucket': self.bucket, 'Key': self.key, 'CopySource': self.copy_source, 'UploadId': self.upload_id, 'PartNumber': self.part_number, 'CopySourceRange': self.copy_source_range } ) task = self.get_copy_task() self.assertEqual( task(), {'PartNumber': self.part_number, 'ETag': self.result_etag}) self.stubber.assert_no_pending_responses() self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
def test_progress_subscribers_on_copy(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) key = '20mb.txt' new_key = '20mb-copy.txt' filename = self.files.create_file_with_size( key, filesize=20 * 1024 * 1024) self.upload_file(filename, key) future = transfer_manager.copy( copy_source={'Bucket': self.bucket_name, 'Key': key}, bucket=self.bucket_name, key=new_key, subscribers=[subscriber] ) future.result() # The callback should have been called enough times such that # the total amount of bytes we've seen (via the "amount" # arg to the callback function) should be the size # of the file we uploaded. self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_copy(self): subscriber = RecordingSubscriber() transfer_manager = self.create_transfer_manager(self.config) key = '20mb.txt' new_key = '20mb-copy.txt' filename = self.files.create_file_with_size(key, filesize=20 * 1024 * 1024) self.upload_file(filename, key) future = transfer_manager.copy(copy_source={ 'Bucket': self.bucket_name, 'Key': key }, bucket=self.bucket_name, key=new_key, subscribers=[subscriber]) future.result() # The callback should have been called enough times such that # the total amount of bytes we've seen (via the "amount" # arg to the callback function) should be the size # of the file we uploaded. self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_retry_rewinds_callbacks(self): self.add_head_object_response() # Insert a response that will trigger a retry after one read of the # stream has been made. self.add_n_retryable_get_object_responses(1, num_reads=1) # Add the normal responses to simulate the download proceeding # as normal after the retry. self.add_successful_get_object_responses() recorder_subscriber = RecordingSubscriber() # Set the streaming to a size that is smaller than the data we # currently provide to it to simulate rewinds of callbacks. self.config.io_chunksize = 3 future = self.manager.download(subscribers=[recorder_subscriber], **self.create_call_kwargs()) future.result() # Ensure that there is no more remaining responses and that contents # are correct. self.stubber.assert_no_pending_responses() with open(self.filename, 'rb') as f: self.assertEqual(self.content, f.read()) # Assert that the number of bytes seen is equal to the length of # downloaded content. self.assertEqual(recorder_subscriber.calculate_bytes_seen(), len(self.content)) # Also ensure that the second progress invocation was negative three # becasue a retry happened on the second read of the stream and we # know that the chunk size for each read is 3. progress_byte_amts = [ call['bytes_transferred'] for call in recorder_subscriber.on_progress_calls ] self.assertEqual(-3, progress_byte_amts[1])
def test_retry_rewinds_callbacks(self): self.add_head_object_response() # Insert a response that will trigger a retry after one read of the # stream has been made. self.add_n_retryable_get_object_responses(1, num_reads=1) # Add the normal responses to simulate the download proceeding # as normal after the retry. self.add_successful_get_object_responses() recorder_subscriber = RecordingSubscriber() # Set the streaming to a size that is smaller than the data we # currently provide to it to simulate rewinds of callbacks. self.config.io_chunksize = 3 future = self.manager.download( subscribers=[recorder_subscriber], **self.create_call_kwargs()) future.result() # Ensure that there is no more remaining responses and that contents # are correct. self.stubber.assert_no_pending_responses() with open(self.filename, 'rb') as f: self.assertEqual(self.content, f.read()) # Assert that the number of bytes seen is equal to the length of # downloaded content. self.assertEqual( recorder_subscriber.calculate_bytes_seen(), len(self.content)) # Also ensure that the second progress invocation was negative three # becasue a retry happened on the second read of the stream and we # know that the chunk size for each read is 3. progress_byte_amts = [ call['bytes_transferred'] for call in recorder_subscriber.on_progress_calls ] self.assertEqual(-3, progress_byte_amts[1])
def test_invoke_progress_callbacks(self): recording_subscriber = RecordingSubscriber() invoke_progress_callbacks([recording_subscriber.on_progress], 2) self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)
def test_invoke_progress_callbacks(self): recording_subscriber = RecordingSubscriber() invoke_progress_callbacks([recording_subscriber.on_progress], 2) self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)