def setUp(self):
self.subscriber = RecordingSubscriber()
self.second_subscriber = RecordingSubscriber()
self.call_args = CallArgs(
subscribers=[self.subscriber, self.second_subscriber])
self.transfer_meta = TransferMeta(self.call_args)
self.transfer_future = TransferFuture(self.transfer_meta)
def test_progress_subscribers_on_upload(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt')
future = transfer_manager.upload(
file, self.bucket_name, '20mb.txt',
subscribers=[subscriber])
self.addCleanup(self.delete_object, '20mb.txt')
future.result()
# The callback should have been called enough times such that
# the total amount of bytes we've seen (via the "amount"
# arg to the callback function) should be the size
# of the file we uploaded.
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_upload(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
file = self.get_input_fileobj(size=20 * 1024 * 1024, name='20mb.txt')
future = transfer_manager.upload(
file, self.bucket_name, '20mb.txt',
subscribers=[subscriber])
self.addCleanup(self.delete_object, '20mb.txt')
future.result()
# The callback should have been called enough times such that
# the total amount of bytes we've seen (via the "amount"
# arg to the callback function) should be the size
# of the file we uploaded.
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_download(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
filename = self.files.create_file_with_size(
'foo.txt', filesize=20 * 1024 * 1024)
self.upload_file(filename, '20mb.txt')
download_path = os.path.join(self.files.rootdir, '20mb.txt')
future = transfer_manager.download(
self.bucket_name, '20mb.txt', download_path,
subscribers=[subscriber])
future.result()
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_callbacks_invoked(self):
subscriber = RecordingSubscriber()
self.callbacks.append(subscriber.on_progress)
self.stubber.add_response(
'copy_object', service_response={},
expected_params={
'Bucket': self.bucket, 'Key': self.key,
'CopySource': self.copy_source
}
)
task = self.get_copy_task()
task()
self.stubber.assert_no_pending_responses()
self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
def test_progress_subscribers_on_download(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
filename = self.files.create_file_with_size(
'foo.txt', filesize=20 * 1024 * 1024)
self.upload_file(filename, '20mb.txt')
download_path = os.path.join(self.files.rootdir, '20mb.txt')
future = transfer_manager.download(
self.bucket_name, '20mb.txt', download_path,
subscribers=[subscriber])
future.result()
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_cleanups_only_ran_once_on_exception(self):
# We want to be able to handle the case where the final task completes
# and anounces done but there is an error in the submission task
# which will cause it to need to anounce done as well. In this case,
# we do not want the done callbacks to be invoke more than once.
final_task = self.get_task(FailureTask, is_final=True)
self.main_kwargs['executor'] = self.executor
self.main_kwargs['tasks_to_submit'] = [final_task]
submission_task = self.get_task(ExceptionSubmissionTask,
main_kwargs=self.main_kwargs)
subscriber = RecordingSubscriber()
self.call_args.subscribers.append(subscriber)
# Add the done callback to the callbacks to be invoked when the
# transfer is done.
done_callbacks = get_callbacks(self.transfer_future, 'done')
for done_callback in done_callbacks:
self.transfer_coordinator.add_done_callback(done_callback)
submission_task()
# Make sure the task failed to start
self.assertEqual(self.transfer_coordinator.status, 'failed')
# Make sure the on_done callback of the subscriber is called only once.
self.assertEqual(subscriber.on_done_calls,
[{
'future': self.transfer_future
}])
def test_on_queued_callbacks(self):
submission_task = self.get_task(
NOOPSubmissionTask, main_kwargs=self.main_kwargs)
subscriber = RecordingSubscriber()
self.call_args.subscribers.append(subscriber)
submission_task()
# Make sure the on_queued callback of the subscriber is called.
self.assertEqual(
subscriber.on_queued_calls, [{'future': self.transfer_future}])
def test_sigv4_progress_callbacks_invoked_once(self):
# Reset the client and manager to use sigv4
self.reset_stubber_with_new_client(
{'config': Config(signature_version='s3v4')})
self.client.meta.events.register(
'before-parameter-build.s3.*', self.collect_body)
self._manager = TransferManager(self.client, self.config)
# Add the stubbed response.
self.add_put_object_response_with_default_expected_params()
subscriber = RecordingSubscriber()
future = self.manager.upload(
self.filename, self.bucket, self.key, subscribers=[subscriber])
future.result()
self.assert_expected_client_calls_were_correct()
# The amount of bytes seen should be the same as the file size
self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def test_sigv4_progress_callbacks_invoked_once(self):
# Reset the client and manager to use sigv4
self.reset_stubber_with_new_client(
{'config': Config(signature_version='s3v4')})
self.client.meta.events.register(
'before-parameter-build.s3.*', self.collect_body)
self._manager = TransferManager(self.client, self.config)
# Add the stubbed response.
self.add_put_object_response_with_default_expected_params()
subscriber = RecordingSubscriber()
future = self.manager.upload(
self.filename, self.bucket, self.key, subscribers=[subscriber])
future.result()
self.assert_expected_client_calls_were_correct()
# The amount of bytes seen should be the same as the file size
self.assertEqual(subscriber.calculate_bytes_seen(), len(self.content))
def test_callbacks_invoked(self):
subscriber = RecordingSubscriber()
self.callbacks.append(subscriber.on_progress)
self.stubber.add_response(
'upload_part_copy', service_response={
'CopyPartResult': {
'ETag': self.result_etag
}
},
expected_params={
'Bucket': self.bucket, 'Key': self.key,
'CopySource': self.copy_source, 'UploadId': self.upload_id,
'PartNumber': self.part_number,
'CopySourceRange': self.copy_source_range
}
)
task = self.get_copy_task()
self.assertEqual(
task(), {'PartNumber': self.part_number, 'ETag': self.result_etag})
self.stubber.assert_no_pending_responses()
self.assertEqual(subscriber.calculate_bytes_seen(), self.size)
def test_progress_subscribers_on_copy(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
key = '20mb.txt'
new_key = '20mb-copy.txt'
filename = self.files.create_file_with_size(
key, filesize=20 * 1024 * 1024)
self.upload_file(filename, key)
future = transfer_manager.copy(
copy_source={'Bucket': self.bucket_name, 'Key': key},
bucket=self.bucket_name,
key=new_key,
subscribers=[subscriber]
)
future.result()
# The callback should have been called enough times such that
# the total amount of bytes we've seen (via the "amount"
# arg to the callback function) should be the size
# of the file we uploaded.
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_progress_subscribers_on_copy(self):
subscriber = RecordingSubscriber()
transfer_manager = self.create_transfer_manager(self.config)
key = '20mb.txt'
new_key = '20mb-copy.txt'
filename = self.files.create_file_with_size(key,
filesize=20 * 1024 * 1024)
self.upload_file(filename, key)
future = transfer_manager.copy(copy_source={
'Bucket': self.bucket_name,
'Key': key
},
bucket=self.bucket_name,
key=new_key,
subscribers=[subscriber])
future.result()
# The callback should have been called enough times such that
# the total amount of bytes we've seen (via the "amount"
# arg to the callback function) should be the size
# of the file we uploaded.
self.assertEqual(subscriber.calculate_bytes_seen(), 20 * 1024 * 1024)
def test_retry_rewinds_callbacks(self):
self.add_head_object_response()
# Insert a response that will trigger a retry after one read of the
# stream has been made.
self.add_n_retryable_get_object_responses(1, num_reads=1)
# Add the normal responses to simulate the download proceeding
# as normal after the retry.
self.add_successful_get_object_responses()
recorder_subscriber = RecordingSubscriber()
# Set the streaming to a size that is smaller than the data we
# currently provide to it to simulate rewinds of callbacks.
self.config.io_chunksize = 3
future = self.manager.download(subscribers=[recorder_subscriber],
**self.create_call_kwargs())
future.result()
# Ensure that there is no more remaining responses and that contents
# are correct.
self.stubber.assert_no_pending_responses()
with open(self.filename, 'rb') as f:
self.assertEqual(self.content, f.read())
# Assert that the number of bytes seen is equal to the length of
# downloaded content.
self.assertEqual(recorder_subscriber.calculate_bytes_seen(),
len(self.content))
# Also ensure that the second progress invocation was negative three
# becasue a retry happened on the second read of the stream and we
# know that the chunk size for each read is 3.
progress_byte_amts = [
call['bytes_transferred']
for call in recorder_subscriber.on_progress_calls
]
self.assertEqual(-3, progress_byte_amts[1])
def test_retry_rewinds_callbacks(self):
self.add_head_object_response()
# Insert a response that will trigger a retry after one read of the
# stream has been made.
self.add_n_retryable_get_object_responses(1, num_reads=1)
# Add the normal responses to simulate the download proceeding
# as normal after the retry.
self.add_successful_get_object_responses()
recorder_subscriber = RecordingSubscriber()
# Set the streaming to a size that is smaller than the data we
# currently provide to it to simulate rewinds of callbacks.
self.config.io_chunksize = 3
future = self.manager.download(
subscribers=[recorder_subscriber], **self.create_call_kwargs())
future.result()
# Ensure that there is no more remaining responses and that contents
# are correct.
self.stubber.assert_no_pending_responses()
with open(self.filename, 'rb') as f:
self.assertEqual(self.content, f.read())
# Assert that the number of bytes seen is equal to the length of
# downloaded content.
self.assertEqual(
recorder_subscriber.calculate_bytes_seen(), len(self.content))
# Also ensure that the second progress invocation was negative three
# becasue a retry happened on the second read of the stream and we
# know that the chunk size for each read is 3.
progress_byte_amts = [
call['bytes_transferred'] for call in
recorder_subscriber.on_progress_calls
]
self.assertEqual(-3, progress_byte_amts[1])
def test_calls_done_callbacks_on_exception(self):
submission_task = self.get_task(
ExceptionSubmissionTask, main_kwargs=self.main_kwargs)
subscriber = RecordingSubscriber()
self.call_args.subscribers.append(subscriber)
# Add the done callback to the callbacks to be invoked when the
# transfer is done.
done_callbacks = get_callbacks(self.transfer_future, 'done')
for done_callback in done_callbacks:
self.transfer_coordinator.add_done_callback(done_callback)
submission_task()
# Make sure the task failed to start
self.assertEqual(self.transfer_coordinator.status, 'failed')
# Make sure the on_done callback of the subscriber is called.
self.assertEqual(
subscriber.on_done_calls, [{'future': self.transfer_future}])
def test_invoke_progress_callbacks_with_no_progress(self):
recording_subscriber = RecordingSubscriber()
invoke_progress_callbacks([recording_subscriber.on_progress], 0)
self.assertEqual(len(recording_subscriber.on_progress_calls), 0)
def test_invoke_progress_callbacks(self):
recording_subscriber = RecordingSubscriber()
invoke_progress_callbacks([recording_subscriber.on_progress], 2)
self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)
def setUp(self):
super(BaseUploadInputManagerTest, self).setUp()
self.osutil = OSUtils()
self.config = TransferConfig()
self.recording_subscriber = RecordingSubscriber()
self.subscribers.append(self.recording_subscriber)
def test_invoke_progress_callbacks(self):
recording_subscriber = RecordingSubscriber()
invoke_progress_callbacks([recording_subscriber.on_progress], 2)
self.assertEqual(recording_subscriber.calculate_bytes_seen(), 2)