def test_multipart_upload__retry_success(self):
"""Verify we recover on a failed upload if a subsequent
retry succeeds."""
syn = mock.Mock()
md5_hex = 'ab123'
file_size = 1234
part_size = 567
dest_file_name = 'foo'
content_type = 'text/plain'
storage_location_id = 3210
result_file_handle_id = 'foo'
max_threads = 5
upload_side_effect = [
SynapseUploadFailedException(),
SynapseUploadFailedException(),
mock.Mock(
return_value={'resultFileHandleId': result_file_handle_id}
)
]
expected_upload_request = {
'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest',
'contentType': content_type,
'contentMD5Hex': md5_hex,
'fileName': dest_file_name,
'fileSizeBytes': file_size,
'generatePreview': True,
'storageLocationId': storage_location_id,
'partSizeBytes': part_size,
}
result, upload_mock = self._multipart_upload_test(
upload_side_effect,
syn,
dest_file_name,
expected_upload_request,
mock.ANY, # part_fn
mock.ANY, # md5_fn,
max_threads,
False,
)
# should have been called multiple times but returned
# the result in the end.
assert result_file_handle_id == result
assert len(upload_side_effect) == upload_mock.call_count
def test_multipart_upload__retry_failure(self):
"""Verify if we run out of upload attempts we give up
and raise the failure."""
syn = mock.Mock()
chunk_fn = mock.Mock()
md5_hex = 'ab123'
file_size = 1234
dest_file_name = 'foo'
content_type = 'text/plain'
storage_location_id = 3210
upload_side_effect = SynapseUploadFailedException()
with pytest.raises(SynapseUploadFailedException):
self._multipart_upload_test(
upload_side_effect,
syn,
chunk_fn,
file_size,
None, # part_size
dest_file_name,
md5_hex,
content_type,
storage_location_id,
None, # max_threads
)
def test_multipart_upload__retry_success(self):
"""Verify we recover on a failed upload if a subsequent
retry succeeds."""
syn = mock.Mock()
chunk_fn = mock.Mock()
md5_hex = 'ab123'
file_size = 1234
dest_file_name = 'foo'
content_type = 'text/plain'
storage_location_id = 3210
result_file_handle_id = 'foo'
upload_side_effect = [
SynapseUploadFailedException(),
SynapseUploadFailedException(),
mock.Mock(
return_value={'resultFileHandleId': result_file_handle_id}
)
]
result, upload_mock = self._multipart_upload_test(
upload_side_effect,
syn,
chunk_fn,
file_size,
None, # part_size
dest_file_name,
md5_hex,
content_type,
storage_location_id,
None, # max_threads
)
# should have been called multiple times but returned
# the result in the end.
assert result_file_handle_id == result
assert len(upload_side_effect) == upload_mock.call_count
def _complete_upload(self):
upload_status_response = self._syn.restPUT(
"/file/multipart/{upload_id}/complete".format(
upload_id=self._upload_id, ),
requests_session=self._get_thread_session(),
endpoint=self._syn.fileHandleEndpoint,
)
upload_state = upload_status_response.get('state')
if upload_state != 'COMPLETED':
# at this point we think successfully uploaded all the parts
# but the upload status isn't complete, we'll throw an error
# and let a subsequent attempt try to reconcile
raise SynapseUploadFailedException(
"Upload status has an unexpected state {}".format(
upload_state))
return upload_status_response
def test_multipart_upload__retry_failure(self):
"""Verify if we run out of upload attempts we give up
and raise the failure."""
syn = mock.Mock()
md5_hex = 'ab123'
file_size = 1234
part_size = 567
dest_file_name = 'foo'
content_type = 'text/plain'
storage_location_id = 3210
max_threads = 5
upload_side_effect = SynapseUploadFailedException()
expected_upload_request = {
'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest',
'contentType': content_type,
'contentMD5Hex': md5_hex,
'fileName': dest_file_name,
'fileSizeBytes': file_size,
'generatePreview': True,
'storageLocationId': storage_location_id,
'partSizeBytes': part_size
}
with pytest.raises(SynapseUploadFailedException):
self._multipart_upload_test(
upload_side_effect,
syn,
dest_file_name,
expected_upload_request,
mock.ANY, # part_fn
mock.ANY, # md5_fn,
max_threads,
False,
)
def _upload_parts(self, part_count, remaining_part_numbers):
time_upload_started = time.time()
completed_part_count = part_count - len(remaining_part_numbers)
file_size = self._upload_request_payload.get('fileSizeBytes')
if not self._is_copy():
# we won't have bytes to measure during a copy so the byte oriented progress bar is not useful
progress = previously_transferred = min(
completed_part_count * self._part_size,
file_size,
)
self._syn._print_transfer_progress(
progress,
file_size,
prefix='Uploading',
postfix=self._dest_file_name,
previouslyTransferred=previously_transferred,
)
self._pre_signed_part_urls = self._fetch_pre_signed_part_urls(
self._upload_id,
remaining_part_numbers,
)
futures = []
with _executor(self._max_threads, False) as executor:
# we don't wait on the shutdown since we do so ourselves below
for part_number in remaining_part_numbers:
futures.append(
executor.submit(
self._handle_part,
part_number,
)
)
for result in concurrent.futures.as_completed(futures):
try:
_, part_size = result.result()
if part_size and not self._is_copy():
progress += part_size
self._syn._print_transfer_progress(
min(progress, file_size),
file_size,
prefix='Uploading',
postfix=self._dest_file_name,
dt=time.time() - time_upload_started,
previouslyTransferred=previously_transferred,
)
except (Exception, KeyboardInterrupt) as cause:
with self._lock:
self._aborted = True
# wait for all threads to complete before
# raising the exception, we don't want to return
# control while there are still threads from this
# upload attempt running
concurrent.futures.wait(futures)
if isinstance(cause, KeyboardInterrupt):
raise SynapseUploadAbortedException(
"User interrupted upload"
)
raise SynapseUploadFailedException(
"Part upload failed"
) from cause
def _upload_parts(self, part_count, remaining_part_numbers):
time_upload_started = time.time()
completed_part_count = part_count - len(remaining_part_numbers)
# note this is an estimate, may not be exact since the final part
# may be smaller and might be included in the completed parts.
# it's good enough though.
progress = previously_transferred = min(
completed_part_count * self._part_size, self._file_size)
printTransferProgress(
progress,
self._file_size,
prefix='Uploading',
postfix=self._dest_file_name,
previouslyTransferred=previously_transferred,
)
self._pre_signed_part_urls = self._fetch_pre_signed_part_urls(
self._upload_id,
remaining_part_numbers,
)
futures = []
with _executor(self._max_threads, False) as executor:
# we don't wait on the shutdown since we do so ourselves below
for part_number in remaining_part_numbers:
futures.append(
executor.submit(
self._handle_part,
part_number,
))
for result in concurrent.futures.as_completed(futures):
try:
_, part_size = result.result()
progress += part_size
printTransferProgress(
min(progress, self._file_size),
self._file_size,
prefix='Uploading',
postfix=self._dest_file_name,
dt=time.time() - time_upload_started,
previouslyTransferred=previously_transferred,
)
except (Exception, KeyboardInterrupt) as cause:
with self._lock:
self._aborted = True
# wait for all threads to complete before
# raising the exception, we don't want to return
# control while there are still threads from this
# upload attempt running
concurrent.futures.wait(futures)
if isinstance(cause, KeyboardInterrupt):
raise SynapseUploadAbortedException(
"User interrupted upload")
raise SynapseUploadFailedException(
"Part upload failed") from cause