def test_single_thread_upload(syn):
synapseclient.core.config.single_threaded = True
try:
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + 1)
assert multipart_upload_file(syn, filepath) is not None
finally:
synapseclient.core.config.single_threaded = False
def test_round_trip(syn, project, schedule_for_cleanup):
fhid = None
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771)
try:
fhid = multipart_upload_file(syn, filepath)
# Download the file and compare it with the original
junk = File(parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity',
tmp_path)
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def upload_synapse_s3(syn, file_path, storageLocationId=None, mimetype=None):
file_handle_id = multipart_upload_file(syn,
file_path,
contentType=mimetype,
storageLocationId=storageLocationId)
syn.cache.add(file_handle_id, file_path)
return syn._getFileHandle(file_handle_id)
def test_randomly_failing_parts(syn, project, schedule_for_cleanup):
"""Verify that we can recover gracefully with some randomly inserted errors
while uploading parts."""
fail_every = 3 # fail every nth request
fail_cycle = random.randint(
0,
fail_every - 1) # randomly vary which n of the request cycle we fail
fhid = None
filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 +
(MIN_PART_SIZE / 2))
put_count = 0
normal_put = requests.Session.put
def _put_chunk_or_fail_randomly(self, url, *args, **kwargs):
# fail every nth put to aws s3
if 's3.amazonaws.com' not in url:
return normal_put(self, url, *args, **kwargs)
nonlocal put_count
put_count += 1
if (put_count + fail_cycle) % fail_every == 0:
raise IOError("Ooops! Artificial upload failure for testing.")
return normal_put(self, url, *args, **kwargs)
with mock.patch('requests.Session.put',
side_effect=_put_chunk_or_fail_randomly,
autospec=True):
try:
fhid = multipart_upload_file(syn,
filepath,
part_size=MIN_PART_SIZE)
# Download the file and compare it with the original
junk = File(parent=project, dataFileHandleId=fhid)
junk.properties.update(syn._createEntity(junk.properties))
(tmp_f, tmp_path) = tempfile.mkstemp()
schedule_for_cleanup(tmp_path)
junk['path'] = syn._downloadFileHandle(fhid, junk['id'],
'FileEntity', tmp_path)
assert filecmp.cmp(filepath, junk.path)
finally:
try:
if 'junk' in locals():
syn.delete(junk)
except Exception:
print(traceback.format_exc())
try:
os.remove(filepath)
except Exception:
print(traceback.format_exc())
def upload_synapse_s3(syn, file_path, storageLocationId=None, mimetype=None, max_threads=None):
file_handle_id = multipart_upload_file(
syn,
file_path,
content_type=mimetype,
storage_location_id=storageLocationId,
max_threads=max_threads,
)
syn.cache.add(file_handle_id, file_path)
return syn._get_file_handle_as_creator(file_handle_id)
def test_multipart_upload_file(self):
"""Verify multipart_upload_file passes through its
args, validating and supplying defaults as expected."""
syn = mock.Mock()
file_path = '/foo/bar/baz'
file_size = 1234
md5_hex = 'abc123'
with mock.patch('os.path.exists') as os_path_exists,\
mock.patch('os.path.isdir') as os_path_is_dir,\
mock.patch('os.path.getsize') as os_path_getsize,\
mock.patch.object(
synapseclient.core.upload.multipart_upload,
'md5_for_file',
) as md5_for_file,\
mock.patch.object(
synapseclient.core.upload.multipart_upload,
'_multipart_upload',
) as mock_multipart_upload:
os_path_getsize.return_value = file_size
md5_for_file.return_value.hexdigest.return_value = md5_hex
os_path_exists.return_value = False
# bad file
with pytest.raises(IOError):
multipart_upload_file(syn, file_path)
os_path_exists.return_value = True
os_path_is_dir.return_value = True
with pytest.raises(IOError):
multipart_upload_file(syn, file_path)
os_path_is_dir.return_value = False
# call w/ defaults
multipart_upload_file(syn, file_path)
mock_multipart_upload.assert_called_once_with(
syn,
mock.ANY, # lambda chunk function
file_size,
None, # part_size
'baz',
md5_hex,
'application/octet-stream', # content_type
None, # storage_location_id
True, # preview
False, # force_restart
None, # max_threads
)
mock_multipart_upload.reset_mock()
# call specifying all optional kwargs
kwargs = {
'dest_file_name': 'blort',
'content_type': 'text/plain',
'part_size': 9876,
'storage_location_id': 5432,
'preview': False,
'force_restart': True,
'max_threads': 8,
}
multipart_upload_file(
syn,
file_path,
**kwargs
)
mock_multipart_upload.assert_called_once_with(
syn,
mock.ANY, # lambda chunk function
file_size,
kwargs['part_size'],
kwargs['dest_file_name'],
md5_hex,
kwargs['content_type'],
kwargs['storage_location_id'],
kwargs['preview'],
kwargs['force_restart'],
kwargs['max_threads'],
)
def test_multipart_upload_file(self):
"""Verify multipart_upload_file passes through its
args, validating and supplying defaults as expected."""
syn = mock.Mock()
file_path = '/foo/bar/baz'
file_size = 1234
md5_hex = 'abc123'
storage_location_id = 5432
with mock.patch('os.path.exists') as os_path_exists,\
mock.patch('os.path.isdir') as os_path_is_dir,\
mock.patch('os.path.getsize') as os_path_getsize,\
mock.patch.object(
multipart_upload,
'md5_for_file',
) as md5_for_file,\
mock.patch.object(
multipart_upload,
'_multipart_upload',
) as mock_multipart_upload,\
mock.patch.object(multipart_upload, 'Spinner') as mock_spinner:
os_path_getsize.return_value = file_size
md5_for_file.return_value.hexdigest.return_value = md5_hex
os_path_exists.return_value = False
# bad file
with pytest.raises(IOError):
multipart_upload_file(syn, file_path, storage_location_id=storage_location_id)
os_path_exists.return_value = True
os_path_is_dir.return_value = True
with pytest.raises(IOError):
multipart_upload_file(syn, file_path, storage_location_id=storage_location_id)
os_path_is_dir.return_value = False
expected_upload_request = {
'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest',
'contentType': 'application/octet-stream',
'contentMD5Hex': md5_hex,
'fileName': 'baz',
'fileSizeBytes': file_size,
'generatePreview': True,
'storageLocationId': storage_location_id,
'partSizeBytes': DEFAULT_PART_SIZE,
}
# call w/ defaults
multipart_upload_file(
syn,
file_path,
storage_location_id=storage_location_id,
)
mock_multipart_upload.assert_called_once_with(
syn,
'baz',
expected_upload_request,
mock.ANY, # part_fn
mock.ANY, # md5_fn,
force_restart=False,
max_threads=None,
)
# Test when call the multipart_upload_file, md5_for_file pass in the correct callback function
syn_with_silent_mode = Synapse(silent=True, skip_checks=True)
multipart_upload_file(
syn_with_silent_mode,
file_path,
storage_location_id=storage_location_id,
)
md5_for_file.assert_called_with(file_path, callback=None)
syn_with_no_silent_mode = Synapse(debug=False, skip_checks=True)
multipart_upload_file(
syn_with_no_silent_mode,
file_path,
storage_location_id=storage_location_id,
)
md5_for_file.assert_called_with(file_path, callback=mock_spinner.return_value.print_tick)
mock_multipart_upload.reset_mock()
# call specifying all optional kwargs
kwargs = {
'dest_file_name': 'blort',
'content_type': 'text/plain',
'part_size': MIN_PART_SIZE * 2,
'preview': False,
'force_restart': True,
'max_threads': 8,
}
expected_upload_request = {
'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest',
'contentType': kwargs['content_type'],
'contentMD5Hex': md5_hex,
'fileName': kwargs['dest_file_name'],
'fileSizeBytes': file_size,
'generatePreview': kwargs['preview'],
'storageLocationId': storage_location_id,
'partSizeBytes': kwargs['part_size'],
}
multipart_upload_file(
syn,
file_path,
storage_location_id=storage_location_id,
**kwargs
)
mock_multipart_upload.assert_called_once_with(
syn,
kwargs['dest_file_name'],
expected_upload_request,
mock.ANY, # part_fn
mock.ANY, # md5_fn,
force_restart=kwargs['force_restart'],
max_threads=kwargs['max_threads'],
)
