def test_single_thread_upload(syn): synapseclient.core.config.single_threaded = True try: filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + 1) assert multipart_upload_file(syn, filepath) is not None finally: synapseclient.core.config.single_threaded = False
def test_round_trip(syn, project, schedule_for_cleanup): fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771) try: fhid = multipart_upload_file(syn, filepath) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert filecmp.cmp(filepath, junk.path) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def upload_synapse_s3(syn, file_path, storageLocationId=None, mimetype=None): file_handle_id = multipart_upload_file(syn, file_path, contentType=mimetype, storageLocationId=storageLocationId) syn.cache.add(file_handle_id, file_path) return syn._getFileHandle(file_handle_id)
def test_randomly_failing_parts(syn, project, schedule_for_cleanup): """Verify that we can recover gracefully with some randomly inserted errors while uploading parts.""" fail_every = 3 # fail every nth request fail_cycle = random.randint( 0, fail_every - 1) # randomly vary which n of the request cycle we fail fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + (MIN_PART_SIZE / 2)) put_count = 0 normal_put = requests.Session.put def _put_chunk_or_fail_randomly(self, url, *args, **kwargs): # fail every nth put to aws s3 if 's3.amazonaws.com' not in url: return normal_put(self, url, *args, **kwargs) nonlocal put_count put_count += 1 if (put_count + fail_cycle) % fail_every == 0: raise IOError("Ooops! Artificial upload failure for testing.") return normal_put(self, url, *args, **kwargs) with mock.patch('requests.Session.put', side_effect=_put_chunk_or_fail_randomly, autospec=True): try: fhid = multipart_upload_file(syn, filepath, part_size=MIN_PART_SIZE) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert filecmp.cmp(filepath, junk.path) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def upload_synapse_s3(syn, file_path, storageLocationId=None, mimetype=None, max_threads=None): file_handle_id = multipart_upload_file( syn, file_path, content_type=mimetype, storage_location_id=storageLocationId, max_threads=max_threads, ) syn.cache.add(file_handle_id, file_path) return syn._get_file_handle_as_creator(file_handle_id)
def test_multipart_upload_file(self): """Verify multipart_upload_file passes through its args, validating and supplying defaults as expected.""" syn = mock.Mock() file_path = '/foo/bar/baz' file_size = 1234 md5_hex = 'abc123' with mock.patch('os.path.exists') as os_path_exists,\ mock.patch('os.path.isdir') as os_path_is_dir,\ mock.patch('os.path.getsize') as os_path_getsize,\ mock.patch.object( synapseclient.core.upload.multipart_upload, 'md5_for_file', ) as md5_for_file,\ mock.patch.object( synapseclient.core.upload.multipart_upload, '_multipart_upload', ) as mock_multipart_upload: os_path_getsize.return_value = file_size md5_for_file.return_value.hexdigest.return_value = md5_hex os_path_exists.return_value = False # bad file with pytest.raises(IOError): multipart_upload_file(syn, file_path) os_path_exists.return_value = True os_path_is_dir.return_value = True with pytest.raises(IOError): multipart_upload_file(syn, file_path) os_path_is_dir.return_value = False # call w/ defaults multipart_upload_file(syn, file_path) mock_multipart_upload.assert_called_once_with( syn, mock.ANY, # lambda chunk function file_size, None, # part_size 'baz', md5_hex, 'application/octet-stream', # content_type None, # storage_location_id True, # preview False, # force_restart None, # max_threads ) mock_multipart_upload.reset_mock() # call specifying all optional kwargs kwargs = { 'dest_file_name': 'blort', 'content_type': 'text/plain', 'part_size': 9876, 'storage_location_id': 5432, 'preview': False, 'force_restart': True, 'max_threads': 8, } multipart_upload_file( syn, file_path, **kwargs ) mock_multipart_upload.assert_called_once_with( syn, mock.ANY, # lambda chunk function file_size, kwargs['part_size'], kwargs['dest_file_name'], md5_hex, kwargs['content_type'], kwargs['storage_location_id'], kwargs['preview'], kwargs['force_restart'], kwargs['max_threads'], )
def test_multipart_upload_file(self): """Verify multipart_upload_file passes through its args, validating and supplying defaults as expected.""" syn = mock.Mock() file_path = '/foo/bar/baz' file_size = 1234 md5_hex = 'abc123' storage_location_id = 5432 with mock.patch('os.path.exists') as os_path_exists,\ mock.patch('os.path.isdir') as os_path_is_dir,\ mock.patch('os.path.getsize') as os_path_getsize,\ mock.patch.object( multipart_upload, 'md5_for_file', ) as md5_for_file,\ mock.patch.object( multipart_upload, '_multipart_upload', ) as mock_multipart_upload,\ mock.patch.object(multipart_upload, 'Spinner') as mock_spinner: os_path_getsize.return_value = file_size md5_for_file.return_value.hexdigest.return_value = md5_hex os_path_exists.return_value = False # bad file with pytest.raises(IOError): multipart_upload_file(syn, file_path, storage_location_id=storage_location_id) os_path_exists.return_value = True os_path_is_dir.return_value = True with pytest.raises(IOError): multipart_upload_file(syn, file_path, storage_location_id=storage_location_id) os_path_is_dir.return_value = False expected_upload_request = { 'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest', 'contentType': 'application/octet-stream', 'contentMD5Hex': md5_hex, 'fileName': 'baz', 'fileSizeBytes': file_size, 'generatePreview': True, 'storageLocationId': storage_location_id, 'partSizeBytes': DEFAULT_PART_SIZE, } # call w/ defaults multipart_upload_file( syn, file_path, storage_location_id=storage_location_id, ) mock_multipart_upload.assert_called_once_with( syn, 'baz', expected_upload_request, mock.ANY, # part_fn mock.ANY, # md5_fn, force_restart=False, max_threads=None, ) # Test when call the multipart_upload_file, md5_for_file pass in the correct callback function syn_with_silent_mode = Synapse(silent=True, skip_checks=True) multipart_upload_file( syn_with_silent_mode, file_path, storage_location_id=storage_location_id, ) md5_for_file.assert_called_with(file_path, callback=None) syn_with_no_silent_mode = Synapse(debug=False, skip_checks=True) multipart_upload_file( syn_with_no_silent_mode, file_path, storage_location_id=storage_location_id, ) md5_for_file.assert_called_with(file_path, callback=mock_spinner.return_value.print_tick) mock_multipart_upload.reset_mock() # call specifying all optional kwargs kwargs = { 'dest_file_name': 'blort', 'content_type': 'text/plain', 'part_size': MIN_PART_SIZE * 2, 'preview': False, 'force_restart': True, 'max_threads': 8, } expected_upload_request = { 'concreteType': 'org.sagebionetworks.repo.model.file.MultipartUploadRequest', 'contentType': kwargs['content_type'], 'contentMD5Hex': md5_hex, 'fileName': kwargs['dest_file_name'], 'fileSizeBytes': file_size, 'generatePreview': kwargs['preview'], 'storageLocationId': storage_location_id, 'partSizeBytes': kwargs['part_size'], } multipart_upload_file( syn, file_path, storage_location_id=storage_location_id, **kwargs ) mock_multipart_upload.assert_called_once_with( syn, kwargs['dest_file_name'], expected_upload_request, mock.ANY, # part_fn mock.ANY, # md5_fn, force_restart=kwargs['force_restart'], max_threads=kwargs['max_threads'], )