def test_single_thread_upload(syn): synapseclient.core.config.single_threaded = True try: filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + 1) assert multipart_upload_file(syn, filepath) is not None finally: synapseclient.core.config.single_threaded = False
def test_store_activity(syn, project, schedule_for_cleanup): # Create a File and an Activity path = utils.make_bogus_binary_file() schedule_for_cleanup(path) entity = File(path, name='Hinkle horn honking holes', parent=project) honking = Activity(name='Hinkle horn honking', description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.') honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/') honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/') # This doesn't set the ID of the Activity entity = syn.store(entity, activity=honking) # But this does honking = syn.getProvenance(entity.id) # Verify the Activity assert honking['name'] == 'Hinkle horn honking' assert len(honking['used']) == 2 assert honking['used'][0]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL' assert not honking['used'][0]['wasExecuted'] assert honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482') assert honking['used'][1]['concreteType'] == 'org.sagebionetworks.repo.model.provenance.UsedURL' assert not honking['used'][1]['wasExecuted'] # Store another Entity with the same Activity entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg', name='Nettlebed Cave', parent=project, synapseStore=False) entity = syn.store(entity, activity=honking) # The Activities should match honking2 = syn.getProvenance(entity) assert honking['id'] == honking2['id']
def test_round_trip(syn, project, schedule_for_cleanup): fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771) try: fhid = multipart_upload_file(syn, filepath) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert filecmp.cmp(filepath, junk.path) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_randomly_failing_parts(syn, project, schedule_for_cleanup): """Verify that we can recover gracefully with some randomly inserted errors while uploading parts.""" fail_every = 3 # fail every nth request fail_cycle = random.randint( 0, fail_every - 1) # randomly vary which n of the request cycle we fail fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + (MIN_PART_SIZE / 2)) put_count = 0 normal_put = requests.Session.put def _put_chunk_or_fail_randomly(self, url, *args, **kwargs): # fail every nth put to aws s3 if 's3.amazonaws.com' not in url: return normal_put(self, url, *args, **kwargs) nonlocal put_count put_count += 1 if (put_count + fail_cycle) % fail_every == 0: raise IOError("Ooops! Artificial upload failure for testing.") return normal_put(self, url, *args, **kwargs) with mock.patch('requests.Session.put', side_effect=_put_chunk_or_fail_randomly, autospec=True): try: fhid = multipart_upload_file(syn, filepath, part_size=MIN_PART_SIZE) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert filecmp.cmp(filepath, junk.path) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_store_isRestricted_flag(syn, project, schedule_for_cleanup): # Store a file with access requirements path = utils.make_bogus_binary_file() schedule_for_cleanup(path) entity = File(path, name='Secret human data', parent=project) # We don't want to spam ACT with test emails with patch('synapseclient.client.Synapse._createAccessRequirementIfNone') as intercepted: entity = syn.store(entity, isRestricted=True) assert intercepted.called
def test_synGet_sftpIntegration(syn, project): # Create file by uploading directly to sftp and creating entity from URL server_prefix = get_sftp_server_prefix() username, password = syn._getUserCredentials(server_prefix) server_url = server_prefix + get_user_home_path(username) + '/test_synGet_sftpIntegration/' + str(uuid.uuid1()) filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771) url = SFTPWrapper.upload_file(filepath, url=server_url, username=username, password=password) file = syn.store(File(path=url, parent=project, synapseStore=False)) junk = syn.get(file, downloadLocation=os.getcwd(), downloadFile=True) filecmp.cmp(filepath, junk.path)
def test_get_with_downloadLocation_and_ifcollision(syn, project, schedule_for_cleanup): # Store a File and delete it locally filepath = utils.make_bogus_binary_file() bogus = File(filepath, name='Bogus Test File', parent=project) bogus = syn.store(bogus) os.remove(filepath) # Compare stuff to this one normalBogus = syn.get(bogus) # Download to the temp folder, should be the same otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath)) assert otherBogus.id == normalBogus.id assert filecmp.cmp(otherBogus.path, normalBogus.path) # Invalidate the downloaded file's timestamps os.utime(otherBogus.path, (0, 0)) badtimestamps = os.path.getmtime(otherBogus.path) # Download again, should change the modification time overwriteBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="overwrite.local") overwriteModTime = os.path.getmtime(overwriteBogus.path) assert badtimestamps != overwriteModTime # Download again, should not change the modification time otherBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="keep.local") assert overwriteModTime == os.path.getmtime(overwriteBogus.path) # "keep.local" should have made the path invalid since it is keeping a potentially modified version assert otherBogus.path is None assert otherBogus.cacheDir is None assert 0 == len(otherBogus.files) # Invalidate the timestamps again os.utime(overwriteBogus.path, (0, 0)) badtimestamps = os.path.getmtime(overwriteBogus.path) # Download once more, but rename renamedBogus = syn.get(bogus, downloadLocation=os.path.dirname(filepath), ifcollision="keep.both") assert overwriteBogus.path != renamedBogus.path assert filecmp.cmp(overwriteBogus.path, renamedBogus.path) # Clean up os.remove(overwriteBogus.path) os.remove(renamedBogus.path)
def test_utils_sftp_upload_and_download(syn): """Tries to upload a file to an sftp file """ serverURL = SFTP_SERVER_PREFIX + SFTP_USER_HOME_PATH + '/test_utils_sftp_upload_and_download/' + str( uuid.uuid1()) filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771) tempdir = tempfile.mkdtemp() username, password = syn._getUserCredentials(SFTP_SERVER_PREFIX) try: url = SFTPWrapper.upload_file(filepath, url=serverURL, username=username, password=password) # Get with a specified localpath junk = SFTPWrapper.download_file(url, tempdir, username=username, password=password) filecmp.cmp(filepath, junk) # Get without specifying path junk2 = SFTPWrapper.download_file(url, username=username, password=password) filecmp.cmp(filepath, junk2) # Get with a specified localpath as file junk3 = SFTPWrapper.download_file(url, os.path.join(tempdir, 'bar.dat'), username=username, password=password) filecmp.cmp(filepath, junk3) finally: try: if 'junk' in locals(): os.remove(junk) if 'junk2' in locals(): os.remove(junk2) if 'junk3' in locals(): os.remove(junk3) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc()) try: shutil.rmtree(tempdir) except Exception: print(traceback.format_exc())
def test_pool_provider_is_used_in__multipart_upload(): mocked_get_chunk_function = MagicMock(side_effect=[1, 2, 3, 4]) file_size = 1 * MB filepath = make_bogus_binary_file(n=file_size) md5 = md5_for_file(filepath).hexdigest() status = {'partsState': {}, 'uploadId': {}, 'state': 'COMPLETED'} pool = MagicMock() with patch.object(syn, "restPOST", return_value=status),\ patch.object(pool_provider, "get_pool", return_value=pool) as mock_provider: _multipart_upload(syn, filepath, "application/octet-stream", mocked_get_chunk_function, md5, file_size) mock_provider.assert_called() pool.map.assert_called()
def test_chunks(): # Read a file in chunks, write the chunks out, and compare to the original try: file_size = 1 * MB filepath = make_bogus_binary_file(n=file_size) chunksize = 64 * 1024 nchunks = int(math.ceil(float(file_size) / chunksize)) with tempfile.NamedTemporaryFile(mode='wb', delete=False) as out: for i in range(1, nchunks + 1): out.write(get_file_chunk(filepath, i, chunksize)) assert_true(filecmp.cmp(filepath, out.name)) finally: if 'filepath' in locals() and filepath: os.remove(filepath) if 'out' in locals() and out: os.remove(out.name)
def test_synStore_sftpIntegration(syn, project, schedule_for_cleanup): """Creates a File Entity on an sftp server and add the external url. """ filepath = utils.make_bogus_binary_file(1 * utils.MB - 777771) try: file = syn.store(File(filepath, parent=project)) file2 = syn.get(file) assert file.externalURL == file2.externalURL assert urlparse(file2.externalURL).scheme == 'sftp' tmpdir = tempfile.mkdtemp() schedule_for_cleanup(tmpdir) # test that we got an MD5 à la SYNPY-185 assert file2.md5 is not None fh = syn._get_file_handle_as_creator(file2.dataFileHandleId) assert fh['contentMd5'] is not None assert file2.md5 == fh['contentMd5'] finally: try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_download_file_false(syn, project, schedule_for_cleanup): RENAME_SUFFIX = 'blah' # Upload a file filepath = utils.make_bogus_binary_file() schedule_for_cleanup(filepath) schedule_for_cleanup(filepath + RENAME_SUFFIX) file = File(filepath, name='SYNR 619', parent=project) file = syn.store(file) # Now hide the file from the cache and download with downloadFile=False os.rename(filepath, filepath + RENAME_SUFFIX) file = syn.get(file.id, downloadFile=False) # Change something and reupload the file's metadata file.name = "Only change the name, not the file" reupload = syn.store(file) assert reupload.path is None, "Path field should be null: %s" % reupload.path # This should still get the correct file reupload = syn.get(reupload.id) assert filecmp.cmp(filepath + RENAME_SUFFIX, reupload.path) assert reupload.name == file.name
def test_store_with_flags(syn, project, schedule_for_cleanup): # -- CreateOrUpdate flag for Projects -- # If we store a project with the same name, it should become an update projUpdate = Project(project.name) projUpdate.updatedThing = 'Yep, sho\'nuf it\'s updated!' projUpdate = syn.store(projUpdate, createOrUpdate=True) assert project.id == projUpdate.id assert projUpdate.updatedThing == ['Yep, sho\'nuf it\'s updated!'] # Store a File filepath = utils.make_bogus_binary_file() schedule_for_cleanup(filepath) origBogus = File(filepath, name='Bogus Test File', parent=project) origBogus = syn.store(origBogus, createOrUpdate=True) assert origBogus.versionNumber == 1 # Modify existing annotations by createOrUpdate del projUpdate['parentId'] del projUpdate['id'] projUpdate.updatedThing = 'Updated again' projUpdate.addedThing = 'Something new' projUpdate = syn.store(projUpdate, createOrUpdate=True) assert project.id == projUpdate.id assert projUpdate.updatedThing == ['Updated again'] # -- ForceVersion flag -- # Re-store the same thing and don't up the version mutaBogus = syn.store(origBogus, forceVersion=False) assert mutaBogus.versionNumber == 1 # Re-store again, essentially the same condition mutaBogus = syn.store(mutaBogus, createOrUpdate=True, forceVersion=False) assert mutaBogus.versionNumber == 1, "expected version 1 but got version %s" % mutaBogus.versionNumber # And again, but up the version this time mutaBogus = syn.store(mutaBogus, forceVersion=True) assert mutaBogus.versionNumber == 2 # Create file with different contents and store it with force version false # This should be ignored because contents (and md5) are different different_filepath = utils.make_bogus_binary_file() schedule_for_cleanup(different_filepath) mutaBogus = File(different_filepath, name='Bogus Test File', parent=project) mutaBogus = syn.store(mutaBogus, forceVersion=False) assert mutaBogus.versionNumber == 3 # -- CreateOrUpdate flag for files -- # Store a different file with the same name and parent # Expected behavior is that a new version of the first File will be created new_filepath = utils.make_bogus_binary_file() schedule_for_cleanup(new_filepath) mutaBogus.path = new_filepath mutaBogus = syn.store(mutaBogus, createOrUpdate=True) assert mutaBogus.id == origBogus.id assert mutaBogus.versionNumber == 4 assert not filecmp.cmp(mutaBogus.path, filepath) # Make doubly sure the File was uploaded checkBogus = syn.get(mutaBogus.id) assert checkBogus.id == origBogus.id assert checkBogus.versionNumber == 4 assert filecmp.cmp(mutaBogus.path, checkBogus.path) # Create yet another file with the same name and parent # Expected behavior is raising an exception with a 409 error newer_filepath = utils.make_bogus_binary_file() schedule_for_cleanup(newer_filepath) badBogus = File(newer_filepath, name='Bogus Test File', parent=project) pytest.raises(SynapseHTTPError, syn.store, badBogus, createOrUpdate=False) # -- Storing after syn.get(..., downloadFile=False) -- ephemeralBogus = syn.get(mutaBogus, downloadFile=False) ephemeralBogus.description = 'Snorklewacker' ephemeralBogus.shoe_size = 11.5 ephemeralBogus = syn.store(ephemeralBogus) ephemeralBogus = syn.get(ephemeralBogus, downloadFile=False) assert ephemeralBogus.description == 'Snorklewacker' assert ephemeralBogus.shoe_size == [11.5]