def test_downloadFile(): # See if the a "wget" works filename = utils.download_file( "http://dev-versions.synapse.sagebase.org/sage_bionetworks_logo_274x128.png" ) schedule_for_cleanup(filename) assert_true(os.path.exists(filename))
def test_teams(): name = "My Uniquely Named Team " + str(uuid.uuid4()) team = syn.store(Team(name=name, description="A fake team for testing...")) schedule_for_cleanup(team) found_team = syn.getTeam(team.id) assert_equals(team, found_team) p = syn.getUserProfile() found = None for m in syn.getTeamMembers(team): if m.member.ownerId == p.ownerId: found = m break assert_is_not_none(found, "Couldn't find user {} in team".format(p.userName)) # needs to be retried 'cause appending to the search index is asynchronous tries = 10 found_team = None while tries > 0: try: found_team = syn.getTeam(name) break except ValueError: tries -= 1 if tries > 0: time.sleep(1) assert_equals(team, found_team)
def test_uploadFileEntity(): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = File(name='fooUploadFileEntity', path=fname, parentId=project['id'], description='A test file entity') entity = syn.store(entity) # Download and verify entity = syn.get(entity) assert_equals(entity['files'][0], os.path.basename(fname)) assert_true(filecmp.cmp(fname, entity['path'])) # Check if we upload the wrong type of file handle fh = syn.restGET('/entity/%s/filehandles' % entity.id)['list'][0] assert_equals(fh['concreteType'], 'org.sagebionetworks.repo.model.file.S3FileHandle') # Create a different temporary file fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) # Update existing FileEntity entity.path = fname entity = syn.store(entity) # Download and verify that it is the same file entity = syn.get(entity) assert_equals(entity['files'][0], os.path.basename(fname)) assert_true(filecmp.cmp(fname, entity['path']))
def test_synapseStore_flag(): # Store a path to a local file path = utils.make_bogus_data_file() schedule_for_cleanup(path) bogus = File(path, name='Totally bogus data', parent=project, synapseStore=False) bogus = syn.store(bogus) # Verify the thing can be downloaded as a URL bogus = syn.get(bogus, downloadFile=False) assert_equals(bogus.name, 'Totally bogus data') assert_equals(bogus.path, path, "Path: %s\nExpected: %s" % (bogus.path, path)) assert_false(bogus.synapseStore) # Make sure the test runs on Windows and other OS's if path[0].isalpha() and path[1] == ':': # A Windows file URL looks like this: file:///c:/foo/bar/bat.txt expected_url = 'file:///' + path.replace("\\", "/") else: expected_url = 'file://' + path assert_equals(bogus.externalURL, expected_url, 'URL: %s\nExpected %s' % (bogus.externalURL, expected_url)) # A file path that doesn't exist should still work bogus = File('/path/to/local/file1.xyz', parentId=project.id, synapseStore=False) bogus = syn.store(bogus) assert_raises(IOError, syn.get, bogus) assert_false(bogus.synapseStore) # Try a URL bogus = File('http://dev-versions.synapse.sagebase.org/synapsePythonClient', parent=project, synapseStore=False) bogus = syn.store(bogus) bogus = syn.get(bogus) assert_false(bogus.synapseStore)
def test_getChildren(): # setup a hierarchy for folders # PROJECT # | \ # File Folder # | # File project_name = str(uuid.uuid1()) test_project = syn.store(Project(name=project_name)) folder = syn.store(Folder(name="firstFolder", parent=test_project)) syn.store( File(path="~/doesntMatter.txt", name="file inside folders", parent=folder, synapseStore=False)) project_file = syn.store( File(path="~/doesntMatterAgain.txt", name="file inside project", parent=test_project, synapseStore=False)) schedule_for_cleanup(test_project) expected_id_set = {project_file.id, folder.id} children_id_set = {x['id'] for x in syn.getChildren(test_project.id)} assert_equals(expected_id_set, children_id_set)
def test_get_local_file(): new_path = utils.make_bogus_data_file() schedule_for_cleanup(new_path) folder = Folder('TestFindFileFolder', parent=project, description='A place to put my junk') folder = syn.store(folder) # Get an nonexistent file in Synapse assert_raises(SynapseError, syn.get, new_path) # Get a file really stored in Synapse ent_folder = syn.store(File(new_path, parent=folder)) ent2 = syn.get(new_path) assert_equals(ent_folder.id, ent2.id) assert_equals(ent_folder.versionNumber, ent2.versionNumber) # Get a file stored in Multiple locations #should display warning syn.store(File(new_path, parent=project)) syn.get(new_path) # Get a file stored in multiple locations with limit set ent = syn.get(new_path, limitSearch=folder.id) assert_equals(ent.id, ent_folder.id) assert_equals(ent.versionNumber, ent_folder.versionNumber) # Get a file that exists but such that limitSearch removes them and raises error assert_raises(SynapseError, syn.get, new_path, limitSearch='syn1')
def test_store__changing_from_Synapse_to_externalURL_by_changing_path(): # create a temp file temp_path = utils.make_bogus_data_file() schedule_for_cleanup(temp_path) ext = syn.store(File(temp_path, parent=project, synapseStore=True)) ext = syn.get(ext) assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle", ext._file_handle.concreteType) ext.synapseStore = False ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert_equal("org.sagebionetworks.repo.model.file.ExternalFileHandle", ext._file_handle.concreteType) assert_equal(utils.as_url(temp_path), ext.externalURL) assert_equal(False, ext.synapseStore) # swap back to synapse storage ext.synapseStore = True ext = syn.store(ext) # do a get to make sure filehandle has been updated correctly ext = syn.get(ext.id, downloadFile=True) assert_equal("org.sagebionetworks.repo.model.file.S3FileHandle", ext._file_handle.concreteType) assert_equal(None, ext.externalURL) assert_equal(True, ext.synapseStore)
def _makeManifest(content): with tempfile.NamedTemporaryFile(mode='w', suffix=".dat", delete=False) as f: f.write(content) filepath = utils.normalize_path(f.name) schedule_for_cleanup(filepath) return filepath
def test_store_activity(): # Create a File and an Activity path = utils.make_bogus_binary_file() schedule_for_cleanup(path) entity = File(path, name='Hinkle horn honking holes', parent=project) honking = Activity(name='Hinkle horn honking', description='Nettlebed Cave is a limestone cave located on the South Island of New Zealand.') honking.used('http://www.flickr.com/photos/bevanbfree/3482259379/') honking.used('http://www.flickr.com/photos/bevanbfree/3482185673/') # This doesn't set the ID of the Activity entity = syn.store(entity, activity=honking) # But this does honking = syn.getProvenance(entity.id) # Verify the Activity assert_equals(honking['name'], 'Hinkle horn honking') assert_equals(len(honking['used']), 2) assert_equals(honking['used'][0]['concreteType'], 'org.sagebionetworks.repo.model.provenance.UsedURL') assert_false(honking['used'][0]['wasExecuted']) assert_true(honking['used'][0]['url'].startswith('http://www.flickr.com/photos/bevanbfree/3482')) assert_equals(honking['used'][1]['concreteType'], 'org.sagebionetworks.repo.model.provenance.UsedURL') assert_false(honking['used'][1]['wasExecuted']) # Store another Entity with the same Activity entity = File('http://en.wikipedia.org/wiki/File:Nettlebed_cave.jpg', name='Nettlebed Cave', parent=project, synapseStore=False) entity = syn.store(entity, activity=honking) # The Activities should match honking2 = syn.getProvenance(entity) assert_equals(honking['id'], honking2['id'])
def test_uploadFile_given_dictionary(): # Make a Folder Entity the old fashioned way folder = { 'concreteType': Folder._synapse_entity_type, 'parentId': project['id'], 'name': 'fooDictionary', 'foo': 334455 } entity = syn.store(folder) # Download and verify that it is the same file entity = syn.get(entity) assert_equals(entity.parentId, project.id) assert_equals(entity.foo[0], 334455) # Update via a dictionary path = utils.make_bogus_data_file() schedule_for_cleanup(path) rareCase = {} rareCase.update(entity.annotations) rareCase.update(entity.properties) rareCase.update(entity.local_state()) rareCase[ 'description'] = 'Updating with a plain dictionary should be rare.' # Verify it works entity = syn.store(rareCase) assert_equals(entity.description, rareCase['description']) assert_equals(entity.name, 'fooDictionary') syn.get(entity['id'])
def test_round_trip(): fhid = None filepath = utils.make_bogus_binary_file(MIN_PART_SIZE + 777771) try: fhid = multipart_upload_file(syn, filepath) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert_true(filecmp.cmp(filepath, junk.path)) finally: try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_multipart_upload_big_string(): cities = ["Seattle", "Portland", "Vancouver", "Victoria", "San Francisco", "Los Angeles", "New York", "Oaxaca", "Cancún", "Curaçao", "जोधपुर", "অসম", "ལྷ་ས།", "ཐིམ་ཕུ་", "دبي", "አዲስ አበባ", "São Paulo", "Buenos Aires", "Cartagena", "Amsterdam", "Venice", "Rome", "Dubrovnik", "Sarajevo", "Madrid", "Barcelona", "Paris", "Αθήνα", "Ρόδος", "København", "Zürich", "金沢市", "서울", "แม่ฮ่องสอน", "Москва"] text = "Places I wanna go:\n" while len(text.encode('utf-8')) < MIN_PART_SIZE: text += ", ".join(random.choice(cities) for i in range(5000)) + "\n" fhid = multipart_upload_string(syn, text) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], "FileEntity", tmp_path) with open(junk.path, encoding='utf-8') as f: retrieved_text = f.read() assert_equals(retrieved_text, text)
def test_synapse_integer_columns_with_missing_values_from_dataframe(): # SYNPY-267 cols = [ Column(name='x', columnType='STRING'), Column(name='y', columnType='INTEGER'), Column(name='z', columnType='DOUBLE') ] schema = syn.store(Schema(name='Big Table', columns=cols, parent=project)) line_terminator = str(os.linesep) # write rows to CSV file with tempfile.NamedTemporaryFile(mode="w", suffix=".csv", delete=False) as temp: schedule_for_cleanup(temp.name) # 2nd row is missing a value in its integer column temp.write('x,y,z' + line_terminator + 'a,1,0.9' + line_terminator + 'b,,0.8' + line_terminator + 'c,3,0.7' + line_terminator) temp.flush() filename = temp.name # create a table from csv table = Table(schema, filename) df = table.asDataFrame() table_from_dataframe = Table(schema, df) assert_not_equal(table.filepath, table_from_dataframe.filepath) df2 = table_from_dataframe.asDataFrame() assert_frame_equal(df, df2)
def dontruntest_big_csvs(): cols = [ Column(name='name', columnType='STRING', maximumSize=1000), Column(name='foo', columnType='STRING', enumValues=['foo', 'bar', 'bat']), Column(name='x', columnType='DOUBLE'), Column(name='n', columnType='INTEGER'), Column(name='is_bogus', columnType='BOOLEAN') ] schema1 = syn.store(Schema(name='Big Table', columns=cols, parent=project)) # write rows to CSV file with tempfile.NamedTemporaryFile(delete=False) as temp: schedule_for_cleanup(temp.name) filename = temp.name with io.open(filename, mode='w', encoding="utf-8", newline='') as temp: writer = csv.writer(temp, quoting=csv.QUOTE_NONNUMERIC, lineterminator=str(os.linesep)) writer.writerow([col.name for col in cols]) for i in range(10): for j in range(100): foo = cols[1].enumValues[random.randint(0, 2)] writer.writerow( ('Robot ' + str(i * 100 + j), foo, random.random() * 200.0, random.randint(0, 100), random.random() >= 0.5)) # upload CSV syn._uploadCsv(filepath=temp.name, schema=schema1) from synapseclient.table import CsvFileTable CsvFileTable.from_table_query(syn, "select * from %s" % schema1.id)
def test_store_file_handle_update_metadata(): original_file_path = utils.make_bogus_data_file() schedule_for_cleanup(original_file_path) # upload the project entity = syn.store(File(original_file_path, parent=project)) old_file_handle = entity._file_handle # create file handle to replace the old one replacement_file_path = utils.make_bogus_data_file() schedule_for_cleanup(replacement_file_path) new_file_handle = syn.uploadFileHandle(replacement_file_path, parent=project) entity.dataFileHandleId = new_file_handle['id'] new_entity = syn.store(entity) # make sure _file_handle info was changed # (_file_handle values are all changed at once so just verifying id change is sufficient) assert_equal(new_file_handle['id'], new_entity._file_handle['id']) assert_not_equal(old_file_handle['id'], new_entity._file_handle['id']) # check that local_state was updated assert_equal(replacement_file_path, new_entity.path) assert_equal(os.path.dirname(replacement_file_path), new_entity.cacheDir) assert_equal([os.path.basename(replacement_file_path)], new_entity.files)
def testMoveProject(): proj1 = syn.store(Project(name=str(uuid.uuid4()) + "testMoveProject-child")) proj2 = syn.store( Project(name=str(uuid.uuid4()) + "testMoveProject-newParent")) assert_raises(SynapseHTTPError, syn.move, proj1, proj2) schedule_for_cleanup(proj1) schedule_for_cleanup(proj2)
def test_download_local_file_URL_path(): path = utils.make_bogus_data_file() schedule_for_cleanup(path) filehandle = create_external_file_handle(syn, path, mimetype=None, file_size=None) localFileEntity = syn.store(File(dataFileHandleId=filehandle['id'], parent=project)) e = syn.get(localFileEntity.id) assert_equal(path, e.path)
def _create_temp_file_with_cleanup(specific_file_text=None): if specific_file_text: with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as file: file.write(specific_file_text) filename = file.name else: filename = utils.make_bogus_data_file() schedule_for_cleanup(filename) return filename
def test_store_isRestricted_flag(): # Store a file with access requirements path = utils.make_bogus_binary_file() schedule_for_cleanup(path) entity = File(path, name='Secret human data', parent=project) # We don't want to spam ACT with test emails with patch('synapseclient.client.Synapse._createAccessRequirementIfNone') as intercepted: entity = syn.store(entity, isRestricted=True) assert_true(intercepted.called)
def test_syncToSynapse(): # Test upload of accurate manifest manifest = _makeManifest(header + row1 + row2 + row3) synapseutils.syncToSynapse(syn, manifest, sendMessages=False, retries=2) # syn.getChildren() used by syncFromSynapse() may intermittently have timing issues time.sleep(3) # Download using syncFromSynapse tmpdir = tempfile.mkdtemp() schedule_for_cleanup(tmpdir) entities = synapseutils.syncFromSynapse(syn, project, path=tmpdir) orig_df = pd.read_csv(manifest, sep='\t') orig_df.index = [os.path.basename(p) for p in orig_df.path] new_df = pd.read_csv(os.path.join(tmpdir, synapseutils.sync.MANIFEST_FILENAME), sep='\t') new_df.index = [os.path.basename(p) for p in new_df.path] assert_equals(len(orig_df), len(new_df)) new_df = new_df.loc[orig_df.index] # Validate what was uploaded is in right location assert_true(new_df.parent.equals(orig_df.parent), 'Downloaded files not stored in same location') # Validate that annotations were set cols = synapseutils.sync.REQUIRED_FIELDS + synapseutils.sync.FILE_CONSTRUCTOR_FIELDS\ + synapseutils.sync.STORE_FUNCTION_FIELDS orig_anots = orig_df.drop(cols, axis=1, errors='ignore') new_anots = new_df.drop(cols, axis=1, errors='ignore') assert_equals( orig_anots.shape[1], new_anots.shape[1]) # Verify that we have the same number of cols assert_true(new_anots.equals(orig_anots.loc[:, new_anots.columns]), 'Annotations different') # Validate that provenance is correct for provenanceType in ['executed', 'used']: # Go through each row for orig, new in zip(orig_df[provenanceType], new_df[provenanceType]): if not pd.isnull(orig) and not pd.isnull(new): # Convert local file paths into synId.versionNumber strings orig_list = [ '%s.%s' % (i.id, i.versionNumber) if isinstance(i, Entity) else i for i in syn._convertProvenanceList(orig.split(';')) ] new_list = [ '%s.%s' % (i.id, i.versionNumber) if isinstance(i, Entity) else i for i in syn._convertProvenanceList(new.split(';')) ] assert_equals(set(orig_list), set(new_list))
def test_syncFromSynapse__given_file_id(): file_path = utils.make_bogus_data_file() schedule_for_cleanup(file_path) file = syn.store( File(file_path, name=str(uuid.uuid4()), parent=project, synapseStore=False)) all_files = synapseutils.syncFromSynapse(syn, file.id) assert_equals(1, len(all_files)) assert_equals(file, all_files[0])
def testCustomConfigFile(): if os.path.isfile(client.CONFIG_FILE): configPath = './CONFIGFILE' shutil.copyfile(client.CONFIG_FILE, configPath) schedule_for_cleanup(configPath) syn2 = Synapse(configPath=configPath) syn2.login() else: raise ValueError( "Please supply a username and password in the configuration file.")
def setup(module): module.syn = integration.syn module.project = integration.project # Use the module-level syn object to communicate between main and child threads # - Read-only objects (for the children) module.syn.test_parent = module.syn.store(Project(name=str(uuid.uuid4()))) schedule_for_cleanup(module.syn.test_parent) module.syn.test_keepRunning = True # - Child writeable objects module.syn.test_errors = Queue() module.syn.test_runCountMutex = Lock() module.syn.test_threadsRunning = 0
def test_entity_view_add_annotation_columns(): folder1 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj1', parent=project, annotations={ 'strAnno': 'str1', 'intAnno': 1, 'floatAnno': 1.1 })) folder2 = syn.store( Folder(name=str(uuid.uuid4()) + 'test_entity_view_add_annotation_columns_proj2', parent=project, annotations={ 'dateAnno': datetime.now(), 'strAnno': 'str2', 'intAnno': 2 })) schedule_for_cleanup(folder1) schedule_for_cleanup(folder2) scopeIds = [utils.id_of(folder1), utils.id_of(folder2)] # This test is to ensure that user code which use the deprecated field `type` continue to work # TODO: remove this test case in Synapse Python client 2.0 entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='project', parent=project) syn.store(entity_view) # This test is to ensure that user code which use the deprecated field `type` continue to work # TODO: remove this test case in Synapse Python client 2.0 entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, type='file', includeEntityTypes=[EntityViewType.PROJECT], parent=project) syn.store(entity_view) entity_view = EntityViewSchema(name=str(uuid.uuid4()), scopeIds=scopeIds, addDefaultViewColumns=False, addAnnotationColumns=True, includeEntityTypes=[EntityViewType.PROJECT], parent=project) syn.store(entity_view)
def testSetStorageLocation(): proj = syn.store( Project(name=str(uuid.uuid4()) + "testSetStorageLocation__existing_storage_location")) schedule_for_cleanup(proj) endpoint = "https://url.doesnt.matter.com" bucket = "fake-bucket-name" storage_location = syn.createStorageLocationSetting( "ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket) storage_setting = syn.setStorageLocation( proj, storage_location['storageLocationId']) retrieved_setting = syn.getProjectSetting(proj, 'upload') assert_equals(storage_setting, retrieved_setting)
def setup(self): # Create a Project self.project_entity = syn.store(Project(name=str(uuid.uuid4()))) filename = utils.make_bogus_data_file() attachname = utils.make_bogus_data_file() file_entity = syn.store(File(filename, parent=self.project_entity)) schedule_for_cleanup(self.project_entity.id) schedule_for_cleanup(filename) schedule_for_cleanup(file_entity.id) # Create mock wiki md = """ This is a test wiki ======================= Blabber jabber blah blah boo. syn123 syn456 """ wiki = Wiki(owner=self.project_entity, title='A Test Wiki', markdown=md, attachments=[attachname]) wiki = syn.store(wiki) # Create a Wiki sub-page subwiki = Wiki(owner=self.project_entity, title='A sub-wiki', markdown='%s' % file_entity.id, parentWikiId=wiki.id) self.subwiki = syn.store(subwiki) second_md = """ Testing internal links ====================== [test](#!Synapse:%s/wiki/%s) %s) """ % (self.project_entity.id, self.subwiki.id, file_entity.id) sub_subwiki = Wiki(owner=self.project_entity, title='A sub-sub-wiki', markdown=second_md, parentWikiId=self.subwiki.id, attachments=[attachname]) self.sub_subwiki = syn.store(sub_subwiki) # Set up the second project self.second_project = syn.store(Project(name=str(uuid.uuid4()))) schedule_for_cleanup(self.second_project.id) self.fileMapping = {'syn123': 'syn12345', 'syn456': 'syn45678'} self.first_headers = syn.getWikiHeaders(self.project_entity)
def test_provenance(): # Create a File Entity fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) data_entity = syn.store(File(fname, parent=project['id'])) # Create a File Entity of Code fd, path = tempfile.mkstemp(suffix=".py") with os.fdopen(fd, 'w') as f: f.write( utils.normalize_lines(""" ## Chris's fabulous random data generator ############################################################ import random random.seed(12345) data = [random.gauss(mu=0.0, sigma=1.0) for i in range(100)] """)) schedule_for_cleanup(path) code_entity = syn.store(File(path, parent=project['id'])) # Create a new Activity asserting that the Code Entity was 'used' activity = Activity(name='random.gauss', description='Generate some random numbers') activity.used(code_entity, wasExecuted=True) activity.used( { 'name': 'Superhack', 'url': 'https://github.com/joe_coder/Superhack' }, wasExecuted=True) activity = syn.setProvenance(data_entity, activity) # Retrieve and verify the saved Provenance record retrieved_activity = syn.getProvenance(data_entity) assert_equals(retrieved_activity, activity) # Test Activity update new_description = 'Generate random numbers like a gangsta' retrieved_activity['description'] = new_description updated_activity = syn.updateActivity(retrieved_activity) assert_equals(updated_activity['name'], retrieved_activity['name']) assert_equals(updated_activity['description'], new_description) # Test delete syn.deleteProvenance(data_entity) assert_raises(SynapseHTTPError, syn.getProvenance, data_entity['id'])
def test_entity_version(): # Make an Entity and make sure the version is one entity = File(parent=project['id']) entity['path'] = utils.make_bogus_data_file() schedule_for_cleanup(entity['path']) entity = syn.store(entity) syn.setAnnotations(entity, {'fizzbuzz': 111222}) entity = syn.get(entity) assert_equals(entity.versionNumber, 1) # Update the Entity and make sure the version is incremented entity.foo = 998877 entity['name'] = 'foobarbat' entity['description'] = 'This is a test entity...' entity = syn.store(entity, incrementVersion=True, versionLabel="Prada remix") assert_equals(entity.versionNumber, 2) # Get the older data and verify the random stuff is still there annotations = syn.getAnnotations(entity, version=1) assert_equals(annotations['fizzbuzz'][0], 111222) returnEntity = syn.get(entity, version=1) assert_equals(returnEntity.versionNumber, 1) assert_equals(returnEntity['fizzbuzz'][0], 111222) assert_not_in('foo', returnEntity) # Try the newer Entity returnEntity = syn.get(entity) assert_equals(returnEntity.versionNumber, 2) assert_equals(returnEntity['foo'][0], 998877) assert_equals(returnEntity['name'], 'foobarbat') assert_equals(returnEntity['description'], 'This is a test entity...') assert_equals(returnEntity['versionLabel'], 'Prada remix') # Try the older Entity again returnEntity = syn.get(entity, version=1) assert_equals(returnEntity.versionNumber, 1) assert_equals(returnEntity['fizzbuzz'][0], 111222) assert_not_in('foo', returnEntity) # Delete version 2 syn.delete(entity, version=2) returnEntity = syn.get(entity) assert_equals(returnEntity.versionNumber, 1)
def test_randomly_failing_parts(): FAILURE_RATE = 1.0/3.0 fhid = None MIN_PART_SIZE = 5 * MB MAX_RETRIES = 20 filepath = utils.make_bogus_binary_file(MIN_PART_SIZE * 2 + 777771) normal_put_chunk = None def _put_chunk_or_fail_randomly(url, chunk, verbose=False): if random.random() < FAILURE_RATE: raise IOError("Ooops! Artificial upload failure for testing.") else: return normal_put_chunk(url, chunk, verbose) # Mock _put_chunk to fail randomly normal_put_chunk = multipart_upload._put_chunk multipart_upload._put_chunk = _put_chunk_or_fail_randomly try: fhid = multipart_upload_file(syn, filepath) # Download the file and compare it with the original junk = File(parent=project, dataFileHandleId=fhid) junk.properties.update(syn._createEntity(junk.properties)) (tmp_f, tmp_path) = tempfile.mkstemp() schedule_for_cleanup(tmp_path) junk['path'] = syn._downloadFileHandle(fhid, junk['id'], 'FileEntity', tmp_path) assert_true(filecmp.cmp(filepath, junk.path)) finally: # Un-mock _put_chunk if normal_put_chunk: multipart_upload._put_chunk = normal_put_chunk try: if 'junk' in locals(): syn.delete(junk) except Exception: print(traceback.format_exc()) try: os.remove(filepath) except Exception: print(traceback.format_exc())
def test_download_multithreaded(): # Create a FileEntity # Dictionaries default to FileEntity as a type fname = utils.make_bogus_data_file() schedule_for_cleanup(fname) entity = File(name='testMultiThreadDownload' + str(uuid.uuid4()), path=fname, parentId=project['id']) entity = syn.store(entity) # Download and verify syn.multi_threaded = True entity = syn.get(entity) assert_equals(entity['files'][0], os.path.basename(fname)) assert_true(filecmp.cmp(fname, entity['path'])) syn.multi_threaded = False