def testMoveProject(): proj1 = syn.store(Project(name=str(uuid.uuid4()) + "testMoveProject-child")) proj2 = syn.store( Project(name=str(uuid.uuid4()) + "testMoveProject-newParent")) assert_raises(SynapseHTTPError, syn.move, proj1, proj2) schedule_for_cleanup(proj1) schedule_for_cleanup(proj2)
def _set_up_external_s3_project(): """ creates a project and links it to an external s3 storage :return: synapse id of the created project, and storageLocationId of the project """ EXTERNAL_S3_BUCKET = 'python-client-integration-test.sagebase.org' project_ext_s3 = syn.store(Project(name=str(uuid.uuid4()))) destination = { 'uploadType': 'S3', 'concreteType': 'org.sagebionetworks.repo.model.project.ExternalS3StorageLocationSetting', 'bucket': EXTERNAL_S3_BUCKET } destination = syn.restPOST('/storageLocation', body=json.dumps(destination)) project_destination = { 'concreteType': 'org.sagebionetworks.repo.model.project.UploadDestinationListSetting', 'settingsType': 'upload', 'locations': [destination['storageLocationId']], 'projectId': project_ext_s3.id } syn.restPOST('/projectSettings', body=json.dumps(project_destination)) schedule_for_cleanup(project_ext_s3) return project_ext_s3.id, destination['storageLocationId']
def test_getChildren(): # setup a hierarchy for folders # PROJECT # | \ # File Folder # | # File project_name = str(uuid.uuid1()) test_project = syn.store(Project(name=project_name)) folder = syn.store(Folder(name="firstFolder", parent=test_project)) syn.store( File(path="~/doesntMatter.txt", name="file inside folders", parent=folder, synapseStore=False)) project_file = syn.store( File(path="~/doesntMatterAgain.txt", name="file inside project", parent=test_project, synapseStore=False)) schedule_for_cleanup(test_project) expected_id_set = {project_file.id, folder.id} children_id_set = {x['id'] for x in syn.getChildren(test_project.id)} assert_equals(expected_id_set, children_id_set)
def test_ExternalObjectStore_roundtrip(): endpoint = "https://s3.amazonaws.com" bucket = "test-client-auth-s3" profile_name = syn._get_client_authenticated_s3_profile(endpoint, bucket) if profile_name != 'client-auth-s3-test': raise SkipTest( "This test only works on travis because it requires AWS credentials to a specific S3 bucket" ) proj = syn.store( Project(name=str(uuid.uuid4()) + "ExternalObjStoreProject")) schedule_for_cleanup(proj) storage_location = syn.createStorageLocationSetting( "ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket) syn.setStorageLocation(proj, storage_location['storageLocationId']) file_path = utils.make_bogus_data_file() file_entity = File(file_path, name="TestName", parent=proj) file_entity = syn.store(file_entity) syn.cache.purge(time.time()) assert_is_none(syn.cache.get(file_entity['dataFileHandleId'])) # verify key is in s3 import boto3 boto_session = boto3.session.Session(profile_name=profile_name) s3 = boto_session.resource('s3', endpoint_url=endpoint) try: s3_file = s3.Object(file_entity._file_handle.bucket, file_entity._file_handle.fileKey) s3_file.load() except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == "404": raise Exception("The file was not uploaded to S3") file_entity_downloaded = syn.get(file_entity['id']) file_handle = file_entity_downloaded['_file_handle'] # verify file_handle metadata assert_equals(endpoint, file_handle['endpointUrl']) assert_equals(bucket, file_handle['bucket']) assert_equals( utils.md5_for_file(file_path).hexdigest(), file_handle['contentMd5']) assert_equals(os.stat(file_path).st_size, file_handle['contentSize']) assert_equals('text/plain', file_handle['contentType']) assert_not_equal(utils.normalize_path(file_path), utils.normalize_path(file_entity_downloaded['path'])) assert_true(filecmp.cmp(file_path, file_entity_downloaded['path'])) # clean up s3_file.delete()
def test_entity_constructors(): project = Project(name=str(uuid.uuid4()), description='Testing 123') folder = Folder(name='Musicians', parent=project, genre='Jazz', datatype='personnel') personnel_file = File(fname, parentId=folder.id, group='Miles Davis Quintet', album='Stockholm 1960 Complete')
def testSetStorageLocation(): proj = syn.store( Project(name=str(uuid.uuid4()) + "testSetStorageLocation__existing_storage_location")) schedule_for_cleanup(proj) endpoint = "https://url.doesnt.matter.com" bucket = "fake-bucket-name" storage_location = syn.createStorageLocationSetting( "ExternalObjectStorage", endpointUrl=endpoint, bucket=bucket) storage_setting = syn.setStorageLocation( proj, storage_location['storageLocationId']) retrieved_setting = syn.getProjectSetting(proj, 'upload') assert_equals(storage_setting, retrieved_setting)
def test_entity_constructors(): project = Project('TestProject', id='syn1001', foo='bar') assert project.name == 'TestProject' assert project['foo'] == 'bar' folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002') assert folder.name == 'MyFolder' assert folder.foo == 'bat' assert folder.parentId == 'syn1001' a_file = File('/path/to/fabulous_things.zzz', parent=folder, foo='biz') #assert a_file.name == 'fabulous_things.zzz' assert a_file.entityType == 'org.sagebionetworks.repo.model.FileEntity' assert a_file.path == '/path/to/fabulous_things.zzz' assert a_file.foo == 'biz' assert a_file.parentId == 'syn1002'
def test_entity_constructors(): project = Project('TestProject', id='syn1001', foo='bar') assert project.name == 'TestProject' assert project['foo'] == 'bar' folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002') assert folder.name == 'MyFolder' assert folder.foo == 'bat' assert folder.parentId == 'syn1001' a_file = File('/path/to/fabulous_things.zzz', parent=folder, foo='biz', contentType='application/cattywampus') #assert a_file.name == 'fabulous_things.zzz' assert a_file.concreteType == 'org.sagebionetworks.repo.model.FileEntity' assert a_file.path == '/path/to/fabulous_things.zzz' assert a_file.foo == 'biz' assert a_file.parentId == 'syn1002' assert a_file.contentType == 'application/cattywampus' assert 'contentType' in a_file.__dict__
def test_entity_constructors(): project = Project('TestProject', id='syn1001', foo='bar') assert_equals(project.name, 'TestProject') assert_equals(project['foo'], 'bar') folder = Folder('MyFolder', parent=project, foo='bat', id='syn1002') assert_equals(folder.name, 'MyFolder') assert_equals(folder.foo, 'bat') assert_equals(folder.parentId, 'syn1001') a_file = File('/path/to/fabulous_things.zzz', parent=folder, foo='biz', contentType='application/cattywampus') assert_equals(a_file.concreteType, 'org.sagebionetworks.repo.model.FileEntity') assert_equals(a_file.path, '/path/to/fabulous_things.zzz') assert_equals(a_file.foo, 'biz') assert_equals(a_file.parentId, 'syn1002') assert_equals(a_file.contentType, 'application/cattywampus') assert_in('contentType', a_file._file_handle)
def test_is_container(): ## result from a Synapse entity annotation query ## Note: prefix may be capitalized or not, depending on the from clause of the query result = { 'entity.versionNumber': 1, 'entity.nodeType': 'project', 'entity.concreteType': ['org.sagebionetworks.repo.model.Project'], 'entity.createdOn': 1451512703905, 'entity.id': 'syn5570912', 'entity.name': 'blah' } assert is_container(result) result = { 'Entity.nodeType': 'project', 'Entity.id': 'syn5570912', 'Entity.name': 'blah' } assert is_container(result) result = { 'entity.concreteType': ['org.sagebionetworks.repo.model.Folder'], 'entity.id': 'syn5570914', 'entity.name': 'flapdoodle' } assert is_container(result) result = { 'File.concreteType': ['org.sagebionetworks.repo.model.FileEntity'], 'File.id': 'syn5570914', 'File.name': 'flapdoodle' } assert not is_container(result) assert is_container(Folder("Stuff", parentId="syn12345")) assert is_container(Project("My Project", parentId="syn12345")) assert not is_container(File("asdf.png", parentId="syn12345"))
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert ev['contentSource'] == evalNamed['contentSource'] assert ev['createdOn'] == evalNamed['createdOn'] assert ev['description'] == evalNamed['description'] assert ev['etag'] == evalNamed['etag'] assert ev['id'] == evalNamed['id'] assert ev['name'] == evalNamed['name'] assert ev['ownerId'] == evalNamed['ownerId'] assert ev['status'] == evalNamed['status'] # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = next(evalProj) assert ev['contentSource'] == evalProj['contentSource'] assert ev['createdOn'] == evalProj['createdOn'] assert ev['description'] == evalProj['description'] assert ev['etag'] == evalProj['etag'] assert ev['id'] == evalProj['id'] assert ev['name'] == evalProj['name'] assert ev['ownerId'] == evalProj['ownerId'] assert ev['status'] == evalProj['status'] # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert ev.status == 'OPEN' # # Add the current user as a participant myOwnerId = int(syn.getUserProfile()['ownerId']) syn._allowParticipation(ev, myOwnerId) # AUTHENTICATED_USERS = 273948 # PUBLIC = 273949 syn.setPermissions(ev, 273948, accessType=['READ']) syn.setPermissions(ev, 273949, accessType=['READ']) # test getPermissions permissions = syn.getPermissions(ev, 273949) assert ['READ'] == permissions permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId']) assert [ p in permissions for p in [ 'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION' ] ] # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert len(list(submissions)) == 0 # -- Get a Submission attachment belonging to another user (SYNR-541) -- # See if the configuration contains test authentication if other_user['username']: print("Testing SYNR-541") # Login as the test user testSyn = client.Synapse(skip_checks=True) testSyn.login(email=other_user['username'], password=other_user['password']) testOwnerId = int(testSyn.getUserProfile()['ownerId']) # Make a project other_project = Project(name=str(uuid.uuid4())) other_project = testSyn.createEntity(other_project) # Give the test user permission to read and join the evaluation syn._allowParticipation(ev, testOwnerId) # Make a file to submit with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=other_project.id, name='Submission 999', description="Haha! I'm inaccessible...") entity = testSyn.store(f) ## test submission by evaluation ID submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname") # Mess up the cached file so that syn._getWithEntityBundle must download again os.utime(filename, (0, 0)) # Grab the Submission as the original user fetched = syn.getSubmission(submission['id']) assert os.path.exists(fetched['filePath']) # make sure the fetched file is the same as the original (PLFM-2666) assert filecmp.cmp(filename, fetched['filePath']) else: print( 'Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE) # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring print("Creating Submissions") for i in range(num_of_submissions): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity = syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team') # Score the submissions submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1) print("Scoring Submissions") for submission in submissions: assert re.match('Submission \d+', submission['name']) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions print("Annotating Submissions") bogosity = {} submissions = syn.getSubmissions(ev) b = 123 for submission, status in syn.getSubmissionBundles(ev): bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) b += 123 status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck for submission, status in syn.getSubmissionBundles(ev): a = from_submission_status_annotations(status.annotations) assert a['foo'] == 'bar' assert a['bogosity'] == bogosity[submission.id] for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert kvp['isPrivate'] == False # test query by submission annotations # These queries run against an eventually consistent index table which is # populated by an asynchronous worker. Thus, the queries may remain out # of sync for some unbounded, but assumed to be short time. attempts = 2 while attempts > 0: try: print("Querying for submissions") results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id) print(results) assert len(results['rows']) == num_of_submissions + 1 results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id) print(results) assert len(results['rows']) == num_of_submissions except AssertionError as ex1: print("failed query: ", ex1) attempts -= 1 if attempts > 0: print("retrying...") time.sleep(2) else: attempts = 0 ## Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert len(invalid_submissions) == 1, len(invalid_submissions) assert invalid_submissions[0]['name'] == 'Submission 01' finally: # Clean up syn.delete(ev) if 'testSyn' in locals(): if 'other_project' in locals(): # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) if 'team' in locals(): ## remove team testSyn.delete(team) ## Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_findEntityIdByNameAndParent(): project_name = str(uuid.uuid1()) project_id = syn.store(Project(name=project_name))['id'] assert_equals(project_id, syn.findEntityId(project_name))