def test_submission_status_annotations_round_trip(): april_28_1969 = Datetime(1969, 4, 28) a = Annotations( 'syn123', '7bdb83e9-a50a-46e4-987a-4962559f090f', { 'screen_name': 'Bullwinkle', 'species': 'Moose', 'lucky': 13, 'pi': pi, 'birthday': april_28_1969 }) sa = to_submission_status_annotations(a) assert {'screen_name', 'species'} == set([kvp['key'] for kvp in sa['stringAnnos']]) assert {'Bullwinkle', 'Moose'} == set([kvp['value'] for kvp in sa['stringAnnos']]) # test idempotence assert sa == to_submission_status_annotations(sa) assert {'lucky', 'birthday'} == set([kvp['key'] for kvp in sa['longAnnos']]) for kvp in sa['longAnnos']: key = kvp['key'] value = kvp['value'] if key == 'lucky': assert value == 13 if key == 'birthday': assert utils.from_unix_epoch_time(value) == april_28_1969 assert {'pi'} == set([kvp['key'] for kvp in sa['doubleAnnos']]) assert {pi} == set([kvp['value'] for kvp in sa['doubleAnnos']]) set_privacy(sa, key='screen_name', is_private=False) pytest.raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False) for kvp in sa['stringAnnos']: if kvp['key'] == 'screen_name': assert not kvp['isPrivate'] a2 = from_submission_status_annotations(sa) # TODO: is there a way to convert dates back from longs automatically? a2['birthday'] = utils.from_unix_epoch_time(a2['birthday']) assert a == a2 # test idempotence assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip(): april_28_1969 = Datetime(1969, 4, 28) a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969) sa = to_submission_status_annotations(a) print sa assert set(['screen_name', 'species']) == set([kvp['key'] for kvp in sa['stringAnnos']]) assert set(['Bullwinkle', 'Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']]) ## test idempotence assert sa == to_submission_status_annotations(sa) assert set(['lucky', 'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']]) for kvp in sa['longAnnos']: key = kvp['key'] value = kvp['value'] if key == 'lucky': assert value == 13 if key == 'birthday': assert utils.from_unix_epoch_time(value) == april_28_1969 assert set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']]) assert set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']]) set_privacy(sa, key='screen_name', is_private=False) assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False) for kvp in sa['stringAnnos']: if kvp['key'] == 'screen_name': assert kvp['isPrivate'] == False a2 = from_submission_status_annotations(sa) # TODO: is there a way to convert dates back from longs automatically? a2['birthday'] = utils.from_unix_epoch_time(a2['birthday']) assert a == a2 ## test idempotence assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip(): from math import pi april_28_1969 = Datetime(1969,4,28) a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969) sa = to_submission_status_annotations(a) print sa assert set(['screen_name','species']) == set([kvp['key'] for kvp in sa['stringAnnos']]) assert set(['Bullwinkle','Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']]) ## test idempotence assert sa == to_submission_status_annotations(sa) assert set(['lucky', 'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']]) for kvp in sa['longAnnos']: key = kvp['key'] value = kvp['value'] if key=='lucky': assert value == 13 if key=='birthday': assert utils.from_unix_epoch_time(value) == april_28_1969 set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']]) set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']]) set_privacy(sa, key='screen_name', is_private=False) assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False) for kvp in sa['stringAnnos']: if kvp['key'] == 'screen_name': assert kvp['isPrivate'] == False a2 = from_submission_status_annotations(sa) # TODO: is there a way to convert dates back from longs automatically? a2['birthday'] = utils.from_unix_epoch_time(a2['birthday']) assert a == a2 ## test idempotence assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip(): april_28_1969 = Datetime(1969, 4, 28) a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969) sa = to_submission_status_annotations(a) assert_equals({'screen_name', 'species'}, set([kvp['key'] for kvp in sa['stringAnnos']])) assert_equals({'Bullwinkle', 'Moose'}, set([kvp['value'] for kvp in sa['stringAnnos']])) # test idempotence assert_equals(sa, to_submission_status_annotations(sa)) assert_equals({'lucky', 'birthday'}, set([kvp['key'] for kvp in sa['longAnnos']])) for kvp in sa['longAnnos']: key = kvp['key'] value = kvp['value'] if key == 'lucky': assert_equals(value, 13) if key == 'birthday': assert_equals(utils.from_unix_epoch_time(value), april_28_1969) assert_equals({'pi'}, set([kvp['key'] for kvp in sa['doubleAnnos']])) assert_equals({pi}, set([kvp['value'] for kvp in sa['doubleAnnos']])) set_privacy(sa, key='screen_name', is_private=False) assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False) for kvp in sa['stringAnnos']: if kvp['key'] == 'screen_name': assert_false(kvp['isPrivate']) a2 = from_submission_status_annotations(sa) # TODO: is there a way to convert dates back from longs automatically? a2['birthday'] = utils.from_unix_epoch_time(a2['birthday']) assert_equals(a, a2) # test idempotence assert_equals(a, from_submission_status_annotations(a))
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert_equals(ev['contentSource'], evalNamed['contentSource']) assert_equals(ev['createdOn'], evalNamed['createdOn']) assert_equals(ev['description'], evalNamed['description']) assert_equals(ev['etag'], evalNamed['etag']) assert_equals(ev['id'], evalNamed['id']) assert_equals(ev['name'], evalNamed['name']) assert_equals(ev['ownerId'], evalNamed['ownerId']) assert_equals(ev['status'], evalNamed['status']) # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = next(evalProj) assert_equals(ev['contentSource'], evalProj['contentSource']) assert_equals(ev['createdOn'], evalProj['createdOn']) assert_equals(ev['description'], evalProj['description']) assert_equals(ev['etag'], evalProj['etag']) assert_equals(ev['id'], evalProj['id']) assert_equals(ev['name'], evalProj['name']) assert_equals(ev['ownerId'], evalProj['ownerId']) assert_equals(ev['status'], evalProj['status']) # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert_equals(ev.status, 'OPEN') # Add the current user as a participant myOwnerId = int(syn.getUserProfile()['ownerId']) syn._allowParticipation(ev, myOwnerId) # AUTHENTICATED_USERS = 273948 # PUBLIC = 273949 syn.setPermissions(ev, 273948, accessType=['READ']) syn.setPermissions(ev, 273949, accessType=['READ']) # test getPermissions permissions = syn.getPermissions(ev, 273949) assert_equals(['READ'], permissions) permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId']) for p in [ 'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION' ]: assert_in(p, permissions) # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert_equals(len(list(submissions)), 0) # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring for i in range(num_of_submissions): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity = syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team') # Score the submissions submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1) for submission in submissions: assert_true(re.match('Submission \d+', submission['name'])) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions bogosity = {} submissions = syn.getSubmissions(ev) b = 123 for submission, status in syn.getSubmissionBundles(ev): bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) b += 123 status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck for submission, status in syn.getSubmissionBundles(ev): a = from_submission_status_annotations(status.annotations) assert_equals(a['foo'], 'bar') assert_equals(a['bogosity'], bogosity[submission.id]) for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert_false(kvp['isPrivate']) # test query by submission annotations # These queries run against an eventually consistent index table which is # populated by an asynchronous worker. Thus, the queries may remain out # of sync for some unbounded, but assumed to be short time. attempts = 2 while attempts > 0: try: results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id) assert_equals(len(results['rows']), num_of_submissions + 1) results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id) assert_equals(len(results['rows']), num_of_submissions) except AssertionError as ex1: attempts -= 1 time.sleep(2) else: attempts = 0 # Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert_equals(len(invalid_submissions), 1, len(invalid_submissions)) assert_equals(invalid_submissions[0]['name'], 'Submission 01') finally: # Clean up syn.delete(ev) if 'testSyn' in locals(): if 'other_project' in locals(): # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) if 'team' in locals(): # remove team testSyn.delete(team) # Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert ev['contentSource'] == evalNamed['contentSource'] assert ev['createdOn'] == evalNamed['createdOn'] assert ev['description'] == evalNamed['description'] assert ev['etag'] == evalNamed['etag'] assert ev['id'] == evalNamed['id'] assert ev['name'] == evalNamed['name'] assert ev['ownerId'] == evalNamed['ownerId'] assert ev['status'] == evalNamed['status'] # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = next(evalProj) assert ev['contentSource'] == evalProj['contentSource'] assert ev['createdOn'] == evalProj['createdOn'] assert ev['description'] == evalProj['description'] assert ev['etag'] == evalProj['etag'] assert ev['id'] == evalProj['id'] assert ev['name'] == evalProj['name'] assert ev['ownerId'] == evalProj['ownerId'] assert ev['status'] == evalProj['status'] # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert ev.status == 'OPEN' # # Add the current user as a participant myOwnerId = int(syn.getUserProfile()['ownerId']) syn._allowParticipation(ev, myOwnerId) # AUTHENTICATED_USERS = 273948 # PUBLIC = 273949 syn.setPermissions(ev, 273948, accessType=['READ']) syn.setPermissions(ev, 273949, accessType=['READ']) # test getPermissions permissions = syn.getPermissions(ev, 273949) assert ['READ'] == permissions permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId']) assert [p in permissions for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']] # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert len(list(submissions)) == 0 # -- Get a Submission attachment belonging to another user (SYNR-541) -- # See if the configuration contains test authentication try: config = configparser.ConfigParser() config.read(client.CONFIG_FILE) other_user = {} other_user['username'] = config.get('test-authentication', 'username') other_user['password'] = config.get('test-authentication', 'password') print("Testing SYNR-541") # Login as the test user testSyn = client.Synapse(skip_checks=True) testSyn.login(email=other_user['username'], password=other_user['password']) testOwnerId = int(testSyn.getUserProfile()['ownerId']) # Make a project other_project = Project(name=str(uuid.uuid4())) other_project = testSyn.createEntity(other_project) # Give the test user permission to read and join the evaluation syn._allowParticipation(ev, testOwnerId) # Make a file to submit with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0,1)) + '\n') f = File(filename, parentId=other_project.id, name='Submission 999', description ="Haha! I'm inaccessible...") entity = testSyn.store(f) ## test submission by evaluation ID submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname") # Mess up the cached file so that syn._getWithEntityBundle must download again os.utime(filename, (0, 0)) # Grab the Submission as the original user fetched = syn.getSubmission(submission['id']) assert os.path.exists(fetched['filePath']) # make sure the fetched file is the same as the original (PLFM-2666) assert filecmp.cmp(filename, fetched['filePath']) except configparser.Error: print('Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE) # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring print("Creating Submissions") for i in range(num_of_submissions): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0,1)) + '\n') f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity=syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team') # Score the submissions submissions = syn.getSubmissions(ev, limit=num_of_submissions-1) print("Scoring Submissions") for submission in submissions: assert re.match('Submission \d+', submission['name']) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions print("Annotating Submissions") bogosity = {} submissions = syn.getSubmissions(ev) b = 123 for submission, status in syn.getSubmissionBundles(ev): bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) b += 123 status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck for submission, status in syn.getSubmissionBundles(ev): a = from_submission_status_annotations(status.annotations) assert a['foo'] == 'bar' assert a['bogosity'] == bogosity[submission.id] for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert kvp['isPrivate'] == False # test query by submission annotations # These queries run against an eventually consistent index table which is # populated by an asynchronous worker. Thus, the queries may remain out # of sync for some unbounded, but assumed to be short time. attempts = 2 while attempts > 0: try: print("Querying for submissions") results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id) print(results) assert results[u'totalNumberOfResults'] == num_of_submissions+1 results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id) print(results) assert results[u'totalNumberOfResults'] == num_of_submissions except AssertionError as ex1: print("failed query: ", ex1) attempts -= 1 if attempts > 0: print("retrying...") time.sleep(2) else: attempts = 0 ## Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert len(invalid_submissions) == 1, len(invalid_submissions) assert invalid_submissions[0]['name'] == 'Submission 01' finally: # Clean up syn.delete(ev) if 'testSyn' in locals(): if 'other_project' in locals(): # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) if 'team' in locals(): ## remove team testSyn.delete(team) ## Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert ev['contentSource'] == evalNamed['contentSource'] assert ev['createdOn'] == evalNamed['createdOn'] assert ev['description'] == evalNamed['description'] assert ev['etag'] == evalNamed['etag'] assert ev['id'] == evalNamed['id'] assert ev['name'] == evalNamed['name'] assert ev['ownerId'] == evalNamed['ownerId'] assert ev['status'] == evalNamed['status'] # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = next(evalProj) assert ev['contentSource'] == evalProj['contentSource'] assert ev['createdOn'] == evalProj['createdOn'] assert ev['description'] == evalProj['description'] assert ev['etag'] == evalProj['etag'] assert ev['id'] == evalProj['id'] assert ev['name'] == evalProj['name'] assert ev['ownerId'] == evalProj['ownerId'] assert ev['status'] == evalProj['status'] # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert ev.status == 'OPEN' # # Add the current user as a participant myOwnerId = int(syn.getUserProfile()['ownerId']) syn._allowParticipation(ev, myOwnerId) # AUTHENTICATED_USERS = 273948 # PUBLIC = 273949 syn.setPermissions(ev, 273948, accessType=['READ']) syn.setPermissions(ev, 273949, accessType=['READ']) # test getPermissions permissions = syn.getPermissions(ev, 273949) assert ['READ'] == permissions permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId']) assert [ p in permissions for p in [ 'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION' ] ] # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert len(list(submissions)) == 0 # -- Get a Submission attachment belonging to another user (SYNR-541) -- # See if the configuration contains test authentication if other_user['username']: print("Testing SYNR-541") # Login as the test user testSyn = client.Synapse(skip_checks=True) testSyn.login(email=other_user['username'], password=other_user['password']) testOwnerId = int(testSyn.getUserProfile()['ownerId']) # Make a project other_project = Project(name=str(uuid.uuid4())) other_project = testSyn.createEntity(other_project) # Give the test user permission to read and join the evaluation syn._allowParticipation(ev, testOwnerId) # Make a file to submit with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=other_project.id, name='Submission 999', description="Haha! I'm inaccessible...") entity = testSyn.store(f) ## test submission by evaluation ID submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname") # Mess up the cached file so that syn._getWithEntityBundle must download again os.utime(filename, (0, 0)) # Grab the Submission as the original user fetched = syn.getSubmission(submission['id']) assert os.path.exists(fetched['filePath']) # make sure the fetched file is the same as the original (PLFM-2666) assert filecmp.cmp(filename, fetched['filePath']) else: print( 'Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE) # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring print("Creating Submissions") for i in range(num_of_submissions): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity = syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team') # Score the submissions submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1) print("Scoring Submissions") for submission in submissions: assert re.match('Submission \d+', submission['name']) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions print("Annotating Submissions") bogosity = {} submissions = syn.getSubmissions(ev) b = 123 for submission, status in syn.getSubmissionBundles(ev): bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) b += 123 status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck for submission, status in syn.getSubmissionBundles(ev): a = from_submission_status_annotations(status.annotations) assert a['foo'] == 'bar' assert a['bogosity'] == bogosity[submission.id] for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert kvp['isPrivate'] == False # test query by submission annotations # These queries run against an eventually consistent index table which is # populated by an asynchronous worker. Thus, the queries may remain out # of sync for some unbounded, but assumed to be short time. attempts = 2 while attempts > 0: try: print("Querying for submissions") results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id) print(results) assert len(results['rows']) == num_of_submissions + 1 results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id) print(results) assert len(results['rows']) == num_of_submissions except AssertionError as ex1: print("failed query: ", ex1) attempts -= 1 if attempts > 0: print("retrying...") time.sleep(2) else: attempts = 0 ## Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert len(invalid_submissions) == 1, len(invalid_submissions) assert invalid_submissions[0]['name'] == 'Submission 01' finally: # Clean up syn.delete(ev) if 'testSyn' in locals(): if 'other_project' in locals(): # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) if 'team' in locals(): ## remove team testSyn.delete(team) ## Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert ev['contentSource'] == evalNamed['contentSource'] assert ev['createdOn'] == evalNamed['createdOn'] assert ev['description'] == evalNamed['description'] assert ev['etag'] == evalNamed['etag'] assert ev['id'] == evalNamed['id'] assert ev['name'] == evalNamed['name'] assert ev['ownerId'] == evalNamed['ownerId'] assert ev['status'] == evalNamed['status'] # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = evalProj.next() assert ev['contentSource'] == evalProj['contentSource'] assert ev['createdOn'] == evalProj['createdOn'] assert ev['description'] == evalProj['description'] assert ev['etag'] == evalProj['etag'] assert ev['id'] == evalProj['id'] assert ev['name'] == evalProj['name'] assert ev['ownerId'] == evalProj['ownerId'] assert ev['status'] == evalProj['status'] # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert ev.status == 'OPEN' # Add the current user as a participant syn.joinEvaluation(ev) # Find this user in the participant list foundMe = False myOwnerId = int(syn.getUserProfile()['ownerId']) for item in syn.getParticipants(ev): if int(item['userId']) == myOwnerId: foundMe = True break assert foundMe # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert len(list(submissions)) == 0 # -- Get a Submission attachment belonging to another user (SYNR-541) -- # See if the configuration contains test authentication try: config = ConfigParser.ConfigParser() config.read(client.CONFIG_FILE) other_user = {} other_user['username'] = config.get('test-authentication', 'username') other_user['password'] = config.get('test-authentication', 'password') print "Testing SYNR-541" # Login as the test user testSyn = client.Synapse(skip_checks=True) testSyn.login(email=other_user['username'], password=other_user['password']) testOwnerId = int(testSyn.getUserProfile()['ownerId']) # Make a project other_project = Project(name=str(uuid.uuid4())) other_project = testSyn.createEntity(other_project) # Give the test user permission to read and join the evaluation syn._allowParticipation(ev, testOwnerId) syn._allowParticipation(ev, "AUTHENTICATED_USERS") syn._allowParticipation(ev, "PUBLIC") # Have the test user join the evaluation testSyn.joinEvaluation(ev) # Find the test user in the participants list foundMe = False for item in syn.getParticipants(ev): if int(item['userId']) == testOwnerId: foundMe = True break assert foundMe # Make a file to submit fd, filename = tempfile.mkstemp() os.write(fd, str(random.gauss(0,1)) + '\n') os.close(fd) f = File(filename, parentId=other_project.id, name='Submission 999', description ="Haha! I'm inaccessible...") entity = testSyn.store(f) ## test submission by evaluation ID submission = testSyn.submit(ev.id, entity) # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) # Mess up the cached file so that syn._getWithEntityBundle must download again os.utime(filename, (0, 0)) # Grab the Submission as the original user fetched = syn.getSubmission(submission['id']) assert os.path.exists(fetched['filePath']) except ConfigParser.Error: print 'Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring print "Creating Submissions" for i in range(num_of_submissions): fd, filename = tempfile.mkstemp() os.write(fd, str(random.gauss(0,1)) + '\n') os.close(fd) f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity=syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, teamName='My Team') # Score the submissions submissions = syn.getSubmissions(ev) print "Scoring Submissions" for submission in submissions: assert re.match('Submission \d+', submission['name']) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions print "Annotating Submissions" bogosity = {} submissions = syn.getSubmissions(ev) for submission in submissions: status = syn.getSubmissionStatus(submission) b = random.randint(1,100) bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck submissions = syn.getSubmissions(ev) for submission in submissions: status = syn.getSubmissionStatus(submission) a = from_submission_status_annotations(status.annotations) assert a['foo'] == 'bar' assert a['bogosity'] == bogosity[submission.id] for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert kvp['isPrivate'] == False ## Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert len(invalid_submissions) == 1, len(invalid_submissions) assert invalid_submissions[0]['name'] == 'Submission 01' finally: # Clean up syn.delete(ev) ## Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations(): # Create an Evaluation name = 'Test Evaluation %s' % str(uuid.uuid4()) ev = Evaluation(name=name, description='Evaluation for testing', contentSource=project['id'], status='CLOSED') ev = syn.store(ev) try: # -- Get the Evaluation by name evalNamed = syn.getEvaluationByName(name) assert_equals(ev['contentSource'], evalNamed['contentSource']) assert_equals(ev['createdOn'], evalNamed['createdOn']) assert_equals(ev['description'], evalNamed['description']) assert_equals(ev['etag'], evalNamed['etag']) assert_equals(ev['id'], evalNamed['id']) assert_equals(ev['name'], evalNamed['name']) assert_equals(ev['ownerId'], evalNamed['ownerId']) assert_equals(ev['status'], evalNamed['status']) # -- Get the Evaluation by project evalProj = syn.getEvaluationByContentSource(project) evalProj = next(evalProj) assert_equals(ev['contentSource'], evalProj['contentSource']) assert_equals(ev['createdOn'], evalProj['createdOn']) assert_equals(ev['description'], evalProj['description']) assert_equals(ev['etag'], evalProj['etag']) assert_equals(ev['id'], evalProj['id']) assert_equals(ev['name'], evalProj['name']) assert_equals(ev['ownerId'], evalProj['ownerId']) assert_equals(ev['status'], evalProj['status']) # Update the Evaluation ev['status'] = 'OPEN' ev = syn.store(ev, createOrUpdate=True) assert_equals(ev.status, 'OPEN') # Add the current user as a participant myOwnerId = int(syn.getUserProfile()['ownerId']) syn._allowParticipation(ev, myOwnerId) # AUTHENTICATED_USERS = 273948 # PUBLIC = 273949 syn.setPermissions(ev, 273948, accessType=['READ']) syn.setPermissions(ev, 273949, accessType=['READ']) # test getPermissions permissions = syn.getPermissions(ev, 273949) assert_equals(['READ'], permissions) permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId']) for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']: assert_in(p, permissions) # Test getSubmissions with no Submissions (SYNR-453) submissions = syn.getSubmissions(ev) assert_equals(len(list(submissions)), 0) # Increase this to fully test paging by getEvaluationSubmissions # not to be less than 2 num_of_submissions = 2 # Create a bunch of Entities and submit them for scoring for i in range(num_of_submissions): with tempfile.NamedTemporaryFile(mode="w", delete=False) as f: filename = f.name f.write(str(random.gauss(0, 1)) + '\n') f = File(filename, parentId=project.id, name='entry-%02d' % i, description='An entry for testing evaluation') entity = syn.store(f) syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team') # Score the submissions submissions = syn.getSubmissions(ev, limit=num_of_submissions-1) for submission in submissions: assert_true(re.match('Submission \d+', submission['name'])) status = syn.getSubmissionStatus(submission) status.score = random.random() if submission['name'] == 'Submission 01': status.status = 'INVALID' status.report = 'Uh-oh, something went wrong!' else: status.status = 'SCORED' status.report = 'a fabulous effort!' syn.store(status) # Annotate the submissions bogosity = {} submissions = syn.getSubmissions(ev) b = 123 for submission, status in syn.getSubmissionBundles(ev): bogosity[submission.id] = b a = dict(foo='bar', bogosity=b) b += 123 status['annotations'] = to_submission_status_annotations(a) set_privacy(status['annotations'], key='bogosity', is_private=False) syn.store(status) # Test that the annotations stuck for submission, status in syn.getSubmissionBundles(ev): a = from_submission_status_annotations(status.annotations) assert_equals(a['foo'], 'bar') assert_equals(a['bogosity'], bogosity[submission.id]) for kvp in status.annotations['longAnnos']: if kvp['key'] == 'bogosity': assert_false(kvp['isPrivate']) # test query by submission annotations # These queries run against an eventually consistent index table which is # populated by an asynchronous worker. Thus, the queries may remain out # of sync for some unbounded, but assumed to be short time. attempts = 2 while attempts > 0: try: results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id) assert_equals(len(results['rows']), num_of_submissions+1) results = syn.restGET( "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id) assert_equals(len(results['rows']), num_of_submissions) except AssertionError as ex1: attempts -= 1 time.sleep(2) else: attempts = 0 # Test that we can retrieve submissions with a specific status invalid_submissions = list(syn.getSubmissions(ev, status='INVALID')) assert_equals(len(invalid_submissions), 1, len(invalid_submissions)) assert_equals(invalid_submissions[0]['name'], 'Submission 01') finally: # Clean up syn.delete(ev) if 'testSyn' in locals(): if 'other_project' in locals(): # Clean up, since the current user can't access this project # This also removes references to the submitted object :) testSyn.delete(other_project) if 'team' in locals(): # remove team testSyn.delete(team) # Just deleted it. Shouldn't be able to get it. assert_raises(SynapseHTTPError, syn.getEvaluation, ev)