def test_submission_status_annotations_round_trip():
april_28_1969 = Datetime(1969, 4, 28)
a = Annotations(
'syn123', '7bdb83e9-a50a-46e4-987a-4962559f090f', {
'screen_name': 'Bullwinkle',
'species': 'Moose',
'lucky': 13,
'pi': pi,
'birthday': april_28_1969
})
sa = to_submission_status_annotations(a)
assert {'screen_name',
'species'} == set([kvp['key'] for kvp in sa['stringAnnos']])
assert {'Bullwinkle',
'Moose'} == set([kvp['value'] for kvp in sa['stringAnnos']])
# test idempotence
assert sa == to_submission_status_annotations(sa)
assert {'lucky',
'birthday'} == set([kvp['key'] for kvp in sa['longAnnos']])
for kvp in sa['longAnnos']:
key = kvp['key']
value = kvp['value']
if key == 'lucky':
assert value == 13
if key == 'birthday':
assert utils.from_unix_epoch_time(value) == april_28_1969
assert {'pi'} == set([kvp['key'] for kvp in sa['doubleAnnos']])
assert {pi} == set([kvp['value'] for kvp in sa['doubleAnnos']])
set_privacy(sa, key='screen_name', is_private=False)
pytest.raises(KeyError,
set_privacy,
sa,
key='this_key_does_not_exist',
is_private=False)
for kvp in sa['stringAnnos']:
if kvp['key'] == 'screen_name':
assert not kvp['isPrivate']
a2 = from_submission_status_annotations(sa)
# TODO: is there a way to convert dates back from longs automatically?
a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
assert a == a2
# test idempotence
assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip():
april_28_1969 = Datetime(1969, 4, 28)
a = dict(screen_name='Bullwinkle',
species='Moose',
lucky=13,
pi=pi,
birthday=april_28_1969)
sa = to_submission_status_annotations(a)
print sa
assert set(['screen_name',
'species']) == set([kvp['key'] for kvp in sa['stringAnnos']])
assert set(['Bullwinkle',
'Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']])
## test idempotence
assert sa == to_submission_status_annotations(sa)
assert set(['lucky',
'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']])
for kvp in sa['longAnnos']:
key = kvp['key']
value = kvp['value']
if key == 'lucky':
assert value == 13
if key == 'birthday':
assert utils.from_unix_epoch_time(value) == april_28_1969
assert set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']])
assert set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']])
set_privacy(sa, key='screen_name', is_private=False)
assert_raises(KeyError,
set_privacy,
sa,
key='this_key_does_not_exist',
is_private=False)
for kvp in sa['stringAnnos']:
if kvp['key'] == 'screen_name':
assert kvp['isPrivate'] == False
a2 = from_submission_status_annotations(sa)
# TODO: is there a way to convert dates back from longs automatically?
a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
assert a == a2
## test idempotence
assert a == from_submission_status_annotations(a)
def test_submission_status_double_annos():
ssa = {
'longAnnos': [{
'isPrivate': False,
'value': 13,
'key': 'lucky'
}],
'doubleAnnos': [{
'isPrivate': False,
'value': 3,
'key': 'three'
}, {
'isPrivate': False,
'value': pi,
'key': 'pi'
}]
}
## test that the double annotation 'three':3 is interpretted as a floating
## point 3.0 rather than an integer 3
annotations = from_submission_status_annotations(ssa)
assert isinstance(annotations['three'], float)
ssa2 = to_submission_status_annotations(annotations)
assert set(['three',
'pi']) == set([kvp['key'] for kvp in ssa2['doubleAnnos']])
assert set(['lucky']) == set([kvp['key'] for kvp in ssa2['longAnnos']])
def test_submission_status_double_annos():
ssa = {'longAnnos': [{'isPrivate': False, 'value':13, 'key':'lucky'}],
'doubleAnnos': [{'isPrivate': False, 'value':3, 'key': 'three'}, {'isPrivate': False, 'value':pi, 'key': 'pi'}]}
## test that the double annotation 'three':3 is interpretted as a floating
## point 3.0 rather than an integer 3
annotations = from_submission_status_annotations(ssa)
assert isinstance(annotations['three'], float)
ssa2 = to_submission_status_annotations(annotations)
assert set(['three', 'pi']) == set([kvp['key'] for kvp in ssa2['doubleAnnos']])
assert set(['lucky']) == set([kvp['key'] for kvp in ssa2['longAnnos']])
def test_submission_status_annotations_round_trip():
from math import pi
april_28_1969 = Datetime(1969,4,28)
a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969)
sa = to_submission_status_annotations(a)
print sa
assert set(['screen_name','species']) == set([kvp['key'] for kvp in sa['stringAnnos']])
assert set(['Bullwinkle','Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']])
## test idempotence
assert sa == to_submission_status_annotations(sa)
assert set(['lucky', 'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']])
for kvp in sa['longAnnos']:
key = kvp['key']
value = kvp['value']
if key=='lucky':
assert value == 13
if key=='birthday':
assert utils.from_unix_epoch_time(value) == april_28_1969
set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']])
set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']])
set_privacy(sa, key='screen_name', is_private=False)
assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False)
for kvp in sa['stringAnnos']:
if kvp['key'] == 'screen_name':
assert kvp['isPrivate'] == False
a2 = from_submission_status_annotations(sa)
# TODO: is there a way to convert dates back from longs automatically?
a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
assert a == a2
## test idempotence
assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip():
april_28_1969 = Datetime(1969, 4, 28)
a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969)
sa = to_submission_status_annotations(a)
assert_equals({'screen_name', 'species'}, set([kvp['key'] for kvp in sa['stringAnnos']]))
assert_equals({'Bullwinkle', 'Moose'}, set([kvp['value'] for kvp in sa['stringAnnos']]))
# test idempotence
assert_equals(sa, to_submission_status_annotations(sa))
assert_equals({'lucky', 'birthday'}, set([kvp['key'] for kvp in sa['longAnnos']]))
for kvp in sa['longAnnos']:
key = kvp['key']
value = kvp['value']
if key == 'lucky':
assert_equals(value, 13)
if key == 'birthday':
assert_equals(utils.from_unix_epoch_time(value), april_28_1969)
assert_equals({'pi'}, set([kvp['key'] for kvp in sa['doubleAnnos']]))
assert_equals({pi}, set([kvp['value'] for kvp in sa['doubleAnnos']]))
set_privacy(sa, key='screen_name', is_private=False)
assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False)
for kvp in sa['stringAnnos']:
if kvp['key'] == 'screen_name':
assert_false(kvp['isPrivate'])
a2 = from_submission_status_annotations(sa)
# TODO: is there a way to convert dates back from longs automatically?
a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
assert_equals(a, a2)
# test idempotence
assert_equals(a, from_submission_status_annotations(a))
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert_equals(ev['contentSource'], evalNamed['contentSource'])
assert_equals(ev['createdOn'], evalNamed['createdOn'])
assert_equals(ev['description'], evalNamed['description'])
assert_equals(ev['etag'], evalNamed['etag'])
assert_equals(ev['id'], evalNamed['id'])
assert_equals(ev['name'], evalNamed['name'])
assert_equals(ev['ownerId'], evalNamed['ownerId'])
assert_equals(ev['status'], evalNamed['status'])
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert_equals(ev['contentSource'], evalProj['contentSource'])
assert_equals(ev['createdOn'], evalProj['createdOn'])
assert_equals(ev['description'], evalProj['description'])
assert_equals(ev['etag'], evalProj['etag'])
assert_equals(ev['id'], evalProj['id'])
assert_equals(ev['name'], evalProj['name'])
assert_equals(ev['ownerId'], evalProj['ownerId'])
assert_equals(ev['status'], evalProj['status'])
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert_equals(ev.status, 'OPEN')
# Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert_equals(['READ'], permissions)
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
for p in [
'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
'READ_PRIVATE_SUBMISSION'
]:
assert_in(p, permissions)
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert_equals(len(list(submissions)), 0)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev,
entity,
name='Submission %02d' % i,
submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
for submission in submissions:
assert_true(re.match('Submission \d+', submission['name']))
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'],
key='bogosity',
is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert_equals(a['foo'], 'bar')
assert_equals(a['bogosity'], bogosity[submission.id])
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert_false(kvp['isPrivate'])
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
% ev.id)
assert_equals(len(results['rows']), num_of_submissions + 1)
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
% ev.id)
assert_equals(len(results['rows']), num_of_submissions)
except AssertionError as ex1:
attempts -= 1
time.sleep(2)
else:
attempts = 0
# Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert_equals(len(invalid_submissions), 1, len(invalid_submissions))
assert_equals(invalid_submissions[0]['name'], 'Submission 01')
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
# remove team
testSyn.delete(team)
# Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name, description='Evaluation for testing',
contentSource=project['id'], status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# # Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert ['READ'] == permissions
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
assert [p in permissions for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']]
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
try:
config = configparser.ConfigParser()
config.read(client.CONFIG_FILE)
other_user = {}
other_user['username'] = config.get('test-authentication', 'username')
other_user['password'] = config.get('test-authentication', 'password')
print("Testing SYNR-541")
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'], password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
# Make a file to submit
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0,1)) + '\n')
f = File(filename, parentId=other_project.id,
name='Submission 999',
description ="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname")
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
# make sure the fetched file is the same as the original (PLFM-2666)
assert filecmp.cmp(filename, fetched['filePath'])
except configparser.Error:
print('Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print("Creating Submissions")
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0,1)) + '\n')
f = File(filename, parentId=project.id, name='entry-%02d' % i,
description='An entry for testing evaluation')
entity=syn.store(f)
syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions-1)
print("Scoring Submissions")
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print("Annotating Submissions")
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'], key='bogosity', is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
print("Querying for submissions")
results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id)
print(results)
assert results[u'totalNumberOfResults'] == num_of_submissions+1
results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id)
print(results)
assert results[u'totalNumberOfResults'] == num_of_submissions
except AssertionError as ex1:
print("failed query: ", ex1)
attempts -= 1
if attempts > 0: print("retrying...")
time.sleep(2)
else:
attempts = 0
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
## remove team
testSyn.delete(team)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name,
description='Evaluation for testing',
contentSource=project['id'],
status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# # Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert ['READ'] == permissions
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
assert [
p in permissions for p in [
'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
'READ_PRIVATE_SUBMISSION'
]
]
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
if other_user['username']:
print("Testing SYNR-541")
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'],
password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
# Make a file to submit
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=other_project.id,
name='Submission 999',
description="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id,
entity,
submitterAlias="My Nickname")
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
# make sure the fetched file is the same as the original (PLFM-2666)
assert filecmp.cmp(filename, fetched['filePath'])
else:
print(
'Skipping test for SYNR-541: No [test-authentication] in %s' %
client.CONFIG_FILE)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print("Creating Submissions")
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename,
parentId=project.id,
name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev,
entity,
name='Submission %02d' % i,
submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
print("Scoring Submissions")
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print("Annotating Submissions")
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'],
key='bogosity',
is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
print("Querying for submissions")
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions + 1
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
% ev.id)
print(results)
assert len(results['rows']) == num_of_submissions
except AssertionError as ex1:
print("failed query: ", ex1)
attempts -= 1
if attempts > 0: print("retrying...")
time.sleep(2)
else:
attempts = 0
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
## remove team
testSyn.delete(team)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name, description='Evaluation for testing',
contentSource=project['id'], status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert ev['contentSource'] == evalNamed['contentSource']
assert ev['createdOn'] == evalNamed['createdOn']
assert ev['description'] == evalNamed['description']
assert ev['etag'] == evalNamed['etag']
assert ev['id'] == evalNamed['id']
assert ev['name'] == evalNamed['name']
assert ev['ownerId'] == evalNamed['ownerId']
assert ev['status'] == evalNamed['status']
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = evalProj.next()
assert ev['contentSource'] == evalProj['contentSource']
assert ev['createdOn'] == evalProj['createdOn']
assert ev['description'] == evalProj['description']
assert ev['etag'] == evalProj['etag']
assert ev['id'] == evalProj['id']
assert ev['name'] == evalProj['name']
assert ev['ownerId'] == evalProj['ownerId']
assert ev['status'] == evalProj['status']
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert ev.status == 'OPEN'
# Add the current user as a participant
syn.joinEvaluation(ev)
# Find this user in the participant list
foundMe = False
myOwnerId = int(syn.getUserProfile()['ownerId'])
for item in syn.getParticipants(ev):
if int(item['userId']) == myOwnerId:
foundMe = True
break
assert foundMe
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert len(list(submissions)) == 0
# -- Get a Submission attachment belonging to another user (SYNR-541) --
# See if the configuration contains test authentication
try:
config = ConfigParser.ConfigParser()
config.read(client.CONFIG_FILE)
other_user = {}
other_user['username'] = config.get('test-authentication', 'username')
other_user['password'] = config.get('test-authentication', 'password')
print "Testing SYNR-541"
# Login as the test user
testSyn = client.Synapse(skip_checks=True)
testSyn.login(email=other_user['username'], password=other_user['password'])
testOwnerId = int(testSyn.getUserProfile()['ownerId'])
# Make a project
other_project = Project(name=str(uuid.uuid4()))
other_project = testSyn.createEntity(other_project)
# Give the test user permission to read and join the evaluation
syn._allowParticipation(ev, testOwnerId)
syn._allowParticipation(ev, "AUTHENTICATED_USERS")
syn._allowParticipation(ev, "PUBLIC")
# Have the test user join the evaluation
testSyn.joinEvaluation(ev)
# Find the test user in the participants list
foundMe = False
for item in syn.getParticipants(ev):
if int(item['userId']) == testOwnerId:
foundMe = True
break
assert foundMe
# Make a file to submit
fd, filename = tempfile.mkstemp()
os.write(fd, str(random.gauss(0,1)) + '\n')
os.close(fd)
f = File(filename, parentId=other_project.id,
name='Submission 999',
description ="Haha! I'm inaccessible...")
entity = testSyn.store(f)
## test submission by evaluation ID
submission = testSyn.submit(ev.id, entity)
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
# Mess up the cached file so that syn._getWithEntityBundle must download again
os.utime(filename, (0, 0))
# Grab the Submission as the original user
fetched = syn.getSubmission(submission['id'])
assert os.path.exists(fetched['filePath'])
except ConfigParser.Error:
print 'Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
print "Creating Submissions"
for i in range(num_of_submissions):
fd, filename = tempfile.mkstemp()
os.write(fd, str(random.gauss(0,1)) + '\n')
os.close(fd)
f = File(filename, parentId=project.id, name='entry-%02d' % i,
description='An entry for testing evaluation')
entity=syn.store(f)
syn.submit(ev, entity, name='Submission %02d' % i, teamName='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev)
print "Scoring Submissions"
for submission in submissions:
assert re.match('Submission \d+', submission['name'])
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
print "Annotating Submissions"
bogosity = {}
submissions = syn.getSubmissions(ev)
for submission in submissions:
status = syn.getSubmissionStatus(submission)
b = random.randint(1,100)
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'], key='bogosity', is_private=False)
syn.store(status)
# Test that the annotations stuck
submissions = syn.getSubmissions(ev)
for submission in submissions:
status = syn.getSubmissionStatus(submission)
a = from_submission_status_annotations(status.annotations)
assert a['foo'] == 'bar'
assert a['bogosity'] == bogosity[submission.id]
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert kvp['isPrivate'] == False
## Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert len(invalid_submissions) == 1, len(invalid_submissions)
assert invalid_submissions[0]['name'] == 'Submission 01'
finally:
# Clean up
syn.delete(ev)
## Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
# Create an Evaluation
name = 'Test Evaluation %s' % str(uuid.uuid4())
ev = Evaluation(name=name, description='Evaluation for testing',
contentSource=project['id'], status='CLOSED')
ev = syn.store(ev)
try:
# -- Get the Evaluation by name
evalNamed = syn.getEvaluationByName(name)
assert_equals(ev['contentSource'], evalNamed['contentSource'])
assert_equals(ev['createdOn'], evalNamed['createdOn'])
assert_equals(ev['description'], evalNamed['description'])
assert_equals(ev['etag'], evalNamed['etag'])
assert_equals(ev['id'], evalNamed['id'])
assert_equals(ev['name'], evalNamed['name'])
assert_equals(ev['ownerId'], evalNamed['ownerId'])
assert_equals(ev['status'], evalNamed['status'])
# -- Get the Evaluation by project
evalProj = syn.getEvaluationByContentSource(project)
evalProj = next(evalProj)
assert_equals(ev['contentSource'], evalProj['contentSource'])
assert_equals(ev['createdOn'], evalProj['createdOn'])
assert_equals(ev['description'], evalProj['description'])
assert_equals(ev['etag'], evalProj['etag'])
assert_equals(ev['id'], evalProj['id'])
assert_equals(ev['name'], evalProj['name'])
assert_equals(ev['ownerId'], evalProj['ownerId'])
assert_equals(ev['status'], evalProj['status'])
# Update the Evaluation
ev['status'] = 'OPEN'
ev = syn.store(ev, createOrUpdate=True)
assert_equals(ev.status, 'OPEN')
# Add the current user as a participant
myOwnerId = int(syn.getUserProfile()['ownerId'])
syn._allowParticipation(ev, myOwnerId)
# AUTHENTICATED_USERS = 273948
# PUBLIC = 273949
syn.setPermissions(ev, 273948, accessType=['READ'])
syn.setPermissions(ev, 273949, accessType=['READ'])
# test getPermissions
permissions = syn.getPermissions(ev, 273949)
assert_equals(['READ'], permissions)
permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']:
assert_in(p, permissions)
# Test getSubmissions with no Submissions (SYNR-453)
submissions = syn.getSubmissions(ev)
assert_equals(len(list(submissions)), 0)
# Increase this to fully test paging by getEvaluationSubmissions
# not to be less than 2
num_of_submissions = 2
# Create a bunch of Entities and submit them for scoring
for i in range(num_of_submissions):
with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
filename = f.name
f.write(str(random.gauss(0, 1)) + '\n')
f = File(filename, parentId=project.id, name='entry-%02d' % i,
description='An entry for testing evaluation')
entity = syn.store(f)
syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team')
# Score the submissions
submissions = syn.getSubmissions(ev, limit=num_of_submissions-1)
for submission in submissions:
assert_true(re.match('Submission \d+', submission['name']))
status = syn.getSubmissionStatus(submission)
status.score = random.random()
if submission['name'] == 'Submission 01':
status.status = 'INVALID'
status.report = 'Uh-oh, something went wrong!'
else:
status.status = 'SCORED'
status.report = 'a fabulous effort!'
syn.store(status)
# Annotate the submissions
bogosity = {}
submissions = syn.getSubmissions(ev)
b = 123
for submission, status in syn.getSubmissionBundles(ev):
bogosity[submission.id] = b
a = dict(foo='bar', bogosity=b)
b += 123
status['annotations'] = to_submission_status_annotations(a)
set_privacy(status['annotations'], key='bogosity', is_private=False)
syn.store(status)
# Test that the annotations stuck
for submission, status in syn.getSubmissionBundles(ev):
a = from_submission_status_annotations(status.annotations)
assert_equals(a['foo'], 'bar')
assert_equals(a['bogosity'], bogosity[submission.id])
for kvp in status.annotations['longAnnos']:
if kvp['key'] == 'bogosity':
assert_false(kvp['isPrivate'])
# test query by submission annotations
# These queries run against an eventually consistent index table which is
# populated by an asynchronous worker. Thus, the queries may remain out
# of sync for some unbounded, but assumed to be short time.
attempts = 2
while attempts > 0:
try:
results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id)
assert_equals(len(results['rows']), num_of_submissions+1)
results = syn.restGET(
"/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id)
assert_equals(len(results['rows']), num_of_submissions)
except AssertionError as ex1:
attempts -= 1
time.sleep(2)
else:
attempts = 0
# Test that we can retrieve submissions with a specific status
invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
assert_equals(len(invalid_submissions), 1, len(invalid_submissions))
assert_equals(invalid_submissions[0]['name'], 'Submission 01')
finally:
# Clean up
syn.delete(ev)
if 'testSyn' in locals():
if 'other_project' in locals():
# Clean up, since the current user can't access this project
# This also removes references to the submitted object :)
testSyn.delete(other_project)
if 'team' in locals():
# remove team
testSyn.delete(team)
# Just deleted it. Shouldn't be able to get it.
assert_raises(SynapseHTTPError, syn.getEvaluation, ev)