Exemplo n.º 1
0
def test_submission_status_annotations_round_trip():
    april_28_1969 = Datetime(1969, 4, 28)
    a = Annotations(
        'syn123', '7bdb83e9-a50a-46e4-987a-4962559f090f', {
            'screen_name': 'Bullwinkle',
            'species': 'Moose',
            'lucky': 13,
            'pi': pi,
            'birthday': april_28_1969
        })
    sa = to_submission_status_annotations(a)
    assert {'screen_name',
            'species'} == set([kvp['key'] for kvp in sa['stringAnnos']])
    assert {'Bullwinkle',
            'Moose'} == set([kvp['value'] for kvp in sa['stringAnnos']])

    # test idempotence
    assert sa == to_submission_status_annotations(sa)

    assert {'lucky',
            'birthday'} == set([kvp['key'] for kvp in sa['longAnnos']])
    for kvp in sa['longAnnos']:
        key = kvp['key']
        value = kvp['value']
        if key == 'lucky':
            assert value == 13
        if key == 'birthday':
            assert utils.from_unix_epoch_time(value) == april_28_1969

    assert {'pi'} == set([kvp['key'] for kvp in sa['doubleAnnos']])
    assert {pi} == set([kvp['value'] for kvp in sa['doubleAnnos']])

    set_privacy(sa, key='screen_name', is_private=False)
    pytest.raises(KeyError,
                  set_privacy,
                  sa,
                  key='this_key_does_not_exist',
                  is_private=False)

    for kvp in sa['stringAnnos']:
        if kvp['key'] == 'screen_name':
            assert not kvp['isPrivate']

    a2 = from_submission_status_annotations(sa)
    # TODO: is there a way to convert dates back from longs automatically?
    a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
    assert a == a2

    # test idempotence
    assert a == from_submission_status_annotations(a)
Exemplo n.º 2
0
def test_submission_status_annotations_round_trip():
    april_28_1969 = Datetime(1969, 4, 28)
    a = dict(screen_name='Bullwinkle',
             species='Moose',
             lucky=13,
             pi=pi,
             birthday=april_28_1969)
    sa = to_submission_status_annotations(a)
    print sa
    assert set(['screen_name',
                'species']) == set([kvp['key'] for kvp in sa['stringAnnos']])
    assert set(['Bullwinkle',
                'Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']])

    ## test idempotence
    assert sa == to_submission_status_annotations(sa)

    assert set(['lucky',
                'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']])
    for kvp in sa['longAnnos']:
        key = kvp['key']
        value = kvp['value']
        if key == 'lucky':
            assert value == 13
        if key == 'birthday':
            assert utils.from_unix_epoch_time(value) == april_28_1969

    assert set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']])
    assert set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']])

    set_privacy(sa, key='screen_name', is_private=False)
    assert_raises(KeyError,
                  set_privacy,
                  sa,
                  key='this_key_does_not_exist',
                  is_private=False)

    for kvp in sa['stringAnnos']:
        if kvp['key'] == 'screen_name':
            assert kvp['isPrivate'] == False

    a2 = from_submission_status_annotations(sa)
    # TODO: is there a way to convert dates back from longs automatically?
    a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
    assert a == a2

    ## test idempotence
    assert a == from_submission_status_annotations(a)
Exemplo n.º 3
0
def test_submission_status_double_annos():
    ssa = {
        'longAnnos': [{
            'isPrivate': False,
            'value': 13,
            'key': 'lucky'
        }],
        'doubleAnnos': [{
            'isPrivate': False,
            'value': 3,
            'key': 'three'
        }, {
            'isPrivate': False,
            'value': pi,
            'key': 'pi'
        }]
    }
    ## test that the double annotation 'three':3 is interpretted as a floating
    ## point 3.0 rather than an integer 3
    annotations = from_submission_status_annotations(ssa)
    assert isinstance(annotations['three'], float)
    ssa2 = to_submission_status_annotations(annotations)
    assert set(['three',
                'pi']) == set([kvp['key'] for kvp in ssa2['doubleAnnos']])
    assert set(['lucky']) == set([kvp['key'] for kvp in ssa2['longAnnos']])
def test_submission_status_double_annos():
    ssa = {'longAnnos':   [{'isPrivate': False, 'value':13, 'key':'lucky'}],
           'doubleAnnos': [{'isPrivate': False, 'value':3, 'key': 'three'}, {'isPrivate': False, 'value':pi, 'key': 'pi'}]}
    ## test that the double annotation 'three':3 is interpretted as a floating
    ## point 3.0 rather than an integer 3
    annotations = from_submission_status_annotations(ssa)
    assert isinstance(annotations['three'], float)
    ssa2 = to_submission_status_annotations(annotations)
    assert set(['three', 'pi']) == set([kvp['key'] for kvp in ssa2['doubleAnnos']])
    assert set(['lucky']) == set([kvp['key'] for kvp in ssa2['longAnnos']])
def test_submission_status_annotations_round_trip():
    from math import pi
    april_28_1969 = Datetime(1969,4,28)
    a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969)
    sa = to_submission_status_annotations(a)
    print sa
    assert set(['screen_name','species']) == set([kvp['key'] for kvp in sa['stringAnnos']])
    assert set(['Bullwinkle','Moose']) == set([kvp['value'] for kvp in sa['stringAnnos']])

    ## test idempotence
    assert sa == to_submission_status_annotations(sa)

    assert set(['lucky', 'birthday']) == set([kvp['key'] for kvp in sa['longAnnos']])
    for kvp in sa['longAnnos']:
        key = kvp['key']
        value = kvp['value']
        if key=='lucky':
            assert value == 13
        if key=='birthday':
            assert utils.from_unix_epoch_time(value) == april_28_1969

    set(['pi']) == set([kvp['key'] for kvp in sa['doubleAnnos']])
    set([pi]) == set([kvp['value'] for kvp in sa['doubleAnnos']])

    set_privacy(sa, key='screen_name', is_private=False)
    assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False)

    for kvp in sa['stringAnnos']:
        if kvp['key'] == 'screen_name':
            assert kvp['isPrivate'] == False

    a2 = from_submission_status_annotations(sa)
    # TODO: is there a way to convert dates back from longs automatically?
    a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
    assert a == a2

    ## test idempotence
    assert a == from_submission_status_annotations(a)
def test_submission_status_annotations_round_trip():
    april_28_1969 = Datetime(1969, 4, 28)
    a = dict(screen_name='Bullwinkle', species='Moose', lucky=13, pi=pi, birthday=april_28_1969)
    sa = to_submission_status_annotations(a)
    assert_equals({'screen_name', 'species'}, set([kvp['key'] for kvp in sa['stringAnnos']]))
    assert_equals({'Bullwinkle', 'Moose'}, set([kvp['value'] for kvp in sa['stringAnnos']]))

    # test idempotence
    assert_equals(sa, to_submission_status_annotations(sa))

    assert_equals({'lucky', 'birthday'}, set([kvp['key'] for kvp in sa['longAnnos']]))
    for kvp in sa['longAnnos']:
        key = kvp['key']
        value = kvp['value']
        if key == 'lucky':
            assert_equals(value, 13)
        if key == 'birthday':
            assert_equals(utils.from_unix_epoch_time(value), april_28_1969)

    assert_equals({'pi'}, set([kvp['key'] for kvp in sa['doubleAnnos']]))
    assert_equals({pi}, set([kvp['value'] for kvp in sa['doubleAnnos']]))

    set_privacy(sa, key='screen_name', is_private=False)
    assert_raises(KeyError, set_privacy, sa, key='this_key_does_not_exist', is_private=False)

    for kvp in sa['stringAnnos']:
        if kvp['key'] == 'screen_name':
            assert_false(kvp['isPrivate'])

    a2 = from_submission_status_annotations(sa)
    # TODO: is there a way to convert dates back from longs automatically?
    a2['birthday'] = utils.from_unix_epoch_time(a2['birthday'])
    assert_equals(a, a2)

    # test idempotence
    assert_equals(a, from_submission_status_annotations(a))
Exemplo n.º 7
0
def test_evaluations():
    # Create an Evaluation
    name = 'Test Evaluation %s' % str(uuid.uuid4())
    ev = Evaluation(name=name,
                    description='Evaluation for testing',
                    contentSource=project['id'],
                    status='CLOSED')
    ev = syn.store(ev)

    try:

        # -- Get the Evaluation by name
        evalNamed = syn.getEvaluationByName(name)
        assert_equals(ev['contentSource'], evalNamed['contentSource'])
        assert_equals(ev['createdOn'], evalNamed['createdOn'])
        assert_equals(ev['description'], evalNamed['description'])
        assert_equals(ev['etag'], evalNamed['etag'])
        assert_equals(ev['id'], evalNamed['id'])
        assert_equals(ev['name'], evalNamed['name'])
        assert_equals(ev['ownerId'], evalNamed['ownerId'])
        assert_equals(ev['status'], evalNamed['status'])

        # -- Get the Evaluation by project
        evalProj = syn.getEvaluationByContentSource(project)
        evalProj = next(evalProj)
        assert_equals(ev['contentSource'], evalProj['contentSource'])
        assert_equals(ev['createdOn'], evalProj['createdOn'])
        assert_equals(ev['description'], evalProj['description'])
        assert_equals(ev['etag'], evalProj['etag'])
        assert_equals(ev['id'], evalProj['id'])
        assert_equals(ev['name'], evalProj['name'])
        assert_equals(ev['ownerId'], evalProj['ownerId'])
        assert_equals(ev['status'], evalProj['status'])

        # Update the Evaluation
        ev['status'] = 'OPEN'
        ev = syn.store(ev, createOrUpdate=True)
        assert_equals(ev.status, 'OPEN')

        # Add the current user as a participant
        myOwnerId = int(syn.getUserProfile()['ownerId'])
        syn._allowParticipation(ev, myOwnerId)

        # AUTHENTICATED_USERS = 273948
        # PUBLIC = 273949
        syn.setPermissions(ev, 273948, accessType=['READ'])
        syn.setPermissions(ev, 273949, accessType=['READ'])

        # test getPermissions
        permissions = syn.getPermissions(ev, 273949)
        assert_equals(['READ'], permissions)

        permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
        for p in [
                'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
                'READ_PRIVATE_SUBMISSION'
        ]:
            assert_in(p, permissions)

        # Test getSubmissions with no Submissions (SYNR-453)
        submissions = syn.getSubmissions(ev)
        assert_equals(len(list(submissions)), 0)

        # Increase this to fully test paging by getEvaluationSubmissions
        # not to be less than 2
        num_of_submissions = 2

        # Create a bunch of Entities and submit them for scoring
        for i in range(num_of_submissions):
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0, 1)) + '\n')

            f = File(filename,
                     parentId=project.id,
                     name='entry-%02d' % i,
                     description='An entry for testing evaluation')
            entity = syn.store(f)
            syn.submit(ev,
                       entity,
                       name='Submission %02d' % i,
                       submitterAlias='My Team')

        # Score the submissions
        submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
        for submission in submissions:
            assert_true(re.match('Submission \d+', submission['name']))
            status = syn.getSubmissionStatus(submission)
            status.score = random.random()
            if submission['name'] == 'Submission 01':
                status.status = 'INVALID'
                status.report = 'Uh-oh, something went wrong!'
            else:
                status.status = 'SCORED'
                status.report = 'a fabulous effort!'
            syn.store(status)

        # Annotate the submissions
        bogosity = {}
        submissions = syn.getSubmissions(ev)
        b = 123
        for submission, status in syn.getSubmissionBundles(ev):
            bogosity[submission.id] = b
            a = dict(foo='bar', bogosity=b)
            b += 123
            status['annotations'] = to_submission_status_annotations(a)
            set_privacy(status['annotations'],
                        key='bogosity',
                        is_private=False)
            syn.store(status)

        # Test that the annotations stuck
        for submission, status in syn.getSubmissionBundles(ev):
            a = from_submission_status_annotations(status.annotations)
            assert_equals(a['foo'], 'bar')
            assert_equals(a['bogosity'], bogosity[submission.id])
            for kvp in status.annotations['longAnnos']:
                if kvp['key'] == 'bogosity':
                    assert_false(kvp['isPrivate'])

        # test query by submission annotations
        # These queries run against an eventually consistent index table which is
        # populated by an asynchronous worker. Thus, the queries may remain out
        # of sync for some unbounded, but assumed to be short time.
        attempts = 2
        while attempts > 0:
            try:
                results = syn.restGET(
                    "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
                    % ev.id)
                assert_equals(len(results['rows']), num_of_submissions + 1)

                results = syn.restGET(
                    "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
                    % ev.id)
                assert_equals(len(results['rows']), num_of_submissions)
            except AssertionError as ex1:
                attempts -= 1
                time.sleep(2)
            else:
                attempts = 0

        # Test that we can retrieve submissions with a specific status
        invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
        assert_equals(len(invalid_submissions), 1, len(invalid_submissions))
        assert_equals(invalid_submissions[0]['name'], 'Submission 01')

    finally:
        # Clean up
        syn.delete(ev)
        if 'testSyn' in locals():
            if 'other_project' in locals():
                # Clean up, since the current user can't access this project
                # This also removes references to the submitted object :)
                testSyn.delete(other_project)
            if 'team' in locals():
                # remove team
                testSyn.delete(team)

    # Just deleted it. Shouldn't be able to get it.
    assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
    # Create an Evaluation
    name = 'Test Evaluation %s' % str(uuid.uuid4())
    ev = Evaluation(name=name, description='Evaluation for testing', 
                    contentSource=project['id'], status='CLOSED')
    ev = syn.store(ev)

    try:
        
        # -- Get the Evaluation by name
        evalNamed = syn.getEvaluationByName(name)
        assert ev['contentSource'] == evalNamed['contentSource']
        assert ev['createdOn'] == evalNamed['createdOn']
        assert ev['description'] == evalNamed['description']
        assert ev['etag'] == evalNamed['etag']
        assert ev['id'] == evalNamed['id']
        assert ev['name'] == evalNamed['name']
        assert ev['ownerId'] == evalNamed['ownerId']
        assert ev['status'] == evalNamed['status']
        
        # -- Get the Evaluation by project
        evalProj = syn.getEvaluationByContentSource(project)
        evalProj = next(evalProj)
        assert ev['contentSource'] == evalProj['contentSource']
        assert ev['createdOn'] == evalProj['createdOn']
        assert ev['description'] == evalProj['description']
        assert ev['etag'] == evalProj['etag']
        assert ev['id'] == evalProj['id']
        assert ev['name'] == evalProj['name']
        assert ev['ownerId'] == evalProj['ownerId']
        assert ev['status'] == evalProj['status']
        
        # Update the Evaluation
        ev['status'] = 'OPEN'
        ev = syn.store(ev, createOrUpdate=True)
        assert ev.status == 'OPEN'

        # # Add the current user as a participant
        myOwnerId = int(syn.getUserProfile()['ownerId'])
        syn._allowParticipation(ev, myOwnerId)

        # AUTHENTICATED_USERS = 273948
        # PUBLIC = 273949
        syn.setPermissions(ev, 273948, accessType=['READ'])
        syn.setPermissions(ev, 273949, accessType=['READ'])

        # test getPermissions
        permissions = syn.getPermissions(ev, 273949)
        assert ['READ'] == permissions

        permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
        assert [p in permissions for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']]

        # Test getSubmissions with no Submissions (SYNR-453)
        submissions = syn.getSubmissions(ev)
        assert len(list(submissions)) == 0

        # -- Get a Submission attachment belonging to another user (SYNR-541) --
        # See if the configuration contains test authentication
        try:
            config = configparser.ConfigParser()
            config.read(client.CONFIG_FILE)
            other_user = {}
            other_user['username'] = config.get('test-authentication', 'username')
            other_user['password'] = config.get('test-authentication', 'password')
            print("Testing SYNR-541")

            # Login as the test user
            testSyn = client.Synapse(skip_checks=True)
            testSyn.login(email=other_user['username'], password=other_user['password'])
            testOwnerId = int(testSyn.getUserProfile()['ownerId'])

            # Make a project
            other_project = Project(name=str(uuid.uuid4()))
            other_project = testSyn.createEntity(other_project)

            # Give the test user permission to read and join the evaluation
            syn._allowParticipation(ev, testOwnerId)

            # Make a file to submit
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0,1)) + '\n')

            f = File(filename, parentId=other_project.id,
                     name='Submission 999',
                     description ="Haha!  I'm inaccessible...")
            entity = testSyn.store(f)

            ## test submission by evaluation ID
            submission = testSyn.submit(ev.id, entity, submitterAlias="My Nickname")

            # Mess up the cached file so that syn._getWithEntityBundle must download again
            os.utime(filename, (0, 0))

            # Grab the Submission as the original user
            fetched = syn.getSubmission(submission['id'])
            assert os.path.exists(fetched['filePath'])

            # make sure the fetched file is the same as the original (PLFM-2666)
            assert filecmp.cmp(filename, fetched['filePath'])


        except configparser.Error:
            print('Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE)

        # Increase this to fully test paging by getEvaluationSubmissions
        # not to be less than 2
        num_of_submissions = 2

        # Create a bunch of Entities and submit them for scoring
        print("Creating Submissions")
        for i in range(num_of_submissions):
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0,1)) + '\n')

            f = File(filename, parentId=project.id, name='entry-%02d' % i,
                     description='An entry for testing evaluation')
            entity=syn.store(f)
            syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team')

        # Score the submissions
        submissions = syn.getSubmissions(ev, limit=num_of_submissions-1)
        print("Scoring Submissions")
        for submission in submissions:
            assert re.match('Submission \d+', submission['name'])
            status = syn.getSubmissionStatus(submission)
            status.score = random.random()
            if submission['name'] == 'Submission 01':
                status.status = 'INVALID'
                status.report = 'Uh-oh, something went wrong!'
            else:
                status.status = 'SCORED'
                status.report = 'a fabulous effort!'
            syn.store(status)

        # Annotate the submissions
        print("Annotating Submissions")
        bogosity = {}
        submissions = syn.getSubmissions(ev)
        b = 123
        for submission, status in syn.getSubmissionBundles(ev):
            bogosity[submission.id] = b
            a = dict(foo='bar', bogosity=b)
            b += 123
            status['annotations'] = to_submission_status_annotations(a)
            set_privacy(status['annotations'], key='bogosity', is_private=False)
            syn.store(status)

        # Test that the annotations stuck
        for submission, status in syn.getSubmissionBundles(ev):
            a = from_submission_status_annotations(status.annotations)
            assert a['foo'] == 'bar'
            assert a['bogosity'] == bogosity[submission.id]
            for kvp in status.annotations['longAnnos']:
                if kvp['key'] == 'bogosity':
                    assert kvp['isPrivate'] == False

        # test query by submission annotations
        # These queries run against an eventually consistent index table which is
        # populated by an asynchronous worker. Thus, the queries may remain out
        # of sync for some unbounded, but assumed to be short time.
        attempts = 2
        while attempts > 0:
            try:
                print("Querying for submissions")
                results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id)
                print(results)
                assert results[u'totalNumberOfResults'] == num_of_submissions+1

                results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id)
                print(results)
                assert results[u'totalNumberOfResults'] == num_of_submissions
            except AssertionError as ex1:
                print("failed query: ", ex1)
                attempts -= 1
                if attempts > 0: print("retrying...")
                time.sleep(2)
            else:
                attempts = 0

        ## Test that we can retrieve submissions with a specific status
        invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
        assert len(invalid_submissions) == 1, len(invalid_submissions)
        assert invalid_submissions[0]['name'] == 'Submission 01'

    finally:
        # Clean up
        syn.delete(ev)
        if 'testSyn' in locals():
            if 'other_project' in locals():
                # Clean up, since the current user can't access this project
                # This also removes references to the submitted object :)
                testSyn.delete(other_project)
            if 'team' in locals():
                ## remove team
                testSyn.delete(team)

    ## Just deleted it. Shouldn't be able to get it.
    assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
Exemplo n.º 9
0
def test_evaluations():
    # Create an Evaluation
    name = 'Test Evaluation %s' % str(uuid.uuid4())
    ev = Evaluation(name=name,
                    description='Evaluation for testing',
                    contentSource=project['id'],
                    status='CLOSED')
    ev = syn.store(ev)

    try:

        # -- Get the Evaluation by name
        evalNamed = syn.getEvaluationByName(name)
        assert ev['contentSource'] == evalNamed['contentSource']
        assert ev['createdOn'] == evalNamed['createdOn']
        assert ev['description'] == evalNamed['description']
        assert ev['etag'] == evalNamed['etag']
        assert ev['id'] == evalNamed['id']
        assert ev['name'] == evalNamed['name']
        assert ev['ownerId'] == evalNamed['ownerId']
        assert ev['status'] == evalNamed['status']

        # -- Get the Evaluation by project
        evalProj = syn.getEvaluationByContentSource(project)
        evalProj = next(evalProj)
        assert ev['contentSource'] == evalProj['contentSource']
        assert ev['createdOn'] == evalProj['createdOn']
        assert ev['description'] == evalProj['description']
        assert ev['etag'] == evalProj['etag']
        assert ev['id'] == evalProj['id']
        assert ev['name'] == evalProj['name']
        assert ev['ownerId'] == evalProj['ownerId']
        assert ev['status'] == evalProj['status']

        # Update the Evaluation
        ev['status'] = 'OPEN'
        ev = syn.store(ev, createOrUpdate=True)
        assert ev.status == 'OPEN'

        # # Add the current user as a participant
        myOwnerId = int(syn.getUserProfile()['ownerId'])
        syn._allowParticipation(ev, myOwnerId)

        # AUTHENTICATED_USERS = 273948
        # PUBLIC = 273949
        syn.setPermissions(ev, 273948, accessType=['READ'])
        syn.setPermissions(ev, 273949, accessType=['READ'])

        # test getPermissions
        permissions = syn.getPermissions(ev, 273949)
        assert ['READ'] == permissions

        permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
        assert [
            p in permissions for p in [
                'READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS',
                'READ_PRIVATE_SUBMISSION'
            ]
        ]

        # Test getSubmissions with no Submissions (SYNR-453)
        submissions = syn.getSubmissions(ev)
        assert len(list(submissions)) == 0

        # -- Get a Submission attachment belonging to another user (SYNR-541) --
        # See if the configuration contains test authentication
        if other_user['username']:
            print("Testing SYNR-541")

            # Login as the test user
            testSyn = client.Synapse(skip_checks=True)
            testSyn.login(email=other_user['username'],
                          password=other_user['password'])
            testOwnerId = int(testSyn.getUserProfile()['ownerId'])

            # Make a project
            other_project = Project(name=str(uuid.uuid4()))
            other_project = testSyn.createEntity(other_project)

            # Give the test user permission to read and join the evaluation
            syn._allowParticipation(ev, testOwnerId)

            # Make a file to submit
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0, 1)) + '\n')

            f = File(filename,
                     parentId=other_project.id,
                     name='Submission 999',
                     description="Haha!  I'm inaccessible...")
            entity = testSyn.store(f)

            ## test submission by evaluation ID
            submission = testSyn.submit(ev.id,
                                        entity,
                                        submitterAlias="My Nickname")

            # Mess up the cached file so that syn._getWithEntityBundle must download again
            os.utime(filename, (0, 0))

            # Grab the Submission as the original user
            fetched = syn.getSubmission(submission['id'])
            assert os.path.exists(fetched['filePath'])

            # make sure the fetched file is the same as the original (PLFM-2666)
            assert filecmp.cmp(filename, fetched['filePath'])
        else:
            print(
                'Skipping test for SYNR-541: No [test-authentication] in %s' %
                client.CONFIG_FILE)

        # Increase this to fully test paging by getEvaluationSubmissions
        # not to be less than 2
        num_of_submissions = 2

        # Create a bunch of Entities and submit them for scoring
        print("Creating Submissions")
        for i in range(num_of_submissions):
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0, 1)) + '\n')

            f = File(filename,
                     parentId=project.id,
                     name='entry-%02d' % i,
                     description='An entry for testing evaluation')
            entity = syn.store(f)
            syn.submit(ev,
                       entity,
                       name='Submission %02d' % i,
                       submitterAlias='My Team')

        # Score the submissions
        submissions = syn.getSubmissions(ev, limit=num_of_submissions - 1)
        print("Scoring Submissions")
        for submission in submissions:
            assert re.match('Submission \d+', submission['name'])
            status = syn.getSubmissionStatus(submission)
            status.score = random.random()
            if submission['name'] == 'Submission 01':
                status.status = 'INVALID'
                status.report = 'Uh-oh, something went wrong!'
            else:
                status.status = 'SCORED'
                status.report = 'a fabulous effort!'
            syn.store(status)

        # Annotate the submissions
        print("Annotating Submissions")
        bogosity = {}
        submissions = syn.getSubmissions(ev)
        b = 123
        for submission, status in syn.getSubmissionBundles(ev):
            bogosity[submission.id] = b
            a = dict(foo='bar', bogosity=b)
            b += 123
            status['annotations'] = to_submission_status_annotations(a)
            set_privacy(status['annotations'],
                        key='bogosity',
                        is_private=False)
            syn.store(status)

        # Test that the annotations stuck
        for submission, status in syn.getSubmissionBundles(ev):
            a = from_submission_status_annotations(status.annotations)
            assert a['foo'] == 'bar'
            assert a['bogosity'] == bogosity[submission.id]
            for kvp in status.annotations['longAnnos']:
                if kvp['key'] == 'bogosity':
                    assert kvp['isPrivate'] == False

        # test query by submission annotations
        # These queries run against an eventually consistent index table which is
        # populated by an asynchronous worker. Thus, the queries may remain out
        # of sync for some unbounded, but assumed to be short time.
        attempts = 2
        while attempts > 0:
            try:
                print("Querying for submissions")
                results = syn.restGET(
                    "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s"
                    % ev.id)
                print(results)
                assert len(results['rows']) == num_of_submissions + 1

                results = syn.restGET(
                    "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200"
                    % ev.id)
                print(results)
                assert len(results['rows']) == num_of_submissions
            except AssertionError as ex1:
                print("failed query: ", ex1)
                attempts -= 1
                if attempts > 0: print("retrying...")
                time.sleep(2)
            else:
                attempts = 0

        ## Test that we can retrieve submissions with a specific status
        invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
        assert len(invalid_submissions) == 1, len(invalid_submissions)
        assert invalid_submissions[0]['name'] == 'Submission 01'

    finally:
        # Clean up
        syn.delete(ev)
        if 'testSyn' in locals():
            if 'other_project' in locals():
                # Clean up, since the current user can't access this project
                # This also removes references to the submitted object :)
                testSyn.delete(other_project)
            if 'team' in locals():
                ## remove team
                testSyn.delete(team)

    ## Just deleted it. Shouldn't be able to get it.
    assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
    # Create an Evaluation
    name = 'Test Evaluation %s' % str(uuid.uuid4())
    ev = Evaluation(name=name, description='Evaluation for testing', 
                    contentSource=project['id'], status='CLOSED')
    ev = syn.store(ev)

    try:
        
        # -- Get the Evaluation by name
        evalNamed = syn.getEvaluationByName(name)
        assert ev['contentSource'] == evalNamed['contentSource']
        assert ev['createdOn'] == evalNamed['createdOn']
        assert ev['description'] == evalNamed['description']
        assert ev['etag'] == evalNamed['etag']
        assert ev['id'] == evalNamed['id']
        assert ev['name'] == evalNamed['name']
        assert ev['ownerId'] == evalNamed['ownerId']
        assert ev['status'] == evalNamed['status']
        
        # -- Get the Evaluation by project
        evalProj = syn.getEvaluationByContentSource(project)
        evalProj = evalProj.next()
        assert ev['contentSource'] == evalProj['contentSource']
        assert ev['createdOn'] == evalProj['createdOn']
        assert ev['description'] == evalProj['description']
        assert ev['etag'] == evalProj['etag']
        assert ev['id'] == evalProj['id']
        assert ev['name'] == evalProj['name']
        assert ev['ownerId'] == evalProj['ownerId']
        assert ev['status'] == evalProj['status']
        
        # Update the Evaluation
        ev['status'] = 'OPEN'
        ev = syn.store(ev, createOrUpdate=True)
        assert ev.status == 'OPEN'

        # Add the current user as a participant
        syn.joinEvaluation(ev)

        # Find this user in the participant list
        foundMe = False
        myOwnerId = int(syn.getUserProfile()['ownerId'])
        for item in syn.getParticipants(ev):
            if int(item['userId']) == myOwnerId:
                foundMe = True
                break
        assert foundMe

        # Test getSubmissions with no Submissions (SYNR-453)
        submissions = syn.getSubmissions(ev)
        assert len(list(submissions)) == 0

        # -- Get a Submission attachment belonging to another user (SYNR-541) --
        # See if the configuration contains test authentication
        try:
            config = ConfigParser.ConfigParser()
            config.read(client.CONFIG_FILE)
            other_user = {}
            other_user['username'] = config.get('test-authentication', 'username')
            other_user['password'] = config.get('test-authentication', 'password')
            print "Testing SYNR-541"

            # Login as the test user
            testSyn = client.Synapse(skip_checks=True)
            testSyn.login(email=other_user['username'], password=other_user['password'])
            testOwnerId = int(testSyn.getUserProfile()['ownerId'])

            # Make a project
            other_project = Project(name=str(uuid.uuid4()))
            other_project = testSyn.createEntity(other_project)

            # Give the test user permission to read and join the evaluation
            syn._allowParticipation(ev, testOwnerId)
            syn._allowParticipation(ev, "AUTHENTICATED_USERS")
            syn._allowParticipation(ev, "PUBLIC")

            # Have the test user join the evaluation
            testSyn.joinEvaluation(ev)

            # Find the test user in the participants list
            foundMe = False
            for item in syn.getParticipants(ev):
                if int(item['userId']) == testOwnerId:
                    foundMe = True
                    break
            assert foundMe

            # Make a file to submit
            fd, filename = tempfile.mkstemp()
            os.write(fd, str(random.gauss(0,1)) + '\n')
            os.close(fd)
            f = File(filename, parentId=other_project.id,
                     name='Submission 999',
                     description ="Haha!  I'm inaccessible...")
            entity = testSyn.store(f)

            ## test submission by evaluation ID
            submission = testSyn.submit(ev.id, entity)

            # Clean up, since the current user can't access this project
            # This also removes references to the submitted object :)
            testSyn.delete(other_project)

            # Mess up the cached file so that syn._getWithEntityBundle must download again
            os.utime(filename, (0, 0))

            # Grab the Submission as the original user
            fetched = syn.getSubmission(submission['id'])
            assert os.path.exists(fetched['filePath'])

        except ConfigParser.Error:
            print 'Skipping test for SYNR-541: No [test-authentication] in %s' % client.CONFIG_FILE

        # Increase this to fully test paging by getEvaluationSubmissions
        # not to be less than 2
        num_of_submissions = 2

        # Create a bunch of Entities and submit them for scoring
        print "Creating Submissions"
        for i in range(num_of_submissions):
            fd, filename = tempfile.mkstemp()
            os.write(fd, str(random.gauss(0,1)) + '\n')
            os.close(fd)

            f = File(filename, parentId=project.id, name='entry-%02d' % i,
                     description='An entry for testing evaluation')
            entity=syn.store(f)
            syn.submit(ev, entity, name='Submission %02d' % i, teamName='My Team')

        # Score the submissions
        submissions = syn.getSubmissions(ev)
        print "Scoring Submissions"
        for submission in submissions:
            assert re.match('Submission \d+', submission['name'])
            status = syn.getSubmissionStatus(submission)
            status.score = random.random()
            if submission['name'] == 'Submission 01':
                status.status = 'INVALID'
                status.report = 'Uh-oh, something went wrong!'
            else:
                status.status = 'SCORED'
                status.report = 'a fabulous effort!'
            syn.store(status)

        # Annotate the submissions
        print "Annotating Submissions"
        bogosity = {}
        submissions = syn.getSubmissions(ev)
        for submission in submissions:
            status = syn.getSubmissionStatus(submission)
            b = random.randint(1,100)
            bogosity[submission.id] = b
            a = dict(foo='bar', bogosity=b)
            status['annotations'] = to_submission_status_annotations(a)
            set_privacy(status['annotations'], key='bogosity', is_private=False)
            syn.store(status)

        # Test that the annotations stuck
        submissions = syn.getSubmissions(ev)
        for submission in submissions:
            status = syn.getSubmissionStatus(submission)
            a = from_submission_status_annotations(status.annotations)
            assert a['foo'] == 'bar'
            assert a['bogosity'] == bogosity[submission.id]
            for kvp in status.annotations['longAnnos']:
                if kvp['key'] == 'bogosity':
                    assert kvp['isPrivate'] == False

        ## Test that we can retrieve submissions with a specific status
        invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
        assert len(invalid_submissions) == 1, len(invalid_submissions)
        assert invalid_submissions[0]['name'] == 'Submission 01'

    finally:
        # Clean up
        syn.delete(ev)

    ## Just deleted it. Shouldn't be able to get it.
    assert_raises(SynapseHTTPError, syn.getEvaluation, ev)
def test_evaluations():
    # Create an Evaluation
    name = 'Test Evaluation %s' % str(uuid.uuid4())
    ev = Evaluation(name=name, description='Evaluation for testing', 
                    contentSource=project['id'], status='CLOSED')
    ev = syn.store(ev)

    try:
        
        # -- Get the Evaluation by name
        evalNamed = syn.getEvaluationByName(name)
        assert_equals(ev['contentSource'], evalNamed['contentSource'])
        assert_equals(ev['createdOn'], evalNamed['createdOn'])
        assert_equals(ev['description'], evalNamed['description'])
        assert_equals(ev['etag'], evalNamed['etag'])
        assert_equals(ev['id'], evalNamed['id'])
        assert_equals(ev['name'], evalNamed['name'])
        assert_equals(ev['ownerId'], evalNamed['ownerId'])
        assert_equals(ev['status'], evalNamed['status'])
        
        # -- Get the Evaluation by project
        evalProj = syn.getEvaluationByContentSource(project)
        evalProj = next(evalProj)
        assert_equals(ev['contentSource'], evalProj['contentSource'])
        assert_equals(ev['createdOn'], evalProj['createdOn'])
        assert_equals(ev['description'], evalProj['description'])
        assert_equals(ev['etag'], evalProj['etag'])
        assert_equals(ev['id'], evalProj['id'])
        assert_equals(ev['name'], evalProj['name'])
        assert_equals(ev['ownerId'], evalProj['ownerId'])
        assert_equals(ev['status'], evalProj['status'])
        
        # Update the Evaluation
        ev['status'] = 'OPEN'
        ev = syn.store(ev, createOrUpdate=True)
        assert_equals(ev.status, 'OPEN')

        # Add the current user as a participant
        myOwnerId = int(syn.getUserProfile()['ownerId'])
        syn._allowParticipation(ev, myOwnerId)

        # AUTHENTICATED_USERS = 273948
        # PUBLIC = 273949
        syn.setPermissions(ev, 273948, accessType=['READ'])
        syn.setPermissions(ev, 273949, accessType=['READ'])

        # test getPermissions
        permissions = syn.getPermissions(ev, 273949)
        assert_equals(['READ'], permissions)

        permissions = syn.getPermissions(ev, syn.getUserProfile()['ownerId'])
        for p in ['READ', 'CREATE', 'DELETE', 'UPDATE', 'CHANGE_PERMISSIONS', 'READ_PRIVATE_SUBMISSION']:
            assert_in(p, permissions)

        # Test getSubmissions with no Submissions (SYNR-453)
        submissions = syn.getSubmissions(ev)
        assert_equals(len(list(submissions)), 0)

        # Increase this to fully test paging by getEvaluationSubmissions
        # not to be less than 2
        num_of_submissions = 2

        # Create a bunch of Entities and submit them for scoring
        for i in range(num_of_submissions):
            with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
                filename = f.name
                f.write(str(random.gauss(0, 1)) + '\n')

            f = File(filename, parentId=project.id, name='entry-%02d' % i,
                     description='An entry for testing evaluation')
            entity = syn.store(f)
            syn.submit(ev, entity, name='Submission %02d' % i, submitterAlias='My Team')

        # Score the submissions
        submissions = syn.getSubmissions(ev, limit=num_of_submissions-1)
        for submission in submissions:
            assert_true(re.match('Submission \d+', submission['name']))
            status = syn.getSubmissionStatus(submission)
            status.score = random.random()
            if submission['name'] == 'Submission 01':
                status.status = 'INVALID'
                status.report = 'Uh-oh, something went wrong!'
            else:
                status.status = 'SCORED'
                status.report = 'a fabulous effort!'
            syn.store(status)

        # Annotate the submissions
        bogosity = {}
        submissions = syn.getSubmissions(ev)
        b = 123
        for submission, status in syn.getSubmissionBundles(ev):
            bogosity[submission.id] = b
            a = dict(foo='bar', bogosity=b)
            b += 123
            status['annotations'] = to_submission_status_annotations(a)
            set_privacy(status['annotations'], key='bogosity', is_private=False)
            syn.store(status)

        # Test that the annotations stuck
        for submission, status in syn.getSubmissionBundles(ev):
            a = from_submission_status_annotations(status.annotations)
            assert_equals(a['foo'], 'bar')
            assert_equals(a['bogosity'], bogosity[submission.id])
            for kvp in status.annotations['longAnnos']:
                if kvp['key'] == 'bogosity':
                    assert_false(kvp['isPrivate'])

        # test query by submission annotations
        # These queries run against an eventually consistent index table which is
        # populated by an asynchronous worker. Thus, the queries may remain out
        # of sync for some unbounded, but assumed to be short time.
        attempts = 2
        while attempts > 0:
            try:
                results = syn.restGET("/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s" % ev.id)
                assert_equals(len(results['rows']), num_of_submissions+1)

                results = syn.restGET(
                    "/evaluation/submission/query?query=SELECT+*+FROM+evaluation_%s where bogosity > 200" % ev.id)
                assert_equals(len(results['rows']), num_of_submissions)
            except AssertionError as ex1:
                attempts -= 1
                time.sleep(2)
            else:
                attempts = 0

        # Test that we can retrieve submissions with a specific status
        invalid_submissions = list(syn.getSubmissions(ev, status='INVALID'))
        assert_equals(len(invalid_submissions), 1, len(invalid_submissions))
        assert_equals(invalid_submissions[0]['name'], 'Submission 01')

    finally:
        # Clean up
        syn.delete(ev)
        if 'testSyn' in locals():
            if 'other_project' in locals():
                # Clean up, since the current user can't access this project
                # This also removes references to the submitted object :)
                testSyn.delete(other_project)
            if 'team' in locals():
                # remove team
                testSyn.delete(team)

    # Just deleted it. Shouldn't be able to get it.
    assert_raises(SynapseHTTPError, syn.getEvaluation, ev)