def test_task_post_api_exceptions(self, inv_field, dup):
"""Get a list of tasks using a list of project_ids."""
[admin, subadminowner] = UserFactory.create_batch(2)
make_admin(admin)
make_subadmin(subadminowner)
project = ProjectFactory.create(owner=subadminowner)
admin_headers = dict(Authorization=admin.api_key)
task_info = dict(field_1='one', field_2='two')
gold_answers = dict(field_3='some ans')
data = dict(project_id=project.id,
info=task_info,
gold_answers=gold_answers,
n_answers=2)
dup.return_value = True
res = self.app.post('/api/task',
data=json.dumps(data),
headers=admin_headers)
res_data = json.loads(res.data)
assert json.loads(
res_data['exception_msg'])['reason'] == 'DUPLICATE_TASK', res
dup.return_value = False
inv_field = True
res = self.app.post('/api/task',
data=json.dumps(data),
headers=admin_headers)
res_data = json.loads(res.data)
assert res_data[
'exception_msg'] == 'Missing or incorrect required fields: ', res
def test_locked_sched_gold_task(self):
""" Test gold tasks presented with locked scheduler """
[admin, owner, user] = UserFactory.create_batch(3)
make_admin(admin)
make_subadmin(owner)
project = ProjectFactory.create(owner=owner)
project.info['sched'] = Schedulers.locked
project.set_gold_task_probability(1.0)
project_repo.save(project)
tasks = TaskFactory.create_batch(4, project=project, n_answers=1)
gold_task = tasks[3]
gold_task.calibration = 1; gold_task.gold_answers = dict(field_3='someans')
# user #1
self.set_proj_passwd_cookie(project, user)
res = self.app.get('api/project/{}/newtask?api_key={}'
.format(project.id, user.api_key))
assert res.status_code == 200, res.status_code
resp = json.loads(res.data)
assert resp['id'] == gold_task.id, \
'task presented to regular user under locked sched should be gold task'
# submit answer for gold task
task_run = dict(project_id=project.id, task_id=gold_task.id, info='hi there!')
res = self.app.post('api/taskrun?api_key={}'.format(user.api_key),
data=json.dumps(task_run))
assert res.status_code == 200, res.status_code
# user #2 also gets gold_task even when redundancy was set to 1
res = self.app.get('api/project/{}/newtask?api_key={}'
.format(project.id, owner.api_key))
assert res.status_code == 200, res.status_code
resp = json.loads(res.data)
assert resp['id'] == gold_task.id, \
'task presented to owner under locked sched should be gold task'
# after two task run submissions for gold task, state is unchanged to ongoing
task_run = dict(project_id=project.id, task_id=gold_task.id, info='hi there!')
res = self.app.post('api/taskrun?api_key={}'.format(owner.api_key),
data=json.dumps(task_run))
assert res.status_code == 200, res.status_code
res = self.app.get('api/task/{}?api_key={}'
.format(gold_task.id, admin.api_key))
assert res.status_code == 200, res.status_code
resp = json.loads(res.data)
assert resp['id'] == gold_task.id and resp['state'] == 'ongoing', \
'gold task state should be unchanged to ongoing'
project.set_gold_task_probability(0.0)
res = self.app.get('api/project/{}/newtask?api_key={}'
.format(project.id, admin.api_key))
assert res.status_code == 200, res.status_code
resp = json.loads(res.data)
assert resp['id'] == tasks[0].id, \
'task presented should not be gold task'
def test_newtask_unpublish_project(self):
"""Test user obtains newtask with published projects only; 404 with unpublished projects"""
project = ProjectFactory.create(
info=dict(published=True, task_presenter='task presenter'))
admin, subadmin_coowner, regular_coowner, user = UserFactory.create_batch(
4)
make_admin(admin)
make_subadmin(subadmin_coowner)
tasks = TaskFactory.create_batch(10, project=project, n_answers=1)
self.set_proj_passwd_cookie(project, user)
res = self.app.get('api/project/{}/newtask?api_key={}'.format(
project.id, user.api_key))
task = json.loads(res.data)
assert res.status_code == 200 and task['id'] == tasks[
0].id, 'User should have obtained new task'
# submit answer for the task
task_run = dict(project_id=project.id,
task_id=task['id'],
info='hi there!')
res = self.app.post('api/taskrun?api_key={}'.format(user.api_key),
data=json.dumps(task_run))
assert res.status_code == 200, res.status_code
#unpublish project
project.published = False
project.owners_ids.append(regular_coowner.id)
project.owners_ids.append(subadmin_coowner.id)
project_repo.save(project)
project = project_repo.get(project.id)
assert not project.published, 'Project should not be published'
# for unpublished project, obtaining newtask would succeed for admin
res = self.app.get('api/project/{}/newtask?api_key={}'.format(
project.id, admin.api_key))
assert res.status_code == 200, 'newtask to unpublished project should succeed for admin'
# for unpublished project, obtaining newtask would succeed for subadmin coowner
res = self.app.get('api/project/{}/newtask?api_key={}'.format(
project.id, subadmin_coowner.api_key))
assert res.status_code == 200, 'newtask to unpublished project should succeed for admin'
# for unpublished project, obtaining newtask would succeed for regular coowner
res = self.app.get('api/project/{}/newtask?api_key={}'.format(
project.id, regular_coowner.api_key))
assert res.status_code == 200, 'newtask to unpublished project should succeed for admin'
# for unpublished project, obtaining newtask would fail for regular user
res = self.app.get('api/project/{}/newtask?api_key={}'.format(
project.id, user.api_key))
assert res.status_code == 404, 'newtask to unpublished project should return 404'
def test_create_update_gold_answers(self):
[admin, subadminowner, subadmin, reguser] = UserFactory.create_batch(4)
make_admin(admin)
make_subadmin(subadminowner)
make_subadmin(subadmin)
project = ProjectFactory.create(owner=subadminowner)
admin_headers = dict(Authorization=admin.api_key)
task_info = dict(field_1='one', field_2='two')
gold_answers = dict(field_3='some ans', field_4='some ans 2')
# POST gold_answers successful
data = dict(project_id=project.id,
info=task_info,
gold_answers=gold_answers,
n_answers=2)
res = self.app.post('/api/task',
data=json.dumps(data),
headers=admin_headers)
assert res.data, res
jdata = json.loads(res.data)
assert_equal(jdata['info'], task_info), jdata
assert_equal(jdata['gold_answers'], gold_answers), jdata
assert jdata[
'calibration'] == 1, 'calibration should have been set with updating gold_answers'
assert jdata[
'exported'] == True, 'exported should be True with new gold task'
# GET task by subadmin not owner user does not get gold answers,
# whereas admin/subadmin gets gold answers
subadminowner_headers = dict(Authorization=subadminowner.api_key)
subadmin_headers = dict(Authorization=subadmin.api_key)
reguser_headers = dict(Authorization=reguser.api_key)
res = self.app.get('/api/task/1', headers=admin_headers)
jdata = json.loads(res.data)
assert_equal(jdata['gold_answers'], gold_answers), jdata
res = self.app.get('/api/task/1', headers=subadminowner_headers)
jdata = json.loads(res.data)
assert_equal(jdata['gold_answers'], gold_answers), jdata
res = self.app.get('/api/task/1', headers=subadmin_headers)
jdata = json.loads(res.data)
assert 'gold_answers' not in jdata, jdata
assert 'calibration' not in jdata, jdata
# regular users should not receive gold_answers and calibration info
res = self.app.get('/api/task/1', headers=reguser_headers)
jdata = json.loads(res.data)
assert 'gold_answers' not in jdata, jdata
assert 'calibration' not in jdata, jdata
# PUT request updates gold answers
updated_gold_answers = dict(field_3='some ans - updated',
field_5='one more ans')
data = dict(project_id=project.id, gold_answers=updated_gold_answers)
res = self.app.put('/api/task/1',
data=json.dumps(data),
headers=subadminowner_headers)
jdata = json.loads(res.data)
gold_answers.update(updated_gold_answers)
assert_equal(jdata['gold_answers'], updated_gold_answers), jdata
# Beyond redundancy, submitting task runs to task with gold_answers
# is permitted. such task run submissions should not mark task as complete
task = task_repo.get_task(jdata['id'])
n_taskruns = 8
taskruns = TaskRunFactory.create_batch(n_taskruns,
project=project,
task=task)
assert task.state == 'ongoing', 'Gold task state should be ongoing beyond task submissions > task redundancy'
assert len(
taskruns
) == n_taskruns, 'For gold task, number of task runs submissions can be beyond task redundancy'
assert task.exported == False, 'Gold tasks to be marked exported as False upon task run submission'
task.exported = True
taskruns = TaskRunFactory.create_batch(1, project=project, task=task)
assert task.exported == False, 'Gold tasks to be marked exported as False upon task run submission'
# reset gold answer
data = dict(project_id=project.id, gold_answers={})
res = self.app.put('/api/task/1',
data=json.dumps(data),
headers=subadminowner_headers)
jdata = json.loads(res.data)
assert_equal(jdata['gold_answers'], {}), jdata
assert jdata[
'calibration'] == 0, 'Calibration should be reset upon gold_answers reset'
def test_completedtask_api(self):
"""Test API query for completedtask works"""
project = ProjectFactory.create()
admin = UserFactory.create()
make_admin(admin)
# create 2 tasks to be completed, 1 gold task, remaining 2 would be ongoing
otask, ctask, gtask, ctask2, _ = TaskFactory.create_batch(
5, project=project, n_answers=2, state='ongoing')
gtask.calibration = 1
task_repo.update(gtask)
# submit task runs
otask_runs = TaskRunFactory.create_batch(1,
task=otask) # ongoing taskrun
ctask_runs = TaskRunFactory.create_batch(
2, task=ctask) # completed taskruns
ctask_runs2 = TaskRunFactory.create_batch(
2, task=ctask2) # completed taskruns 2
gtask_runs = TaskRunFactory.create_batch(5,
task=gtask) # gold taskruns
all_expected_taskruns = ctask_runs + ctask_runs2 + gtask_runs
all_expected_taskruns_ids = [
taskruns.id for taskruns in all_expected_taskruns
]
gtask.calibration = 1
gtask.gold_answers = dict(somefield='g ans')
task_repo.update(gtask)
# get all tasks; confirm tasks 2,4 are complete and 3 is gold task
url = '/api/task?project_id=1&api_key={}&all=1'.format(admin.api_key)
res = self.app.get(url)
data = json.loads(res.data)
assert len(data) == 5, 'There should be total 5 tasks'
assert data[1]['state'] == 'completed' and data[1]['calibration'] == 0, \
'Task should have been completed and not a gold task'
assert data[3]['state'] == 'completed' and data[3]['calibration'] == 0, \
'Task should have been completed and not a gold task'
assert data[2]['state'] == 'ongoing' and data[2]['calibration'] == 1, \
'Task should have been ongoing and a gold task'
assert data[1]['id'] == ctask.id and data[2]['id'] == gtask.id and \
data[3]['id'] == ctask2.id, 'Task ids for completed and gold tasks should match'
# get completed tasks and gold tasks
expected_tasks = [ctask, gtask, ctask2]
url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format(
admin.api_key)
res = self.app.get(url)
data = json.loads(res.data)
assert len(data) == len(expected_tasks), \
'Completedtask api should have returned {} tasks'.format(len(expected_tasks))
for i, task in enumerate(expected_tasks):
assert data[i]['id'] == task.id, \
'completedtask api should have returned task {}, state {}, calibration {}' \
.format(task.id, task.state, task.calibration)
# mark expected_tasks as exported, add 3 more gold tasks
# completedtask api should return new 3 gold tasks
for task in expected_tasks:
task.exported = True
task_repo.save(task)
new_gold_tasks = TaskFactory.create_batch(
3,
project=project,
n_answers=2,
state='ongoing',
calibration=1,
gold_answers=dict(somefield='g ans'))
url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format(
admin.api_key)
res = self.app.get(url)
data = json.loads(res.data)
assert len(data) == len(new_gold_tasks), \
'completedtasks api should have returned {} tasks'.format(len(new_gold_tasks))
for i, task in enumerate(new_gold_tasks):
assert data[i]['id'] == task.id, \
'completedtask api should have returned new gold tasks {}, state {}, calibration {}' \
.format(task.id, task.state, task.calibration)
# set exported to false to all 8 tasks
all_expected_tasks = expected_tasks + new_gold_tasks
all_expected_tasks_ids = [task.id for task in all_expected_tasks]
for task in all_expected_tasks:
task.exported = False
task_repo.save(task)
url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format(
admin.api_key)
res = self.app.get(url)
tasks = json.loads(res.data)
for task in tasks:
assert task['id'] in all_expected_tasks_ids, \
'completedtask api should have returned task {}'.format(task['id'])
# perform api call with last_id
url = '/api/completedtask?project_id=1&api_key={}&exported=False&last_id={}' \
.format(admin.api_key, all_expected_tasks[2].id)
res = self.app.get(url)
tasks = json.loads(res.data)
for task in tasks:
# last id set to all_expected_tasks[2].id. hence returned task ids to be from index 3 onwards
assert task['id'] in all_expected_tasks_ids[3:], \
'completedtask api with last_id should not have returned task {}'.format(task['id'])
# get all completed and gold taskruns
url = '/api/completedtaskrun?project_id=1&api_key={}&exported=False'.format(
admin.api_key)
res = self.app.get(url)
taskruns = json.loads(res.data)
assert len(taskruns) == len(all_expected_taskruns)
for taskrun in taskruns:
assert taskrun['id'] in all_expected_taskruns_ids, \
'completedtaskrun api should have returned taskrun {}'.format(taskrun['id'])
# perform api call with last_id
url = '/api/completedtaskrun?project_id=1&api_key={}&exported=False&last_id={}' \
.format(admin.api_key, all_expected_taskruns_ids[2])
res = self.app.get(url)
taskruns = json.loads(res.data)
for taskrun in taskruns:
# last id set to all_expected_tasks[2].id. hence returned task ids to be from index 3 onwards
assert taskrun['id'] in all_expected_taskruns_ids[3:], \
'completedtaskrun api with last_id should not have returned taskrun {}'.format(taskrun['id'])
