def test_task_post_api_exceptions(self, inv_field, dup): """Get a list of tasks using a list of project_ids.""" [admin, subadminowner] = UserFactory.create_batch(2) make_admin(admin) make_subadmin(subadminowner) project = ProjectFactory.create(owner=subadminowner) admin_headers = dict(Authorization=admin.api_key) task_info = dict(field_1='one', field_2='two') gold_answers = dict(field_3='some ans') data = dict(project_id=project.id, info=task_info, gold_answers=gold_answers, n_answers=2) dup.return_value = True res = self.app.post('/api/task', data=json.dumps(data), headers=admin_headers) res_data = json.loads(res.data) assert json.loads( res_data['exception_msg'])['reason'] == 'DUPLICATE_TASK', res dup.return_value = False inv_field = True res = self.app.post('/api/task', data=json.dumps(data), headers=admin_headers) res_data = json.loads(res.data) assert res_data[ 'exception_msg'] == 'Missing or incorrect required fields: ', res
def test_locked_sched_gold_task(self): """ Test gold tasks presented with locked scheduler """ [admin, owner, user] = UserFactory.create_batch(3) make_admin(admin) make_subadmin(owner) project = ProjectFactory.create(owner=owner) project.info['sched'] = Schedulers.locked project.set_gold_task_probability(1.0) project_repo.save(project) tasks = TaskFactory.create_batch(4, project=project, n_answers=1) gold_task = tasks[3] gold_task.calibration = 1; gold_task.gold_answers = dict(field_3='someans') # user #1 self.set_proj_passwd_cookie(project, user) res = self.app.get('api/project/{}/newtask?api_key={}' .format(project.id, user.api_key)) assert res.status_code == 200, res.status_code resp = json.loads(res.data) assert resp['id'] == gold_task.id, \ 'task presented to regular user under locked sched should be gold task' # submit answer for gold task task_run = dict(project_id=project.id, task_id=gold_task.id, info='hi there!') res = self.app.post('api/taskrun?api_key={}'.format(user.api_key), data=json.dumps(task_run)) assert res.status_code == 200, res.status_code # user #2 also gets gold_task even when redundancy was set to 1 res = self.app.get('api/project/{}/newtask?api_key={}' .format(project.id, owner.api_key)) assert res.status_code == 200, res.status_code resp = json.loads(res.data) assert resp['id'] == gold_task.id, \ 'task presented to owner under locked sched should be gold task' # after two task run submissions for gold task, state is unchanged to ongoing task_run = dict(project_id=project.id, task_id=gold_task.id, info='hi there!') res = self.app.post('api/taskrun?api_key={}'.format(owner.api_key), data=json.dumps(task_run)) assert res.status_code == 200, res.status_code res = self.app.get('api/task/{}?api_key={}' .format(gold_task.id, admin.api_key)) assert res.status_code == 200, res.status_code resp = json.loads(res.data) assert resp['id'] == gold_task.id and resp['state'] == 'ongoing', \ 'gold task state should be unchanged to ongoing' project.set_gold_task_probability(0.0) res = self.app.get('api/project/{}/newtask?api_key={}' .format(project.id, admin.api_key)) assert res.status_code == 200, res.status_code resp = json.loads(res.data) assert resp['id'] == tasks[0].id, \ 'task presented should not be gold task'
def test_newtask_unpublish_project(self): """Test user obtains newtask with published projects only; 404 with unpublished projects""" project = ProjectFactory.create( info=dict(published=True, task_presenter='task presenter')) admin, subadmin_coowner, regular_coowner, user = UserFactory.create_batch( 4) make_admin(admin) make_subadmin(subadmin_coowner) tasks = TaskFactory.create_batch(10, project=project, n_answers=1) self.set_proj_passwd_cookie(project, user) res = self.app.get('api/project/{}/newtask?api_key={}'.format( project.id, user.api_key)) task = json.loads(res.data) assert res.status_code == 200 and task['id'] == tasks[ 0].id, 'User should have obtained new task' # submit answer for the task task_run = dict(project_id=project.id, task_id=task['id'], info='hi there!') res = self.app.post('api/taskrun?api_key={}'.format(user.api_key), data=json.dumps(task_run)) assert res.status_code == 200, res.status_code #unpublish project project.published = False project.owners_ids.append(regular_coowner.id) project.owners_ids.append(subadmin_coowner.id) project_repo.save(project) project = project_repo.get(project.id) assert not project.published, 'Project should not be published' # for unpublished project, obtaining newtask would succeed for admin res = self.app.get('api/project/{}/newtask?api_key={}'.format( project.id, admin.api_key)) assert res.status_code == 200, 'newtask to unpublished project should succeed for admin' # for unpublished project, obtaining newtask would succeed for subadmin coowner res = self.app.get('api/project/{}/newtask?api_key={}'.format( project.id, subadmin_coowner.api_key)) assert res.status_code == 200, 'newtask to unpublished project should succeed for admin' # for unpublished project, obtaining newtask would succeed for regular coowner res = self.app.get('api/project/{}/newtask?api_key={}'.format( project.id, regular_coowner.api_key)) assert res.status_code == 200, 'newtask to unpublished project should succeed for admin' # for unpublished project, obtaining newtask would fail for regular user res = self.app.get('api/project/{}/newtask?api_key={}'.format( project.id, user.api_key)) assert res.status_code == 404, 'newtask to unpublished project should return 404'
def test_create_update_gold_answers(self): [admin, subadminowner, subadmin, reguser] = UserFactory.create_batch(4) make_admin(admin) make_subadmin(subadminowner) make_subadmin(subadmin) project = ProjectFactory.create(owner=subadminowner) admin_headers = dict(Authorization=admin.api_key) task_info = dict(field_1='one', field_2='two') gold_answers = dict(field_3='some ans', field_4='some ans 2') # POST gold_answers successful data = dict(project_id=project.id, info=task_info, gold_answers=gold_answers, n_answers=2) res = self.app.post('/api/task', data=json.dumps(data), headers=admin_headers) assert res.data, res jdata = json.loads(res.data) assert_equal(jdata['info'], task_info), jdata assert_equal(jdata['gold_answers'], gold_answers), jdata assert jdata[ 'calibration'] == 1, 'calibration should have been set with updating gold_answers' assert jdata[ 'exported'] == True, 'exported should be True with new gold task' # GET task by subadmin not owner user does not get gold answers, # whereas admin/subadmin gets gold answers subadminowner_headers = dict(Authorization=subadminowner.api_key) subadmin_headers = dict(Authorization=subadmin.api_key) reguser_headers = dict(Authorization=reguser.api_key) res = self.app.get('/api/task/1', headers=admin_headers) jdata = json.loads(res.data) assert_equal(jdata['gold_answers'], gold_answers), jdata res = self.app.get('/api/task/1', headers=subadminowner_headers) jdata = json.loads(res.data) assert_equal(jdata['gold_answers'], gold_answers), jdata res = self.app.get('/api/task/1', headers=subadmin_headers) jdata = json.loads(res.data) assert 'gold_answers' not in jdata, jdata assert 'calibration' not in jdata, jdata # regular users should not receive gold_answers and calibration info res = self.app.get('/api/task/1', headers=reguser_headers) jdata = json.loads(res.data) assert 'gold_answers' not in jdata, jdata assert 'calibration' not in jdata, jdata # PUT request updates gold answers updated_gold_answers = dict(field_3='some ans - updated', field_5='one more ans') data = dict(project_id=project.id, gold_answers=updated_gold_answers) res = self.app.put('/api/task/1', data=json.dumps(data), headers=subadminowner_headers) jdata = json.loads(res.data) gold_answers.update(updated_gold_answers) assert_equal(jdata['gold_answers'], updated_gold_answers), jdata # Beyond redundancy, submitting task runs to task with gold_answers # is permitted. such task run submissions should not mark task as complete task = task_repo.get_task(jdata['id']) n_taskruns = 8 taskruns = TaskRunFactory.create_batch(n_taskruns, project=project, task=task) assert task.state == 'ongoing', 'Gold task state should be ongoing beyond task submissions > task redundancy' assert len( taskruns ) == n_taskruns, 'For gold task, number of task runs submissions can be beyond task redundancy' assert task.exported == False, 'Gold tasks to be marked exported as False upon task run submission' task.exported = True taskruns = TaskRunFactory.create_batch(1, project=project, task=task) assert task.exported == False, 'Gold tasks to be marked exported as False upon task run submission' # reset gold answer data = dict(project_id=project.id, gold_answers={}) res = self.app.put('/api/task/1', data=json.dumps(data), headers=subadminowner_headers) jdata = json.loads(res.data) assert_equal(jdata['gold_answers'], {}), jdata assert jdata[ 'calibration'] == 0, 'Calibration should be reset upon gold_answers reset'
def test_completedtask_api(self): """Test API query for completedtask works""" project = ProjectFactory.create() admin = UserFactory.create() make_admin(admin) # create 2 tasks to be completed, 1 gold task, remaining 2 would be ongoing otask, ctask, gtask, ctask2, _ = TaskFactory.create_batch( 5, project=project, n_answers=2, state='ongoing') gtask.calibration = 1 task_repo.update(gtask) # submit task runs otask_runs = TaskRunFactory.create_batch(1, task=otask) # ongoing taskrun ctask_runs = TaskRunFactory.create_batch( 2, task=ctask) # completed taskruns ctask_runs2 = TaskRunFactory.create_batch( 2, task=ctask2) # completed taskruns 2 gtask_runs = TaskRunFactory.create_batch(5, task=gtask) # gold taskruns all_expected_taskruns = ctask_runs + ctask_runs2 + gtask_runs all_expected_taskruns_ids = [ taskruns.id for taskruns in all_expected_taskruns ] gtask.calibration = 1 gtask.gold_answers = dict(somefield='g ans') task_repo.update(gtask) # get all tasks; confirm tasks 2,4 are complete and 3 is gold task url = '/api/task?project_id=1&api_key={}&all=1'.format(admin.api_key) res = self.app.get(url) data = json.loads(res.data) assert len(data) == 5, 'There should be total 5 tasks' assert data[1]['state'] == 'completed' and data[1]['calibration'] == 0, \ 'Task should have been completed and not a gold task' assert data[3]['state'] == 'completed' and data[3]['calibration'] == 0, \ 'Task should have been completed and not a gold task' assert data[2]['state'] == 'ongoing' and data[2]['calibration'] == 1, \ 'Task should have been ongoing and a gold task' assert data[1]['id'] == ctask.id and data[2]['id'] == gtask.id and \ data[3]['id'] == ctask2.id, 'Task ids for completed and gold tasks should match' # get completed tasks and gold tasks expected_tasks = [ctask, gtask, ctask2] url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format( admin.api_key) res = self.app.get(url) data = json.loads(res.data) assert len(data) == len(expected_tasks), \ 'Completedtask api should have returned {} tasks'.format(len(expected_tasks)) for i, task in enumerate(expected_tasks): assert data[i]['id'] == task.id, \ 'completedtask api should have returned task {}, state {}, calibration {}' \ .format(task.id, task.state, task.calibration) # mark expected_tasks as exported, add 3 more gold tasks # completedtask api should return new 3 gold tasks for task in expected_tasks: task.exported = True task_repo.save(task) new_gold_tasks = TaskFactory.create_batch( 3, project=project, n_answers=2, state='ongoing', calibration=1, gold_answers=dict(somefield='g ans')) url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format( admin.api_key) res = self.app.get(url) data = json.loads(res.data) assert len(data) == len(new_gold_tasks), \ 'completedtasks api should have returned {} tasks'.format(len(new_gold_tasks)) for i, task in enumerate(new_gold_tasks): assert data[i]['id'] == task.id, \ 'completedtask api should have returned new gold tasks {}, state {}, calibration {}' \ .format(task.id, task.state, task.calibration) # set exported to false to all 8 tasks all_expected_tasks = expected_tasks + new_gold_tasks all_expected_tasks_ids = [task.id for task in all_expected_tasks] for task in all_expected_tasks: task.exported = False task_repo.save(task) url = '/api/completedtask?project_id=1&api_key={}&exported=False'.format( admin.api_key) res = self.app.get(url) tasks = json.loads(res.data) for task in tasks: assert task['id'] in all_expected_tasks_ids, \ 'completedtask api should have returned task {}'.format(task['id']) # perform api call with last_id url = '/api/completedtask?project_id=1&api_key={}&exported=False&last_id={}' \ .format(admin.api_key, all_expected_tasks[2].id) res = self.app.get(url) tasks = json.loads(res.data) for task in tasks: # last id set to all_expected_tasks[2].id. hence returned task ids to be from index 3 onwards assert task['id'] in all_expected_tasks_ids[3:], \ 'completedtask api with last_id should not have returned task {}'.format(task['id']) # get all completed and gold taskruns url = '/api/completedtaskrun?project_id=1&api_key={}&exported=False'.format( admin.api_key) res = self.app.get(url) taskruns = json.loads(res.data) assert len(taskruns) == len(all_expected_taskruns) for taskrun in taskruns: assert taskrun['id'] in all_expected_taskruns_ids, \ 'completedtaskrun api should have returned taskrun {}'.format(taskrun['id']) # perform api call with last_id url = '/api/completedtaskrun?project_id=1&api_key={}&exported=False&last_id={}' \ .format(admin.api_key, all_expected_taskruns_ids[2]) res = self.app.get(url) taskruns = json.loads(res.data) for taskrun in taskruns: # last id set to all_expected_tasks[2].id. hence returned task ids to be from index 3 onwards assert taskrun['id'] in all_expected_taskruns_ids[3:], \ 'completedtaskrun api with last_id should not have returned taskrun {}'.format(taskrun['id'])