def test_create_annotation_task_with_task_group(self): self.before_each() # make an AnnotationTaskTopicGroup that contains first two annotation tasks in self.tasks task_group = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'first task group', 'annotation_task_ids': [self.tasks[0].id, self.tasks[1].id], 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1, 'gold_annotator_user_ids': [1, 2] }) self.task_groups = [task_group] db_session_users.add_all(self.task_groups) db_session_users.commit() for group in self.task_groups: db_session_users.refresh(group) # create two new annotation tasks that point point to this existing annotation_task_topic_group request_body_1 = json.dumps({ 'name': 'wat_1', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'} }, 'user_ids': [self.user.id], 'term_sampling_group_ids': [1], 'annotation_task_topic_group_id': self.task_groups[0].id }) request_body_2 = json.dumps({ 'name': 'wat_2', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'} }, 'user_ids': [self.user.id], 'term_sampling_group_ids': [1], 'annotation_task_topic_group_id': self.task_groups[0].id }) response_1 = self.client.post("/annotation_tasks", headers={'Authorization': self.admin_user_token}, data=request_body_1) task_result_1 = response_1.json["annotation_task"] response_2 = self.client.post("/annotation_tasks", headers={'Authorization': self.admin_user_token}, data=request_body_2) task_result_2 = response_2.json["annotation_task"] for group in self.task_groups: db_session_users.refresh(group) # check that these two tasks point to task group correctly task_from_db_1 = db_session_users.query(AnnotationTask).filter_by(id=task_result_1['id']).first() task_from_db_2 = db_session_users.query(AnnotationTask).filter_by(id=task_result_2['id']).first() self.assertEqual(task_from_db_1.annotation_task_topic_group_id, self.task_groups[0].id) self.assertEqual(task_from_db_2.annotation_task_topic_group_id, self.task_groups[0].id) # check that ORM query on the annotation_task_topic_group now gives the these two annotation tasks task_group_from_db = db_session_users.query(AnnotationTaskTopicGroup)\ .filter_by(id=self.task_groups[0].id)\ .first() self.assertEqual({task_from_db_1, task_from_db_2}, set(task_group_from_db.annotation_tasks))
def test_get_annotation_tasks_by_annotation_type(self): self.before_each() first_task = self.tasks[0] first_task.type = "contributor" first_task.is_contributor_task = True db_session_users.add(first_task) second_task = self.tasks[1] second_task.type = AnnotationTask.TOPIC_ANNOTATION_TYPE second_task.is_contributor_task = True second_task.user_ids = [self.contributor_user.id] db_session_users.add(second_task) new_task_no_flag = AnnotationTask({ 'name': 'watman', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'}, }, 'type': AnnotationTask.TOPIC_ANNOTATION_TYPE, 'is_contributor_task': False, 'user_ids': [self.contributor_user.id] }) db_session_users.add(new_task_no_flag) db_session_users.commit() db_session_users.refresh(new_task_no_flag) response = self.client.get("/annotation_tasks?type=topic_annotation", headers={'Authorization': self.contributor_user_token}) self.assert200(response) self.assertIn("annotation_tasks", response.json) result_tasks = response.json["annotation_tasks"] self.assertIsInstance(result_tasks, list) self.assertEqual(len(result_tasks), 2) self.assertEqual(result_tasks[0]['id'], second_task.id) self.assertEqual(result_tasks[1]['id'], new_task_no_flag.id)
def test_skip_flagged_document(self): doc_id = 3 # first write a doc for review ufd = UserFlaggedDocument({ 'issue_severity': 'review', 'issue_type': 'technical', 'field': 'title', 'notes': 'typo in title', 'user_id': self.user.id, 'doc_id': doc_id }) db_session_users.add(ufd) db_session_users.commit() db_session_users.refresh(ufd) # then post a status update to skipped flagged_data = json.dumps( {'flagged': { 'id': ufd.id, 'status': 'skipped' }}) response = self.client.post( "/documents/{}".format(doc_id), headers={'Authorization': self.qa_user_token}, data=flagged_data) self.assert200(response) self.assertIn('document', response.json) self.assertEquals(response.json['document']['id'], doc_id) self.assertEquals(response.json['document']['issue_severity'], 'review') self.assertEquals(response.json['document']['issue_type'], 'technical') self.assertEquals(response.json['document']['status'], 'skipped') db_session_users.refresh(ufd) self.assertEquals(ufd.status, 'skipped')
def test_update_user_folder_with_new_documents(self): # create a folder user_folder = UserFolderFactory(user_id=self.user.id) db_session_users.add(user_folder) db_session_users.commit() db_session_users.refresh(user_folder) # add single doc to folder folder = db_session_users.query(UserFolder).filter_by( user_id=self.user.id).first() request_body = json.dumps({ 'document_ids': [1, 2, 3], 'folder_id': folder.id, }) response = self.client.post("/documents", headers={'Authorization': self.token}, data=request_body) self.assert200(response) self.assertIn('documents', response.json) self.assertEquals(response.json['documents'], { 'ids': [1, 2, 3], 'documents_added_to_folder': True })
def test_remove_positive_tag_from_document(self): doc_id = 3 self.before_each_tags() user_doc_tag = UserDocumentTag({ 'user_tag_id': self.user_tags[0].id, 'is_positive': True, 'display_style': 'modal', 'doc_id': doc_id, 'user_id': self.user.id }) db_session_users.add(user_doc_tag) db_session_users.commit() request_data = { 'tag': { 'id': self.user_tags[0].id, 'is_positive': False, 'display_style': 'modal' } } response = self.client.post("/documents/{}".format(doc_id), headers={'Authorization': self.token}, data=json.dumps(request_data)) self.assert200(response) self.assertIn('document', response.json) for key in [ 'doc_id', 'user_id', 'user_tag_id', 'is_positive', 'display_style' ]: self.assertIn(key, response.json['document']) self.assertEquals(response.json['document']['id'], doc_id) self.assertEquals(response.json['document']['user_tag_id'], self.user_tags[0].id) self.assertEquals(response.json['document']['user_id'], self.user.id) self.assertFalse(response.json['document']['is_positive']) self.assertEquals(response.json['document']['display_style'], 'modal') # make sure it is updated (not new) in the db too db_session_users.refresh(user_doc_tag) self.assertFalse(user_doc_tag.is_positive) # now check that it does not show up in the get_document route response = self.client.get( "/documents/{}".format(doc_id), headers={'Authorization': self.token}, ) self.assert200(response) tag_ids = [t[0] for t in response.json['document']['tags']] self.assertNotIn(self.user_tags[0].id, tag_ids)
def before_each(self): # n.b. cleaning this out due to other test interference db_session_users.query(UserSharedFolder).delete() db_session_users.query(UserFolderDocument).delete() db_session_users.query(UserFolder).delete() # create a folder self.user_folder = UserFolder({ "name": 'test_folder', 'user_id': self.user.id, }) db_session_users.add(self.user_folder) db_session_users.commit() db_session_users.refresh(self.user_folder)
def before_each(self, skip_user_entries=False): db_session_users.query(UserContributorPoint).delete() db_session_users.query(ContributorPointType).delete() self.contributor_point_types = [] for contributor_type in CONTRIBUTOR_TYPE_TEMPLATES: self.contributor_point_types.append( ContributorPointType(contributor_type)) if not skip_user_entries: # add one event for each type for s in self.contributor_point_types: s.user_contributor_points.append( UserContributorPoint({ "user_id": self.user.id, 'num_points': s.points_per_action })) # for weekly/anytime frequencies, add another basic option if s.frequency != 'onboarding': # add another recent option s.user_contributor_points.append( UserContributorPoint({ "user_id": self.user.id, 'num_points': s.points_per_action })) db_session_users.add_all(self.contributor_point_types) db_session_users.commit() if not skip_user_entries: # add another option from a year ago for anytime/weekly types that will get ignored for weekly but returned for anytime for s in self.contributor_point_types: if s.frequency != 'onboarding': one_year_ago = dt.datetime.now() - dt.timedelta(weeks=52) ucp = UserContributorPoint({ "user_id": self.user.id, 'num_points': s.points_per_action, 'contributor_point_type_id': s.id }) db_session_users.add(ucp) db_session_users.commit() ucp.created_at = one_year_ago # n.b. need to do it like this to fudge the date db_session_users.add(ucp) db_session_users.commit() db_session_users.refresh(ucp)
def test_get_annotation_task_group_tags_for_task(self): self.before_each() # create AnnotationTaskGroups objects # self.tasks[0] is contained in both task groups task_group_1 = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'first task group', 'annotation_task_ids': [self.tasks[0].id], 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1 }) task_group_2 = AnnotationTaskTopicGroup({ 'name': 'task_group_2', 'description': 'second task group', 'annotation_task_ids': [self.tasks[0].id, self.tasks[1].id], 'arbitrary_tags': ["tag2", "tag3"], 'topic_id': 2 }) self.task_groups = [task_group_1, task_group_2] db_session_users.add_all(self.task_groups) db_session_users.commit() for tg in self.task_groups: db_session_users.refresh(tg) # make request for task contained in one task group response = self.client.get('/annotation_tasks/' + str(self.tasks[1].id) + '/task_group_labels', headers={'Authorization': self.admin_user_token}) self.assert200(response) self.assertIn("annotation_task_group_tags", response.json) self.assertEqual(set(response.json["annotation_task_group_tags"]), {"tag2", "tag3"}) # make request for task contained in two task groups response = self.client.get('/annotation_tasks/' + str(self.tasks[0].id) + '/task_group_labels', headers={'Authorization': self.admin_user_token}) self.assert200(response) self.assertIn("annotation_task_group_tags", response.json) self.assertEqual(set(response.json["annotation_task_group_tags"]), {"tag1", "tag2", "tag3", "WARNING: MORE THAN ONE ANNOTATION TASK GROUP CONTAINS THIS TASK"})
def before_each(self): # n.b. cleaning this out due to other test interference db_session_users.query(TeamMember).delete() db_session_users.query(Team).delete() self.teams = [ UserTeamFactory(name='test_team'), UserTeamFactory(name='foo_team_2') ] db_session_users.add_all(self.teams) db_session_users.commit() for team in self.teams: db_session_users.refresh(team) self.teamMembers = [ UserTeamMemberFactory(user_id=self.user.id, team_id=self.teams[0].id), UserTeamMemberFactory(user_id=self.user.id, team_id=self.teams[1].id), ] db_session_users.add_all(self.teamMembers) db_session_users.commit() for tm in self.teamMembers: db_session_users.refresh(tm)
def test_update_user_folder_with_existing_documents(self): # create a folder user_folder = UserFolderFactory(user_id=self.user.id) db_session_users.add(user_folder) db_session_users.commit() db_session_users.refresh(user_folder) # add a document to the folder folder = db_session_users.query(UserFolder).filter_by( user_id=self.user.id).first() user_folder_documents = [ UserFolderDocument({ "user_folder_id": folder.id, 'doc_id': 1, }), UserFolderDocument({ "user_folder_id": folder.id, 'doc_id': 2, }), UserFolderDocument({ "user_folder_id": folder.id, 'doc_id': 3, }), ] db_session_users.add_all(user_folder_documents) db_session_users.commit() # add the doc that already exists in the folder request_body = json.dumps({ 'document_ids': [1, 2, 3], 'folder_id': folder.id, }) response = self.client.post("/documents", headers={'Authorization': self.token}, data=request_body) self.assertStatus(response, 409) self.assertIn('errors', response.json)
def test_update_annotation_task_include_gold_annotations(self): self.before_each() ## test update to is_training_task ## # check original annotation task value for include_gold_annotations self.assertTrue(self.tasks[0].include_gold_annotations) # make update request_body = json.dumps({ 'include_gold_annotations': False }) response = self.client.post('/annotation_tasks/' + str(self.tasks[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertFalse(task_result['include_gold_annotations']) self.assertNotEqual(task_result['id'], self.tasks[0].id) # check that new annotation_task was created db_session_users.refresh(self.tasks[0]) self.assertEqual(self.tasks[0].status, 'inactive') # check that old task is no longer active self.assertEqual(self.tasks[0].active_task_id, task_result['id']) # check that old task points to new task
def before_each(self): db_session_users.query(MarketingCampaignUsers).delete() db_session_users.query(MarketingCampaign).delete() m1 = MarketingCampaign({ 'name': 'foo', 'start_date': '01/01/2017', 'end_date': '01/06/2017', 'created_by_user_id': self.user.id, 'notes': 'yada yada yada' }) m1.gen_token() m1.users.append(self.user) m2 = MarketingCampaign({ 'name': 'bar', 'start_date': '01/01/2015', 'end_date': '01/06/2015', 'created_by_user_id': self.user.id }) m2.gen_token() self.marketing_campaigns = [m1, m2] db_session_users.add_all(self.marketing_campaigns) db_session_users.commit() for m in self.marketing_campaigns: db_session_users.refresh(m)
def before_each(self): db_session_users.query(TopicAnnotation).delete() db_session_users.query(AnnotationJob).delete() db_session_users.query(AnnotationTask).delete() t1 = AnnotationTask({ 'name': 'foo', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'}, } }) self.tasks = [t1] self.jobs = [] self.annotations = [] for i in xrange(1, 6): j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.COMPLETE_STATUS, "user_id": self.user.id, "was_skipped": True if i % 2 == 0 else None }) j1.annotation_task = t1 self.jobs.append(j1) j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.COMPLETE_STATUS, "user_id": self.admin_user.id, "was_skipped": None }) j1.annotation_task = t1 self.jobs.append(j1) a1 = TopicAnnotation({ "doc_id": i, "is_positive": True if i % 2 == 0 else False, "user_id": self.user.id, "topic_name": "Lending" }) a1.annotation_task = t1 self.annotations.append(a1) a1 = TopicAnnotation({ "doc_id": i, "is_positive": False if i % 2 == 0 else True, "user_id": self.admin_user.id, "topic_name": "Lending" }) a1.annotation_task = t1 self.annotations.append(a1) db_session_users.add_all(self.tasks) db_session_users.add_all(self.jobs) db_session_users.add_all(self.annotations) db_session_users.commit() for t in self.tasks: db_session_users.refresh(t) for j in self.jobs: db_session_users.refresh(j)
def test_update_annotation_task_is_onboarding_task(self): self.before_each() db_session_users.query(AnnotationJob).delete() # make annotation task group to contain the onboarding task task_group = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'onboarding task group', 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1, 'gold_annotator_user_ids': [1, 2] }) self.task_groups = [task_group] db_session_users.add_all(self.task_groups) db_session_users.commit() for group in self.task_groups: db_session_users.refresh(group) # create onboarding annotation task onboarding_task = AnnotationTask({ 'name': 'onboarding task', 'topics': { 'lending': {'topic_id': 2, 'topic_table': 'topics'}, }, 'type': AnnotationTask.TOPIC_ANNOTATION_TYPE, 'is_training_task': True, 'annotation_task_topic_group_id': self.task_groups[0].id, 'user_ids': [[self.user.id, self.new_user.id, self.qa_user.id]] }) db_session_users.add(onboarding_task) db_session_users.commit() db_session_users.refresh(onboarding_task) # create job for this onboarding task (so that if this is not an onboarding task, new task will be created) onboarding_job = AnnotationJob({ "doc_id": 1, "priority": 1.0, "status": AnnotationJob.QUEUED_STATUS, "is_gold_evaluation": True }) onboarding_job.annotation_task = onboarding_task db_session_users.add(onboarding_job) db_session_users.commit() db_session_users.refresh(onboarding_job) # make update to users request_body = json.dumps({ 'user_ids': [self.user.id] }) response = self.client.post('/annotation_tasks/' + str(onboarding_task.id), headers={'Authorization': self.admin_user_token}, data=request_body) # check result - no new task should be created, because this is an onboarding task self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertEqual(task_result['user_ids'], [self.user.id]) self.assertEqual(task_result['id'], onboarding_task.id) # check that no new annotation_task was created # update task to be not-onboarding onboarding_task.is_training_task = False db_session_users.add(onboarding_task) db_session_users.commit() db_session_users.refresh(onboarding_task) # make another user_ids update request_body = json.dumps({ 'user_ids': [self.user.id, self.new_user.id] }) response = self.client.post('/annotation_tasks/' + str(onboarding_task.id), headers={'Authorization': self.admin_user_token}, data=request_body) # confirm that new task was created this time self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertEqual(task_result['user_ids'], [self.user.id, self.new_user.id]) self.assertNotEqual(task_result['id'], onboarding_task.id) # check that new annotation_task was created
def before_each(self, with_annotations=False): db_session_users.query(AnnotationTaskTermSamplingGroup).delete() db_session_users.query(TopicAnnotation).delete() db_session_users.query(AnnotationJob).delete() db_session_users.query(AnnotationTask).delete() db_session_users.query(AnnotationTaskTopicGroup).delete() t1 = AnnotationTask({ 'name': 'bar', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'}, }, 'type': 'topic_annotation' }) t2 = AnnotationTask({ 'name': 'foo', 'topics': { 'fintech': {'topic_id': 3, 'topic_table': 'topics'}, 'payments': {'topic_id': 4, 'topic_table': 'topics'}, }, "user_ids": [self.user.id, self.new_user.id, self.qa_user.id], "config": { "dummy-value": 1 } }) # n.b. add a third task that will be totally ignored t3 = AnnotationTask({ 'name': 'annotate any doc', 'topics': { 'banking': {'topic_id': 1, 'topic_table': 'topics'}, 'lending': {'topic_id': 2, 'topic_table': 'topics'}, }, 'type': AnnotationTask.ANNOTATE_ANY_DOCUMENT_TYPE }) self.tasks = [t1, t2] self.jobs = [] for i in xrange(1, 5): j1 = AnnotationJob({ "doc_id": 1, "priority": 1.0, "status": AnnotationJob.QUEUED_STATUS }) j1.annotation_task = t1 j2 = AnnotationJob({ "doc_id": 1, "priority": 1.0, "status": AnnotationJob.QUEUED_STATUS }) j2.annotation_task = t2 self.jobs.append(j1) self.jobs.append(j2) if with_annotations: ta1 = TopicAnnotation({"doc_id": 1, "topic_name": "test", "is_positive": True, "user_id": self.user.id}) ta1.annotation_job = j1 ta1.annotation_task = t1 ta2 = TopicAnnotation({"doc_id": 3, "topic_name": "test2", "is_positive": True, "user_id": self.user.id}) ta2.annotation_job = j2 ta2.annotation_task = t2 db_session_users.add_all([ta1, ta2]) db_session_users.add_all(self.tasks) db_session_users.add(t3) db_session_users.add_all(self.jobs) db_session_users.commit() for t in self.tasks: db_session_users.refresh(t)
def test_update_annotation_task_is_training_task(self): # TODO: this behavior creates new task even in case when illegal is_training_task update occurs; # need to either make this impossible in API side or enforce that this does not happen on frontend self.before_each() ## test update to is_training_task ## # check original annotation task value for is_training_task self.assertFalse(self.tasks[0].is_training_task) # make update when annotation_task_topic_group_id is NOT set (so is_training_task cannot be True) request_body = json.dumps({ 'is_training_task': True }) response = self.client.post('/annotation_tasks/' + str(self.tasks[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertFalse(task_result['is_training_task']) # since annotation_task_topic_group_id is NOT set self.assertNotEqual(task_result['id'], self.tasks[0].id) # check that new annotation_task was created db_session_users.refresh(self.tasks[0]) self.assertEqual(self.tasks[0].status, 'inactive') # check that old task is no longer active self.assertEqual(self.tasks[0].active_task_id, task_result['id']) # check that old task points to new task # update task to have an annotation_task_topic_group_id and do update again # make an AnnotationTaskTopicGroup that contains the second annotation task in self.tasks task_group = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'first task group', 'annotation_task_ids': [self.tasks[1].id], 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1, 'gold_annotator_user_ids': [1, 2] }) self.task_groups = [task_group] db_session_users.add_all(self.task_groups) db_session_users.commit() for group in self.task_groups: db_session_users.refresh(group) # make second task point to this annotation_task_topic_group self.tasks[1].annotation_task_topic_group_id = self.task_groups[0].id db_session_users.commit() # make update when annotation_task_topic_group_id IS set (so is_training_task can be True) request_body = json.dumps({ 'is_training_task': True }) response = self.client.post('/annotation_tasks/' + str(self.tasks[1].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertTrue(task_result['is_training_task']) # since annotation_task_topic_group_id IS set self.assertNotEqual(task_result['id'], self.tasks[1].id) # check that new annotation_task was created db_session_users.refresh(self.tasks[1]) self.assertEqual(self.tasks[1].status, 'inactive') # check that old task is no longer active self.assertEqual(self.tasks[1].active_task_id, task_result['id']) # check that old task points to new task
def test_update_annotation_task_is_training_task_include_gold_annotations_but_no_jobs(self): self.before_each() db_session_users.query(AnnotationJob).delete() # make an AnnotationTaskTopicGroup that contains the first annotation task in self.tasks task_group = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'first task group', 'annotation_task_ids': [self.tasks[0].id], 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1, 'gold_annotator_user_ids': [1, 2] }) self.task_groups = [task_group] db_session_users.add_all(self.task_groups) db_session_users.commit() for group in self.task_groups: db_session_users.refresh(group) ## test update to is_training_task ## # check original annotation task value for is_training_task self.assertFalse(self.tasks[0].is_training_task) # make update when annotation_task_topic_group_id is NOT set (so is_training_task cannot be True) request_body = json.dumps({ 'is_training_task': True }) response = self.client.post('/annotation_tasks/' + str(self.tasks[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response - update should NOT happen because task not in an annotation_task_topic_group self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertIsNone(task_result['annotation_task_topic_group_id']) self.assertFalse(task_result['is_training_task']) # because task does not have annotation_task_topic_group_id self.assertEqual(task_result['id'], self.tasks[0].id) # check that no new annotation_task was created # make update when annotation_task_topic_group_id is IS set (update should now go through) # NOTE: the annotation_task_topic_group_id here does not link to an actual annotation_task_topic_group self.tasks[0].annotation_task_topic_group_id = self.task_groups[0].id db_session_users.commit() request_body = json.dumps({ 'is_training_task': True }) response = self.client.post('/annotation_tasks/' + str(self.tasks[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response - update should NOT happen because task not in an annotation_task_topic_group self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertEqual(task_result['annotation_task_topic_group_id'], self.task_groups[0].id) self.assertTrue(task_result['is_training_task']) # because task DOES have annotation_task_topic_group_id self.assertEqual(task_result['id'], self.tasks[0].id) # check that no new annotation_task was created ## test update to include_gold_annotations ## # check original annotation task value for include_gold_annotations self.assertTrue(self.tasks[0].include_gold_annotations) # make update request_body = json.dumps({ 'include_gold_annotations': False }) response = self.client.post('/annotation_tasks/' + str(self.tasks[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertIn('annotation_task', response.json) self.assertIsInstance(response.json['annotation_task'], dict) task_result = response.json['annotation_task'] self.assertFalse(task_result['include_gold_annotations']) self.assertEqual(task_result['id'], self.tasks[0].id) # check that no new annotation_task was created
def before_each(self): # must delete tables in correct order to avoid key constraint error # see necessary ordering in test_annotation_tasks.py and test_annotation_jobs.py db_session_users.query(AggregatedAnnotations).delete() db_session_users.query(AnnotationTaskTermSamplingGroup).delete() db_session_users.query(TopicAnnotation).delete() db_session_users.query(AnnotationJob).delete() db_session_users.query(AnnotationTask).delete() db_session_users.query(AnnotationTaskTopicGroup).delete() self.valid_doc_ids = [3, 8, 12, 13, 14] # valid doc_ids in ES index as of Nov 2017 # (update if any of these become invalid) # more ids: [18, 21, 22, 160, 161] ########################################### # create AnnotationTask objects (3 total) ########################################### t1 = AnnotationTask({ 'name': 'bar', 'topics': { 'BSA/AML': { 'topic_id': 2, 'topic_table': 'topics' }, 'Lending': { 'topic_id': 1, 'topic_table': 'topics' }, } }) t2 = AnnotationTask({ 'name': 'foo', 'topics': { 'BSA/AML': { 'topic_id': 2, 'topic_table': 'topics' }, 'Lending': { 'topic_id': 1, 'topic_table': 'topics' }, } }) t3 = AnnotationTask({ 'name': 'fubar', 'topics': { 'BSA/AML': { 'topic_id': 2, 'topic_table': 'topics' }, 'Lending': { 'topic_id': 1, 'topic_table': 'topics' }, } }) self.tasks = [t1, t2, t3] db_session_users.add_all(self.tasks) db_session_users.commit() # do refresh to get current primary key ids of tasks for t in self.tasks: db_session_users.refresh(t) ########################################## # create AnnotationJob objects (10 total) ########################################## self.jobs = [] for i in xrange(10): j1 = AnnotationJob({ "doc_id": self.valid_doc_ids[i % len(self.valid_doc_ids)], # round robin "priority": 1.0, "arbitrary_tags": ["tag_{}".format(str(i))] if i <= 2 else [], "status": AnnotationJob.QUEUED_STATUS }) j1.annotation_task = self.tasks[i % len(self.tasks)] # round robin self.jobs.append(j1) db_session_users.add_all(self.jobs) db_session_users.commit() # refresh to get current primary key ids of jobs for j in self.jobs: db_session_users.refresh(j) ############################################### # create AnnotationTaskGroups object (1 total) ############################################### task_group = AnnotationTaskTopicGroup({ 'name': 'task_group_1', 'description': 'first task group', 'annotation_task_ids': [self.tasks[0].id, self.tasks[1].id], 'arbitrary_tags': ["tag1", "tag2"], 'topic_id': 1 }) self.task_groups = [task_group] db_session_users.add_all(self.task_groups) db_session_users.commit() for tg in self.task_groups: db_session_users.refresh(tg) ################################### # create User objects (3 total) ################################### # for testing retrieval of user names as judges # only creates users if they don't already exist # N.B. in future use a factory for creating users for testing if db_session_users.query(User)\ .filter_by(email='*****@*****.**')\ .first() is not None: self.users = [ db_session_users.query(User)\ .filter_by(email='*****@*****.**')\ .first(), db_session_users.query(User) \ .filter_by(email='*****@*****.**') \ .first(), db_session_users.query(User)\ .filter_by(email='*****@*****.**')\ .first() ] else: self.users = [] user_1 = User({ 'first_name': 'Alice', 'last_name': 'Smith', 'email': '*****@*****.**', 'password': '******' }) self.users.append(user_1) user_2 = User({ 'first_name': 'Bob', 'last_name': 'Doe', 'email': '*****@*****.**', 'password': '******' }) self.users.append(user_2) user_3 = User({ 'first_name': 'Cindy', 'last_name': 'Zeta', 'email': '*****@*****.**', 'password': '******' }) self.users.append(user_3) db_session_users.add_all(self.users) db_session_users.commit() for us in self.users: db_session_users.refresh(us) ######################################################################## # create TopicAnnotation objects (10 total, one for each AnnotationJob) ######################################################################## # here there is a 1-to-1 correspondence between TopicAnnotations and AnnotationJobs self.topic_annotations = [] # connect TopicAnnotation values with AnnotationJob values for i, job in enumerate(self.jobs): ta = TopicAnnotation({ "doc_id": job.doc_id, "is_positive": True if i % 2 == 0 else False, "user_id": self.users[i % 3].id, "topic_name": "Lending", "annotation_task_id": self.tasks[i % len(self.tasks)].id }) ta.annotation_job = job self.topic_annotations.append(ta) db_session_users.add_all(self.topic_annotations) db_session_users.commit() for ta in self.topic_annotations: db_session_users.refresh(ta) ################################################## # create AggregatedAnnotations objects (3 total) ################################################## # NB: AggregatedAnnotations object has foreign keys for annotation_task_group and topic_annotation # NB: this table is populated by a daemon job in actual database # all have same topic_id (1 for "Lending") self.aggregated_annotations = [] # self.valid_doc_ids[0] includes jobs 1 and 6 agg_annotation_1 = AggregatedAnnotations({ 'annotation_task_group_id': self.task_groups[0].id, 'doc_id': self.valid_doc_ids[0], 'topic_id': 1, 'is_gold_standard': True, 'gold_topic_annotation_id': self.topic_annotations[0].id, 'is_in_agreement': True, 'is_active_for_gold_annotation': True, 'gold_difficulty': "easy", 'arbitrary_tags': ["tag_1", "tag_2"], 'notes': "text_1" }) # self.valid_doc_ids[0] includes jobs 2 and 7 agg_annotation_2 = AggregatedAnnotations({ 'annotation_task_group_id': self.task_groups[0].id, 'doc_id': self.valid_doc_ids[1], 'topic_id': 1, 'is_gold_standard': True, 'gold_topic_annotation_id': self.topic_annotations[1].id, 'is_in_agreement': False, 'is_active_for_gold_annotation': True, 'gold_difficulty': "medium", 'arbitrary_tags': ["tag_1", "tag_3"], 'notes': "text_1" }) # self.valid_doc_ids[1] includes jobs 3 and 8 agg_annotation_3 = AggregatedAnnotations({ 'annotation_task_group_id': self.task_groups[0].id, 'doc_id': self.valid_doc_ids[2], 'topic_id': 1, 'is_gold_standard': True, 'gold_topic_annotation_id': self.topic_annotations[2].id, 'is_active_for_gold_annotation': True, 'gold_difficulty': "hard", 'arbitrary_tags': ["tag_1", "tag_2"], 'notes': "text_1" }) self.aggregated_annotations = [ agg_annotation_1, agg_annotation_2, agg_annotation_3 ] db_session_users.add_all(self.aggregated_annotations) db_session_users.commit() # do refresh to get current primary key ids of aggregated annotations for agt in self.aggregated_annotations: db_session_users.refresh(agt)
def test_update_research_mode_expanded_view(self): self.before_each() # check initial topic annotation initial_topic_ant = db_session_users.query(TopicAnnotation)\ .filter_by(id=self.topic_annotations[0].id)\ .first() self.assertTrue(initial_topic_ant.is_positive) self.assertIsNone(initial_topic_ant.admin_notes) self.assertEqual(initial_topic_ant.user_id, self.users[0].id) self.assertEqual(initial_topic_ant.details, {}) # make update to topic_annotation request_body = json.dumps({ 'topic_annotation_id': self.topic_annotations[0].id, 'topic_annotation_updates': { "is_positive": False, "user_id": self.users[1].id, "admin_notes": 'this tation was SUPER hard bruh' } }) response = self.client.post( '/aggregated_annotations/research/' + str(self.aggregated_annotations[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertTrue(response.json['success']) # check that update occurred updated_topic_ant = db_session_users.query(TopicAnnotation)\ .filter_by(id=self.topic_annotations[0].id)\ .first() updated_ant_job = db_session_users.query(AnnotationJob)\ .filter_by(id=self.jobs[0].id)\ .first() self.assertFalse(updated_topic_ant.is_positive) self.assertEqual(updated_topic_ant.admin_notes, 'this tation was SUPER hard bruh') self.assertEqual(updated_topic_ant.user_id, self.users[1].id) self.assertEqual(updated_ant_job.user_id, self.users[1].id) self.assertNotEqual(updated_topic_ant.details, {}) self.assertEqual( updated_topic_ant.details['previous_annotators'][0]['is_positive'], True) self.assertEqual( updated_topic_ant.details['previous_annotators'][0]['user_id'], self.users[0].id) # check initial annotation job (gold standard annotation_job) and initial agg_annotation containing this job initial_ant_job = db_session_users.query(AnnotationJob)\ .filter_by(id=self.jobs[0].id)\ .first() self.assertEqual(initial_ant_job.arbitrary_tags, ['tag_0']) self.assertIsNone(initial_ant_job.user_difficulty) self.assertEqual(initial_ant_job.annotation_task_id, self.tasks[0].id) self.assertEqual(self.aggregated_annotations[0].arbitrary_tags, ["tag_1", "tag_2"]) self.assertEqual(self.aggregated_annotations[0].gold_difficulty, 'easy') # make update to annotation job for arbitrary_tags request_body = json.dumps({ 'topic_annotation_id': self.topic_annotations[0].id, 'annotation_job_updates': { 'arbitrary_tags': ['tag2', 'NOT ALLOWED TAG'] } }) response = self.client.post( '/aggregated_annotations/research/' + str(self.aggregated_annotations[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertTrue(response.json['success']) # check that update occurred updated_ant_job = db_session_users.query(AnnotationJob)\ .filter_by(id=self.jobs[0].id)\ .first() self.assertEqual( updated_ant_job.arbitrary_tags, ['tag2']) # since other tag not in AnnotationTaskTopicGroup self.assertIsNone(updated_ant_job.user_difficulty) # check that aggregated_annotation[0] had arbitrary_tags updated as well db_session_users.refresh(self.aggregated_annotations[0]) self.assertEqual(self.aggregated_annotations[0].arbitrary_tags, ['tag2']) # make second update to same annotation job for user_difficulty request_body = json.dumps({ 'topic_annotation_id': self.topic_annotations[0].id, 'annotation_job_updates': { 'user_difficulty': 'medium' } }) response = self.client.post( '/aggregated_annotations/research/' + str(self.aggregated_annotations[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertTrue(response.json['success']) # check that update occurred updated_ant_job = db_session_users.query(AnnotationJob)\ .filter_by(id=self.jobs[0].id)\ .first() self.assertEqual(updated_ant_job.arbitrary_tags, ['tag2']) # same as before self.assertEqual(updated_ant_job.user_difficulty, 'medium') # update that occurred here # check that aggregated_annotation[0] had gold_difficulty updated as well db_session_users.refresh(self.aggregated_annotations[0]) self.assertEqual(self.aggregated_annotations[0].gold_difficulty, 'medium') # make task that contains this job point to new task as its active task (which is in same task group) self.tasks[0].active_task_id = self.tasks[1].id db_session_users.add(self.tasks[0]) db_session_users.commit() db_session_users.refresh(self.tasks[0]) # update task group to no longer contain task that contains initial_ant_job (but does contain its active task) self.task_groups[0].annotation_task_ids = [self.tasks[1].id] db_session_users.add(self.task_groups[0]) db_session_users.commit() db_session_users.refresh(self.task_groups[0]) # do update to tags for job whose parent task no longer in task group (but whose active task is) request_body = json.dumps({ 'topic_annotation_id': self.topic_annotations[0].id, 'annotation_job_updates': { 'arbitrary_tags': ['tag1', 'NOT ALLOWED TAG'] } }) response = self.client.post( '/aggregated_annotations/research/' + str(self.aggregated_annotations[0].id), headers={'Authorization': self.admin_user_token}, data=request_body) # check response self.assert200(response) self.assertTrue(response.json['success']) # check that update occurred updated_ant_job = db_session_users.query(AnnotationJob)\ .filter_by(id=self.jobs[0].id)\ .first() self.assertEqual( updated_ant_job.arbitrary_tags, ['tag1']) # since other tag not in AnnotationTaskTopicGroup
def before_each(self): db_session_users.query(AnnotationJob).delete() db_session_users.query(AnnotationTask).delete() db_session_users.query(UserFlaggedDocument).delete() t1 = AnnotationTask({ 'name': 'foo', 'type': 'contributor', }) self.tasks = [t1] self.jobs = [] self.docs = [] for i in xrange(1, 6): j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.COMPLETE_STATUS, "user_id": self.user.id }) j1.annotation_task = t1 self.jobs.append(j1) if i % 2 == 0: flagged_doc = UserFlaggedDocument({ 'user_id': self.user.id, 'doc_id': i, 'issue_severity': UserFlaggedDocument.REVIEW_SEVERITY, 'issue_type': UserFlaggedDocument.CONTRIBUTOR_ISSUE_TYPE, 'multiple_field': {"field1": "test notes"}, }) self.docs.append(flagged_doc) j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.COMPLETE_STATUS, "user_id": self.contributor_user.id }) j1.annotation_task = t1 self.jobs.append(j1) flagged_doc = UserFlaggedDocument({ 'user_id': self.contributor_user.id, 'doc_id': i, 'issue_severity': UserFlaggedDocument.REVIEW_SEVERITY, 'issue_type': UserFlaggedDocument.CONTRIBUTOR_ISSUE_TYPE, 'multiple_field': {"field1": "test notes"}, 'status': UserFlaggedDocument.FIXED_STATUS if i % 2 == 0 else UserFlaggedDocument.PROCESSED_STATUS }) self.docs.append(flagged_doc) j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.COMPLETE_STATUS, "user_id": self.qa_user.id }) j1.annotation_task = t1 self.jobs.append(j1) j1 = AnnotationJob({ "doc_id": i, "priority": 1.0 if i % 2 == 0 else 0.5, "status": AnnotationJob.QUEUED_STATUS, "user_id": self.internal_user.id }) j1.annotation_task = t1 self.jobs.append(j1) db_session_users.add_all(self.tasks) db_session_users.add_all(self.jobs) db_session_users.add_all(self.docs) db_session_users.commit() for t in self.tasks: db_session_users.refresh(t) for j in self.jobs: db_session_users.refresh(j) for d in self.docs: db_session_users.refresh(d)