def test_csv_export(now, client, answer_type, answer): im = ImageFactory() rs = ReaderStudyFactory() rs.images.add(im) rs.save() editor = UserFactory() rs.add_editor(editor) reader = UserFactory() rs.add_reader(reader) q = QuestionFactory( question_text="foo", reader_study=rs, answer_type=answer_type ) a = AnswerFactory(question=q, answer=answer) a.images.add(im) a.save() response = get_view_for_user( viewname="api:reader-study-export-answers", reverse_kwargs={"pk": rs.pk}, user=editor, client=client, method=client.get, content_type="application/json", ) headers = str(response.serialize_headers()) content = str(response.content) assert response.status_code == 200 assert "Content-Type: text/csv" in headers assert ( f'filename="{rs.slug}-answers-2020-01-01T00:00:00+00:00.csv"' in headers ) assert a.question.question_text in content assert a.question.get_answer_type_display() in content assert str(a.question.required) in content assert a.question.get_image_port_display() in content if isinstance(answer, dict): for key in answer: assert key in content else: assert re.sub(r"[\n\r\t]", " ", str(a.answer)) in content assert im.name in content assert a.creator.username in content response = get_view_for_user( viewname="api:reader-study-export-answers", reverse_kwargs={"pk": rs.pk}, user=reader, client=client, method=client.get, content_type="application/json", ) assert response.status_code == 404
def test_ground_truth_is_excluded(client): im = ImageFactory() rs = ReaderStudyFactory() rs.images.add(im) editor = UserFactory() rs.add_editor(editor) rs.add_reader(editor) q = QuestionFactory(reader_study=rs, answer_type=Question.AnswerType.BOOL) a1 = AnswerFactory(question=q, creator=editor, answer=True, is_ground_truth=True) a1.images.add(im) a2 = AnswerFactory(question=q, creator=editor, answer=True, is_ground_truth=False) a2.images.add(im) response = get_view_for_user( viewname="api:reader-studies-answer-mine", user=editor, client=client, method=client.get, content_type="application/json", ) results = response.json()["results"] assert len(results) == 1 assert results[0]["pk"] == str(a2.pk)
def test_leaderboard(reader_study_with_gt, settings): # noqa: C901 settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = reader_study_with_gt r1, r2 = rs.readers_group.user_set.all() e = rs.editors_group.user_set.first() for question in rs.questions.all(): for im in rs.images.all(): ans = AnswerFactory(question=question, creator=r1, answer=True) ans.images.add(im) leaderboard = rs.leaderboard assert Answer.objects.filter(is_ground_truth=False).count() == 6 assert leaderboard["question_count"] == 6.0 scores = leaderboard["grouped_scores"] assert len(scores) == 1 user_score = scores[0] assert user_score["creator__username"] == r1.username assert user_score["score__sum"] == 6.0 assert user_score["score__avg"] == 1.0 for i, question in enumerate(rs.questions.all()): for j, im in enumerate(rs.images.all()): ans = AnswerFactory(question=question, creator=r2, answer=(i + j) % 2 == 0) ans.images.add(im) del rs.scores_by_user del rs.leaderboard leaderboard = rs.leaderboard assert Answer.objects.filter(is_ground_truth=False).count() == 12 assert leaderboard["question_count"] == 6.0 scores = leaderboard["grouped_scores"] assert len(scores) == 2 for user_score in scores: if user_score["creator__username"] != r2.username: continue assert user_score["score__sum"] == 3.0 assert user_score["score__avg"] == 0.5 for question in rs.questions.all(): for im in rs.images.all(): ans = AnswerFactory(question=question, creator=e, answer=True) ans.images.add(im) del rs.scores_by_user del rs.leaderboard leaderboard = rs.leaderboard assert Answer.objects.filter(is_ground_truth=False).count() == 18 assert leaderboard["question_count"] == 6.0 scores = leaderboard["grouped_scores"] assert len(scores) == 3 for user_score in scores: if user_score["creator__username"] != e.username: continue assert user_score["score__sum"] == 6.0 assert user_score["score__avg"] == 1.0
def test_mine(client): im1, im2 = ImageFactory(), ImageFactory() rs1, rs2 = ReaderStudyFactory(), ReaderStudyFactory() rs1.images.add(im1) rs2.images.add(im2) reader = UserFactory() rs1.add_reader(reader) rs2.add_reader(reader) q1 = QuestionFactory( reader_study=rs1, answer_type=Question.ANSWER_TYPE_BOOL ) q2 = QuestionFactory( reader_study=rs2, answer_type=Question.ANSWER_TYPE_BOOL ) a1 = AnswerFactory(question=q1, creator=reader, answer=True) a1.images.add(im1) a2 = AnswerFactory(question=q2, creator=reader, answer=True) a2.images.add(im2) response = get_view_for_user( viewname="api:reader-studies-answer-mine", user=reader, client=client, method=client.get, content_type="application/json", ) response = response.json() assert response["count"] == 2 response = get_view_for_user( viewname="api:reader-studies-answer-mine", user=reader, client=client, method=client.get, data={"question__reader_study": rs1.pk}, content_type="application/json", ) response = response.json() assert response["count"] == 1 assert response["results"][0]["pk"] == str(a1.pk) response = get_view_for_user( viewname="api:reader-studies-answer-mine", user=reader, client=client, method=client.get, data={"question__reader_study": rs2.pk}, content_type="application/json", ) response = response.json() assert response["count"] == 1 assert response["results"][0]["pk"] == str(a2.pk)
def test_api_rs_answer_mine_list_permissions(client): """ For the "mine" endpoint the list should be filtered by the users own answers """ rs_set = TwoReaderStudies() q1, q2 = ( QuestionFactory(reader_study=rs_set.rs1), QuestionFactory(reader_study=rs_set.rs2), ) reader11 = UserFactory() rs_set.rs1.add_reader(reader11) a1, a11, a2 = ( AnswerFactory(question=q1, creator=rs_set.reader1, answer=""), AnswerFactory(question=q1, creator=reader11, answer=""), AnswerFactory(question=q2, creator=rs_set.reader2, answer=""), ) tests = ( (None, 401, []), (rs_set.creator, 200, []), (rs_set.editor1, 200, []), (rs_set.reader1, 200, [a1.pk]), (reader11, 200, [a11.pk]), (rs_set.editor2, 200, []), (rs_set.reader2, 200, [a2.pk]), (rs_set.u, 200, []), ) for test in tests: response = get_view_for_user( viewname="api:reader-studies-answer-mine", client=client, user=test[0], content_type="application/json", ) assert response.status_code == test[1] if test[1] != 401: # We provided auth details and get a response assert response.json()["count"] == len(test[2]) pks = [obj["pk"] for obj in response.json()["results"]] for pk in test[2]: assert str(pk) in pks
def test_assign_answer_image(client, settings): settings.task_eager_propagates = (True,) settings.task_always_eager = (True,) rs = ReaderStudyFactory() im = ImageFactory() editor, reader = UserFactory(), UserFactory() rs.images.add(im) rs.add_editor(editor) rs.add_reader(reader) question = QuestionFactory( reader_study=rs, answer_type=Question.ANSWER_TYPE_POLYGON_IMAGE ) us = RawImageUploadSessionFactory(creator=reader) answer = AnswerFactory( creator=reader, question=question, answer={"upload_session_pk": str(us.pk)}, ) f = StagedFileFactory( file__from_path=Path(__file__).parent.parent / "cases_tests" / "resources" / "image10x10x10.mha" ) RawImageFileFactory(upload_session=us, staged_file_id=f.file_id) response = get_view_for_user( viewname="api:upload-session-process-images", reverse_kwargs={"pk": us.pk}, user=reader, client=client, method=client.patch, data={"answer": str(answer.pk)}, content_type="application/json", ) assert response.status_code == 200 answer.refresh_from_db() image = us.image_set.first() assert answer.answer_image == image assert reader.has_perm("view_image", image) assert editor.has_perm("view_image", image)
def test_assert_modification_allowed(): rs = ReaderStudyFactory(use_display_sets=False) ci = ComponentInterfaceFactory( kind=InterfaceKind.InterfaceKindChoices.BOOL) civ = ComponentInterfaceValueFactory(interface=ci, value=True) ds = DisplaySetFactory(reader_study=rs) ds.values.add(civ) del ds.is_editable civ2 = ComponentInterfaceValueFactory(interface=ci, value=True) ds.values.remove(civ) ds.values.add(civ2) assert ds.values.count() == 1 assert ds.values.first() == civ2 q = QuestionFactory(reader_study=rs) AnswerFactory(question=q, display_set=ds) del ds.is_editable with pytest.raises(ValidationError): with transaction.atomic(): ds.values.remove(civ2) assert ds.values.count() == 1 assert ds.values.first() == civ2
def test_api_rs_answer_detail_permissions(client): rs_set = TwoReaderStudies() q1 = QuestionFactory(reader_study=rs_set.rs1) reader11 = UserFactory() rs_set.rs1.add_reader(reader11) a1 = AnswerFactory(question=q1, creator=rs_set.reader1, answer="") tests = ( (None, 401), (rs_set.creator, 404), (rs_set.editor1, 200), (rs_set.reader1, 200), (reader11, 404), (rs_set.editor2, 404), (rs_set.reader2, 404), (rs_set.u, 404), ) for test in tests: response = get_view_for_user( viewname="api:reader-studies-answer-detail", reverse_kwargs={"pk": a1.pk}, client=client, user=test[0], content_type="application/json", ) assert response.status_code == test[1]
def test_api_list_is_filtered(client): rs1, rs2 = ReaderStudyFactory(), ReaderStudyFactory() rs1_editor = UserFactory() rs1.add_editor(rs1_editor) q1, q2 = ( QuestionFactory(reader_study=rs1), QuestionFactory(reader_study=rs2), ) a1, _ = ( AnswerFactory(question=q1, answer=True), AnswerFactory(question=q2, answer=False), ) response = get_view_for_user(viewname="api:reader-study-list", user=rs1_editor, client=client) assert response.status_code == 200 assert response.json()["count"] == 1 response = get_view_for_user( viewname="api:reader-study-detail", reverse_kwargs={"pk": rs1.pk}, user=rs1_editor, client=client, ) assert response.status_code == 200 assert len(response.json()["questions"]) == 1 response = get_view_for_user( viewname="api:reader-studies-question-list", user=rs1_editor, client=client, ) assert response.status_code == 200 assert response.json()["count"] == 1 assert response.json()["results"][0]["pk"] == str(q1.pk) response = get_view_for_user( viewname="api:reader-studies-answer-list", user=rs1_editor, client=client, ) assert response.status_code == 200 assert response.json()["count"] == 1 assert response.json()["results"][0]["pk"] == str(a1.pk)
def test_answer_remove(client): rs = ReaderStudyFactory() r1, r2, editor = UserFactory(), UserFactory(), UserFactory() rs.add_reader(r1) rs.add_reader(r2) rs.add_editor(editor) q = QuestionFactory( reader_study=rs, question_text="q1", answer_type=Question.ANSWER_TYPE_BOOL, ) im = ImageFactory() a1 = AnswerFactory(creator=r1, question=q, answer=True) a1.images.set([im]) a2 = AnswerFactory(creator=r2, question=q, answer=True) a2.images.set([im]) assert Answer.objects.count() == 2 response = get_view_for_user( viewname="reader-studies:answers-remove", client=client, method=client.post, reverse_kwargs={"slug": rs.slug}, data={"user": r1.id}, follow=True, user=r1, ) assert response.status_code == 403 response = get_view_for_user( viewname="reader-studies:answers-remove", client=client, method=client.post, reverse_kwargs={"slug": rs.slug}, data={"user": r1.id}, follow=True, user=editor, ) assert response.status_code == 200 assert Answer.objects.count() == 1 assert Answer.objects.filter(creator=r1).count() == 0 assert Answer.objects.filter(creator=r2).count() == 1
def test_question_delete_disabled_for_questions_with_answers(client): rs = ReaderStudyFactory(use_display_sets=False) r1, editor = UserFactory(), UserFactory() rs.add_reader(r1) rs.add_editor(editor) q = QuestionFactory( reader_study=rs, question_text="q1", answer_type=Question.AnswerType.BOOL, ) AnswerFactory(creator=r1, question=q, answer=True) assert Answer.objects.count() == 1 assert Question.objects.count() == 1 assert not q.is_fully_editable response = get_view_for_user( viewname="reader-studies:question-delete", client=client, method=client.post, reverse_kwargs={ "slug": rs.slug, "pk": q.pk }, user=editor, ) assert response.status_code == 403 assert Question.objects.count() == 1 # if answer is deleted, deletion of the question is possible again get_view_for_user( viewname="reader-studies:answers-remove", client=client, method=client.post, reverse_kwargs={"slug": rs.slug}, data={"user": r1.id}, follow=True, user=editor, ) assert Answer.objects.count() == 0 response = get_view_for_user( viewname="reader-studies:question-delete", client=client, method=client.post, reverse_kwargs={ "slug": rs.slug, "pk": q.pk }, user=editor, ) assert response.status_code == 302 assert Question.objects.count() == 0
def test_read_only_fields(): rs = ReaderStudyFactory(use_display_sets=False) q = QuestionFactory(reader_study=rs) assert q.is_fully_editable is True assert q.read_only_fields == [] AnswerFactory(question=q, answer="true") assert q.is_fully_editable is False assert q.read_only_fields == [ "question_text", "answer_type", "image_port", "required", ]
def test_validate_answer(): u = UserFactory() im1, im2, im3 = ImageFactory(), ImageFactory(), ImageFactory() rs = ReaderStudyFactory(hanging_list=[ { "main": im1.name, "main-overlay": im3.name }, { "main": im2.name, "main-overlay": im3.name }, ]) rs.images.set([im1, im2, im3]) rs.add_reader(u) q = QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.BOOL, question_text="q1", ) answer = AnswerFactory( creator=u, question=q, answer=True, ) answer.images.set([im1, im3]) with pytest.raises(ValidationError) as e: Answer.validate( creator=u, question=q, answer=True, images=[im1, im3], ) assert ( e.value.message == f"User {u} has already answered this question for this set of images." ) assert (Answer.validate( creator=u, question=q, answer=True, images=[im2, im3], ) is None)
def test_score_for_user(reader_study_with_gt, settings): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = reader_study_with_gt r1 = rs.readers_group.user_set.first() for i, question in enumerate(rs.questions.all()): for j, im in enumerate(rs.images.all()): ans = AnswerFactory(question=question, creator=r1, answer=(i + j) % 2 == 0) ans.images.add(im) score = rs.score_for_user(r1) assert Answer.objects.filter(is_ground_truth=False).count() == 6 assert score["score__sum"] == 3.0 assert score["score__avg"] == 0.5
def reader_study_with_mc_gt(reader_study_with_gt): rs = reader_study_with_gt q_choice = QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.CHOICE, question_text="C", ) q_multiple_choice = QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.MULTIPLE_CHOICE, question_text="MC", ) c_options = [ CategoricalOptionFactory(question=q_choice, title="fee"), CategoricalOptionFactory(question=q_choice, title="foh"), CategoricalOptionFactory(question=q_choice, title="fum"), ] mc_options = [ CategoricalOptionFactory(question=q_multiple_choice, title="fee"), CategoricalOptionFactory(question=q_multiple_choice, title="foh"), CategoricalOptionFactory(question=q_multiple_choice, title="fum"), ] editor = rs.editors_group.user_set.first() images = reader_study_with_gt.images.all() for question, answer in [ (q_choice, c_options[0].id), (q_multiple_choice, [mc_options[0].id, mc_options[1].id]), ]: ans = AnswerFactory( question=question, creator=editor, answer=answer, is_ground_truth=True, ) for im in images: ans.images.add(im) return rs
def test_upload_session_owned_by_answer_creator(client, settings, answer_type): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = ReaderStudyFactory() im = ImageFactory() editor, reader = UserFactory(), UserFactory() rs.images.add(im) rs.add_editor(editor) rs.add_reader(reader) question = QuestionFactory(reader_study=rs, answer_type=answer_type) us1 = RawImageUploadSessionFactory(creator=reader) us2 = RawImageUploadSessionFactory(creator=editor) answer1 = AnswerFactory( creator=reader, question=question, answer={"upload_session_pk": str(us1.pk)}, ) f = StagedFileFactory(file__from_path=Path(__file__).parent.parent / "cases_tests" / "resources" / "image10x10x10.mha") RawImageFileFactory(upload_session=us1, staged_file_id=f.file_id) response = get_view_for_user( viewname="api:upload-session-process-images", reverse_kwargs={"pk": us2.pk}, user=editor, client=client, method=client.patch, data={"answer": str(answer1.pk)}, content_type="application/json", ) assert response.status_code == 400 assert (b"User does not have permission to add an image to this answer" in response.rendered_content)
def reader_study_with_gt(): rs = ReaderStudyFactory() im1, im2 = ImageFactory(name="im1"), ImageFactory(name="im2") q1, q2, q3 = [ QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.BOOL, question_text="q1", ), QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.BOOL, question_text="q2", ), QuestionFactory( reader_study=rs, answer_type=Question.AnswerType.BOOL, question_text="q3", ), ] r1, r2, editor = UserFactory(), UserFactory(), UserFactory() rs.add_reader(r1) rs.add_reader(r2) rs.add_editor(editor) rs.images.set([im1, im2]) rs.hanging_list = [{"main": im1.name}, {"main": im2.name}] rs.save() for question in [q1, q2, q3]: for im in [im1, im2]: ans = AnswerFactory( question=question, creator=editor, answer=True, is_ground_truth=True, ) ans.images.add(im) return rs
def test_question_accepts_image_type_answers(client, settings): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = ReaderStudyFactory() im = ImageFactory() reader = UserFactory() rs.images.add(im) rs.add_reader(reader) question = QuestionFactory(reader_study=rs, answer_type=Question.AnswerType.BOOL) us = RawImageUploadSessionFactory(creator=reader) answer = AnswerFactory( creator=reader, question=question, answer={"upload_session_pk": str(us.pk)}, ) f = StagedFileFactory(file__from_path=Path(__file__).parent.parent / "cases_tests" / "resources" / "image10x10x10.mha") RawImageFileFactory(upload_session=us, staged_file_id=f.file_id) response = get_view_for_user( viewname="api:upload-session-process-images", reverse_kwargs={"pk": us.pk}, user=reader, client=client, method=client.patch, data={"answer": str(answer.pk)}, content_type="application/json", ) assert response.status_code == 400 assert (b"This question does not accept image type answers" in response.rendered_content)
def test_progress_for_user(settings): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = ReaderStudyFactory() im1, im2 = ImageFactory(name="im1"), ImageFactory(name="im2") q1, q2, q3 = [ QuestionFactory(reader_study=rs), QuestionFactory(reader_study=rs), QuestionFactory(reader_study=rs), ] reader = UserFactory() rs.add_reader(reader) question_perc = 100 / 6 assert rs.get_progress_for_user(reader) == { "diff": 0.0, "hangings": 0.0, "questions": 0.0, } rs.images.set([im1, im2]) rs.hanging_list = [{"main": im1.name}, {"main": im2.name}] rs.save() progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == 0 a11 = AnswerFactory(question=q1, answer="foo", creator=reader) a11.images.add(im1) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == pytest.approx(question_perc) a21 = AnswerFactory(question=q1, answer="foo", creator=reader) a21.images.add(im2) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == pytest.approx(question_perc * 2) a12 = AnswerFactory(question=q2, answer="foo", creator=reader) a12.images.add(im1) a13 = AnswerFactory(question=q3, answer="foo", creator=reader) a13.images.add(im1) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 50 assert progress["questions"] == pytest.approx(question_perc * 4) editor = UserFactory() rs.add_reader(editor) rs.add_editor(editor) for q in [q1, q2, q3]: for im in [im1, im2]: a = AnswerFactory(question=q, answer="foo", creator=editor, is_ground_truth=True) a.images.add(im) progress = rs.get_progress_for_user(editor) assert progress["hangings"] == 0 assert progress["questions"] == 0 for q in [q1, q2, q3]: for im in [im1, im2]: a = AnswerFactory(question=q, answer="foo", creator=editor, is_ground_truth=False) a.images.add(im) progress = rs.get_progress_for_user(editor) assert progress["hangings"] == 100.0 assert progress["questions"] == 100.0
def test_csv_export(client, answer_type, answer): im = ImageFactory() rs = ReaderStudyFactory() rs.images.add(im) rs.save() editor = UserFactory() rs.add_editor(editor) reader = UserFactory() rs.add_reader(reader) q = QuestionFactory(question_text="foo", reader_study=rs, answer_type=answer_type) a = AnswerFactory(question=q, answer=answer) a.images.add(im) a.save() response = get_view_for_user( viewname="api:reader-studies-answer-list", params={"question__reader_study": str(rs.pk)}, user=editor, client=client, method=client.get, HTTP_ACCEPT="text/csv", ) headers = str(response.serialize_headers()) content = str(response.content) assert response.status_code == 200 assert "Content-Type: text/csv" in headers if isinstance(answer, dict): for key in answer: assert key in content else: assert re.sub(r"\n", r"\\n", str(a.answer)) in content assert a.creator.username in content response = get_view_for_user( viewname="api:reader-studies-question-list", params={"reader_study": str(rs.pk)}, user=editor, client=client, method=client.get, HTTP_ACCEPT="text/csv", ) headers = str(response.serialize_headers()) content = str(response.content) assert response.status_code == 200 assert "Content-Type: text/csv" in headers assert a.question.question_text in content assert a.question.get_answer_type_display() in content assert str(a.question.required) in content assert a.question.get_image_port_display() in content response = get_view_for_user( viewname="api:image-list", params={"readerstudies": str(rs.pk)}, user=editor, client=client, method=client.get, HTTP_ACCEPT="text/csv", ) headers = str(response.serialize_headers()) content = str(response.content) assert response.status_code == 200 assert "Content-Type: text/csv" in headers assert im.name in content
def test_progress_for_user(settings, use_display_sets): # noqa: C901 settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = ReaderStudyFactory(use_display_sets=use_display_sets) im1, im2 = ImageFactory(name="im1"), ImageFactory(name="im2") q1, q2, q3 = [ QuestionFactory(reader_study=rs), QuestionFactory(reader_study=rs), QuestionFactory(reader_study=rs), ] reader = UserFactory() rs.add_reader(reader) question_perc = 100 / 6 assert rs.get_progress_for_user(reader) == { "diff": 0.0, "hangings": 0.0, "questions": 0.0, } if use_display_sets: ci = ComponentInterface.objects.get(slug="generic-medical-image") civ1 = ComponentInterfaceValueFactory(image=im1, interface=ci) civ2 = ComponentInterfaceValueFactory(image=im2, interface=ci) ds1, ds2 = DisplaySetFactory(reader_study=rs), DisplaySetFactory( reader_study=rs) ds1.values.add(civ1) ds2.values.add(civ2) else: rs.images.set([im1, im2]) rs.hanging_list = [{"main": im1.name}, {"main": im2.name}] rs.save() progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == 0 a11 = AnswerFactory(question=q1, answer="foo", creator=reader) if use_display_sets: a11.display_set = ds1 a11.save() else: a11.images.add(im1) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == pytest.approx(question_perc) a21 = AnswerFactory(question=q1, answer="foo", creator=reader) if use_display_sets: a21.display_set = ds2 a21.save() else: a21.images.add(im2) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 0 assert progress["questions"] == pytest.approx(question_perc * 2) a12 = AnswerFactory(question=q2, answer="foo", creator=reader) a13 = AnswerFactory(question=q3, answer="foo", creator=reader) if use_display_sets: a12.display_set = ds1 a12.save() a13.display_set = ds1 a13.save() else: a12.images.add(im1) a13.images.add(im1) progress = rs.get_progress_for_user(reader) assert progress["hangings"] == 50 assert progress["questions"] == pytest.approx(question_perc * 4) editor = UserFactory() rs.add_reader(editor) rs.add_editor(editor) for q in [q1, q2, q3]: if use_display_sets: for ds in [ds1, ds2]: a = AnswerFactory( question=q, answer="foo", creator=editor, is_ground_truth=True, display_set=ds, ) else: for im in [im1, im2]: a = AnswerFactory( question=q, answer="foo", creator=editor, is_ground_truth=True, ) a.images.add(im) progress = rs.get_progress_for_user(editor) assert progress["hangings"] == 0 assert progress["questions"] == 0 for q in [q1, q2, q3]: if use_display_sets: for ds in [ds1, ds2]: a = AnswerFactory( question=q, answer="foo", creator=editor, is_ground_truth=False, display_set=ds, ) else: for im in [im1, im2]: a = AnswerFactory( question=q, answer="foo", creator=editor, is_ground_truth=False, ) a.images.add(im) progress = rs.get_progress_for_user(editor) assert progress["hangings"] == 100.0 assert progress["questions"] == 100.0
def test_assign_score(settings): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = ReaderStudyFactory(use_display_sets=False) im = ImageFactory() q1 = QuestionFactory(reader_study=rs) q2 = QuestionFactory(reader_study=rs, answer_type=Question.AnswerType.MULTIPLE_CHOICE) e, r1, r2 = UserFactory(), UserFactory(), UserFactory() rs.images.add(im) rs.add_editor(e) rs.add_reader(r1) rs.add_reader(r2) with capture_on_commit_callbacks(execute=True): a1 = AnswerFactory(question=q1, creator=r1, answer="foo") a1.images.add(im) assert a1.score is None with capture_on_commit_callbacks(execute=True): gt = AnswerFactory(question=q1, creator=e, answer="foo", is_ground_truth=True) gt.images.add(im) a1.refresh_from_db() assert a1.score == 1.0 with capture_on_commit_callbacks(execute=True): a2 = AnswerFactory(question=q1, creator=r2, answer="foo") a2.images.add(im) a2.refresh_from_db() assert a2.score == 1.0 with capture_on_commit_callbacks(execute=True): a1 = AnswerFactory(question=q2, creator=r1, answer=[]) a1.images.add(im) a1.refresh_from_db() assert a1.score is None with capture_on_commit_callbacks(execute=True): gt = AnswerFactory(question=q2, creator=e, answer=[], is_ground_truth=True) gt.images.add(im) a1.refresh_from_db() assert a1.score == 1.0 with capture_on_commit_callbacks(execute=True): a2 = AnswerFactory(question=q2, creator=r2, answer=[]) a2.images.add(im) a2.refresh_from_db() assert a2.score == 1.0
def test_ground_truth(client): rs = ReaderStudyFactory(is_educational=True) reader = UserFactory() rs.add_reader(reader) q1 = QuestionFactory(answer_type=Question.AnswerType.CHOICE, reader_study=rs) q2 = QuestionFactory(answer_type=Question.AnswerType.MULTIPLE_CHOICE, reader_study=rs) q3 = QuestionFactory( answer_type=Question.AnswerType.MULTIPLE_CHOICE_DROPDOWN, reader_study=rs, ) op1 = CategoricalOptionFactory(question=q1, title="option1") op2 = CategoricalOptionFactory(question=q2, title="option1") op3 = CategoricalOptionFactory(question=q2, title="option1") op4 = CategoricalOptionFactory(question=q3, title="option1") op5 = CategoricalOptionFactory(question=q3, title="option1") im = ImageFactory() rs.images.add(im) a1 = AnswerFactory(question=q1, answer=op1.pk, is_ground_truth=True) a1.images.add(im) a2 = AnswerFactory(question=q2, answer=[op2.pk, op3.pk], is_ground_truth=True) a2.images.add(im) a3 = AnswerFactory(question=q3, answer=[op4.pk, op5.pk], is_ground_truth=True) a3.images.add(im) response = get_view_for_user( viewname="api:reader-study-ground-truth", reverse_kwargs={ "pk": rs.pk, "case_pk": im.pk }, user=reader, client=client, content_type="application/json", follow=True, ) assert response.status_code == 200 response = response.json() assert response[str(q1.pk)] == { "answer": op1.pk, "answer_text": op1.title, "question_text": q1.question_text, "options": { str(op1.pk): op1.title }, "explanation": "", } assert response[str(q2.pk)] == { "answer": [op2.pk, op3.pk], "answer_text": f"{op2.title}, {op3.title}", "question_text": q2.question_text, "options": { str(op2.pk): op2.title, str(op3.pk): op3.title }, "explanation": "", } assert response[str(q3.pk)] == { "answer": [op4.pk, op5.pk], "answer_text": f"{op4.title}, {op5.title}", "question_text": q3.question_text, "options": { str(op4.pk): op4.title, str(op5.pk): op5.title }, "explanation": "", }
def test_question_update(client): rs = ReaderStudyFactory(use_display_sets=False) editor = UserFactory() reader = UserFactory() rs.editors_group.user_set.add(editor) rs.readers_group.user_set.add(reader) question = QuestionFactory( question_text="foo", reader_study=rs, answer_type=Question.AnswerType.SINGLE_LINE_TEXT, direction=Question.Direction.HORIZONTAL, order=100, ) response = get_view_for_user( viewname="reader-studies:question-update", client=client, method=client.get, reverse_kwargs={"slug": rs.slug, "pk": question.pk}, follow=True, user=reader, ) assert response.status_code == 403 response = get_view_for_user( viewname="reader-studies:question-update", client=client, method=client.get, reverse_kwargs={"slug": rs.slug, "pk": question.pk}, follow=True, user=editor, ) assert response.status_code == 200 assert question.question_text == "foo" assert question.answer_type == Question.AnswerType.SINGLE_LINE_TEXT assert question.direction == Question.Direction.HORIZONTAL assert question.order == 100 get_view_for_user( viewname="reader-studies:question-update", client=client, method=client.post, data={ "question_text": "bar", "answer_type": Question.AnswerType.BOOL, "direction": Question.Direction.VERTICAL, "order": 200, "options-TOTAL_FORMS": 2, "options-INITIAL_FORMS": 1, "options-MIN_NUM_FORMS": 0, "options-MAX_NUM_FORMS": 1000, }, reverse_kwargs={"slug": rs.slug, "pk": question.pk}, follow=True, user=editor, ) question.refresh_from_db() assert question.question_text == "bar" assert question.answer_type == Question.AnswerType.BOOL assert question.direction == Question.Direction.VERTICAL assert question.order == 200 AnswerFactory(question=question, answer="true") # An answer is added, so changing the question text should no longer be possible get_view_for_user( viewname="reader-studies:question-update", client=client, method=client.post, data={ "question_text": "foo", "answer_type": Question.AnswerType.SINGLE_LINE_TEXT, "direction": Question.Direction.HORIZONTAL, "order": 100, "options-TOTAL_FORMS": 2, "options-INITIAL_FORMS": 1, "options-MIN_NUM_FORMS": 0, "options-MAX_NUM_FORMS": 1000, }, reverse_kwargs={"slug": rs.slug, "pk": question.pk}, follow=True, user=editor, ) question.refresh_from_db() assert question.question_text == "bar" assert question.answer_type == Question.AnswerType.BOOL assert question.direction == Question.Direction.HORIZONTAL assert question.order == 100
def test_assign_score(settings): settings.task_eager_propagates = (True,) settings.task_always_eager = (True,) rs = ReaderStudyFactory() im = ImageFactory() q1 = QuestionFactory(reader_study=rs) q2 = QuestionFactory( reader_study=rs, answer_type=Question.ANSWER_TYPE_MULTIPLE_CHOICE ) e, r1, r2 = UserFactory(), UserFactory(), UserFactory() rs.images.add(im) rs.add_editor(e) rs.add_reader(r1) rs.add_reader(r2) a1 = AnswerFactory(question=q1, creator=r1, answer="foo") a1.images.add(im) assert a1.score is None gt = AnswerFactory( question=q1, creator=e, answer="foo", is_ground_truth=True ) gt.images.add(im) a1.refresh_from_db() assert a1.score == 1.0 a2 = AnswerFactory(question=q1, creator=r2, answer="foo") a2.images.add(im) a2.refresh_from_db() assert a2.score == 1.0 a1 = AnswerFactory(question=q2, creator=r1, answer=[]) a1.images.add(im) assert a1.score is None gt = AnswerFactory(question=q2, creator=e, answer=[], is_ground_truth=True) gt.images.add(im) a1.refresh_from_db() assert a1.score == 1.0 a2 = AnswerFactory(question=q2, creator=r2, answer=[]) a2.images.add(im) a2.refresh_from_db() assert a2.score == 1.0
def test_statistics(reader_study_with_gt, settings): settings.task_eager_propagates = (True, ) settings.task_always_eager = (True, ) rs = reader_study_with_gt r1, r2 = rs.readers_group.user_set.all() rs_questions = rs.questions.values_list("question_text", flat=True) with capture_on_commit_callbacks(execute=True): for question in rs.questions.all(): for im in rs.images.all(): ans = AnswerFactory(question=question, creator=r1, answer=True) ans.images.add(im) statistics = rs.statistics assert Answer.objects.filter(is_ground_truth=False).count() == 6 assert statistics["max_score_questions"] == 2.0 scores = statistics["scores_by_question"] assert len(scores) == rs.questions.count() questions = set(rs_questions) for score in scores: questions -= {score["question__question_text"]} assert score["score__sum"] == 2.0 assert score["score__avg"] == 1.0 assert questions == set() scores = statistics["scores_by_case"] assert len(scores) == rs.images.count() images = set(rs.images.values_list("name", flat=True)) for score in scores: images -= {score["images__name"]} assert score["score__sum"] == 3.0 assert score["score__avg"] == 1.0 assert images == set() with capture_on_commit_callbacks(execute=True): for question in rs.questions.all(): for im in rs.images.all(): answer = question.question_text == "q1" and im.name == "im1" ans = AnswerFactory(question=question, creator=r2, answer=answer) ans.images.add(im) del rs.statistics statistics = rs.statistics assert Answer.objects.filter(is_ground_truth=False).count() == 12 assert statistics["max_score_cases"] == 6.0 scores = statistics["scores_by_question"] assert len(scores) == rs.questions.count() questions = set(rs_questions) for score in scores: questions -= {score["question__question_text"]} if score["question__question_text"] == "q1": assert score["score__sum"] == 3.0 assert score["score__avg"] == 0.75 else: assert score["score__sum"] == 2.0 assert score["score__avg"] == 0.5 assert questions == set() assert sorted(statistics["questions"]) == sorted(rs_questions) for im in rs.images.all(): assert sorted(statistics["ground_truths"][im.name].keys()) == sorted( rs_questions)