def test_on_review_saved_bad_code(self): """ Simulate calling on_review_saved() with an attempt code that cannot be found """ provider = get_backend_provider() review = ProctoredExamSoftwareSecureReview() review.attempt_code = "foo" self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def test_on_review_saved_bad_code(self): """ Simulate calling on_review_saved() with an attempt code that cannot be found """ provider = get_backend_provider() review = ProctoredExamSoftwareSecureReview() review.attempt_code = 'foo' self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ test_payload = self.get_review_payload(ReviewStatus.passed) # now delete the attempt, which puts it into the archive table remove_exam_attempt(self.attempt_id, requesting_user=self.user) # now process the report ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, SoftwareSecureReviewStatus.clean) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 3)
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ test_payload = self.get_review_payload(ReviewStatus.passed) # now delete the attempt, which puts it into the archive table remove_exam_attempt(self.attempt_id, requesting_user=self.user) # now process the report ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, SoftwareSecureReviewStatus.clean) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 3)
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) provider.on_review_callback(json.loads(test_payload)) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload = test_payload.replace("Clean", "Suspicious") provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"]) self.assertIsNotNone(review) self.assertEqual(review.review_status, "Suspicious") self.assertEqual( review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo" ) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, "Clean") # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, "Clean") self.assertEqual(records[1].review_status, "Suspicious")
def test_psi_review_callback(self, psi_review_status, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ test_payload = json.loads( create_test_review_payload( attempt_code=self.attempt['attempt_code'], external_id=self.attempt['external_id'], review_status=psi_review_status)) self.attempt['proctored_exam']['backend'] = 'software_secure' if review_status is None: with self.assertRaises(ProctoredExamBadReviewStatus): ProctoredExamReviewCallback().make_review( self.attempt, test_payload) else: ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state( self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status) instructor_service = get_runtime_service('instructor') notifications = instructor_service.notifications if psi_review_status == SoftwareSecureReviewStatus.suspicious: # check to see whether the zendesk ticket was created self.assertEqual(len(notifications), 1) exam = self.attempt['proctored_exam'] review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid' self.assertEqual(notifications, [(exam['course_id'], exam['exam_name'], self.attempt['user']['username'], review.review_status, review_url)]) else: self.assertEqual(len(notifications), 0)
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) test_payload = test_payload.replace('Clean', review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status )
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) test_payload = test_payload.replace('Clean', review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state( self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status)
def test_psi_review_callback(self, psi_review_status, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ test_payload = json.loads(create_test_review_payload( attempt_code=self.attempt['attempt_code'], external_id=self.attempt['external_id'], review_status=psi_review_status )) self.attempt['proctored_exam']['backend'] = 'software_secure' if review_status is None: with self.assertRaises(ProctoredExamBadReviewStatus): ProctoredExamReviewCallback().make_review(self.attempt, test_payload) else: ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status ) instructor_service = get_runtime_service('instructor') notifications = instructor_service.notifications if psi_review_status == SoftwareSecureReviewStatus.suspicious: # check to see whether the zendesk ticket was created self.assertEqual(len(notifications), 1) exam = self.attempt['proctored_exam'] review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid' self.assertEqual(notifications, [(exam['course_id'], exam['exam_name'], self.attempt['user']['username'], review.review_status, review_url)]) else: self.assertEqual(len(notifications), 0)
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Clean') self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6)
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Clean') self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6)
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) test_payload = test_payload.replace("Clean", review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"]) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service("credit") credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz") self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def test_review_update_attempt_active_field(self): """ Make sure we update the is_active_attempt field when an attempt is archived """ test_payload = self.get_review_payload(ReviewStatus.passed) ProctoredExamReviewCallback().make_review(self.attempt, test_payload) review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertTrue(review.is_attempt_active) # now delete the attempt, which puts it into the archive table with mock.patch('edx_proctoring.api.update_attempt_status' ) as mock_update_status: remove_exam_attempt(self.attempt_id, requesting_user=self.user) # check that the field has been updated review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertFalse(review.is_attempt_active) # check that update_attempt_status has not been called, as the attempt has been archived mock_update_status.assert_not_called()
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id, requesting_user=self.user) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"]) self.assertIsNotNone(review) self.assertEqual(review.review_status, "Clean") self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6)
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ test_payload = self.get_review_payload(ReviewStatus.passed) ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload['status'] = ReviewStatus.suspicious ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, SoftwareSecureReviewStatus.suspicious) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, 'Clean') # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, SoftwareSecureReviewStatus.clean) self.assertEqual(records[1].review_status, SoftwareSecureReviewStatus.suspicious)
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ test_payload = self.get_review_payload(ReviewStatus.passed) ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload['status'] = ReviewStatus.suspicious ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, SoftwareSecureReviewStatus.suspicious) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, 'Clean') # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=self.attempt['attempt_code']) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, SoftwareSecureReviewStatus.clean) self.assertEqual(records[1].review_status, SoftwareSecureReviewStatus.suspicious)
def on_review_callback(self, payload): """ Called when the reviewing 3rd party service posts back the results Documentation on the data format can be found from SoftwareSecure's documentation named "Reviewer Data Transfer" """ # redact the videoReviewLink from the payload if "videoReviewLink" in payload: del payload["videoReviewLink"] log_msg = "Received callback from SoftwareSecure with review data: {payload}".format(payload=payload) log.info(log_msg) # what we consider the external_id is SoftwareSecure's 'ssiRecordLocator' external_id = payload["examMetaData"]["ssiRecordLocator"] # what we consider the attempt_code is SoftwareSecure's 'examCode' attempt_code = payload["examMetaData"]["examCode"] # get the SoftwareSecure status on this attempt review_status = payload["reviewStatus"] bad_status = review_status not in self.passing_review_status + self.failing_review_status if bad_status: err_msg = "Received unexpected reviewStatus field calue from payload. " "Was {review_status}.".format( review_status=review_status ) raise ProctoredExamBadReviewStatus(err_msg) # do a lookup on the attempt by examCode, and compare the # passed in ssiRecordLocator and make sure it matches # what we recorded as the external_id. We need to look in both # the attempt table as well as the archive table (attempt_obj, is_archived_attempt) = locate_attempt_by_attempt_code(attempt_code) if not attempt_obj: # still can't find, error out err_msg = "Could not locate attempt_code: {attempt_code}".format(attempt_code=attempt_code) raise StudentExamAttemptDoesNotExistsException(err_msg) # then make sure we have the right external_id # note that SoftwareSecure might send a case insensitive # ssiRecordLocator than what it returned when we registered the # exam match = attempt_obj.external_id.lower() == external_id.lower() or settings.PROCTORING_SETTINGS.get( "ALLOW_CALLBACK_SIMULATION", False ) if not match: err_msg = ( "Found attempt_code {attempt_code}, but the recorded external_id did not " "match the ssiRecordLocator that had been recorded previously. Has {existing} " "but received {received}!".format( attempt_code=attempt_code, existing=attempt_obj.external_id, received=external_id ) ) raise ProctoredExamSuspiciousLookup(err_msg) # do some limited parsing of the JSON payload review_status = payload["reviewStatus"] # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( "We already have a review submitted from SoftwareSecure regarding " "attempt_code {attempt_code}. We do not allow for updates!".format(attempt_code=attempt_code) ) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( "We already have a review submitted from SoftwareSecure regarding " "attempt_code {attempt_code}. We have been configured to allow for " "updates and will continue...".format(attempt_code=attempt_code) ) log.warn(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() review.attempt_code = attempt_code review.raw_data = json.dumps(payload) review.review_status = review_status review.student = attempt_obj.user review.exam = attempt_obj.proctored_exam # set reviewed_by to None because it was reviewed by our 3rd party # service provider, not a user in our database review.reviewed_by = None review.save() # go through and populate all of the specific comments for comment in payload.get("webCamComments", []): self._save_review_comment(review, comment) for comment in payload.get("desktopComments", []): self._save_review_comment(review, comment) # we could have gotten a review for an archived attempt # this should *not* cause an update in our credit # eligibility table if not is_archived_attempt: # update our attempt status, note we have to import api.py here because # api.py imports software_secure.py, so we'll get an import circular reference allow_rejects = not constants.REQUIRE_FAILURE_SECOND_REVIEWS self.on_review_saved(review, allow_rejects=allow_rejects) # emit an event for 'review-received' data = { "review_attempt_code": review.attempt_code, "review_raw_data": review.raw_data, "review_status": review.review_status, } serialized_attempt_obj = ProctoredExamStudentAttemptSerializer(attempt_obj) attempt = serialized_attempt_obj.data serialized_exam_object = ProctoredExamSerializer(attempt_obj.proctored_exam) exam = serialized_exam_object.data emit_event(exam, "review-received", attempt=attempt, override_data=data)
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) provider.on_review_callback(json.loads(test_payload)) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload = test_payload.replace('Clean', 'Suspicious') provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Suspicious') self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, 'Clean') # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, 'Clean') self.assertEqual(records[1].review_status, 'Suspicious')
def make_review(self, attempt, data, backend=None): """ Save the review and review comments """ attempt_code = attempt['attempt_code'] if not backend: backend = get_backend_provider(attempt['proctored_exam']) # this method should convert the payload into a normalized format backend_review = backend.on_review_callback(attempt, data) # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( u'We already have a review submitted regarding ' u'attempt_code {attempt_code}. We do not allow for updates!'.format( attempt_code=attempt_code ) ) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( u'We already have a review submitted from our proctoring provider regarding ' u'attempt_code {attempt_code}. We have been configured to allow for ' u'updates and will continue...'.format( attempt_code=attempt_code ) ) LOG.warning(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() # first, validate that the backend review status is valid ReviewStatus.validate(backend_review['status']) # For now, we'll convert the standard review status to the old # software secure review status. # In the future, the old data should be standardized. review.review_status = SoftwareSecureReviewStatus.from_standard_status.get(backend_review['status']) review.attempt_code = attempt_code review.raw_data = json.dumps(data) review.student_id = attempt['user']['id'] review.exam_id = attempt['proctored_exam']['id'] try: review.reviewed_by = get_user_model().objects.get(email=data['reviewed_by']) except (ObjectDoesNotExist, KeyError): review.reviewed_by = None # If the reviewing user is a user in the system (user may be None for automated reviews) and does # not have permission to submit a review, log a warning. course_id = attempt['proctored_exam']['course_id'] if review.reviewed_by is not None and not is_user_course_or_global_staff(review.reviewed_by, course_id): LOG.warning( u'User %(user)s does not have the required permissions to submit ' u'a review for attempt_code %(attempt_code)s.', {'user': review.reviewed_by, 'attempt_code': attempt_code} ) review.save() # go through and populate all of the specific comments for comment in backend_review.get('comments', []): comment = ProctoredExamSoftwareSecureComment( review=review, start_time=comment.get('start', 0), stop_time=comment.get('stop', 0), duration=comment.get('duration', 0), comment=comment['comment'], status=comment['status'] ) comment.save() if review.should_notify: instructor_service = get_runtime_service('instructor') request = get_current_request() if instructor_service and request: course_id = attempt['proctored_exam']['course_id'] exam_id = attempt['proctored_exam']['id'] review_url = request.build_absolute_uri( u'{}?attempt={}'.format( reverse('edx_proctoring:instructor_dashboard_exam', args=[course_id, exam_id]), attempt['external_id'] )) instructor_service.send_support_notification( course_id=attempt['proctored_exam']['course_id'], exam_name=attempt['proctored_exam']['exam_name'], student_username=attempt['user']['username'], review_status=review.review_status, review_url=review_url, )
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) provider.on_review_callback(json.loads(test_payload)) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload = test_payload.replace('Clean', 'Suspicious') provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Suspicious') self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, 'Clean') # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, 'Clean') self.assertEqual(records[1].review_status, 'Suspicious')
def make_review(self, attempt, data, backend=None): """ Save the review and review comments """ attempt_code = attempt['attempt_code'] if not backend: backend = get_backend_provider(attempt['proctored_exam']) # this method should convert the payload into a normalized format backend_review = backend.on_review_callback(attempt, data) # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( 'We already have a review submitted regarding ' 'attempt_code {attempt_code}. We do not allow for updates!'.format( attempt_code=attempt_code ) ) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( 'We already have a review submitted from our proctoring provider regarding ' 'attempt_code {attempt_code}. We have been configured to allow for ' 'updates and will continue...'.format( attempt_code=attempt_code ) ) LOG.warning(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() # first, validate that the backend review status is valid ReviewStatus.validate(backend_review['status']) # For now, we'll convert the standard review status to the old # software secure review status. # In the future, the old data should be standardized. review.review_status = SoftwareSecureReviewStatus.from_standard_status.get(backend_review['status']) review.attempt_code = attempt_code review.raw_data = json.dumps(backend_review) review.student_id = attempt['user']['id'] review.exam_id = attempt['proctored_exam']['id'] # set reviewed_by to None because it was reviewed by our 3rd party # service provider, not a user in our database review.reviewed_by = None review.save() # go through and populate all of the specific comments for comment in backend_review.get('comments', []): comment = ProctoredExamSoftwareSecureComment( review=review, start_time=comment.get('start', 0), stop_time=comment.get('stop', 0), duration=comment.get('duration', 0), comment=comment['comment'], status=comment['status'] ) comment.save()
def on_review_callback(self, payload): """ Called when the reviewing 3rd party service posts back the results Documentation on the data format can be found from ProctorWebassistant's documentation named "Reviewer Data Transfer" """ log_msg = ( 'Received callback from ProctorWebassistant with review data: {payload}'.format( payload=payload ) ) log.info(log_msg) # what we consider the external_id is ProctorWebassistant's 'ssiRecordLocator' external_id = payload['examMetaData']['ssiRecordLocator'] # what we consider the attempt_code is ProctorWebassistant's 'examCode' attempt_code = payload['examMetaData']['examCode'] # get the ProctorWebassistant status on this attempt review_status = payload['reviewStatus'] bad_status = review_status not in self.passing_review_status + self.failing_review_status if bad_status: err_msg = ( 'Received unexpected reviewStatus field calue from payload. ' 'Was {review_status}.'.format(review_status=review_status) ) raise ProctoredExamBadReviewStatus(err_msg) # do a lookup on the attempt by examCode, and compare the # passed in ssiRecordLocator and make sure it matches # what we recorded as the external_id. We need to look in both # the attempt table as well as the archive table (attempt_obj, is_archived_attempt) = locate_attempt_by_attempt_code(attempt_code) if not attempt_obj: # still can't find, error out err_msg = ( 'Could not locate attempt_code: {attempt_code}'.format(attempt_code=attempt_code) ) raise StudentExamAttemptDoesNotExistsException(err_msg) # then make sure we have the right external_id # note that ProctorWebassistant might send a case insensitive # ssiRecordLocator than what it returned when we registered the # exam match = ( attempt_obj.external_id.lower() == external_id.lower() or settings.PROCTORING_SETTINGS.get('ALLOW_CALLBACK_SIMULATION', False) ) if not match: err_msg = ( 'Found attempt_code {attempt_code}, but the recorded external_id did not ' 'match the ssiRecordLocator that had been recorded previously. Has {existing} ' 'but received {received}!'.format( attempt_code=attempt_code, existing=attempt_obj.external_id, received=external_id ) ) raise ProctoredExamSuspiciousLookup(err_msg) # do some limited parsing of the JSON payload review_status = payload['reviewStatus'] video_review_link = payload['videoReviewLink'] # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( 'We already have a review submitted from ProctorWebassistant regarding ' 'attempt_code {attempt_code}. We do not allow for updates!'.format( attempt_code=attempt_code ) ) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( 'We already have a review submitted from ProctorWebassistant regarding ' 'attempt_code {attempt_code}. We have been configured to allow for ' 'updates and will continue...'.format( attempt_code=attempt_code ) ) log.warn(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() review.attempt_code = attempt_code review.raw_data = json.dumps(payload) review.review_status = review_status review.video_url = video_review_link review.student = attempt_obj.user review.exam = attempt_obj.proctored_exam # set reviewed_by to None because it was reviewed by our 3rd party # service provider, not a user in our database try: reviewer_username = payload['examMetaData']['proctor_username'] reviewer = User.objects.get(username=reviewer_username) except (User.DoesNotExist, KeyError): reviewer = None review.reviewed_by = reviewer review.save() # go through and populate all of the specific comments for comment in payload.get('webCamComments', []): self._save_review_comment(review, comment) for comment in payload.get('desktopComments', []): self._save_review_comment(review, comment) # we could have gotten a review for an archived attempt # this should *not* cause an update in our credit # eligibility table if not is_archived_attempt: allow_status_update_on_fail = not constants.REQUIRE_FAILURE_SECOND_REVIEWS self.on_review_saved(review, allow_status_update_on_fail=allow_status_update_on_fail)
def make_review(self, attempt, data, backend=None): """ Save the review and review comments """ attempt_code = attempt['attempt_code'] if not backend: backend = get_backend_provider(attempt['proctored_exam']) # this method should convert the payload into a normalized format backend_review = backend.on_review_callback(attempt, data) # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( 'We already have a review submitted regarding ' 'attempt_code {attempt_code}. We do not allow for updates!'.format( attempt_code=attempt_code ) ) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( 'We already have a review submitted from our proctoring provider regarding ' 'attempt_code {attempt_code}. We have been configured to allow for ' 'updates and will continue...'.format( attempt_code=attempt_code ) ) LOG.warning(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() # first, validate that the backend review status is valid ReviewStatus.validate(backend_review['status']) # For now, we'll convert the standard review status to the old # software secure review status. # In the future, the old data should be standardized. review.review_status = SoftwareSecureReviewStatus.from_standard_status.get(backend_review['status']) review.attempt_code = attempt_code review.raw_data = json.dumps(data) review.student_id = attempt['user']['id'] review.exam_id = attempt['proctored_exam']['id'] try: review.reviewed_by = get_user_model().objects.get(email=data['reviewed_by']) except (ObjectDoesNotExist, KeyError): review.reviewed_by = None # If the reviewing user is a user in the system (user may be None for automated reviews) and does # not have permission to submit a review, log a warning. course_id = attempt['proctored_exam']['course_id'] if review.reviewed_by is not None and not is_user_course_or_global_staff(review.reviewed_by, course_id): LOG.warning( 'User %(user)s does not have the required permissions to submit ' 'a review for attempt_code %(attempt_code)s.', {'user': review.reviewed_by, 'attempt_code': attempt_code} ) review.save() # go through and populate all of the specific comments for comment in backend_review.get('comments', []): comment = ProctoredExamSoftwareSecureComment( review=review, start_time=comment.get('start', 0), stop_time=comment.get('stop', 0), duration=comment.get('duration', 0), comment=comment['comment'], status=comment['status'] ) comment.save() if review.should_notify: instructor_service = get_runtime_service('instructor') request = get_current_request() if instructor_service and request: course_id = attempt['proctored_exam']['course_id'] exam_id = attempt['proctored_exam']['id'] review_url = request.build_absolute_uri( u'{}?attempt={}'.format( reverse('edx_proctoring:instructor_dashboard_exam', args=[course_id, exam_id]), attempt['external_id'] )) instructor_service.send_support_notification( course_id=attempt['proctored_exam']['course_id'], exam_name=attempt['proctored_exam']['exam_name'], student_username=attempt['user']['username'], review_status=review.review_status, review_url=review_url, )
def on_review_callback(self, payload): """ Called when the reviewing 3rd party service posts back the results Documentation on the data format can be found from SoftwareSecure's documentation named "Reviewer Data Transfer" """ # redact the videoReviewLink from the payload if 'videoReviewLink' in payload: del payload['videoReviewLink'] log_msg = ( 'Received callback from SoftwareSecure with review data: {payload}' .format(payload=payload)) log.info(log_msg) # what we consider the external_id is SoftwareSecure's 'ssiRecordLocator' external_id = payload['examMetaData']['ssiRecordLocator'] # what we consider the attempt_code is SoftwareSecure's 'examCode' attempt_code = payload['examMetaData']['examCode'] # get the SoftwareSecure status on this attempt review_status = payload['reviewStatus'] bad_status = review_status not in self.passing_review_status + self.failing_review_status if bad_status: err_msg = ( 'Received unexpected reviewStatus field calue from payload. ' 'Was {review_status}.'.format(review_status=review_status)) raise ProctoredExamBadReviewStatus(err_msg) # do a lookup on the attempt by examCode, and compare the # passed in ssiRecordLocator and make sure it matches # what we recorded as the external_id. We need to look in both # the attempt table as well as the archive table (attempt_obj, is_archived_attempt) = locate_attempt_by_attempt_code(attempt_code) if not attempt_obj: # still can't find, error out err_msg = ('Could not locate attempt_code: {attempt_code}'.format( attempt_code=attempt_code)) raise StudentExamAttemptDoesNotExistsException(err_msg) # then make sure we have the right external_id # note that SoftwareSecure might send a case insensitive # ssiRecordLocator than what it returned when we registered the # exam match = (attempt_obj.external_id.lower() == external_id.lower() or settings.PROCTORING_SETTINGS.get( 'ALLOW_CALLBACK_SIMULATION', False)) if not match: err_msg = ( 'Found attempt_code {attempt_code}, but the recorded external_id did not ' 'match the ssiRecordLocator that had been recorded previously. Has {existing} ' 'but received {received}!'.format( attempt_code=attempt_code, existing=attempt_obj.external_id, received=external_id)) raise ProctoredExamSuspiciousLookup(err_msg) # do we already have a review for this attempt?!? We may not allow updates review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt_code) if review: if not constants.ALLOW_REVIEW_UPDATES: err_msg = ( 'We already have a review submitted from SoftwareSecure regarding ' 'attempt_code {attempt_code}. We do not allow for updates!' .format(attempt_code=attempt_code)) raise ProctoredExamReviewAlreadyExists(err_msg) # we allow updates warn_msg = ( 'We already have a review submitted from SoftwareSecure regarding ' 'attempt_code {attempt_code}. We have been configured to allow for ' 'updates and will continue...'.format( attempt_code=attempt_code)) log.warn(warn_msg) else: # this is first time we've received this attempt_code, so # make a new record in the review table review = ProctoredExamSoftwareSecureReview() review.attempt_code = attempt_code review.raw_data = json.dumps(payload) review.review_status = review_status review.student = attempt_obj.user review.exam = attempt_obj.proctored_exam # set reviewed_by to None because it was reviewed by our 3rd party # service provider, not a user in our database review.reviewed_by = None review.save() # go through and populate all of the specific comments for comment in payload.get('webCamComments', []): self._save_review_comment(review, comment) for comment in payload.get('desktopComments', []): self._save_review_comment(review, comment) # we could have gotten a review for an archived attempt # this should *not* cause an update in our credit # eligibility table if not is_archived_attempt: # update our attempt status, note we have to import api.py here because # api.py imports software_secure.py, so we'll get an import circular reference allow_rejects = not constants.REQUIRE_FAILURE_SECOND_REVIEWS self.on_review_saved(review, allow_rejects=allow_rejects) # emit an event for 'review_received' data = { 'review_attempt_code': review.attempt_code, 'review_status': review.review_status, } attempt = ProctoredExamStudentAttemptSerializer(attempt_obj).data exam = ProctoredExamSerializer(attempt_obj.proctored_exam).data emit_event(exam, 'review_received', attempt=attempt, override_data=data) self._create_zendesk_ticket(review, exam, attempt)
def on_review_callback(self, payload): """ Called when the reviewing 3rd party service posts back the results Documentation on the data format can be found from SoftwareSecure's documentation named "Reviewer Data Transfer" """ log_msg = ( 'Received callback from SoftwareSecure with review data: {payload}'.format( payload=payload ) ) log.info(log_msg) # what we consider the external_id is SoftwareSecure's 'ssiRecordLocator' external_id = payload['examMetaData']['ssiRecordLocator'] # what we consider the attempt_code is SoftwareSecure's 'examCode' attempt_code = payload['examMetaData']['examCode'] # get the SoftwareSecure status on this attempt review_status = payload['reviewStatus'] bad_status = review_status not in [ 'Not Reviewed', 'Suspicious', 'Rules Violation', 'Clean' ] if bad_status: err_msg = ( 'Received unexpected reviewStatus field calue from payload. ' 'Was {review_status}.'.format(review_status=review_status) ) raise ProctoredExamBadReviewStatus(err_msg) # do a lookup on the attempt by examCode, and compare the # passed in ssiRecordLocator and make sure it matches # what we recorded as the external_id. We need to look in both # the attempt table as well as the archive table attempt_obj = ProctoredExamStudentAttempt.objects.get_exam_attempt_by_code(attempt_code) is_archived_attempt = False if not attempt_obj: # try archive table attempt_obj = ProctoredExamStudentAttemptHistory.get_exam_attempt_by_code(attempt_code) is_archived_attempt = True if not attempt_obj: # still can't find, error out err_msg = ( 'Could not locate attempt_code: {attempt_code}'.format(attempt_code=attempt_code) ) raise StudentExamAttemptDoesNotExistsException(err_msg) # then make sure we have the right external_id # note that SoftwareSecure might send a case insensitive # ssiRecordLocator than what it returned when we registered the # exam match = ( attempt_obj.external_id.lower() == external_id.lower() or settings.PROCTORING_SETTINGS.get('ALLOW_CALLBACK_SIMULATION', False) ) if not match: err_msg = ( 'Found attempt_code {attempt_code}, but the recorded external_id did not ' 'match the ssiRecordLocator that had been recorded previously. Has {existing} ' 'but received {received}!'.format( attempt_code=attempt_code, existing=attempt_obj.external_id, received=external_id ) ) raise ProctoredExamSuspiciousLookup(err_msg) # do we already have a review for this attempt?!? It should not be updated! review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt_code) if review: err_msg = ( 'We already have a review submitted from SoftwareSecure regarding ' 'attempt_code {attempt_code}. We do not allow for updates!'.format( attempt_code=attempt_code ) ) raise ProctoredExamReviewAlreadyExists(err_msg) # do some limited parsing of the JSON payload review_status = payload['reviewStatus'] video_review_link = payload['videoReviewLink'] # make a new record in the review table review = ProctoredExamSoftwareSecureReview( attempt_code=attempt_code, raw_data=json.dumps(payload), review_status=review_status, video_url=video_review_link, ) review.save() # go through and populate all of the specific comments for comment in payload.get('webCamComments', []): self._save_review_comment(review, comment) for comment in payload.get('desktopComments', []): self._save_review_comment(review, comment) # we could have gottent a review for an archived attempt # this should *not* cause an update in our credit # eligibility table if not is_archived_attempt: # update our attempt status, note we have to import api.py here because # api.py imports software_secure.py, so we'll get an import circular reference from edx_proctoring.api import update_attempt_status # only 'Clean' and 'Rules Violation' could as passing status = ( ProctoredExamStudentAttemptStatus.verified if review_status in ['Clean', 'Suspicious'] else ProctoredExamStudentAttemptStatus.rejected ) update_attempt_status( attempt_obj.proctored_exam_id, attempt_obj.user_id, status )