def test_missing_attempt_code(self): """ Test that bad attept codes return errors """ exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True, backend='software_secure', ) with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertIsNotNone(attempt_id) test_payload = create_test_review_payload(attempt_code='bag code', external_id='bogus') response = self.client.post( reverse('edx_proctoring:anonymous.proctoring_review_callback'), data=test_payload, content_type='application/json') self.assertEqual(response.status_code, 400) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_review_mistmatched_tokens(self): """ Asserts raising of an exception if we get a report for an attempt code which has a external_id which does not match the report """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus') with self.assertRaises(ProctoredExamSuspiciousLookup): provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self): """ Verify that the configuration switch to not do confirmation of external_id/ssiRecordLocators """ exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True, backend='software_secure', ) # this should not raise an exception since we have # the ALLOW_CALLBACK_SIMULATION override with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertIsNotNone(attempt_id) attempt = get_exam_attempt_by_id(attempt_id) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus') response = self.client.post( reverse('edx_proctoring:anonymous.proctoring_review_callback'), data=test_payload, content_type='application/json') self.assertEqual(response.status_code, 200) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_disallow_review_resubmission(self): """ Tests that an exception is raised if a review report is resubmitted for the same attempt """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) provider.on_review_callback(json.loads(test_payload)) # now call again with self.assertRaises(ProctoredExamReviewAlreadyExists): provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self): """ Verify that the configuration switch to not do confirmation of external_id/ssiRecordLocators """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus') # this should not raise an exception since we have # the ALLOW_CALLBACK_SIMULATION override provider.on_review_callback(json.loads(test_payload)) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_missing_attempt_code(self): """ Test that bad attept codes return errors """ exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True, backend='software_secure', ) with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertIsNotNone(attempt_id) test_payload = create_test_review_payload( attempt_code='bag code', external_id='bogus' ) response = self.client.post( reverse('edx_proctoring:anonymous.proctoring_review_callback'), data=test_payload, content_type='application/json' ) self.assertEqual(response.status_code, 400) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_allow_simulated_callbacks(self): """ Verify that the configuration switch to not do confirmation of external_id/ssiRecordLocators """ exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True, backend='software_secure', ) # this should not raise an exception since we have # the ALLOW_CALLBACK_SIMULATION override with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertIsNotNone(attempt_id) attempt = get_exam_attempt_by_id(attempt_id) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus' ) response = self.client.post( reverse('edx_proctoring:anonymous.proctoring_review_callback'), data=test_payload, content_type='application/json' ) self.assertEqual(response.status_code, 200) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_psi_review_callback(self, psi_review_status, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ test_payload = json.loads( create_test_review_payload( attempt_code=self.attempt['attempt_code'], external_id=self.attempt['external_id'], review_status=psi_review_status)) self.attempt['proctored_exam']['backend'] = 'software_secure' if review_status is None: with self.assertRaises(ProctoredExamBadReviewStatus): ProctoredExamReviewCallback().make_review( self.attempt, test_payload) else: ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state( self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status) instructor_service = get_runtime_service('instructor') notifications = instructor_service.notifications if psi_review_status == SoftwareSecureReviewStatus.suspicious: # check to see whether the zendesk ticket was created self.assertEqual(len(notifications), 1) exam = self.attempt['proctored_exam'] review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid' self.assertEqual(notifications, [(exam['course_id'], exam['exam_name'], self.attempt['user']['username'], review.review_status, review_url)]) else: self.assertEqual(len(notifications), 0)
def test_psi_review_callback(self, psi_review_status, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ test_payload = json.loads(create_test_review_payload( attempt_code=self.attempt['attempt_code'], external_id=self.attempt['external_id'], review_status=psi_review_status )) self.attempt['proctored_exam']['backend'] = 'software_secure' if review_status is None: with self.assertRaises(ProctoredExamBadReviewStatus): ProctoredExamReviewCallback().make_review(self.attempt, test_payload) else: ProctoredExamReviewCallback().make_review(self.attempt, test_payload) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(self.attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status ) instructor_service = get_runtime_service('instructor') notifications = instructor_service.notifications if psi_review_status == SoftwareSecureReviewStatus.suspicious: # check to see whether the zendesk ticket was created self.assertEqual(len(notifications), 1) exam = self.attempt['proctored_exam'] review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid' self.assertEqual(notifications, [(exam['course_id'], exam['exam_name'], self.attempt['user']['username'], review.review_status, review_url)]) else: self.assertEqual(len(notifications), 0)
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) test_payload = test_payload.replace('Clean', review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state( self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status)
def test_failure_submission(self, allow_rejects): """ Tests that a submission of a failed test and make sure that we don't automatically update the status to failure """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) test_payload = test_payload.replace('Clean', 'Suspicious') # submit a Suspicious review payload provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected) review = ProctoredExamSoftwareSecureReview.objects.get( attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status (as well as trigger) # other workflow provider.on_review_saved(review, allow_rejects=allow_rejects) attempt = get_exam_attempt_by_id(attempt_id) # if we don't allow rejects to be stored in attempt status # then we should expect a 'second_review_required' status expected_status = ( ProctoredExamStudentAttemptStatus.rejected if allow_rejects else ProctoredExamStudentAttemptStatus.second_review_required) self.assertEqual(attempt['status'], expected_status)
def test_review_bad_code(self): """ Asserts raising of an exception if we get a report for an attempt code which does not exist """ provider = get_backend_provider() test_payload = create_test_review_payload(attempt_code='not-here', external_id='also-not-here') with self.assertRaises(StudentExamAttemptDoesNotExistsException): provider.on_review_callback(json.loads(test_payload))
def test_update_archived_attempt(self): """ Test calling the on_review_saved interface point with an attempt_code that was archived """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) # now process the report provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], attempt['status']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id, requesting_user=self.user) review = ProctoredExamSoftwareSecureReview.objects.get( attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status but # as this is an archived attempt, we don't do anything provider.on_review_saved(review, allow_rejects=True) # look at the attempt again, since it moved into Archived state # then it should still remain unchanged archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter( attempt_code=attempt['attempt_code']).latest('created') self.assertEqual(archived_attempt.status, attempt['status'])
def test_review_status_code(self): """ Asserts raising of an exception if we get a report with a reviewStatus which is unexpected """ provider = get_backend_provider() test_payload = create_test_review_payload(attempt_code='not-here', external_id='also-not-here') test_payload = test_payload.replace('Clean', 'Unexpected') with self.assertRaises(ProctoredExamBadReviewStatus): provider.on_review_callback(json.loads(test_payload))
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id, requesting_user=self.user) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Clean') self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6)
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) provider.on_review_callback(json.loads(test_payload)) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload = test_payload.replace('Clean', 'Suspicious') provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Suspicious') self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, 'Clean') # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter( attempt_code=attempt['attempt_code']) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, 'Clean') self.assertEqual(records[1].review_status, 'Suspicious')