def test_attempt_with_unicode_characters(self): """ test that the unicode characters are removed from exam names before registering with software secure. """ def is_ascii(value): """ returns True if string is ascii and False otherwise. """ try: value.encode('ascii') return True except UnicodeEncodeError: return False def assert_get_payload_mock_unicode_characters(exam, context): """ Add a mock so we can assert that the _get_payload call removes unicode characters. """ # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context) self.assertFalse(isinstance(result['examName'], unicode)) self.assertTrue(is_ascii(result['examName'])) self.assertGreater(len(result['examName']), 0) return result with HTTMock(mock_response_content): exam_id = create_exam( course_id='foo/bar/baz', content_id='content with unicode characters', exam_name=u'Klüft skräms inför på fédéral électoral große', time_limit_mins=10, is_proctored=True) # patch the _get_payload method on the backend provider with patch.object(get_backend_provider(), '_get_payload', assert_get_payload_mock_unicode_characters): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0) # now try with an eastern language (Chinese) exam_id = create_exam(course_id='foo/bar/baz', content_id='content with chinese characters', exam_name=u'到处群魔乱舞', time_limit_mins=10, is_proctored=True) # patch the _get_payload method on the backend provider with patch.object(get_backend_provider(), '_get_payload', assert_get_payload_mock_unicode_characters): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0)
def test_get_different_backend(self): """ Test that passing in a backend name returns the right backend """ backend = get_backend_provider({'backend': 'null'}) self.assertIsInstance(backend, NullBackendProvider) backend = get_backend_provider(name='test') self.assertIsInstance(backend, TestBackendProvider)
def test_get_different_backend(self): """ Test that passing in a backend name returns the right backend """ backend = get_backend_provider({'backend': 'null'}) self.assertIsInstance(backend, NullBackendProvider) backend = get_backend_provider(name='test') self.assertIsInstance(backend, TestBackendProvider)
def save_model(self, request, review, form, change): # pylint: disable=arguments-differ """ Override callback so that we can inject the user_id that made the change """ review.reviewed_by = request.user review.save() # call the review saved and since it's coming from # the Django admin will we accept failures get_backend_provider().on_review_saved(review, allow_rejects=True)
def save_model(self, request, review, form, change): """ Override callback so that we can inject the user_id that made the change """ review.reviewed_by = request.user review.save() # call the review saved and since it's coming from # the Django admin will we accept failures get_backend_provider().on_review_saved(review, allow_rejects=True)
def save_model(self, request, review, form, change): """ Override callback so that we can inject the user_id that made the change """ review.reviewed_by = request.user review.save() # call the review saved and since it's coming from # the Django admin will we accept failures course_id = review.exam['course_id'] course_key = CourseKey.from_string(course_id) course = modulestore().get_course(course_key) provider_name = course.proctoring_service get_backend_provider(provider_name).on_review_saved(review, allow_status_update_on_fail=True)
def save_model(self, request, review, form, change): """ Override callback so that we can inject the user_id that made the change """ review.reviewed_by = request.user review.save() # call the review saved and since it's coming from # the Django admin will we accept failures course_id = review.exam.course_id course_key = CourseKey.from_string(course_id) course = modulestore().get_course(course_key) provider_name = course.proctoring_service get_backend_provider(provider_name).on_review_saved(review, allow_status_update_on_fail=True)
def test_provider_instance(self): """ Makes sure the instance of the proctoring module can be created """ provider = get_backend_provider() self.assertIsNotNone(provider)
def on_attempt_changed(sender, instance, signal, **kwargs): # pylint: disable=unused-argument """ Archive the exam attempt whenever the attempt status is about to be modified. Make a new entry with the previous value of the status in the ProctoredExamStudentAttemptHistory table. """ if signal is pre_save: if instance.id: # on an update case, get the original # and see if the status has changed, if so, then we need # to archive it original = sender.objects.get(id=instance.id) if original.status != instance.status: instance = original else: return else: return else: # remove the attempt on the backend # timed exams have no backend backend = get_backend_provider(name=instance.proctored_exam.backend) if backend: result = backend.remove_exam_attempt( instance.proctored_exam.external_id, instance.external_id) if not result: log.error(u'Failed to remove attempt %d from %s', instance.id, backend.verbose_name) models.archive_model(models.ProctoredExamStudentAttemptHistory, instance, id='attempt_id')
def test_allow_simulated_callbacks(self): """ Verify that the configuration switch to not do confirmation of external_id/ssiRecordLocators """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus') # this should not raise an exception since we have # the ALLOW_CALLBACK_SIMULATION override provider.on_review_callback(json.loads(test_payload)) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_get_software_download_url(self): """ Makes sure we get the expected download url """ provider = get_backend_provider() self.assertEqual(provider.get_software_download_url(), 'http://example.com')
def test_review_mistmatched_tokens(self): """ Asserts raising of an exception if we get a report for an attempt code which has a external_id which does not match the report """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id="bogus" ) with self.assertRaises(ProctoredExamSuspiciousLookup): provider.on_review_callback(json.loads(test_payload))
def test_provider_instance(self): """ Makes sure the instance of the proctoring module can be created """ provider = get_backend_provider() self.assertIsNotNone(provider)
def test_get_software_download_url(self): """ Makes sure we get the expected download url """ provider = get_backend_provider() self.assertEqual(provider.get_software_download_url(), 'http://example.com')
def test_disallow_review_resubmission(self): """ Tests that an exception is raised if a review report is resubmitted for the same attempt """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) provider.on_review_callback(json.loads(test_payload)) # now call again with self.assertRaises(ProctoredExamReviewAlreadyExists): provider.on_review_callback(json.loads(test_payload))
class ExamReviewCallback(APIView): """ This endpoint is called by a 3rd party proctoring review service when there are results available for us to record IMPORTANT: This is an unauthenticated endpoint, so be VERY CAREFUL about extending this endpoint """ content_negotiation_class = IgnoreClientContentNegotiation def post(self, request): """ Post callback handler """ try: attempt_code = request.DATA['examMetaData']['examCode'] except KeyError, ex: log.exception(ex) return Response(data={'reason': unicode(ex)}, status=400) attempt_obj, is_archived_attempt = locate_attempt_by_attempt_code( attempt_code) course_id = attempt_obj.proctored_exam.course_id provider_name = get_provider_name_by_course_id(course_id) provider = get_backend_provider(provider_name) # call down into the underlying provider code try: provider.on_review_callback(request.DATA) except ProctoredBaseException, ex: log.exception(ex) return Response(data={'reason': unicode(ex)}, status=400)
def test_review_mistmatched_tokens(self): """ Asserts raising of an exception if we get a report for an attempt code which has a external_id which does not match the report """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id='bogus') with self.assertRaises(ProctoredExamSuspiciousLookup): provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self): """ Verify that the configuration switch to not do confirmation of external_id/ssiRecordLocators """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id="bogus" ) # this should not raise an exception since we have # the ALLOW_CALLBACK_SIMULATION override provider.on_review_callback(json.loads(test_payload)) attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.verified)
def test_disallow_review_resubmission(self): """ Tests that an exception is raised if a review report is resubmitted for the same attempt """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) provider.on_review_callback(json.loads(test_payload)) # now call again with self.assertRaises(ProctoredExamReviewAlreadyExists): provider.on_review_callback(json.loads(test_payload))
def test_no_backend_for_timed_exams(self): """ Timed exams should not return a backend, even if one has accidentally been set """ exam = {'is_proctored': False, 'backend': 'test'} backend = get_backend_provider(exam) self.assertIsNone(backend)
def on_attempt_changed(sender, instance, signal, **kwargs): # pylint: disable=unused-argument """ Archive the exam attempt whenever the attempt status is about to be modified. Make a new entry with the previous value of the status in the ProctoredExamStudentAttemptHistory table. """ if signal is pre_save: if instance.id: # on an update case, get the original # and see if the status has changed, if so, then we need # to archive it original = sender.objects.get(id=instance.id) if original.status != instance.status: instance = original else: return else: return else: # remove the attempt on the backend # timed exams have no backend backend = get_backend_provider(name=instance.proctored_exam.backend) if backend: result = backend.remove_exam_attempt(instance.proctored_exam.external_id, instance.external_id) if not result: log.error('Failed to remove attempt %d from %s', instance.id, backend.verbose_name) models.archive_model(models.ProctoredExamStudentAttemptHistory, instance, id='attempt_id')
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) test_payload = test_payload.replace('Clean', review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status )
def test_allow_review_resubmission(self): """ Tests that an resubmission is allowed """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) provider.on_review_callback(json.loads(test_payload)) # make sure history table is empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 0) # now call again, this will not throw exception test_payload = test_payload.replace("Clean", "Suspicious") provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"]) self.assertIsNotNone(review) self.assertEqual(review.review_status, "Suspicious") self.assertEqual( review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo" ) self.assertIsNotNone(review.raw_data) # make sure history table is no longer empty records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 1) self.assertEqual(records[0].review_status, "Clean") # now try to delete the record and make sure it was archived review.delete() records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"]) self.assertEqual(len(records), 2) self.assertEqual(records[0].review_status, "Clean") self.assertEqual(records[1].review_status, "Suspicious")
def test_mark_erroneous_proctored_exam(self): """ Test that SoftwareSecure's implementation returns None, because there is no work that needs to happen right now """ provider = get_backend_provider() self.assertIsNone(provider.mark_erroneous_exam_attempt(None, None))
def test_mark_erroneous_proctored_exam(self): """ Test that SoftwareSecure's implementation returns None, because there is no work that needs to happen right now """ provider = get_backend_provider() self.assertIsNone(provider.mark_erroneous_exam_attempt(None, None))
def is_passing(self): """ Returns whether the review should be considered "passing" """ backend = get_backend_provider(name=self.exam.backend) # if the backend defines `passing_statuses`, use that statuses = getattr(backend, 'passing_statuses', []) or SoftwareSecureReviewStatus.passing_statuses return self.review_status in statuses
def is_passing(self): """ Returns whether the review should be considered "passing" """ backend = get_backend_provider(name=self.exam.backend) # if the backend defines `passing_statuses`, use that statuses = getattr(backend, 'passing_statuses', []) or SoftwareSecureReviewStatus.passing_statuses return self.review_status in statuses
def test_attempt_with_review_policy(self, review_policy_exception): """ Create an unstarted proctoring attempt with a review policy associated with it. """ exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) if review_policy_exception: add_allowance_for_user( exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception) policy = ProctoredExamReviewPolicy.objects.create( set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy='Foo Policy') def assert_get_payload_mock(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertIn('review_policy', context) self.assertEqual(policy.review_policy, context['review_policy']) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context) # assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure expected = context['review_policy'] if review_policy_exception: expected = '{base}; {exception}'.format( base=expected, exception=review_policy_exception) self.assertEqual(result['reviewerNotes'], expected) return result with HTTMock(mock_response_content): # patch the _get_payload method on the backend provider # so that we can assert that we are called with the review policy # as well as asserting that _get_payload includes that review policy # that was passed in with patch.object(get_backend_provider(), '_get_payload', assert_get_payload_mock): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0) # make sure we recorded the policy id at the time this was created attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['review_policy_id'], policy.id)
def test_failure_submission(self, allow_rejects): """ Tests that a submission of a failed test and make sure that we don't automatically update the status to failure """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) test_payload = test_payload.replace('Clean', 'Suspicious') # submit a Suspicious review payload provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected) review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status (as well as trigger) # other workflow provider.on_review_saved(review, allow_rejects=allow_rejects) attempt = get_exam_attempt_by_id(attempt_id) # if we don't allow rejects to be stored in attempt status # then we should expect a 'second_review_required' status expected_status = ( ProctoredExamStudentAttemptStatus.rejected if allow_rejects else ProctoredExamStudentAttemptStatus.second_review_required ) self.assertEqual(attempt['status'], expected_status)
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) test_payload = test_payload.replace('Clean', review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service('credit') credit_status = credit_service.get_credit_state( self.user.id, 'foo/bar/baz') self.assertEqual( credit_status['credit_requirement_status'][0]['status'], credit_requirement_status)
def test_update_archived_attempt(self): """ Test calling the on_review_saved interface point with an attempt_code that was archived """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) # now process the report provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], attempt['status']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id, requesting_user=self.user) review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status but # as this is an archived attempt, we don't do anything provider.on_review_saved(review, allow_rejects=True) # look at the attempt again, since it moved into Archived state # then it should still remain unchanged archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter( attempt_code=attempt['attempt_code'] ).latest('created') self.assertEqual(archived_attempt.status, attempt['status'])
def test_no_backend_for_timed_exams(self): """ Timed exams should not return a backend, even if one has accidentally been set """ exam = { 'is_proctored': False, 'backend': 'test' } backend = get_backend_provider(exam) self.assertIsNone(backend)
def test_review_bad_code(self): """ Asserts raising of an exception if we get a report for an attempt code which does not exist """ provider = get_backend_provider() test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(attempt_code="not-here", external_id="also-not-here") with self.assertRaises(StudentExamAttemptDoesNotExistsException): provider.on_review_callback(json.loads(test_payload))
def test_on_review_saved_bad_code(self): """ Simulate calling on_review_saved() with an attempt code that cannot be found """ provider = get_backend_provider() review = ProctoredExamSoftwareSecureReview() review.attempt_code = "foo" self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def assert_get_payload_mock_unicode_characters(exam, context): """ Add a mock so we can assert that the _get_payload call removes unicode characters. """ # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload(exam, context) # pylint: disable=protected-access self.assertFalse(isinstance(result['examName'], unicode)) self.assertTrue(is_ascii(result['examName'])) self.assertGreater(len(result['examName']), 0) return result
def test_should_block_access_to_exam_material(self, cookie_present, resultant_boolean, mocked_get_current_request): """ Test that conditions applied for blocking user from accessing course content are correct """ provider = get_backend_provider() mocked_get_current_request.return_value.get_signed_cookie.return_value = cookie_present assert bool(provider.should_block_access_to_exam_material() ) == resultant_boolean
def test_clean_status(self): """ Test that defining `passing_statuses` on the backend works """ test_backend = get_backend_provider(name='test') with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True): test_payload = self.get_review_payload(status=ReviewStatus.violation) ProctoredExamReviewCallback().make_review(self.attempt, test_payload) attempt = get_exam_attempt_by_id(self.attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_clean_status(self): """ Test that defining `passing_statuses` on the backend works """ test_backend = get_backend_provider(name='test') with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True): test_payload = self.get_review_payload(status=ReviewStatus.violation) ProctoredExamReviewCallback().make_review(self.attempt, test_payload) attempt = get_exam_attempt_by_id(self.attempt_id) self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_on_review_saved_bad_code(self): """ Simulate calling on_review_saved() with an attempt code that cannot be found """ provider = get_backend_provider() review = ProctoredExamSoftwareSecureReview() review.attempt_code = 'foo' self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def test_attempt_with_review_policy(self, review_policy_exception): """ Create an unstarted proctoring attempt with a review policy associated with it. """ exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) if review_policy_exception: add_allowance_for_user( exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception ) policy = ProctoredExamReviewPolicy.objects.create( set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy="Foo Policy" ) def assert_get_payload_mock(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertIn("review_policy", context) self.assertEqual(policy.review_policy, context["review_policy"]) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload(exam, context) # assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure expected = context["review_policy"] if review_policy_exception: expected = "{base}; {exception}".format(base=expected, exception=review_policy_exception) self.assertEqual(result["reviewerNotes"], expected) return result with HTTMock(mock_response_content): # patch the _get_payload method on the backend provider # so that we can assert that we are called with the review policy # as well as asserting that _get_payload includes that review policy # that was passed in with patch.object(get_backend_provider(), "_get_payload", assert_get_payload_mock): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0) # make sure we recorded the policy id at the time this was created attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt["review_policy_id"], policy.id)
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam( course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt( exam_id, self.user.id, taking_as_proctored=True ) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id'] ) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Clean') self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6)
def test_review_bad_code(self): """ Asserts raising of an exception if we get a report for an attempt code which does not exist """ provider = get_backend_provider() test_payload = create_test_review_payload(attempt_code='not-here', external_id='also-not-here') with self.assertRaises(StudentExamAttemptDoesNotExistsException): provider.on_review_callback(json.loads(test_payload))
def assert_get_payload_mock_unicode_characters(exam, context): """ Add a mock so we can assert that the _get_payload call removes unicode characters. """ # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context) self.assertFalse(isinstance(result['examName'], unicode)) self.assertTrue(is_ascii(result['examName'])) self.assertGreater(len(result['examName']), 0) return result
def test_failure_submission(self, allow_rejects): """ Tests that a submission of a failed test and make sure that we don't automatically update the status to failure """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) test_payload = test_payload.replace('Clean', 'Suspicious') # submit a Suspicious review payload provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected) review = ProctoredExamSoftwareSecureReview.objects.get( attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status (as well as trigger) # other workflow provider.on_review_saved(review, allow_rejects=allow_rejects) attempt = get_exam_attempt_by_id(attempt_id) # if we don't allow rejects to be stored in attempt status # then we should expect a 'second_review_required' status expected_status = ( ProctoredExamStudentAttemptStatus.rejected if allow_rejects else ProctoredExamStudentAttemptStatus.second_review_required) self.assertEqual(attempt['status'], expected_status)
def test_review_status_code(self): """ Asserts raising of an exception if we get a report with a reviewStatus which is unexpected """ provider = get_backend_provider() test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(attempt_code="not-here", external_id="also-not-here") test_payload = test_payload.replace("Clean", "Unexpected") with self.assertRaises(ProctoredExamBadReviewStatus): provider.on_review_callback(json.loads(test_payload))
def post(self, request): """ Post callback handler """ provider = get_backend_provider() # call down into the underlying provider code try: provider.on_review_callback(request.data) except ProctoredBaseException, ex: log.exception(ex) return Response(data={'reason': unicode(ex)}, status=400)
def test_update_archived_attempt(self): """ Test calling the on_review_saved interface point with an attempt_code that was archived """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = create_test_review_payload( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) # now process the report provider.on_review_callback(json.loads(test_payload)) # now look at the attempt and make sure it did not # transition to failure on the callback, # as we'll need a manual confirmation via Django Admin pages attempt = get_exam_attempt_by_id(attempt_id) self.assertEqual(attempt['status'], attempt['status']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id, requesting_user=self.user) review = ProctoredExamSoftwareSecureReview.objects.get( attempt_code=attempt['attempt_code']) # now simulate a update via Django Admin table which will actually # push through the failure into our attempt status but # as this is an archived attempt, we don't do anything provider.on_review_saved(review, allow_rejects=True) # look at the attempt again, since it moved into Archived state # then it should still remain unchanged archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter( attempt_code=attempt['attempt_code']).latest('created') self.assertEqual(archived_attempt.status, attempt['status'])
def test_review_status_code(self): """ Asserts raising of an exception if we get a report with a reviewStatus which is unexpected """ provider = get_backend_provider() test_payload = create_test_review_payload(attempt_code='not-here', external_id='also-not-here') test_payload = test_payload.replace('Clean', 'Unexpected') with self.assertRaises(ProctoredExamBadReviewStatus): provider.on_review_callback(json.loads(test_payload))
def check_for_category_switch(sender, instance, **kwargs): # pylint: disable=unused-argument """ If the exam switches from proctored to timed, notify the backend """ if instance.id: original = sender.objects.get(pk=instance.id) if original.is_proctored and instance.is_proctored != original.is_proctored: from edx_proctoring.serializers import ProctoredExamJSONSafeSerializer exam = ProctoredExamJSONSafeSerializer(instance).data # from the perspective of the backend, the exam is now inactive. exam['is_active'] = False backend = get_backend_provider(name=exam['backend']) backend.on_exam_saved(exam)
def check_for_category_switch(sender, instance, **kwargs): # pylint: disable=unused-argument """ If the exam switches from proctored to timed, notify the backend """ if instance.id: original = sender.objects.get(pk=instance.id) if original.is_proctored and instance.is_proctored != original.is_proctored: from edx_proctoring.serializers import ProctoredExamSerializer exam = ProctoredExamSerializer(instance).data # from the perspective of the backend, the exam is now inactive. exam['is_active'] = False backend = get_backend_provider(name=exam['backend']) backend.on_exam_saved(exam)
def test_attempt_with_no_review_policy(self): """ Create an unstarted proctoring attempt with no review policy associated with it. """ def assert_get_payload_mock_no_policy(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertNotIn("review_policy", context) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context ) # pylint: disable=protected-access # assert that we use the default that is defined in system configuration self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY) # the check that if a colon was passed in for the exam name, then the colon was changed to # a dash. This is because SoftwareSecure cannot handle a colon in the exam name for illegal_char in SOFTWARE_SECURE_INVALID_CHARS: if illegal_char in exam["exam_name"]: self.assertNotIn(illegal_char, result["examName"]) self.assertIn("_", result["examName"]) return result for illegal_char in SOFTWARE_SECURE_INVALID_CHARS: exam_id = create_exam( course_id="foo/bar/baz", content_id="content with {}".format(illegal_char), exam_name="Sample Exam with {} character".format(illegal_char), time_limit_mins=10, is_proctored=True, ) with HTTMock(mock_response_content): # patch the _get_payload method on the backend provider # so that we can assert that we are called with the review policy # undefined and that we use the system default with patch.object( get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy ): # pylint: disable=protected-access attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0) # make sure we recorded that there is no review policy attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNone(attempt["review_policy_id"])
def test_payload_construction(self): """ Calls directly into the SoftwareSecure payload construction """ provider = get_backend_provider() body = provider._body_string({"foo": False, "none": None}) # pylint: disable=protected-access self.assertIn("false", body) self.assertIn("null", body) body = provider._body_string({"foo": ["first", {"here": "yes"}]}) # pylint: disable=protected-access self.assertIn("first", body) self.assertIn("here", body) self.assertIn("yes", body)
def test_review_on_archived_attempt(self): """ Make sure we can process a review report for an attempt which has been archived """ provider = get_backend_provider() exam_id = create_exam(course_id='foo/bar/baz', content_id='content', exam_name='Sample Exam', time_limit_mins=10, is_proctored=True) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt['external_id']) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt['attempt_code'], external_id=attempt['external_id']) # now delete the attempt, which puts it into the archive table remove_exam_attempt(attempt_id) # now process the report provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code( attempt['attempt_code']) self.assertIsNotNone(review) self.assertEqual(review.review_status, 'Clean') self.assertEqual( review.video_url, 'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo' ) self.assertIsNotNone(review.raw_data) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter( review_id=review.id) self.assertEqual(len(comments), 6)
def test_attempt_with_no_review_policy(self): """ Create an unstarted proctoring attempt with no review policy associated with it. """ def assert_get_payload_mock_no_policy(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertNotIn('review_policy', context) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context) # assert that we use the default that is defined in system configuration self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY) # the check that if a colon was passed in for the exam name, then the colon was changed to # a dash. This is because SoftwareSecure cannot handle a colon in the exam name for illegal_char in SOFTWARE_SECURE_INVALID_CHARS: if illegal_char in exam['exam_name']: self.assertNotIn(illegal_char, result['examName']) self.assertIn('_', result['examName']) return result for illegal_char in SOFTWARE_SECURE_INVALID_CHARS: exam_id = create_exam( course_id='foo/bar/baz', content_id='content with {}'.format(illegal_char), exam_name='Sample Exam with {} character'.format(illegal_char), time_limit_mins=10, is_proctored=True) with HTTMock(mock_response_content): # patch the _get_payload method on the backend provider # so that we can assert that we are called with the review policy # undefined and that we use the system default with patch.object(get_backend_provider(), '_get_payload', assert_get_payload_mock_no_policy): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) self.assertGreater(attempt_id, 0) # make sure we recorded that there is no review policy attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNone(attempt['review_policy_id'])
def test_review_callback(self, review_status, credit_requirement_status): """ Simulates callbacks from SoftwareSecure with various statuses """ provider = get_backend_provider() exam_id = create_exam( course_id="foo/bar/baz", content_id="content", exam_name="Sample Exam", time_limit_mins=10, is_proctored=True, ) # be sure to use the mocked out SoftwareSecure handlers with HTTMock(mock_response_content): attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True) attempt = get_exam_attempt_by_id(attempt_id) self.assertIsNotNone(attempt["external_id"]) test_payload = Template(TEST_REVIEW_PAYLOAD).substitute( attempt_code=attempt["attempt_code"], external_id=attempt["external_id"] ) test_payload = test_payload.replace("Clean", review_status) provider.on_review_callback(json.loads(test_payload)) # make sure that what we have in the Database matches what we expect review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"]) self.assertIsNotNone(review) self.assertEqual(review.review_status, review_status) self.assertFalse(review.video_url) self.assertIsNotNone(review.raw_data) self.assertIsNone(review.reviewed_by) # now check the comments that were stored comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id) self.assertEqual(len(comments), 6) # check that we got credit requirement set appropriately credit_service = get_runtime_service("credit") credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz") self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def finish_review_workflow(sender, instance, signal, **kwargs): # pylint: disable=unused-argument """ Updates the attempt status based on the review status """ review = instance attempt_obj, is_archived = locate_attempt_by_attempt_code( review.attempt_code) attempt = api.ProctoredExamStudentAttemptSerializer(attempt_obj).data backend = get_backend_provider(attempt['proctored_exam']) # we could have gotten a review for an archived attempt # this should *not* cause an update in our credit # eligibility table if review.is_passing: attempt_status = ProctoredExamStudentAttemptStatus.verified elif review.review_status == SoftwareSecureReviewStatus.not_reviewed: attempt_status = ProctoredExamStudentAttemptStatus.error elif review.reviewed_by or not constants.REQUIRE_FAILURE_SECOND_REVIEWS: # reviews from the django admin have a reviewer set. They should be allowed to # reject an attempt attempt_status = ProctoredExamStudentAttemptStatus.rejected elif backend and backend.supports_onboarding and attempt[ 'is_sample_attempt']: attempt_status = ProctoredExamStudentAttemptStatus.rejected else: # if we are not allowed to store 'rejected' on this # code path, then put status into 'second_review_required' attempt_status = ProctoredExamStudentAttemptStatus.second_review_required if not is_archived: # updating attempt status will trigger workflow # (i.e. updating credit eligibility table) # archived attempts should not trigger the workflow api.update_attempt_status(attempt['proctored_exam']['id'], attempt['user']['id'], attempt_status, raise_if_not_found=False, update_attributable_to=review.reviewed_by or None) # emit an event for 'review_received' data = { 'review_attempt_code': review.attempt_code, 'review_status': review.review_status, } emit_event(attempt['proctored_exam'], 'review_received', attempt=attempt, override_data=data)
def assert_get_payload_mock_no_policy(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertNotIn('review_policy', context) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context) # pylint: disable=protected-access # assert that we use the default that is defined in system configuration self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY) return result
def assert_get_payload_mock_no_policy(exam, context): """ Add a mock shim so we can assert that the _get_payload has been called with the right review policy """ self.assertNotIn("review_policy", context) # call into real implementation result = get_backend_provider(emphemeral=True)._get_payload( exam, context ) # pylint: disable=protected-access # assert that we use the default that is defined in system configuration self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY) return result
def test_should_block_access_to_exam_material( self, cookie_present, switch_active, resultant_boolean, mocked_get_current_request, mocked_switch_is_active ): """ Test that conditions applied for blocking user from accessing course content are correct """ provider = get_backend_provider() mocked_get_current_request.return_value.get_signed_cookie.return_value = cookie_present mocked_switch_is_active.return_value = switch_active assert bool(provider.should_block_access_to_exam_material()) == resultant_boolean
def post(self, request): """ Post callback handler """ provider = get_backend_provider() # call down into the underlying provider code try: provider.on_review_callback(request.DATA) except ProctoredBaseException, ex: log.exception(ex) return Response( data={ 'reason': unicode(ex) }, status=400 )