def test_attempt_with_unicode_characters(self):
"""
test that the unicode characters are removed from exam names before registering with
software secure.
"""
def is_ascii(value):
"""
returns True if string is ascii and False otherwise.
"""
try:
value.encode('ascii')
return True
except UnicodeEncodeError:
return False
def assert_get_payload_mock_unicode_characters(exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
self.assertFalse(isinstance(result['examName'], unicode))
self.assertTrue(is_ascii(result['examName']))
self.assertGreater(len(result['examName']), 0)
return result
with HTTMock(mock_response_content):
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with unicode characters',
exam_name=u'Klüft skräms inför på fédéral électoral große',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# now try with an eastern language (Chinese)
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content with chinese characters',
exam_name=u'到处群魔乱舞',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
def test_get_different_backend(self):
"""
Test that passing in a backend name returns the right backend
"""
backend = get_backend_provider({'backend': 'null'})
self.assertIsInstance(backend, NullBackendProvider)
backend = get_backend_provider(name='test')
self.assertIsInstance(backend, TestBackendProvider)
def test_get_different_backend(self):
"""
Test that passing in a backend name returns the right backend
"""
backend = get_backend_provider({'backend': 'null'})
self.assertIsInstance(backend, NullBackendProvider)
backend = get_backend_provider(name='test')
self.assertIsInstance(backend, TestBackendProvider)
def save_model(self, request, review, form, change): # pylint: disable=arguments-differ
"""
Override callback so that we can inject the user_id that made the change
"""
review.reviewed_by = request.user
review.save()
# call the review saved and since it's coming from
# the Django admin will we accept failures
get_backend_provider().on_review_saved(review, allow_rejects=True)
def save_model(self, request, review, form, change):
"""
Override callback so that we can inject the user_id that made the change
"""
review.reviewed_by = request.user
review.save()
# call the review saved and since it's coming from
# the Django admin will we accept failures
get_backend_provider().on_review_saved(review, allow_rejects=True)
def save_model(self, request, review, form, change):
"""
Override callback so that we can inject the user_id that made the change
"""
review.reviewed_by = request.user
review.save()
# call the review saved and since it's coming from
# the Django admin will we accept failures
course_id = review.exam['course_id']
course_key = CourseKey.from_string(course_id)
course = modulestore().get_course(course_key)
provider_name = course.proctoring_service
get_backend_provider(provider_name).on_review_saved(review, allow_status_update_on_fail=True)
def save_model(self, request, review, form, change):
"""
Override callback so that we can inject the user_id that made the change
"""
review.reviewed_by = request.user
review.save()
# call the review saved and since it's coming from
# the Django admin will we accept failures
course_id = review.exam.course_id
course_key = CourseKey.from_string(course_id)
course = modulestore().get_course(course_key)
provider_name = course.proctoring_service
get_backend_provider(provider_name).on_review_saved(review, allow_status_update_on_fail=True)
def test_provider_instance(self):
"""
Makes sure the instance of the proctoring module can be created
"""
provider = get_backend_provider()
self.assertIsNotNone(provider)
def on_attempt_changed(sender, instance, signal, **kwargs): # pylint: disable=unused-argument
"""
Archive the exam attempt whenever the attempt status is about to be
modified. Make a new entry with the previous value of the status in the
ProctoredExamStudentAttemptHistory table.
"""
if signal is pre_save:
if instance.id:
# on an update case, get the original
# and see if the status has changed, if so, then we need
# to archive it
original = sender.objects.get(id=instance.id)
if original.status != instance.status:
instance = original
else:
return
else:
return
else:
# remove the attempt on the backend
# timed exams have no backend
backend = get_backend_provider(name=instance.proctored_exam.backend)
if backend:
result = backend.remove_exam_attempt(
instance.proctored_exam.external_id, instance.external_id)
if not result:
log.error(u'Failed to remove attempt %d from %s', instance.id,
backend.verbose_name)
models.archive_model(models.ProctoredExamStudentAttemptHistory,
instance,
id='attempt_id')
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_get_software_download_url(self):
"""
Makes sure we get the expected download url
"""
provider = get_backend_provider()
self.assertEqual(provider.get_software_download_url(), 'http://example.com')
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_provider_instance(self):
"""
Makes sure the instance of the proctoring module can be created
"""
provider = get_backend_provider()
self.assertIsNotNone(provider)
def test_get_software_download_url(self):
"""
Makes sure we get the expected download url
"""
provider = get_backend_provider()
self.assertEqual(provider.get_software_download_url(), 'http://example.com')
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
class ExamReviewCallback(APIView):
"""
This endpoint is called by a 3rd party proctoring review service when
there are results available for us to record
IMPORTANT: This is an unauthenticated endpoint, so be VERY CAREFUL about extending
this endpoint
"""
content_negotiation_class = IgnoreClientContentNegotiation
def post(self, request):
"""
Post callback handler
"""
try:
attempt_code = request.DATA['examMetaData']['examCode']
except KeyError, ex:
log.exception(ex)
return Response(data={'reason': unicode(ex)}, status=400)
attempt_obj, is_archived_attempt = locate_attempt_by_attempt_code(
attempt_code)
course_id = attempt_obj.proctored_exam.course_id
provider_name = get_provider_name_by_course_id(course_id)
provider = get_backend_provider(provider_name)
# call down into the underlying provider code
try:
provider.on_review_callback(request.DATA)
except ProctoredBaseException, ex:
log.exception(ex)
return Response(data={'reason': unicode(ex)}, status=400)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.verified)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_no_backend_for_timed_exams(self):
"""
Timed exams should not return a backend, even if one has accidentally been set
"""
exam = {'is_proctored': False, 'backend': 'test'}
backend = get_backend_provider(exam)
self.assertIsNone(backend)
def on_attempt_changed(sender, instance, signal, **kwargs): # pylint: disable=unused-argument
"""
Archive the exam attempt whenever the attempt status is about to be
modified. Make a new entry with the previous value of the status in the
ProctoredExamStudentAttemptHistory table.
"""
if signal is pre_save:
if instance.id:
# on an update case, get the original
# and see if the status has changed, if so, then we need
# to archive it
original = sender.objects.get(id=instance.id)
if original.status != instance.status:
instance = original
else:
return
else:
return
else:
# remove the attempt on the backend
# timed exams have no backend
backend = get_backend_provider(name=instance.proctored_exam.backend)
if backend:
result = backend.remove_exam_attempt(instance.proctored_exam.external_id, instance.external_id)
if not result:
log.error('Failed to remove attempt %d from %s', instance.id, backend.verbose_name)
models.archive_model(models.ProctoredExamStudentAttemptHistory, instance, id='attempt_id')
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status
)
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace("Clean", "Suspicious")
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Suspicious")
self.assertEqual(
review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo"
)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, "Clean")
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, "Clean")
self.assertEqual(records[1].review_status, "Suspicious")
def test_mark_erroneous_proctored_exam(self):
"""
Test that SoftwareSecure's implementation returns None, because there is no
work that needs to happen right now
"""
provider = get_backend_provider()
self.assertIsNone(provider.mark_erroneous_exam_attempt(None, None))
def test_mark_erroneous_proctored_exam(self):
"""
Test that SoftwareSecure's implementation returns None, because there is no
work that needs to happen right now
"""
provider = get_backend_provider()
self.assertIsNone(provider.mark_erroneous_exam_attempt(None, None))
def is_passing(self):
"""
Returns whether the review should be considered "passing"
"""
backend = get_backend_provider(name=self.exam.backend)
# if the backend defines `passing_statuses`, use that
statuses = getattr(backend, 'passing_statuses', []) or SoftwareSecureReviewStatus.passing_statuses
return self.review_status in statuses
def is_passing(self):
"""
Returns whether the review should be considered "passing"
"""
backend = get_backend_provider(name=self.exam.backend)
# if the backend defines `passing_statuses`, use that
statuses = getattr(backend, 'passing_statuses', []) or SoftwareSecureReviewStatus.passing_statuses
return self.review_status in statuses
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id,
ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION,
review_policy_exception)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id,
proctored_exam_id=exam_id,
review_policy='Foo Policy')
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn('review_policy', context)
self.assertEqual(policy.review_policy, context['review_policy'])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context['review_policy']
if review_policy_exception:
expected = '{base}; {exception}'.format(
base=expected, exception=review_policy_exception)
self.assertEqual(result['reviewerNotes'], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['review_policy_id'], policy.id)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required
)
self.assertEqual(attempt['status'], expected_status)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']
).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_no_backend_for_timed_exams(self):
"""
Timed exams should not return a backend, even if one has accidentally been set
"""
exam = {
'is_proctored': False,
'backend': 'test'
}
backend = get_backend_provider(exam)
self.assertIsNone(backend)
def test_review_bad_code(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which does not exist
"""
provider = get_backend_provider()
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(attempt_code="not-here", external_id="also-not-here")
with self.assertRaises(StudentExamAttemptDoesNotExistsException):
provider.on_review_callback(json.loads(test_payload))
def test_on_review_saved_bad_code(self):
"""
Simulate calling on_review_saved() with an attempt code that cannot be found
"""
provider = get_backend_provider()
review = ProctoredExamSoftwareSecureReview()
review.attempt_code = "foo"
self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def assert_get_payload_mock_unicode_characters(exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(exam, context) # pylint: disable=protected-access
self.assertFalse(isinstance(result['examName'], unicode))
self.assertTrue(is_ascii(result['examName']))
self.assertGreater(len(result['examName']), 0)
return result
def test_should_block_access_to_exam_material(self, cookie_present,
resultant_boolean,
mocked_get_current_request):
"""
Test that conditions applied for blocking user from accessing
course content are correct
"""
provider = get_backend_provider()
mocked_get_current_request.return_value.get_signed_cookie.return_value = cookie_present
assert bool(provider.should_block_access_to_exam_material()
) == resultant_boolean
def test_clean_status(self):
"""
Test that defining `passing_statuses` on the backend works
"""
test_backend = get_backend_provider(name='test')
with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True):
test_payload = self.get_review_payload(status=ReviewStatus.violation)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_clean_status(self):
"""
Test that defining `passing_statuses` on the backend works
"""
test_backend = get_backend_provider(name='test')
with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True):
test_payload = self.get_review_payload(status=ReviewStatus.violation)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_on_review_saved_bad_code(self):
"""
Simulate calling on_review_saved() with an attempt code that cannot be found
"""
provider = get_backend_provider()
review = ProctoredExamSoftwareSecureReview()
review.attempt_code = 'foo'
self.assertIsNone(provider.on_review_saved(review, allow_rejects=True))
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception
)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy="Foo Policy"
)
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn("review_policy", context)
self.assertEqual(policy.review_policy, context["review_policy"])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context["review_policy"]
if review_policy_exception:
expected = "{base}; {exception}".format(base=expected, exception=review_policy_exception)
self.assertEqual(result["reviewerNotes"], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), "_get_payload", assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["review_policy_id"], policy.id)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_review_bad_code(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which does not exist
"""
provider = get_backend_provider()
test_payload = create_test_review_payload(attempt_code='not-here',
external_id='also-not-here')
with self.assertRaises(StudentExamAttemptDoesNotExistsException):
provider.on_review_callback(json.loads(test_payload))
def assert_get_payload_mock_unicode_characters(exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
self.assertFalse(isinstance(result['examName'], unicode))
self.assertTrue(is_ascii(result['examName']))
self.assertGreater(len(result['examName']), 0)
return result
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
def test_review_status_code(self):
"""
Asserts raising of an exception if we get a report
with a reviewStatus which is unexpected
"""
provider = get_backend_provider()
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(attempt_code="not-here", external_id="also-not-here")
test_payload = test_payload.replace("Clean", "Unexpected")
with self.assertRaises(ProctoredExamBadReviewStatus):
provider.on_review_callback(json.loads(test_payload))
def post(self, request):
"""
Post callback handler
"""
provider = get_backend_provider()
# call down into the underlying provider code
try:
provider.on_review_callback(request.data)
except ProctoredBaseException, ex:
log.exception(ex)
return Response(data={'reason': unicode(ex)}, status=400)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_review_status_code(self):
"""
Asserts raising of an exception if we get a report
with a reviewStatus which is unexpected
"""
provider = get_backend_provider()
test_payload = create_test_review_payload(attempt_code='not-here',
external_id='also-not-here')
test_payload = test_payload.replace('Clean', 'Unexpected')
with self.assertRaises(ProctoredExamBadReviewStatus):
provider.on_review_callback(json.loads(test_payload))
def check_for_category_switch(sender, instance, **kwargs): # pylint: disable=unused-argument
"""
If the exam switches from proctored to timed, notify the backend
"""
if instance.id:
original = sender.objects.get(pk=instance.id)
if original.is_proctored and instance.is_proctored != original.is_proctored:
from edx_proctoring.serializers import ProctoredExamJSONSafeSerializer
exam = ProctoredExamJSONSafeSerializer(instance).data
# from the perspective of the backend, the exam is now inactive.
exam['is_active'] = False
backend = get_backend_provider(name=exam['backend'])
backend.on_exam_saved(exam)
def check_for_category_switch(sender, instance, **kwargs): # pylint: disable=unused-argument
"""
If the exam switches from proctored to timed, notify the backend
"""
if instance.id:
original = sender.objects.get(pk=instance.id)
if original.is_proctored and instance.is_proctored != original.is_proctored:
from edx_proctoring.serializers import ProctoredExamSerializer
exam = ProctoredExamSerializer(instance).data
# from the perspective of the backend, the exam is now inactive.
exam['is_active'] = False
backend = get_backend_provider(name=exam['backend'])
backend.on_exam_saved(exam)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam["exam_name"]:
self.assertNotIn(illegal_char, result["examName"])
self.assertIn("_", result["examName"])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content with {}".format(illegal_char),
exam_name="Sample Exam with {} character".format(illegal_char),
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def test_payload_construction(self):
"""
Calls directly into the SoftwareSecure payload construction
"""
provider = get_backend_provider()
body = provider._body_string({"foo": False, "none": None}) # pylint: disable=protected-access
self.assertIn("false", body)
self.assertIn("null", body)
body = provider._body_string({"foo": ["first", {"here": "yes"}]}) # pylint: disable=protected-access
self.assertIn("first", body)
self.assertIn("here", body)
self.assertIn("yes", body)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn('review_policy', context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
# assert that we use the default that is defined in system configuration
self.assertEqual(result['reviewerNotes'],
constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
self.assertNotIn(illegal_char, result['examName'])
self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_no_policy):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service("credit")
credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz")
self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def finish_review_workflow(sender, instance, signal, **kwargs): # pylint: disable=unused-argument
"""
Updates the attempt status based on the review status
"""
review = instance
attempt_obj, is_archived = locate_attempt_by_attempt_code(
review.attempt_code)
attempt = api.ProctoredExamStudentAttemptSerializer(attempt_obj).data
backend = get_backend_provider(attempt['proctored_exam'])
# we could have gotten a review for an archived attempt
# this should *not* cause an update in our credit
# eligibility table
if review.is_passing:
attempt_status = ProctoredExamStudentAttemptStatus.verified
elif review.review_status == SoftwareSecureReviewStatus.not_reviewed:
attempt_status = ProctoredExamStudentAttemptStatus.error
elif review.reviewed_by or not constants.REQUIRE_FAILURE_SECOND_REVIEWS:
# reviews from the django admin have a reviewer set. They should be allowed to
# reject an attempt
attempt_status = ProctoredExamStudentAttemptStatus.rejected
elif backend and backend.supports_onboarding and attempt[
'is_sample_attempt']:
attempt_status = ProctoredExamStudentAttemptStatus.rejected
else:
# if we are not allowed to store 'rejected' on this
# code path, then put status into 'second_review_required'
attempt_status = ProctoredExamStudentAttemptStatus.second_review_required
if not is_archived:
# updating attempt status will trigger workflow
# (i.e. updating credit eligibility table)
# archived attempts should not trigger the workflow
api.update_attempt_status(attempt['proctored_exam']['id'],
attempt['user']['id'],
attempt_status,
raise_if_not_found=False,
update_attributable_to=review.reviewed_by
or None)
# emit an event for 'review_received'
data = {
'review_attempt_code': review.attempt_code,
'review_status': review.review_status,
}
emit_event(attempt['proctored_exam'],
'review_received',
attempt=attempt,
override_data=data)
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn('review_policy', context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result['reviewerNotes'],
constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
def test_should_block_access_to_exam_material(
self,
cookie_present,
switch_active,
resultant_boolean,
mocked_get_current_request,
mocked_switch_is_active
):
"""
Test that conditions applied for blocking user from accessing
course content are correct
"""
provider = get_backend_provider()
mocked_get_current_request.return_value.get_signed_cookie.return_value = cookie_present
mocked_switch_is_active.return_value = switch_active
assert bool(provider.should_block_access_to_exam_material()) == resultant_boolean
def post(self, request):
"""
Post callback handler
"""
provider = get_backend_provider()
# call down into the underlying provider code
try:
provider.on_review_callback(request.DATA)
except ProctoredBaseException, ex:
log.exception(ex)
return Response(
data={
'reason': unicode(ex)
},
status=400
)