def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(attempt_code='bag code',
external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.created)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(
attempt_code='bag code',
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_psi_review_callback(self, psi_review_status, review_status,
credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
test_payload = json.loads(
create_test_review_payload(
attempt_code=self.attempt['attempt_code'],
external_id=self.attempt['external_id'],
review_status=psi_review_status))
self.attempt['proctored_exam']['backend'] = 'software_secure'
if review_status is None:
with self.assertRaises(ProctoredExamBadReviewStatus):
ProctoredExamReviewCallback().make_review(
self.attempt, test_payload)
else:
ProctoredExamReviewCallback().make_review(self.attempt,
test_payload)
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
self.attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
instructor_service = get_runtime_service('instructor')
notifications = instructor_service.notifications
if psi_review_status == SoftwareSecureReviewStatus.suspicious:
# check to see whether the zendesk ticket was created
self.assertEqual(len(notifications), 1)
exam = self.attempt['proctored_exam']
review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid'
self.assertEqual(notifications,
[(exam['course_id'], exam['exam_name'],
self.attempt['user']['username'],
review.review_status, review_url)])
else:
self.assertEqual(len(notifications), 0)
def test_psi_review_callback(self, psi_review_status, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
test_payload = json.loads(create_test_review_payload(
attempt_code=self.attempt['attempt_code'],
external_id=self.attempt['external_id'],
review_status=psi_review_status
))
self.attempt['proctored_exam']['backend'] = 'software_secure'
if review_status is None:
with self.assertRaises(ProctoredExamBadReviewStatus):
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
else:
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(self.attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status
)
instructor_service = get_runtime_service('instructor')
notifications = instructor_service.notifications
if psi_review_status == SoftwareSecureReviewStatus.suspicious:
# check to see whether the zendesk ticket was created
self.assertEqual(len(notifications), 1)
exam = self.attempt['proctored_exam']
review_url = 'http://testserver/edx_proctoring/v1/instructor/foo/bar/baz/1?attempt=testexternalid'
self.assertEqual(notifications,
[(exam['course_id'],
exam['exam_name'],
self.attempt['user']['username'],
review.review_status,
review_url)])
else:
self.assertEqual(len(notifications), 0)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
def test_review_bad_code(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which does not exist
"""
provider = get_backend_provider()
test_payload = create_test_review_payload(attempt_code='not-here',
external_id='also-not-here')
with self.assertRaises(StudentExamAttemptDoesNotExistsException):
provider.on_review_callback(json.loads(test_payload))
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_review_status_code(self):
"""
Asserts raising of an exception if we get a report
with a reviewStatus which is unexpected
"""
provider = get_backend_provider()
test_payload = create_test_review_payload(attempt_code='not-here',
external_id='also-not-here')
test_payload = test_payload.replace('Clean', 'Unexpected')
with self.assertRaises(ProctoredExamBadReviewStatus):
provider.on_review_callback(json.loads(test_payload))
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace('Clean', 'Suspicious')
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Suspicious')
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, 'Clean')
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, 'Clean')
self.assertEqual(records[1].review_status, 'Suspicious')
