def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_failure_submission_rejected(self):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
test_payload = self.get_review_payload(ReviewStatus.suspicious)
allow_rejects = not constants.REQUIRE_FAILURE_SECOND_REVIEWS
# submit a Suspicious review payload
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=self.attempt['attempt_code'])
attempt = get_exam_attempt_by_id(self.attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required
)
self.assertEqual(attempt['status'], expected_status)
self.assertEqual(review.review_status, SoftwareSecureReviewStatus.suspicious)
def test_failure_submission_rejected(self):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
test_payload = self.get_review_payload(ReviewStatus.suspicious)
allow_rejects = not constants.REQUIRE_FAILURE_SECOND_REVIEWS
# submit a Suspicious review payload
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=self.attempt['attempt_code'])
attempt = get_exam_attempt_by_id(self.attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
self.assertEqual(review.review_status,
SoftwareSecureReviewStatus.suspicious)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.verified)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']
).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required
)
self.assertEqual(attempt['status'], expected_status)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def get(self, request, attempt_id):
"""
HTTP GET Handler. Returns the status of the exam attempt.
"""
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'it does not exist.'.format(attempt_id=attempt_id))
raise StudentExamAttemptDoesNotExistsException(err_msg)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'does not have access to it.'.format(
attempt_id=attempt_id))
raise ProctoredExamPermissionDenied(err_msg)
# add in the computed time remaining as a helper
time_remaining_seconds = get_time_remaining_for_attempt(attempt)
attempt['time_remaining_seconds'] = time_remaining_seconds
accessibility_time_string = _(
u'you have {remaining_time} remaining').format(
remaining_time=humanized_time(
int(round(time_remaining_seconds / 60.0, 0))))
# special case if we are less than a minute, since we don't produce
# text translations of granularity at the seconds range
if time_remaining_seconds < 60:
accessibility_time_string = _(
u'you have less than a minute remaining')
attempt['accessibility_time_string'] = accessibility_time_string
return Response(attempt)
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(attempt_code='bag code',
external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.created)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(
attempt_code='bag code',
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def put(self, request, attempt_id): # pylint: disable=unused-argument
"""
Update the is_status_acknowledge flag for the specific attempt
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
# make sure the the attempt belongs to the calling user_id
if attempt and attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
update_exam_attempt(attempt_id, is_status_acknowledged=True)
return Response(
status=status.HTTP_200_OK
)
except (StudentExamAttemptDoesNotExistsException, ProctoredExamPermissionDenied) as ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def put(self, request, attempt_id): # pylint: disable=unused-argument
"""
Update the is_status_acknowledge flag for the specific attempt
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
# make sure the the attempt belongs to the calling user_id
if attempt and attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
update_exam_attempt(attempt_id, is_status_acknowledged=True)
return Response(
status=status.HTTP_200_OK
)
except (StudentExamAttemptDoesNotExistsException, ProctoredExamPermissionDenied) as ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def delete(self, request, attempt_id): # pylint: disable=unused-argument
"""
HTTP DELETE handler. Removes an exam attempt.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
raise StudentExamAttemptDoesNotExistsException(err_msg)
remove_exam_attempt(attempt_id)
return Response(
status=status.HTTP_200_OK,
data={}
)
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def delete(self, request, attempt_id): # pylint: disable=unused-argument
"""
HTTP DELETE handler. Removes an exam attempt.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
raise StudentExamAttemptDoesNotExistsException(err_msg)
remove_exam_attempt(attempt_id, request.user)
return Response()
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def put(self, request, attempt_id):
"""
HTTP POST handler. To stop an exam.
"""
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'it does not exist.'.format(attempt_id=attempt_id))
raise StudentExamAttemptDoesNotExistsException(err_msg)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'does not have access to it.'.format(
attempt_id=attempt_id))
raise ProctoredExamPermissionDenied(err_msg)
action = request.data.get('action')
if action == 'stop':
exam_attempt_id = stop_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id)
elif action == 'start':
exam_attempt_id = start_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id)
elif action == 'submit':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.submitted)
elif action == 'click_download_software':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.download_software_clicked)
elif action == 'error':
backend = attempt['proctored_exam']['backend']
waffle_name = PING_FAILURE_PASSTHROUGH_TEMPLATE.format(backend)
should_block_user = not (
backend and waffle.switch_is_active(waffle_name)) and (
not attempt['status']
== ProctoredExamStudentAttemptStatus.submitted)
if should_block_user:
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.error)
else:
exam_attempt_id = False
LOG.warning(
u'Browser JS reported problem with proctoring desktop '
u'application. Did block user: %s, for attempt: %s',
should_block_user, attempt['id'])
elif action == 'decline':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.declined)
data = {"exam_attempt_id": exam_attempt_id}
return Response(data)
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace("Clean", "Suspicious")
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Suspicious")
self.assertEqual(
review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo"
)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, "Clean")
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, "Clean")
self.assertEqual(records[1].review_status, "Suspicious")
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status
)
def get(self, request, attempt_id):
"""
HTTP GET Handler. Returns the status of the exam attempt.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = ('Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(attempt_id=attempt_id))
return Response(status=status.HTTP_400_BAD_REQUEST)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = ('Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id))
raise ProctoredExamPermissionDenied(err_msg)
# check if the last_poll_timestamp is not None
# and if it is older than CLIENT_TIMEOUT
# then attempt status should be marked as error.
last_poll_timestamp = attempt['last_poll_timestamp']
if last_poll_timestamp is not None \
and (datetime.now(pytz.UTC) - last_poll_timestamp).total_seconds() > CLIENT_TIMEOUT:
try:
update_attempt_status(
attempt['proctored_exam']['id'], attempt['user']['id'],
ProctoredExamStudentAttemptStatus.error)
attempt['status'] = ProctoredExamStudentAttemptStatus.error
except ProctoredExamIllegalStatusTransition:
# don't transition a completed state to an error state
pass
# add in the computed time remaining as a helper to a client app
time_remaining_seconds = get_time_remaining_for_attempt(attempt)
attempt['time_remaining_seconds'] = time_remaining_seconds
accessibility_time_string = _(
'you have {remaining_time} remaining').format(
remaining_time=humanized_time(
int(round(time_remaining_seconds / 60.0, 0))))
# special case if we are less than a minute, since we don't produce
# text translations of granularity at the seconds range
if time_remaining_seconds < 60:
accessibility_time_string = _(
'you have less than a minute remaining')
attempt['accessibility_time_string'] = accessibility_time_string
return Response(data=attempt, status=status.HTTP_200_OK)
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)})
def get(self, request, attempt_id):
"""
HTTP GET Handler. Returns the status of the exam attempt.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
return Response(
status=status.HTTP_400_BAD_REQUEST
)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
# check if the last_poll_timestamp is not None
# and if it is older than SOFTWARE_SECURE_CLIENT_TIMEOUT
# then attempt status should be marked as error.
last_poll_timestamp = attempt['last_poll_timestamp']
if last_poll_timestamp is not None \
and (datetime.now(pytz.UTC) - last_poll_timestamp).total_seconds() > SOFTWARE_SECURE_CLIENT_TIMEOUT:
attempt['status'] = 'error'
update_attempt_status(
attempt['proctored_exam']['id'],
attempt['user']['id'],
ProctoredExamStudentAttemptStatus.error
)
# add in the computed time remaining as a helper to a client app
time_remaining_seconds = get_time_remaining_for_attempt(attempt)
attempt['time_remaining_seconds'] = time_remaining_seconds
return Response(
data=attempt,
status=status.HTTP_200_OK
)
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id,
ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION,
review_policy_exception)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id,
proctored_exam_id=exam_id,
review_policy='Foo Policy')
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn('review_policy', context)
self.assertEqual(policy.review_policy, context['review_policy'])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context['review_policy']
if review_policy_exception:
expected = '{base}; {exception}'.format(
base=expected, exception=review_policy_exception)
self.assertEqual(result['reviewerNotes'], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['review_policy_id'], policy.id)
def put(self, request, attempt_id):
"""
HTTP POST handler. To stop an exam.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
raise StudentExamAttemptDoesNotExistsException(err_msg)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
action = request.DATA.get('action')
if action == 'stop':
exam_attempt_id = stop_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id
)
elif action == 'start':
exam_attempt_id = start_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id
)
elif action == 'submit':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'],
request.user.id,
ProctoredExamStudentAttemptStatus.submitted
)
elif action == 'decline':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'],
request.user.id,
ProctoredExamStudentAttemptStatus.declined
)
return Response({"exam_attempt_id": exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def test_failure_submission(self):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", "Suspicious")
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt["status"], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt["attempt_code"])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_status_update_on_fail=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.rejected)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
def test_failure_not_reviewed(self):
"""
Tests that a review which comes back as "not reviewed"
transitions to an error state
"""
test_payload = self.get_review_payload(ReviewStatus.not_reviewed)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.error)
def test_failure_not_reviewed(self):
"""
Tests that a review which comes back as "not reviewed"
transitions to an error state
"""
test_payload = self.get_review_payload(ReviewStatus.not_reviewed)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.error)
def get(self, request, attempt_id):
"""
HTTP GET Handler. Returns the status of the exam attempt.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
return Response(
status=status.HTTP_400_BAD_REQUEST
)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
# add in the computed time remaining as a helper
time_remaining_seconds = get_time_remaining_for_attempt(attempt)
attempt['time_remaining_seconds'] = time_remaining_seconds
accessibility_time_string = _('you have {remaining_time} remaining').format(
remaining_time=humanized_time(int(round(time_remaining_seconds / 60.0, 0))))
# special case if we are less than a minute, since we don't produce
# text translations of granularity at the seconds range
if time_remaining_seconds < 60:
accessibility_time_string = _('you have less than a minute remaining')
attempt['accessibility_time_string'] = accessibility_time_string
return Response(
data=attempt,
status=status.HTTP_200_OK
)
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)}
)
def test_clean_status(self):
"""
Test that defining `passing_statuses` on the backend works
"""
test_backend = get_backend_provider(name='test')
with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True):
test_payload = self.get_review_payload(status=ReviewStatus.violation)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_clean_status(self):
"""
Test that defining `passing_statuses` on the backend works
"""
test_backend = get_backend_provider(name='test')
with patch.object(test_backend, 'passing_statuses', [SoftwareSecureReviewStatus.clean], create=True):
test_payload = self.get_review_payload(status=ReviewStatus.violation)
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.second_review_required)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_onboarding_attempts_no_second_review_necessary(self):
"""
Test that onboarding exams do not require a manual pass of review before they land in rejected
"""
exam_creation_params = self.exam_creation_params.copy()
exam_creation_params.update({
'is_practice_exam': True,
'content_id': 'onboarding_content',
})
onboarding_exam_id = create_exam(**exam_creation_params)
onboarding_attempt_id = create_exam_attempt(
onboarding_exam_id,
self.user.id,
taking_as_proctored=True,
)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
test_payload = self.get_review_payload(ReviewStatus.suspicious)
ProctoredExamReviewCallback().make_review(onboarding_attempt, test_payload)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
assert onboarding_attempt['status'] != ProctoredExamStudentAttemptStatus.second_review_required
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception
)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy="Foo Policy"
)
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn("review_policy", context)
self.assertEqual(policy.review_policy, context["review_policy"])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context["review_policy"]
if review_policy_exception:
expected = "{base}; {exception}".format(base=expected, exception=review_policy_exception)
self.assertEqual(result["reviewerNotes"], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), "_get_payload", assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["review_policy_id"], policy.id)
def test_onboarding_attempts_no_second_review_necessary(self):
"""
Test that onboarding exams do not require a manual pass of review before they land in rejected
"""
exam_creation_params = self.exam_creation_params.copy()
exam_creation_params.update({
'is_practice_exam': True,
'content_id': 'onboarding_content',
})
onboarding_exam_id = create_exam(**exam_creation_params)
onboarding_attempt_id = create_exam_attempt(
onboarding_exam_id,
self.user.id,
taking_as_proctored=True,
)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
test_payload = self.get_review_payload(ReviewStatus.suspicious)
ProctoredExamReviewCallback().make_review(onboarding_attempt, test_payload)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
assert onboarding_attempt['status'] != ProctoredExamStudentAttemptStatus.second_review_required
def put(self, request, attempt_id):
"""
HTTP POST handler. To stop an exam.
"""
try:
attempt_id = int(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
except:
attempt = get_exam_attempt_by_code(attempt_id)
try:
if not attempt:
err_msg = ('Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(attempt_id=attempt_id))
raise StudentExamAttemptDoesNotExistsException(err_msg)
# make sure the the attempt belongs to the calling user_id
if attempt['user']['id'] != request.user.id:
err_msg = ('Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id))
raise ProctoredExamPermissionDenied(err_msg)
action = request.data.get('action')
if action == 'stop':
exam_attempt_id = stop_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id)
elif action == 'start':
exam_attempt_id = start_exam_attempt(
exam_id=attempt['proctored_exam']['id'],
user_id=request.user.id)
elif action == 'submit':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.submitted)
elif action == 'click_download_software':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.download_software_clicked
)
elif action == 'decline':
exam_attempt_id = update_attempt_status(
attempt['proctored_exam']['id'], request.user.id,
ProctoredExamStudentAttemptStatus.declined)
return Response({"exam_attempt_id": exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(status=status.HTTP_400_BAD_REQUEST,
data={"detail": str(ex)})
def delete(self, request, attempt_id): # pylint: disable=unused-argument
"""
HTTP DELETE handler. Removes an exam attempt.
"""
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'it does not exist.'.format(attempt_id=attempt_id))
raise StudentExamAttemptDoesNotExistsException(err_msg)
remove_exam_attempt(attempt_id, request.user)
return Response()
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam["exam_name"]:
self.assertNotIn(illegal_char, result["examName"])
self.assertIn("_", result["examName"])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content with {}".format(illegal_char),
exam_name="Sample Exam with {} character".format(illegal_char),
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service("credit")
credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz")
self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def put(self, request, attempt_id): # pylint: disable=unused-argument
"""
Update the is_status_acknowledge flag for the specific attempt
"""
attempt = get_exam_attempt_by_id(attempt_id)
# make sure the the attempt belongs to the calling user_id
if attempt and attempt['user']['id'] != request.user.id:
err_msg = (u'Attempted to access attempt_id {attempt_id} but '
u'does not have access to it.'.format(
attempt_id=attempt_id))
raise ProctoredExamPermissionDenied(err_msg)
update_exam_attempt(attempt_id, is_status_acknowledged=True)
return Response()
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Clean")
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def delete(self, request, attempt_id): # pylint: disable=unused-argument
"""
HTTP DELETE handler. Removes an exam attempt.
"""
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'it does not exist.'.format(
attempt_id=attempt_id
)
)
raise StudentExamAttemptDoesNotExistsException(err_msg)
remove_exam_attempt(attempt_id, request.user)
return Response()
def put(self, request, attempt_id):
"""
HTTP POST handler. To stop an exam.
"""
try:
attempt = get_exam_attempt_by_id(attempt_id)
if not attempt:
err_msg = "Attempted to access attempt_id {attempt_id} but " "it does not exist.".format(
attempt_id=attempt_id
)
raise StudentExamAttemptDoesNotExistsException(err_msg)
# make sure the the attempt belongs to the calling user_id
if attempt["user"]["id"] != request.user.id:
err_msg = "Attempted to access attempt_id {attempt_id} but " "does not have access to it.".format(
attempt_id=attempt_id
)
raise ProctoredExamPermissionDenied(err_msg)
action = request.data.get("action")
if action == "stop":
exam_attempt_id = stop_exam_attempt(exam_id=attempt["proctored_exam"]["id"], user_id=request.user.id)
elif action == "start":
exam_attempt_id = start_exam_attempt(exam_id=attempt["proctored_exam"]["id"], user_id=request.user.id)
elif action == "submit":
exam_attempt_id = update_attempt_status(
attempt["proctored_exam"]["id"], request.user.id, ProctoredExamStudentAttemptStatus.submitted
)
elif action == "click_download_software":
exam_attempt_id = update_attempt_status(
attempt["proctored_exam"]["id"],
request.user.id,
ProctoredExamStudentAttemptStatus.download_software_clicked,
)
elif action == "decline":
exam_attempt_id = update_attempt_status(
attempt["proctored_exam"]["id"], request.user.id, ProctoredExamStudentAttemptStatus.declined
)
return Response({"exam_attempt_id": exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(status=status.HTTP_400_BAD_REQUEST, data={"detail": str(ex)})
def put(self, request, attempt_id): # pylint: disable=unused-argument
"""
Update the is_status_acknowledge flag for the specific attempt
"""
attempt = get_exam_attempt_by_id(attempt_id)
# make sure the the attempt belongs to the calling user_id
if attempt and attempt['user']['id'] != request.user.id:
err_msg = (
'Attempted to access attempt_id {attempt_id} but '
'does not have access to it.'.format(
attempt_id=attempt_id
)
)
raise ProctoredExamPermissionDenied(err_msg)
update_exam_attempt(attempt_id, is_status_acknowledged=True)
return Response()
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='test')
self.attempt_id = create_exam_attempt(self.exam_id,
self.user.id,
taking_as_proctored=True)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def test_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["external_id"], "foobar")
self.assertIsNone(attempt["started_at"])
def test_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['external_id'], 'foobar')
self.assertIsNone(attempt['started_at'])
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn('review_policy', context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result['reviewerNotes'],
constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_no_policy): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
def test_update_archived_attempt(self):
"""
Test calling the interface point with an attempt_code that was archived
"""
test_payload = self.get_review_payload()
# now process the report
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(self.attempt_id)
self.assertEqual(attempt['status'], 'verified')
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(self.attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=self.attempt['attempt_code'])
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=self.attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
self.assertEqual(review.review_status,
SoftwareSecureReviewStatus.clean)
# now we'll make another review for the archived attempt. It should NOT update the status
test_payload = self.get_review_payload(ReviewStatus.suspicious)
self.attempt['is_archived'] = True
ProctoredExamReviewCallback().make_review(self.attempt, test_payload)
attempt, is_archived = locate_attempt_by_attempt_code(
self.attempt['attempt_code'])
self.assertTrue(is_archived)
self.assertEqual(attempt.status, 'verified')
def setUp(self):
"""
Build up test data
"""
super().setUp()
set_runtime_service('credit', MockCreditService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
self.exam_id = create_exam(course_id='foo',
content_id='bar',
exam_name='Test Exam',
time_limit_mins=90)
self.attempt_id = create_exam_attempt(self.exam_id,
self.user.id,
taking_as_proctored=True)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
ProctoredExamSoftwareSecureReview.objects.create(
attempt_code=self.attempt['attempt_code'],
exam_id=self.exam_id,
student_id=self.user.id,
)
