def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service('credit', MockCreditService(profile_fullname=u'अआईउऊऋऌ अआईउऊऋऌ'))
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content_unicode_name',
exam_name=u'अआईउऊऋऌ अआईउऊऋऌ',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_attempt_with_unicode_characters(self):
"""
test that the unicode characters are removed from exam names before registering with
software secure.
"""
def is_ascii(value):
"""
returns True if string is ascii and False otherwise.
"""
try:
value.encode('ascii')
return True
except UnicodeEncodeError:
return False
def assert_get_payload_mock_unicode_characters(exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
self.assertFalse(isinstance(result['examName'], unicode))
self.assertTrue(is_ascii(result['examName']))
self.assertGreater(len(result['examName']), 0)
return result
with HTTMock(mock_response_content):
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with unicode characters',
exam_name=u'Klüft skräms inför på fédéral électoral große',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# now try with an eastern language (Chinese)
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content with chinese characters',
exam_name=u'到处群魔乱舞',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service('credit', MockCreditService(profile_fullname=u'अआईउऊऋऌ अआईउऊऋऌ'))
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content_unicode_name',
exam_name=u'अआईउऊऋऌ अआईउऊऋऌ',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service("credit", MockCreditService(profile_fullname=u"अआईउऊऋऌ अआईउऊऋऌ"))
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content_unicode_name",
exam_name=u"अआईउऊऋऌ अआईउऊऋऌ",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(attempt_code='bag code',
external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.created)
def post(self, request):
"""
HTTP POST handler. To create an exam attempt.
"""
start_immediately = request.data.get('start_clock',
'false').lower() == 'true'
exam_id = request.data.get('exam_id', None)
attempt_proctored = request.data.get('attempt_proctored',
'false').lower() == 'true'
try:
exam_attempt_id = create_exam_attempt(
exam_id=exam_id,
user_id=request.user.id,
taking_as_proctored=attempt_proctored)
exam = get_exam_by_id(exam_id)
# if use elected not to take as proctored exam, then
# use must take as open book, and loose credit eligibility
if exam['is_proctored'] and not attempt_proctored:
update_attempt_status(
exam_id, request.user.id,
ProctoredExamStudentAttemptStatus.declined)
elif start_immediately:
start_exam_attempt(exam_id, request.user.id)
return Response({'exam_attempt_id': exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(status=status.HTTP_400_BAD_REQUEST,
data={"detail": unicode(ex)})
def post(self, request):
"""
HTTP POST handler. To create an exam attempt.
"""
start_immediately = request.data.get('start_clock', 'false').lower() == 'true'
exam_id = request.data.get('exam_id', None)
attempt_proctored = request.data.get('attempt_proctored', 'false').lower() == 'true'
exam = get_exam_by_id(exam_id)
# Bypassing the due date check for practice exam
# because student can attempt the practice after the due date
if not exam.get("is_practice_exam") and is_exam_passed_due(exam, request.user):
raise ProctoredExamPermissionDenied(
u'Attempted to access expired exam with exam_id {exam_id}'.format(exam_id=exam_id)
)
exam_attempt_id = create_exam_attempt(
exam_id=exam_id,
user_id=request.user.id,
taking_as_proctored=attempt_proctored
)
# if use elected not to take as proctored exam, then
# use must take as open book, and loose credit eligibility
if exam['is_proctored'] and not attempt_proctored:
update_attempt_status(
exam_id,
request.user.id,
ProctoredExamStudentAttemptStatus.declined
)
elif start_immediately:
start_exam_attempt(exam_id, request.user.id)
data = {'exam_attempt_id': exam_attempt_id}
return Response(data)
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def post(self, request):
"""
HTTP POST handler. To create an exam attempt.
"""
start_immediately = request.DATA.get('start_clock', 'false').lower() == 'true'
exam_id = request.DATA.get('exam_id', None)
attempt_proctored = request.DATA.get('attempt_proctored', 'false').lower() == 'true'
try:
exam_attempt_id = create_exam_attempt(
exam_id=exam_id,
user_id=request.user.id,
taking_as_proctored=attempt_proctored
)
exam = get_exam_by_id(exam_id)
# if use elected not to take as proctored exam, then
# use must take as open book, and loose credit eligibility
if exam['is_proctored'] and not attempt_proctored:
update_attempt_status(
exam_id,
request.user.id,
ProctoredExamStudentAttemptStatus.declined
)
elif start_immediately:
start_exam_attempt(exam_id, request.user.id)
return Response({'exam_attempt_id': exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"detail": unicode(ex)}
)
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(
attempt_code='bag code',
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def post(self, request):
"""
HTTP POST handler. To create an exam attempt.
"""
start_immediately = request.data.get('start_clock', 'false').lower() == 'true'
exam_id = request.data.get('exam_id', None)
attempt_proctored = request.data.get('attempt_proctored', 'false').lower() == 'true'
exam = get_exam_by_id(exam_id)
# Bypassing the due date check for practice exam
# because student can attempt the practice after the due date
if not exam.get("is_practice_exam") and is_exam_passed_due(exam, request.user):
raise ProctoredExamPermissionDenied(
'Attempted to access expired exam with exam_id {exam_id}'.format(exam_id=exam_id)
)
exam_attempt_id = create_exam_attempt(
exam_id=exam_id,
user_id=request.user.id,
taking_as_proctored=attempt_proctored
)
# if use elected not to take as proctored exam, then
# use must take as open book, and loose credit eligibility
if exam['is_proctored'] and not attempt_proctored:
update_attempt_status(
exam_id,
request.user.id,
ProctoredExamStudentAttemptStatus.declined
)
elif start_immediately:
start_exam_attempt(exam_id, request.user.id)
data = {'exam_attempt_id': exam_attempt_id}
return Response(data)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.verified)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status
)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace("Clean", "Suspicious")
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Suspicious")
self.assertEqual(
review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo"
)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, "Clean")
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, "Clean")
self.assertEqual(records[1].review_status, "Suspicious")
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def test_failing_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# now try a failing request
with HTTMock(mock_response_error):
with self.assertRaises(BackendProvideCannotRegisterAttempt):
create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
def test_failing_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# now try a failing request
with HTTMock(mock_response_error):
with self.assertRaises(BackendProvideCannotRegisterAttempt):
create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id,
ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION,
review_policy_exception)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id,
proctored_exam_id=exam_id,
review_policy='Foo Policy')
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn('review_policy', context)
self.assertEqual(policy.review_policy, context['review_policy'])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context['review_policy']
if review_policy_exception:
expected = '{base}; {exception}'.format(
base=expected, exception=review_policy_exception)
self.assertEqual(result['reviewerNotes'], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['review_policy_id'], policy.id)
def setup_proctored_exam(self, block, attempt_status, user_id):
"""
Test helper to configure the given block as a proctored exam.
"""
exam_id = create_exam(
course_id=unicode(block.location.course_key),
content_id=unicode(block.location),
exam_name='foo',
time_limit_mins=10,
is_proctored=True,
is_practice_exam=block.is_practice_exam,
)
set_runtime_service('credit',
MockCreditService(enrollment_mode='verified'))
create_exam_attempt(exam_id, user_id, taking_as_proctored=True)
update_attempt_status(exam_id, user_id, attempt_status)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required
)
self.assertEqual(attempt['status'], expected_status)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']
).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def setup_proctored_exam(self, block, attempt_status, user_id):
"""
Test helper to configure the given block as a proctored exam.
"""
exam_id = create_exam(
course_id=unicode(block.location.course_key),
content_id=unicode(block.location),
exam_name='foo',
time_limit_mins=10,
is_proctored=True,
is_practice_exam=block.is_practice_exam,
)
set_runtime_service(
'credit',
MockCreditService(enrollment_mode='verified')
)
create_exam_attempt(exam_id, user_id, taking_as_proctored=True)
update_attempt_status(exam_id, user_id, attempt_status)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception
)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy="Foo Policy"
)
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn("review_policy", context)
self.assertEqual(policy.review_policy, context["review_policy"])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context["review_policy"]
if review_policy_exception:
expected = "{base}; {exception}".format(base=expected, exception=review_policy_exception)
self.assertEqual(result["reviewerNotes"], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), "_get_payload", assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["review_policy_id"], policy.id)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam["exam_name"]:
self.assertNotIn(illegal_char, result["examName"])
self.assertIn("_", result["examName"])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content with {}".format(illegal_char),
exam_name="Sample Exam with {} character".format(illegal_char),
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service("credit")
credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz")
self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Clean")
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_failure_submission(self):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", "Suspicious")
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt["status"], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt["attempt_code"])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_status_update_on_fail=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.rejected)
def test_single_name_attempt(self):
"""
Tests to make sure we can parse a fullname which does not have any spaces in it
"""
set_runtime_service("credit", MockCreditService())
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_single_name_attempt(self):
"""
Tests to make sure we can parse a fullname which does not have any spaces in it
"""
set_runtime_service('credit', MockCreditService())
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def post(self, request):
"""
HTTP POST handler. To create an exam attempt.
"""
start_immediately = request.data.get('start_clock',
'false').lower() == 'true'
exam_id = request.data.get('exam_id', None)
attempt_proctored = request.data.get('attempt_proctored',
'false').lower() == 'true'
provider_name = request.data.get('provider_name', 'dummy')
try:
exam = get_exam_by_id(exam_id)
# Bypassing the due date check for practice exam
# because student can attempt the practice after the due date
if not exam.get("is_practice_exam") and has_due_date_passed(
exam.get('due_date')):
raise ProctoredExamPermissionDenied(
'Attempted to access expired exam with exam_id {exam_id}'.
format(exam_id=exam_id))
exam_attempt_id = create_exam_attempt(
exam_id=exam_id,
user_id=request.user.id,
taking_as_proctored=attempt_proctored,
provider_name=provider_name)
# if use elected not to take as proctored exam, then
# use must take as open book, and loose credit eligibility
if exam['is_proctored'] and not attempt_proctored:
update_attempt_status(
exam_id, request.user.id,
ProctoredExamStudentAttemptStatus.declined)
elif start_immediately:
start_exam_attempt(exam_id, request.user.id)
return Response({'exam_attempt_id': exam_attempt_id})
except ProctoredBaseException, ex:
LOG.exception(ex)
return Response(status=status.HTTP_400_BAD_REQUEST,
data={"detail": unicode(ex)})
def test_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["external_id"], "foobar")
self.assertIsNone(attempt["started_at"])
def test_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['external_id'], 'foobar')
self.assertIsNone(attempt['started_at'])
def test_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['external_id'], 'foobar')
self.assertIsNone(attempt['started_at'])
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='test')
self.attempt_id = create_exam_attempt(self.exam_id,
self.user.id,
taking_as_proctored=True)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn('review_policy', context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result['reviewerNotes'],
constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_no_policy): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
def test_full_name_without_credit_service(self):
"""
Tests to make sure split doesn't raises AttributeError if credit service is down.
"""
set_runtime_service('credit', MockCreditServiceNone())
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_onboarding_attempts_no_second_review_necessary(self):
"""
Test that onboarding exams do not require a manual pass of review before they land in rejected
"""
exam_creation_params = self.exam_creation_params.copy()
exam_creation_params.update({
'is_practice_exam': True,
'content_id': 'onboarding_content',
})
onboarding_exam_id = create_exam(**exam_creation_params)
onboarding_attempt_id = create_exam_attempt(
onboarding_exam_id,
self.user.id,
taking_as_proctored=True,
)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
test_payload = self.get_review_payload(ReviewStatus.suspicious)
ProctoredExamReviewCallback().make_review(onboarding_attempt, test_payload)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
assert onboarding_attempt['status'] != ProctoredExamStudentAttemptStatus.second_review_required
def test_onboarding_attempts_no_second_review_necessary(self):
"""
Test that onboarding exams do not require a manual pass of review before they land in rejected
"""
exam_creation_params = self.exam_creation_params.copy()
exam_creation_params.update({
'is_practice_exam': True,
'content_id': 'onboarding_content',
})
onboarding_exam_id = create_exam(**exam_creation_params)
onboarding_attempt_id = create_exam_attempt(
onboarding_exam_id,
self.user.id,
taking_as_proctored=True,
)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
test_payload = self.get_review_payload(ReviewStatus.suspicious)
ProctoredExamReviewCallback().make_review(onboarding_attempt, test_payload)
onboarding_attempt = get_exam_attempt_by_id(onboarding_attempt_id)
assert onboarding_attempt['status'] != ProctoredExamStudentAttemptStatus.second_review_required
def setUp(self):
"""
Build up test data
"""
super().setUp()
set_runtime_service('credit', MockCreditService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
self.exam_id = create_exam(course_id='foo',
content_id='bar',
exam_name='Test Exam',
time_limit_mins=90)
self.attempt_id = create_exam_attempt(self.exam_id,
self.user.id,
taking_as_proctored=True)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
ProctoredExamSoftwareSecureReview.objects.create(
attempt_code=self.attempt['attempt_code'],
exam_id=self.exam_id,
student_id=self.user.id,
)
def test_attempt_with_unicode_characters(self):
"""
test that the unicode characters are removed from exam names before registering with
software secure.
"""
def is_ascii(value):
"""
returns True if string is ascii and False otherwise.
"""
try:
value.encode('ascii')
return True
except UnicodeEncodeError: # pragma: no cover (only run if test fails)
return False
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_unicode_characters(self, exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
assert_get_payload_mock_unicode_characters.called = True
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
test_self.assertTrue(is_ascii(result['examName']))
test_self.assertGreater(len(result['examName']), 0)
return result
with HTTMock(mock_response_content):
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with unicode characters',
exam_name=u'Klüft skräms inför på fédéral électoral große',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# patch the _get_payload method on the backend provider
with patch.object(SoftwareSecureBackendProvider, '_get_payload',
assert_get_payload_mock_unicode_characters):
assert_get_payload_mock_unicode_characters.called = False
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
self.assertTrue(
assert_get_payload_mock_unicode_characters.called)
# now try with an eastern language (Chinese)
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with chinese characters',
exam_name=u'到处群魔乱舞',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# patch the _get_payload method on the backend provider
with patch.object(SoftwareSecureBackendProvider, '_get_payload',
assert_get_payload_mock_unicode_characters):
assert_get_payload_mock_unicode_characters.called = False
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
self.assertTrue(
assert_get_payload_mock_unicode_characters.called)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
if review_policy_exception:
add_allowance_for_user(
exam_id,
self.user.id,
ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION,
review_policy_exception
)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id,
proctored_exam_id=exam_id,
review_policy='Foo Policy'
)
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock.called = True
test_self.assertIn('review_policy', context)
test_self.assertEqual(policy.review_policy, context['review_policy'])
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context['review_policy']
if review_policy_exception:
expected = '{base}; {exception}'.format(
base=expected,
exception=review_policy_exception
)
test_self.assertEqual(result['reviewerNotes'], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock):
assert_get_payload_mock.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['review_policy_id'], policy.id)
self.assertTrue(assert_get_payload_mock.called)
def test_attempt_with_unicode_characters(self):
"""
test that the unicode characters are removed from exam names before registering with
software secure.
"""
def is_ascii(value):
"""
returns True if string is ascii and False otherwise.
"""
try:
value.encode('ascii')
return True
except UnicodeEncodeError: # pragma: no cover (only run if test fails)
return False
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_unicode_characters(self, exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
assert_get_payload_mock_unicode_characters.called = True
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
test_self.assertTrue(is_ascii(result['examName']))
test_self.assertGreater(len(result['examName']), 0)
return result
with HTTMock(mock_response_content):
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with unicode characters',
exam_name=u'Klüft skräms inför på fédéral électoral große',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# patch the _get_payload method on the backend provider
with patch.object(SoftwareSecureBackendProvider, '_get_payload',
assert_get_payload_mock_unicode_characters):
assert_get_payload_mock_unicode_characters.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
self.assertTrue(assert_get_payload_mock_unicode_characters.called)
# now try with an eastern language (Chinese)
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with chinese characters',
exam_name=u'到处群魔乱舞',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# patch the _get_payload method on the backend provider
with patch.object(SoftwareSecureBackendProvider, '_get_payload',
assert_get_payload_mock_unicode_characters):
assert_get_payload_mock_unicode_characters.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
self.assertTrue(assert_get_payload_mock_unicode_characters.called)
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace('Clean', 'Suspicious')
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Suspicious')
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, 'Clean')
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(
attempt_code=attempt['attempt_code'])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, 'Clean')
self.assertEqual(records[1].review_status, 'Suspicious')
