def test_proctored_exam_requirements(self):
"""
Make sure that proctored exams are being registered as requirements
"""
self.add_credit_course(self.course.id)
create_exam(
course_id=unicode(self.course.id),
content_id='foo',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
# just inspect the proctored exam requirement
requirements = [
requirement
for requirement in get_credit_requirements(self.course.id)
if requirement['namespace'] == 'proctored_exam'
]
self.assertEqual(len(requirements), 1)
self.assertEqual(requirements[0]['namespace'], 'proctored_exam')
self.assertEqual(requirements[0]['name'], 'proctored_exam_id:1')
self.assertEqual(requirements[0]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[0]['criteria'], {})
def test_attempt_with_unicode_characters(self):
"""
test that the unicode characters are removed from exam names before registering with
software secure.
"""
def is_ascii(value):
"""
returns True if string is ascii and False otherwise.
"""
try:
value.encode('ascii')
return True
except UnicodeEncodeError:
return False
def assert_get_payload_mock_unicode_characters(exam, context):
"""
Add a mock so we can assert that the _get_payload call removes unicode characters.
"""
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
self.assertFalse(isinstance(result['examName'], unicode))
self.assertTrue(is_ascii(result['examName']))
self.assertGreater(len(result['examName']), 0)
return result
with HTTMock(mock_response_content):
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with unicode characters',
exam_name=u'Klüft skräms inför på fédéral électoral große',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# now try with an eastern language (Chinese)
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content with chinese characters',
exam_name=u'到处群魔乱舞',
time_limit_mins=10,
is_proctored=True)
# patch the _get_payload method on the backend provider
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock_unicode_characters):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
def test_proctored_exam_requirements(self):
"""
Make sure that proctored exams are being registered as requirements
"""
self.add_credit_course(self.course.id)
create_exam(
course_id=six.text_type(self.course.id),
content_id=six.text_type(self.subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'], six.text_type(self.subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service("credit", MockCreditService(profile_fullname=u"अआईउऊऋऌ अआईउऊऋऌ"))
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content_unicode_name",
exam_name=u"अआईउऊऋऌ अआईउऊऋऌ",
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service('credit', MockCreditService(profile_fullname=u'अआईउऊऋऌ अआईउऊऋऌ'))
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content_unicode_name',
exam_name=u'अआईउऊऋऌ अआईउऊऋऌ',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_unicode_attempt(self):
"""
Tests to make sure we can handle an attempt when a user's fullname has unicode characters in it
"""
set_runtime_service('credit', MockCreditService(profile_fullname=u'अआईउऊऋऌ अआईउऊऋऌ'))
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
# try unicode exam name, also
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content_unicode_name',
exam_name=u'अआईउऊऋऌ अआईउऊऋऌ',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
def test_proctored_exam_requirements(self):
"""
Make sure that proctored exams are being registered as requirements
"""
self.add_credit_course(self.course.id)
create_exam(course_id=six.text_type(self.course.id),
content_id=six.text_type(self.subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'],
six.text_type(self.subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
def test_proctored_exam_requirements(self):
"""
Make sure that proctored exams are being registered as requirements
"""
self.add_credit_course(self.course.id)
create_exam(
course_id=str(self.course.id),
content_id=str(self.subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
assert len(requirements) == 0
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
assert len(requirements) == 2
assert requirements[1]['namespace'] == 'proctored_exam'
assert requirements[1]['name'] == str(self.subsection.location)
assert requirements[1]['display_name'] == 'A Proctored Exam'
assert requirements[1]['criteria'] == {}
def test_proctored_exam_requirements(self):
"""
Make sure that proctored exams are being registered as requirements
"""
self.add_credit_course(self.course.id)
create_exam(course_id=unicode(self.course.id),
content_id='foo',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
# just inspect the proctored exam requirement
requirements = [
requirement
for requirement in get_credit_requirements(self.course.id)
if requirement['namespace'] == 'proctored_exam'
]
self.assertEqual(len(requirements), 1)
self.assertEqual(requirements[0]['namespace'], 'proctored_exam')
self.assertEqual(requirements[0]['name'], 'proctored_exam_id:1')
self.assertEqual(requirements[0]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[0]['criteria'], {})
def test_proctored_exam_filtering(self):
"""
Make sure that timed or inactive exams do not end up in the requirements table
"""
self.add_credit_course(self.course.id)
create_exam(
course_id=unicode(self.course.id),
content_id='foo',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=False,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 1)
# make sure we don't have a proctoring requirement
self.assertFalse([
requirement
for requirement in requirements
if requirement['namespace'] == 'proctored_exam'
])
create_exam(
course_id=unicode(self.course.id),
content_id='foo2',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=False
)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 1)
# make sure we don't have a proctoring requirement
self.assertFalse([
requirement
for requirement in requirements
if requirement['namespace'] == 'proctored_exam'
])
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_disallow_review_resubmission(self):
"""
Tests that an exception is raised if a review report is resubmitted for the same
attempt
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
provider.on_review_callback(json.loads(test_payload))
# now call again
with self.assertRaises(ProctoredExamReviewAlreadyExists):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["status"], ProctoredExamStudentAttemptStatus.verified)
def test_get_student_exam_attempt_features(self):
query_features = [
'email',
'exam_name',
'allowed_time_limit_mins',
'is_sample_attempt',
'started_at',
'completed_at',
'status',
'Suspicious Count',
'Suspicious Comments',
'Rules Violation Count',
'Rules Violation Comments',
]
proctored_exam_id = create_exam(self.course_key, 'Test Content', 'Test Exam', 1)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[0].id, '',
'Test Code 1', True, False, 'ad13'
)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[1].id, '',
'Test Code 2', True, False, 'ad13'
)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[2].id, '',
'Test Code 3', True, False, 'asd'
)
proctored_exam_attempts = get_proctored_exam_results(self.course_key, query_features)
self.assertEqual(len(proctored_exam_attempts), 3)
for proctored_exam_attempt in proctored_exam_attempts:
self.assertEqual(set(proctored_exam_attempt.keys()), set(query_features))
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(attempt_code='bag code',
external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.created)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json')
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def test_get_student_exam_attempt_features(self):
query_features = [
'user_email',
'exam_name',
'allowed_time_limit_mins',
'is_sample_attempt',
'started_at',
'completed_at',
'status',
]
proctored_exam_id = create_exam(self.course_key, 'Test Content', 'Test Exam', 1)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[0].id, '', 1,
'Test Code 1', True, False, 'ad13'
)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[1].id, '', 2,
'Test Code 2', True, False, 'ad13'
)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[2].id, '', 3,
'Test Code 3', True, False, 'asd'
)
proctored_exam_attempts = get_proctored_exam_results(self.course_key, query_features)
self.assertEqual(len(proctored_exam_attempts), 3)
for proctored_exam_attempt in proctored_exam_attempts:
self.assertEqual(set(proctored_exam_attempt.keys()), set(query_features))
def test_missing_attempt_code(self):
"""
Test that bad attept codes return errors
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
test_payload = create_test_review_payload(
attempt_code='bag code',
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 400)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.created)
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertIsNotNone(attempt_id)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id='bogus'
)
response = self.client.post(
reverse('edx_proctoring:anonymous.proctoring_review_callback'),
data=test_payload,
content_type='application/json'
)
self.assertEqual(response.status_code, 200)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], ProctoredExamStudentAttemptStatus.verified)
def setUp(self):
super(ReviewTests, self).setUp()
self.dummy_request = RequestFactory().get('/')
self.exam_creation_params = {
'course_id': 'foo/bar/baz',
'content_id': 'content',
'exam_name': 'Sample Exam',
'time_limit_mins': 10,
'is_proctored': True,
'backend': 'test'
}
self.exam_id = create_exam(**self.exam_creation_params)
self.attempt_id = create_exam_attempt(
self.exam_id,
self.user.id,
taking_as_proctored=True
)
self.attempt = get_exam_attempt_by_id(self.attempt_id)
set_runtime_service('credit', MockCreditService())
set_runtime_service('instructor', MockInstructorService())
set_runtime_service('grades', MockGradesService())
set_runtime_service('certificates', MockCertificateService())
set_current_request(self.dummy_request)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id="bogus"
)
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_get_student_exam_attempt_features(self):
query_features = [
'email', 'exam_name', 'allowed_time_limit_mins',
'is_sample_attempt', 'started_at', 'completed_at', 'status',
'Suspicious Count', 'Suspicious Comments', 'Rules Violation Count',
'Rules Violation Comments', 'track'
]
proctored_exam_id = create_exam(self.course_key, 'Test Content',
'Test Exam', 1)
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[0].id, 'Test Code 1', True, False,
'ad13')
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[1].id, 'Test Code 2', True, False,
'ad13')
ProctoredExamStudentAttempt.create_exam_attempt(
proctored_exam_id, self.users[2].id, 'Test Code 3', True, False,
'asd')
proctored_exam_attempts = get_proctored_exam_results(
self.course_key, query_features)
assert len(proctored_exam_attempts) == 3
for proctored_exam_attempt in proctored_exam_attempts:
assert set(proctored_exam_attempt.keys()) == set(query_features)
def test_review_mistmatched_tokens(self):
"""
Asserts raising of an exception if we get a report for
an attempt code which has a external_id which does not
match the report
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
with self.assertRaises(ProctoredExamSuspiciousLookup):
provider.on_review_callback(json.loads(test_payload))
def test_allow_simulated_callbacks(self):
"""
Verify that the configuration switch to
not do confirmation of external_id/ssiRecordLocators
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'], external_id='bogus')
# this should not raise an exception since we have
# the ALLOW_CALLBACK_SIMULATION override
provider.on_review_callback(json.loads(test_payload))
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.verified)
def test_credit_requirement_blocks_ordering(self):
"""
Test ordering of the proctoring and ICRV blocks are in proper order.
"""
self.add_credit_course(self.course.id)
subsection = ItemFactory.create(parent=self.section,
category='sequential',
display_name='Dummy Subsection')
create_exam(course_id=unicode(self.course.id),
content_id=unicode(subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'], unicode(subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
# Create multiple ICRV blocks
start = datetime.now(UTC)
self.add_icrv_xblock(related_assessment_name="Midterm A",
start_date=start)
start = start - timedelta(days=1)
self.add_icrv_xblock(related_assessment_name="Midterm B",
start_date=start)
# Primary sort is based on start date
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
# grade requirement is added on publish of the requirements
self.assertEqual(len(requirements), 4)
# check requirements are added in the desired order
# 1st Minimum grade then the blocks with start date than other blocks
self.assertEqual(requirements[0]["display_name"], "Minimum Grade")
self.assertEqual(requirements[1]["display_name"], "A Proctored Exam")
self.assertEqual(requirements[2]["display_name"], "Midterm B")
self.assertEqual(requirements[3]["display_name"], "Midterm A")
def test_allow_review_resubmission(self):
"""
Tests that an resubmission is allowed
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
provider.on_review_callback(json.loads(test_payload))
# make sure history table is empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 0)
# now call again, this will not throw exception
test_payload = test_payload.replace("Clean", "Suspicious")
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Suspicious")
self.assertEqual(
review.video_url, "http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo"
)
self.assertIsNotNone(review.raw_data)
# make sure history table is no longer empty
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 1)
self.assertEqual(records[0].review_status, "Clean")
# now try to delete the record and make sure it was archived
review.delete()
records = ProctoredExamSoftwareSecureReviewHistory.objects.filter(attempt_code=attempt["attempt_code"])
self.assertEqual(len(records), 2)
self.assertEqual(records[0].review_status, "Clean")
self.assertEqual(records[1].review_status, "Suspicious")
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status
)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
test_self = self # So that we can access test methods in nested function.
def assert_get_payload_mock_no_policy(self, exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
assert_get_payload_mock_no_policy.called = True
test_self.assertNotIn('review_policy', context)
# call into real implementation
# pylint: disable=too-many-function-args
result = software_secure_get_payload(self, exam, context)
# assert that we use the default that is defined in system configuration
test_self.assertEqual(result['reviewerNotes'], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam['exam_name']:
test_self.assertNotIn(illegal_char, result['examName'])
test_self.assertIn('_', result['examName'])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content with {}'.format(illegal_char),
exam_name='Sample Exam with {} character'.format(illegal_char),
time_limit_mins=10,
is_proctored=True,
backend='software_secure',
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(SoftwareSecureBackendProvider, '_get_payload', assert_get_payload_mock_no_policy):
assert_get_payload_mock_no_policy.called = False
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt['review_policy_id'])
self.assertTrue(assert_get_payload_mock_no_policy.called)
def _create_proctored_exam(self):
"""
Calls the api's create_exam to create an exam object.
"""
return create_exam(course_id=self.course_id,
content_id=self.content_id,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id,
ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION,
review_policy_exception)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id,
proctored_exam_id=exam_id,
review_policy='Foo Policy')
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn('review_policy', context)
self.assertEqual(policy.review_policy, context['review_policy'])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context['review_policy']
if review_policy_exception:
expected = '{base}; {exception}'.format(
base=expected, exception=review_policy_exception)
self.assertEqual(result['reviewerNotes'], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), '_get_payload',
assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['review_policy_id'], policy.id)
def test_credit_requirement_blocks_ordering(self):
"""
Test ordering of the proctoring and ICRV blocks are in proper order.
"""
self.add_credit_course(self.course.id)
subsection = ItemFactory.create(parent=self.section, category='sequential', display_name='Dummy Subsection')
create_exam(
course_id=unicode(self.course.id),
content_id=unicode(subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'], unicode(subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
# Create multiple ICRV blocks
start = datetime.now(UTC)
self.add_icrv_xblock(related_assessment_name="Midterm A", start_date=start)
start = start - timedelta(days=1)
self.add_icrv_xblock(related_assessment_name="Midterm B", start_date=start)
# Primary sort is based on start date
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
# grade requirement is added on publish of the requirements
self.assertEqual(len(requirements), 4)
# check requirements are added in the desired order
# 1st Minimum grade then the blocks with start date than other blocks
self.assertEqual(requirements[0]["display_name"], "Minimum Grade")
self.assertEqual(requirements[1]["display_name"], "A Proctored Exam")
self.assertEqual(requirements[2]["display_name"], "Midterm B")
self.assertEqual(requirements[3]["display_name"], "Midterm A")
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'], ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required
)
self.assertEqual(attempt['status'], expected_status)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service('credit')
credit_status = credit_service.get_credit_state(
self.user.id, 'foo/bar/baz')
self.assertEqual(
credit_status['credit_requirement_status'][0]['status'],
credit_requirement_status)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']
).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def test_proctored_exam_filtering(self):
"""
Make sure that timed or inactive exams do not end up in the requirements table
"""
self.add_credit_course(self.course.id)
create_exam(course_id=unicode(self.course.id),
content_id='foo',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=False,
is_active=True)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 1)
# make sure we don't have a proctoring requirement
self.assertFalse([
requirement for requirement in requirements
if requirement['namespace'] == 'proctored_exam'
])
create_exam(course_id=unicode(self.course.id),
content_id='foo2',
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=False)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 1)
# make sure we don't have a proctoring requirement
self.assertFalse([
requirement for requirement in requirements
if requirement['namespace'] == 'proctored_exam'
])
def _create_practice_exam(self):
"""
Calls the api's create_exam to create a practice exam object.
"""
return create_exam(course_id=self.course_id,
content_id=self.content_id_practice,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_practice_exam=True,
is_proctored=True)
def _create_disabled_exam(self):
"""
Calls the api's create_exam to create an exam object.
"""
return create_exam(course_id=self.course_id,
is_proctored=False,
content_id=self.disabled_content_id,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_active=False)
def _create_timed_exam(self):
"""
Calls the api's create_exam to create an exam object.
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_timed,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_proctored=False
)
def test_review_dashboard(self):
"""
The exam review dashboard will appear for backends that support the feature
"""
self.setup_course(True, True)
response = self.client.get(self.url)
# the default backend does not support the review dashboard
self.assertNotIn('Review Dashboard', response.content)
backend = TestBackendProvider()
config = apps.get_app_config('edx_proctoring')
with patch.object(config, 'backends', {'test': backend}):
create_exam(
course_id=self.course.id,
content_id='test_content',
exam_name='Final Test Exam',
time_limit_mins=10,
backend='test',
)
response = self.client.get(self.url)
self.assertIn('Review Dashboard', response.content)
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(
exam_id,
self.user.id,
taking_as_proctored=True
)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id']
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_failure_submission(self, allow_rejects):
"""
Tests that a submission of a failed test and make sure that we
don't automatically update the status to failure
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
test_payload = test_payload.replace('Clean', 'Suspicious')
# submit a Suspicious review payload
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertNotEqual(attempt['status'],
ProctoredExamStudentAttemptStatus.rejected)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status (as well as trigger)
# other workflow
provider.on_review_saved(review, allow_rejects=allow_rejects)
attempt = get_exam_attempt_by_id(attempt_id)
# if we don't allow rejects to be stored in attempt status
# then we should expect a 'second_review_required' status
expected_status = (
ProctoredExamStudentAttemptStatus.rejected if allow_rejects else
ProctoredExamStudentAttemptStatus.second_review_required)
self.assertEqual(attempt['status'], expected_status)
def test_review_dashboard(self):
"""
The exam review dashboard will appear for backends that support the feature
"""
self.setup_course(True, True)
response = self.client.get(self.url)
# the default backend does not support the review dashboard
self.assertNotIn('Review Dashboard', response.content)
backend = TestBackendProvider()
config = apps.get_app_config('edx_proctoring')
with patch.object(config, 'backends', {'test': backend}):
create_exam(
course_id=self.course.id,
content_id='test_content',
exam_name='Final Test Exam',
time_limit_mins=10,
backend='test',
)
response = self.client.get(self.url)
self.assertIn('Review Dashboard', response.content)
def test_attempt_with_review_policy(self, review_policy_exception):
"""
Create an unstarted proctoring attempt with a review policy associated with it.
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
if review_policy_exception:
add_allowance_for_user(
exam_id, self.user.id, ProctoredExamStudentAllowance.REVIEW_POLICY_EXCEPTION, review_policy_exception
)
policy = ProctoredExamReviewPolicy.objects.create(
set_by_user_id=self.user.id, proctored_exam_id=exam_id, review_policy="Foo Policy"
)
def assert_get_payload_mock(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertIn("review_policy", context)
self.assertEqual(policy.review_policy, context["review_policy"])
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(exam, context)
# assert that this is in the 'reviewerNotes' field that is passed to SoftwareSecure
expected = context["review_policy"]
if review_policy_exception:
expected = "{base}; {exception}".format(base=expected, exception=review_policy_exception)
self.assertEqual(result["reviewerNotes"], expected)
return result
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# as well as asserting that _get_payload includes that review policy
# that was passed in
with patch.object(get_backend_provider(), "_get_payload", assert_get_payload_mock):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded the policy id at the time this was created
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt["review_policy_id"], policy.id)
def test_update_archived_attempt(self):
"""
Test calling the on_review_saved interface point with an attempt_code that was archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = create_test_review_payload(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now process the report
provider.on_review_callback(json.loads(test_payload))
# now look at the attempt and make sure it did not
# transition to failure on the callback,
# as we'll need a manual confirmation via Django Admin pages
attempt = get_exam_attempt_by_id(attempt_id)
self.assertEqual(attempt['status'], attempt['status'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
review = ProctoredExamSoftwareSecureReview.objects.get(
attempt_code=attempt['attempt_code'])
# now simulate a update via Django Admin table which will actually
# push through the failure into our attempt status but
# as this is an archived attempt, we don't do anything
provider.on_review_saved(review, allow_rejects=True)
# look at the attempt again, since it moved into Archived state
# then it should still remain unchanged
archived_attempt = ProctoredExamStudentAttemptHistory.objects.filter(
attempt_code=attempt['attempt_code']).latest('created')
self.assertEqual(archived_attempt.status, attempt['status'])
def _create_exam_with_due_time(self, is_proctored=True, is_practice_exam=False, due_date=None):
"""
Calls the api's create_exam to create an exam object.
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_for_exam_with_due_date,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_proctored=is_proctored,
is_practice_exam=is_practice_exam,
due_date=due_date
)
def _create_onboarding_exam(self):
"""
Create an onboarding exam
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_onboarding,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_practice_exam=True,
is_proctored=True,
backend='test',
)
def _create_onboarding_exam(self):
"""
Create an onboarding exam
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_onboarding,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_practice_exam=True,
is_proctored=True,
backend='test',
)
def _create_exam_with_due_time(self, is_proctored=True, is_practice_exam=False, due_date=None):
"""
Calls the api's create_exam to create an exam object.
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_for_exam_with_due_date,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_proctored=is_proctored,
is_practice_exam=is_practice_exam,
due_date=due_date
)
def _create_practice_exam(self):
"""
Calls the api's create_exam to create a practice exam object.
"""
return create_exam(
course_id=self.course_id,
content_id=self.content_id_practice,
exam_name=self.exam_name,
time_limit_mins=self.default_time_limit,
is_practice_exam=True,
is_proctored=True,
backend='null',
)
def test_attempt_with_no_review_policy(self):
"""
Create an unstarted proctoring attempt with no review policy associated with it.
"""
def assert_get_payload_mock_no_policy(exam, context):
"""
Add a mock shim so we can assert that the _get_payload has been called with the right
review policy
"""
self.assertNotIn("review_policy", context)
# call into real implementation
result = get_backend_provider(emphemeral=True)._get_payload(
exam, context
) # pylint: disable=protected-access
# assert that we use the default that is defined in system configuration
self.assertEqual(result["reviewerNotes"], constants.DEFAULT_SOFTWARE_SECURE_REVIEW_POLICY)
# the check that if a colon was passed in for the exam name, then the colon was changed to
# a dash. This is because SoftwareSecure cannot handle a colon in the exam name
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
if illegal_char in exam["exam_name"]:
self.assertNotIn(illegal_char, result["examName"])
self.assertIn("_", result["examName"])
return result
for illegal_char in SOFTWARE_SECURE_INVALID_CHARS:
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content with {}".format(illegal_char),
exam_name="Sample Exam with {} character".format(illegal_char),
time_limit_mins=10,
is_proctored=True,
)
with HTTMock(mock_response_content):
# patch the _get_payload method on the backend provider
# so that we can assert that we are called with the review policy
# undefined and that we use the system default
with patch.object(
get_backend_provider(), "_get_payload", assert_get_payload_mock_no_policy
): # pylint: disable=protected-access
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
self.assertGreater(attempt_id, 0)
# make sure we recorded that there is no review policy
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNone(attempt["review_policy_id"])
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt['external_id'])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt['attempt_code'],
external_id=attempt['external_id'])
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(
attempt['attempt_code'])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, 'Clean')
self.assertEqual(
review.video_url,
'http://www.remoteproctor.com/AdminSite/Account/Reviewer/DirectLink-Generic.aspx?ID=foo'
)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(
review_id=review.id)
self.assertEqual(len(comments), 6)
def test_review_callback(self, review_status, credit_requirement_status):
"""
Simulates callbacks from SoftwareSecure with various statuses
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
test_payload = test_payload.replace("Clean", review_status)
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, review_status)
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
self.assertIsNone(review.reviewed_by)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
# check that we got credit requirement set appropriately
credit_service = get_runtime_service("credit")
credit_status = credit_service.get_credit_state(self.user.id, "foo/bar/baz")
self.assertEqual(credit_status["credit_requirement_status"][0]["status"], credit_requirement_status)
def test_credit_requirement_blocks_ordering(self):
"""
Test ordering of proctoring blocks.
"""
self.add_credit_course(self.course.id)
subsection = ItemFactory.create(parent=self.section,
category='sequential',
display_name='Dummy Subsection')
create_exam(course_id=six.text_type(self.course.id),
content_id=six.text_type(subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'],
six.text_type(subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
# Primary sort is based on start date
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
# grade requirement is added on publish of the requirements
self.assertEqual(len(requirements), 2)
# check requirements are added in the desired order
# 1st Minimum grade then the blocks with start date than other blocks
self.assertEqual(requirements[0]["display_name"], "Minimum Grade")
self.assertEqual(requirements[1]["display_name"], "A Proctored Exam")
def test_credit_requirement_blocks_ordering(self):
"""
Test ordering of proctoring blocks.
"""
self.add_credit_course(self.course.id)
subsection = ItemFactory.create(parent=self.section, category='sequential', display_name='Dummy Subsection')
create_exam(
course_id=six.text_type(self.course.id),
content_id=six.text_type(subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 0)
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
self.assertEqual(len(requirements), 2)
self.assertEqual(requirements[1]['namespace'], 'proctored_exam')
self.assertEqual(requirements[1]['name'], six.text_type(subsection.location))
self.assertEqual(requirements[1]['display_name'], 'A Proctored Exam')
self.assertEqual(requirements[1]['criteria'], {})
# Primary sort is based on start date
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
# grade requirement is added on publish of the requirements
self.assertEqual(len(requirements), 2)
# check requirements are added in the desired order
# 1st Minimum grade then the blocks with start date than other blocks
self.assertEqual(requirements[0]["display_name"], "Minimum Grade")
self.assertEqual(requirements[1]["display_name"], "A Proctored Exam")
def test_credit_requirement_blocks_ordering(self):
"""
Test ordering of proctoring blocks.
"""
self.add_credit_course(self.course.id)
subsection = ItemFactory.create(parent=self.section, category='sequential', display_name='Dummy Subsection')
create_exam(
course_id=str(self.course.id),
content_id=str(subsection.location),
exam_name='A Proctored Exam',
time_limit_mins=10,
is_proctored=True,
is_active=True
)
requirements = get_credit_requirements(self.course.id)
assert len(requirements) == 0
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
assert len(requirements) == 2
assert requirements[1]['namespace'] == 'proctored_exam'
assert requirements[1]['name'] == str(subsection.location)
assert requirements[1]['display_name'] == 'A Proctored Exam'
assert requirements[1]['criteria'] == {}
# Primary sort is based on start date
on_course_publish(self.course.id)
requirements = get_credit_requirements(self.course.id)
# grade requirement is added on publish of the requirements
assert len(requirements) == 2
# check requirements are added in the desired order
# 1st Minimum grade then the blocks with start date than other blocks
assert requirements[0]['display_name'] == 'Minimum Grade'
assert requirements[1]['display_name'] == 'A Proctored Exam'
def test_review_on_archived_attempt(self):
"""
Make sure we can process a review report for
an attempt which has been archived
"""
provider = get_backend_provider()
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# be sure to use the mocked out SoftwareSecure handlers
with HTTMock(mock_response_content):
attempt_id = create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
attempt = get_exam_attempt_by_id(attempt_id)
self.assertIsNotNone(attempt["external_id"])
test_payload = Template(TEST_REVIEW_PAYLOAD).substitute(
attempt_code=attempt["attempt_code"], external_id=attempt["external_id"]
)
# now delete the attempt, which puts it into the archive table
remove_exam_attempt(attempt_id, requesting_user=self.user)
# now process the report
provider.on_review_callback(json.loads(test_payload))
# make sure that what we have in the Database matches what we expect
review = ProctoredExamSoftwareSecureReview.get_review_by_attempt_code(attempt["attempt_code"])
self.assertIsNotNone(review)
self.assertEqual(review.review_status, "Clean")
self.assertFalse(review.video_url)
self.assertIsNotNone(review.raw_data)
# now check the comments that were stored
comments = ProctoredExamSoftwareSecureComment.objects.filter(review_id=review.id)
self.assertEqual(len(comments), 6)
def test_failing_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(course_id='foo/bar/baz',
content_id='content',
exam_name='Sample Exam',
time_limit_mins=10,
is_proctored=True)
# now try a failing request
with HTTMock(mock_response_error):
with self.assertRaises(BackendProvideCannotRegisterAttempt):
create_exam_attempt(exam_id,
self.user.id,
taking_as_proctored=True)
def test_failing_register_attempt(self):
"""
Makes sure we can register an attempt
"""
exam_id = create_exam(
course_id="foo/bar/baz",
content_id="content",
exam_name="Sample Exam",
time_limit_mins=10,
is_proctored=True,
)
# now try a failing request
with HTTMock(mock_response_error):
with self.assertRaises(BackendProvideCannotRegisterAttempt):
create_exam_attempt(exam_id, self.user.id, taking_as_proctored=True)
