def setUp(self):
"""
Visits courseware_page and defines self.problem_page.
"""
super(ProblemTypeTestBase, self).setUp()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def setUp(self):
super(ProblemStateOnNavigationTest, self).setUp()
self.courseware_page = CoursewarePage(self.browser, self.course_id)
# Install a course with section, tabs and multiple choice problems.
course_fix = CourseFixture(self.course_info['org'],
self.course_info['number'],
self.course_info['run'],
self.course_info['display_name'])
course_fix.add_children(
XBlockFixtureDesc('chapter', 'Test Section 1').add_children(
XBlockFixtureDesc('sequential',
'Test Subsection 1,1').add_children(
self.create_multiple_choice_problem(
self.problem1_name),
self.create_multiple_choice_problem(
self.problem2_name),
), ), ).install()
# Auto-auth register for the course.
AutoAuthPage(self.browser,
username=self.USERNAME,
email=self.EMAIL,
course_id=self.course_id,
staff=False).visit()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def setUp(self):
"""
Visits courseware_page and defines self.problem_page.
"""
super(ProblemTypeTestBase, self).setUp() # lint-amnesty, pylint: disable=super-with-arguments
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def test_partial_credit(self):
"""
Test that we can see the partial credit value and feedback.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.wait_for_element_visibility(problem_page.CSS_PROBLEM_HEADER, 'wait for problem header')
self.assertEqual(problem_page.problem_name, 'PARTIAL CREDIT TEST PROBLEM')
problem_page.fill_answer_numerical('-1')
problem_page.click_check()
problem_page.wait_for_status_icon()
self.assertTrue(problem_page.simpleprob_is_partially_correct())
def test_conditional_displays_content(self):
self.install_course_fixture()
self.courseware_page.visit()
# Answer the problem
problem_page = ProblemPage(self.browser)
problem_page.fill_answer('correct string')
problem_page.click_submit()
# The conditional does not update on its own, so we need to reload the page.
self.courseware_page.visit()
# Verify that we can see the content.
conditional_page = ConditionalPage(self.browser)
self.assertTrue(conditional_page.is_content_visible())
def test_question_with_description(self):
"""
Scenario: Test that question and description are rendered as expected.
Given I am enrolled in a course.
When I visit a unit page with a CAPA question.
Then label and description should be rendered correctly.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.wait_for_element_visibility(problem_page.CSS_PROBLEM_HEADER, 'wait for problem header')
self.assertEqual(problem_page.problem_name, 'Label with Description')
self.assertEqual(problem_page.problem_question, 'Eggplant is a _____?')
self.assertEqual(problem_page.problem_question_descriptions, self.descriptions)
def test_clarification(self):
"""
Test that we can see the <clarification> tooltips.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TOOLTIP TEST PROBLEM')
problem_page.click_clarification(0)
self.assertIn('"Example PV Installation Costs"', problem_page.visible_tooltip_text)
problem_page.click_clarification(1)
tooltip_text = problem_page.visible_tooltip_text
self.assertIn('Return on Investment', tooltip_text)
self.assertIn('per year', tooltip_text)
self.assertNotIn('strong', tooltip_text)
def test_partial_credit(self):
"""
Test that we can see the partial credit value and feedback.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'PARTIAL CREDIT TEST PROBLEM')
problem_page.fill_answer_numerical('-1')
problem_page.click_submit()
problem_page.wait_for_status_icon()
self.assertTrue(problem_page.simpleprob_is_partially_correct())
def test_question_with_description(self):
"""
Scenario: Test that question and description are rendered as expected.
Given I am enrolled in a course.
When I visit a unit page with a CAPA question.
Then label and description should be rendered correctly.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.wait_for_element_visibility(
problem_page.CSS_PROBLEM_HEADER, 'wait for problem header')
self.assertEqual(problem_page.problem_name, 'Label with Description')
self.assertEqual(problem_page.problem_question, 'Eggplant is a _____?')
self.assertEqual(problem_page.problem_question_descriptions,
self.descriptions)
def setUp(self):
"""
Visits courseware_page and defines self.problem_page.
"""
super(ProblemTypeTestBase, self).setUp()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def verify_check_hint(self, answer, answer_text, expected_events):
"""
Verify clicking Check shows the extended hint in the problem message.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_text[0], u'question text')
problem_page.fill_answer(answer)
problem_page.click_submit()
self.assertEqual(problem_page.message_text, answer_text)
# Check for corresponding tracking event
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.feedback_displayed'},
number_of_matches=1
)
self.assert_events_match(expected_events, actual_events)
def _goto_problem_page(self):
"""
Open problem page with assertion.
"""
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser) # pylint: disable=attribute-defined-outside-init
self.assertEqual(self.problem_page.problem_name, 'Test Problem 1')
def setUp(self):
super(ProblemStateOnNavigationTest, self).setUp()
self.courseware_page = CoursewarePage(self.browser, self.course_id)
# Install a course with section, tabs and multiple choice problems.
course_fix = CourseFixture(
self.course_info['org'], self.course_info['number'],
self.course_info['run'], self.course_info['display_name']
)
course_fix.add_children(
XBlockFixtureDesc('chapter', 'Test Section 1').add_children(
XBlockFixtureDesc('sequential', 'Test Subsection 1,1').add_children(
self.create_multiple_choice_problem(self.problem1_name),
self.create_multiple_choice_problem(self.problem2_name),
),
),
).install()
# Auto-auth register for the course.
AutoAuthPage(
self.browser, username=self.USERNAME, email=self.EMAIL,
course_id=self.course_id, staff=False
).visit()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def test_reset_button_not_rendered_after_correct_submission(self):
"""
Scenario: Verify that formula problem can not be resetted after an incorrect submission.
Given I am attempting a formula response problem type
When I input a correct answer
Then I should be able to see the mathjax generated preview
When I submit the answer
Then the correct status is visible
And reset button is not rendered
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical('R_1*R_2/R_3')
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
self.assertFalse(problem_page.is_reset_button_present())
def test_demand_hint(self):
"""
Test clicking hint button shows the demand hint in its div.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# The hint button rotates through multiple hints
problem_page.click_hint()
self.assertEqual(problem_page.hint_text, u'Hint (1 of 2): aa bb cc')
problem_page.click_hint()
self.assertEqual(problem_page.hint_text, u'Hint (2 of 2): dd ee ff')
problem_page.click_hint()
self.assertEqual(problem_page.hint_text, u'Hint (1 of 2): aa bb cc')
# Check corresponding tracking events
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.demandhint_displayed'},
number_of_matches=3
)
self.assert_events_match(
[
{'event': {u'hint_index': 0, u'hint_len': 2, u'hint_text': u'aa <a href="#">bb</a> cc'}},
{'event': {u'hint_index': 1, u'hint_len': 2, u'hint_text': u'<a href="#">dd ee</a> ff'}},
{'event': {u'hint_index': 0, u'hint_len': 2, u'hint_text': u'aa <a href="#">bb</a> cc'}}
],
actual_events)
def _fulfill_prerequisite(self):
"""
Fulfill the prerequisite needed to see gated content
"""
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.wait_for_page().problem_name, 'HEIGHT OF EIFFEL TOWER')
problem_page.click_choice('choice_1')
problem_page.click_submit()
def test_past_due(self):
"""
Verifies that the Submit button disables when the max number of attempts is reached.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# Should have Submit button disabled on original rendering.
problem_page.wait_for_submit_disabled()
# Select a choice, and make sure that the Submit button remains disabled.
problem_page.click_choice("choice_2")
problem_page.wait_for_submit_disabled()
def test_reset_button_not_rendered_after_correct_submission(self):
"""
Scenario: Verify that formula problem can not be resetted after an incorrect submission.
Given I am attempting a formula response problem type
When I input a correct answer
Then I should be able to see the mathjax generated preview
When I submit the answer
Then the correct status is visible
And reset button is not rendered
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical('R_1*R_2/R_3')
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
self.assertFalse(problem_page.is_reset_button_present())
def test_check_hint(self):
"""
Test clicking Check shows the extended hint in the problem message.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_text[0], u'question text')
problem_page.fill_answer('C')
problem_page.click_check()
self.assertEqual(problem_page.message_text, u'Incorrect: aa bb cc')
# Check for corresponding tracking event
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.feedback_displayed'},
number_of_matches=1)
self.assert_events_match([{
'event': {
'hint_label': u'Incorrect',
'trigger_type': 'single',
'student_answer': [u'C'],
'correctness': False,
'question_type': 'stringresponse',
'hints': [{
'text': '<a href="#">aa bb</a> cc'
}]
}
}], actual_events)
def test_masquerade_and_switch_course(self):
"""
Scenario:
Staff user should be able to access other courses after
masquerading as student in one course
As Staff user, Select a course
When I click to change view from Staff to Learner
Then the first subsection from course outline should be visible as Learner
When I click to select a different course
Then the first subsection from new course outline should be visible as Staff
"""
AutoAuthPage(
self.browser,
username=self.username,
email=self.email,
staff=True
).visit()
self.dashboard_page.visit()
section_title = 'Test Section 1'
subsection_title = 'Test Subsection 1,1'
course_page = CourseHomePage(self.browser, str(self.course_keys['A']))
course_page.visit()
problem_name = u'Test Problem 1'
staff_page = StaffPreviewPage(self.browser)
staff_page.set_staff_view_mode('Learner')
course_page.outline.go_to_section(section_title, subsection_title)
self.assertEqual(staff_page.staff_view_mode, 'Learner')
self.assertEqual(ProblemPage(self.browser).problem_name, problem_name)
course_page.course_id = str(self.course_keys['B'])
course_page.visit()
course_page.outline.go_to_section(section_title, subsection_title)
self.assertNotEqual(staff_page.staff_view_mode, 'Learner')
self.assertEqual(ProblemPage(self.browser).problem_name, problem_name)
def test_a11y(self):
"""
Verifies that there are no accessibility issues for a particular problem type
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# Set the scope to the problem question
problem_page.a11y_audit.config.set_scope(
include=['.wrapper-problem-response'])
# Run the accessibility audit.
problem_page.a11y_audit.check_for_accessibility_errors()
def setUp(self):
super(ProctoredExamTest, self).setUp()
self.courseware_page = CoursewarePage(self.browser, self.course_id)
self.course_outline = CourseOutlinePage(self.browser,
self.course_info['org'],
self.course_info['number'],
self.course_info['run'])
# Install a course with sections/problems, tabs, updates, and handouts
course_fix = CourseFixture(self.course_info['org'],
self.course_info['number'],
self.course_info['run'],
self.course_info['display_name'])
course_fix.add_advanced_settings(
{"enable_proctored_exams": {
"value": "true"
}})
course_fix.add_children(
XBlockFixtureDesc('chapter', 'Test Section 1').add_children(
XBlockFixtureDesc('sequential',
'Test Subsection 1').add_children(
XBlockFixtureDesc(
'problem',
'Test Problem 1')))).install()
self.track_selection_page = TrackSelectionPage(self.browser,
self.course_id)
self.payment_and_verification_flow = PaymentAndVerificationFlow(
self.browser, self.course_id)
self.immediate_verification_page = PaymentAndVerificationFlow(
self.browser, self.course_id, entry_point='verify-now')
self.upgrade_page = PaymentAndVerificationFlow(self.browser,
self.course_id,
entry_point='upgrade')
self.fake_payment_page = FakePaymentPage(self.browser, self.course_id)
self.dashboard_page = DashboardPage(self.browser)
self.problem_page = ProblemPage(self.browser)
# Add a verified mode to the course
ModeCreationPage(self.browser,
self.course_id,
mode_slug=u'verified',
mode_display_name=u'Verified Certificate',
min_price=10,
suggested_prices='10,20').visit()
# Auto-auth register for the course.
self._auto_auth(self.USERNAME, self.EMAIL, False)
def _fulfill_prerequisite(self):
"""
Fulfill the prerequisite needed to see gated content
"""
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.wait_for_page().problem_name, 'HEIGHT OF EIFFEL TOWER')
problem_page.click_choice('choice_1')
problem_page.click_submit()
def test_past_due(self):
"""
Verifies that the Submit button disables when the max number of attempts is reached.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# Should have Submit button disabled on original rendering.
problem_page.wait_for_submit_disabled()
# Select a choice, and make sure that the Submit button remains disabled.
problem_page.click_choice("choice_2")
problem_page.wait_for_submit_disabled()
def test_check_hint(self):
"""
Test clicking Check shows the extended hint in the problem message.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_text[0], u'question text')
problem_page.fill_answer('C')
problem_page.click_check()
self.assertEqual(problem_page.message_text, u'Incorrect: aa bb cc')
# Check for corresponding tracking event
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.feedback_displayed'},
number_of_matches=1
)
self.assert_events_match(
[{'event': {'hint_label': u'Incorrect',
'trigger_type': 'single',
'student_answer': [u'C'],
'correctness': False,
'question_type': 'stringresponse',
'hints': [{'text': '<a href="#">aa bb</a> cc'}]}}],
actual_events)
def test_logout_cancel_no_redirect(self):
"""
1) User goes to a problem page.
2) User fills out an answer to the problem.
3) User is logged out because their session id is invalidated or removed.
4) User clicks "check", and sees a confirmation modal asking them to
re-authenticate, since they've just been logged out.
5) User clicks "cancel".
6) User is not redirected to the login page.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
self.log_user_out()
with problem_page.handle_alert(confirm=False):
problem_page.click_submit()
problem_page.wait_for_page()
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
def test_conditional_displays_content(self):
self.install_course_fixture()
self.courseware_page.visit()
# Answer the problem
problem_page = ProblemPage(self.browser)
problem_page.fill_answer('correct string')
problem_page.click_submit()
# The conditional does not update on its own, so we need to reload the page.
self.courseware_page.visit()
# Verify that we can see the content.
conditional_page = ConditionalPage(self.browser)
self.assertTrue(conditional_page.is_content_visible())
def test_logout_cancel_no_redirect(self):
"""
1) User goes to a problem page.
2) User fills out an answer to the problem.
3) User is logged out because their session id is invalidated or removed.
4) User clicks "check", and sees a confirmation modal asking them to
re-authenticate, since they've just been logged out.
5) User clicks "cancel".
6) User is not redirected to the login page.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
self.log_user_out()
with problem_page.handle_alert(confirm=False):
problem_page.click_submit()
problem_page.wait_for_page()
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
def test_demand_hint(self):
"""
Test clicking hint button shows the demand hint in its div.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# The hint button rotates through multiple hints
problem_page.click_hint()
self.assertEqual(problem_page.hint_text,
u'Hint (1 of 2): demand-hint1')
problem_page.click_hint()
self.assertEqual(problem_page.hint_text,
u'Hint (2 of 2): demand-hint2')
problem_page.click_hint()
self.assertEqual(problem_page.hint_text,
u'Hint (1 of 2): demand-hint1')
# Check corresponding tracking events
actual_events = self.wait_for_events(event_filter={
'event_type':
'edx.problem.hint.demandhint_displayed'
},
number_of_matches=3)
self.assert_events_match([{
'event': {
u'hint_index': 0,
u'hint_len': 2,
u'hint_text': u'demand-hint1'
}
}, {
'event': {
u'hint_index': 1,
u'hint_len': 2,
u'hint_text': u'demand-hint2'
}
}, {
'event': {
u'hint_index': 0,
u'hint_len': 2,
u'hint_text': u'demand-hint1'
}
}], actual_events)
def test_clarification(self):
"""
Test that we can see the <clarification> tooltips.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TOOLTIP TEST PROBLEM')
problem_page.click_clarification(0)
self.assertIn('"Example PV Installation Costs"', problem_page.visible_tooltip_text)
problem_page.click_clarification(1)
tooltip_text = problem_page.visible_tooltip_text
self.assertIn('Return on Investment', tooltip_text)
self.assertIn('per year', tooltip_text)
self.assertNotIn('strong', tooltip_text)
def verify_check_hint(self, answer, answer_text, expected_events):
"""
Verify clicking Check shows the extended hint in the problem message.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_text[0], u'question text')
problem_page.fill_answer(answer)
problem_page.click_submit()
self.assertEqual(problem_page.message_text, answer_text)
# Check for corresponding tracking event
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.feedback_displayed'},
number_of_matches=1)
self.assert_events_match(expected_events, actual_events)
def test_course_is_unblocked_as_soon_as_student_passes_entrance_exam(self):
"""
Scenario: Ensure that entrance exam status message is updated and courseware is unblocked as soon as
student passes entrance exam.
Given I have a course with entrance exam as pre-requisite
When I pass entrance exam
Then I can see complete TOC of course
And I can see message indicating my pass status
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.wait_for_page().problem_name, 'HEIGHT OF EIFFEL TOWER')
self.assertTrue(self.courseware_page.has_entrance_exam_message())
self.assertFalse(self.courseware_page.has_passed_message())
problem_page.click_choice('choice_1')
problem_page.click_check()
self.courseware_page.wait_for_page()
self.assertTrue(self.courseware_page.has_passed_message())
self.assertEqual(self.courseware_page.chapter_count_in_navigation, 2)
def test_notification_updates(self):
"""
Verifies that the notification is removed and not visible when it should be
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_2")
self.assertFalse(problem_page.is_success_notification_visible())
problem_page.click_submit()
problem_page.wait_success_notification()
self.assertEqual('Question 1: correct', problem_page.status_sr_text)
# Clicking Save should clear the submit notification
problem_page.click_save()
self.assertFalse(problem_page.is_success_notification_visible())
problem_page.wait_for_save_notification()
# Changing the answer should clear the save notification
problem_page.click_choice("choice_1")
self.assertFalse(problem_page.is_save_notification_visible())
problem_page.click_save()
problem_page.wait_for_save_notification()
# Submitting the problem again should clear the save notification
problem_page.click_submit()
problem_page.wait_incorrect_notification()
self.assertEqual('Question 1: incorrect', problem_page.status_sr_text)
self.assertFalse(problem_page.is_save_notification_visible())
def verify_demand_hints(self, first_hint, second_hint, expected_events):
"""
Test clicking through the demand hints and verify the events sent.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# The hint notification should not be visible on load
self.assertFalse(problem_page.is_hint_notification_visible())
# The two Hint button should be enabled. One visible, one present, but not visible in the DOM
self.assertEqual([None, None],
problem_page.get_hint_button_disabled_attr())
# The hint button rotates through multiple hints
problem_page.click_hint(hint_index=0)
self.assertTrue(problem_page.is_hint_notification_visible())
self.assertEqual(problem_page.hint_text, first_hint)
# Now there are two "hint" buttons, as there is also one in the hint notification.
self.assertEqual([None, None],
problem_page.get_hint_button_disabled_attr())
problem_page.click_hint(hint_index=1)
self.assertEqual(problem_page.hint_text, second_hint)
# Now both "hint" buttons should be disabled, as there are no more hints.
self.assertEqual(['true', 'true'],
problem_page.get_hint_button_disabled_attr())
# Now click on "Review" and make sure the focus goes to the correct place.
problem_page.click_review_in_notification(notification_type='hint')
problem_page.wait_for_focus_on_problem_meta()
# Check corresponding tracking events
actual_events = self.wait_for_events(event_filter={
'event_type':
'edx.problem.hint.demandhint_displayed'
},
number_of_matches=2)
self.assert_events_match(expected_events, actual_events)
def test_reset_correctness_after_changing_answer(self, input_value,
correctness, next_input):
"""
Scenario: Test that formula problem can be resetted after changing the answer.
Given I am attempting a formula problem type with randomization:always configuration
When I input an answer
Then the mathjax generated preview should be visible
When I submit the problem, I can see the correctness status
When I only input another answer
Then the correctness status is no longer visible
And I am able to see the reset button
And when I click the reset button
Then input pane contents are cleared
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical(input_value)
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertEqual(problem_page.get_simpleprob_correctness(),
correctness)
problem_page.fill_answer_numerical(next_input)
self.assertIsNone(problem_page.get_simpleprob_correctness())
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertEqual(problem_page.get_numerical_input_value, '')
def test_status_removed_after_save_before_submit(self):
"""
Scenario: User should see the status removed when saving after submitting an answer and reloading the page.
Given that I have loaded the problem page
And a choice has been selected and submitted
When I change the choice
And Save the problem
And reload the problem page
Then I should see the save notification and I should not see any indication of problem status
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_1")
problem_page.click_submit()
problem_page.wait_incorrect_notification()
problem_page.wait_for_expected_status('label.choicegroup_incorrect', 'incorrect')
problem_page.click_choice("choice_2")
self.assertFalse(problem_page.is_expected_status_visible('label.choicegroup_incorrect'))
problem_page.click_save()
problem_page.wait_for_save_notification()
# Refresh the page and the status should not be added
self.courseware_page.visit()
self.assertFalse(problem_page.is_expected_status_visible('label.choicegroup_incorrect'))
self.assertTrue(problem_page.is_save_notification_visible())
def test_max_attempts(self):
"""
Verifies that the Submit button disables when the max number of attempts is reached.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# Submit first answer (correct)
problem_page.click_choice("choice_2")
self.assertFalse(problem_page.is_submit_disabled())
problem_page.click_submit()
problem_page.wait_success_notification()
# Submit second and final answer (incorrect)
problem_page.click_choice("choice_1")
problem_page.click_submit()
problem_page.wait_incorrect_notification()
# Make sure that the Submit button disables.
problem_page.wait_for_submit_disabled()
def test_reset_problem_after_changing_correctness(self):
"""
Scenario: Verify that formula problem can be resetted after changing the correctness.
Given I am attempting a formula problem type
When I answer it correctly
Then the correctness status should be visible
And reset button is not rendered
When I change my submission to incorrect
Then the reset button appears and is clickable
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical('R_1*R_2/R_3')
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
self.assertFalse(problem_page.is_reset_button_present())
problem_page.fill_answer_numerical('R_1/R_3')
problem_page.click_submit()
self.assertFalse(problem_page.simpleprob_is_correct())
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertEqual(problem_page.get_numerical_input_value, '')
def test_mathjax_in_hint(self):
"""
Test that MathJax have successfully rendered in problem hint
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, "MATHJAX TEST PROBLEM")
problem_page.verify_mathjax_rendered_in_problem()
# The hint button rotates through multiple hints
problem_page.click_hint(hint_index=0)
self.assertEqual(
["<strong>Hint (1 of 2): </strong>mathjax should work1"],
problem_page.extract_hint_text_from_html)
problem_page.verify_mathjax_rendered_in_hint()
# Rotate the hint and check the problem hint
problem_page.click_hint(hint_index=1)
self.assertEqual([
"<strong>Hint (1 of 2): </strong>mathjax should work1",
"<strong>Hint (2 of 2): </strong>mathjax should work2"
], problem_page.extract_hint_text_from_html)
problem_page.verify_mathjax_rendered_in_hint()
def verify_demand_hints(self, first_hint, second_hint, expected_events):
"""
Test clicking through the demand hints and verify the events sent.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# The hint notification should not be visible on load
self.assertFalse(problem_page.is_hint_notification_visible())
# The two Hint button should be enabled. One visible, one present, but not visible in the DOM
self.assertEqual([None, None], problem_page.get_hint_button_disabled_attr())
# The hint button rotates through multiple hints
problem_page.click_hint(hint_index=0)
self.assertTrue(problem_page.is_hint_notification_visible())
self.assertEqual(problem_page.hint_text, first_hint)
# Now there are two "hint" buttons, as there is also one in the hint notification.
self.assertEqual([None, None], problem_page.get_hint_button_disabled_attr())
problem_page.click_hint(hint_index=1)
self.assertEqual(problem_page.hint_text, second_hint)
# Now both "hint" buttons should be disabled, as there are no more hints.
self.assertEqual(['true', 'true'], problem_page.get_hint_button_disabled_attr())
# Now click on "Review" and make sure the focus goes to the correct place.
problem_page.click_review_in_notification(notification_type='hint')
problem_page.wait_for_focus_on_problem_meta()
# Check corresponding tracking events
actual_events = self.wait_for_events(
event_filter={'event_type': 'edx.problem.hint.demandhint_displayed'},
number_of_matches=2
)
self.assert_events_match(expected_events, actual_events)
def test_logout_after_click_redirect(self):
"""
1) User goes to a problem page.
2) User fills out an answer to the problem.
3) User is logged out because their session id is invalidated or removed.
4) User clicks "check", and sees a confirmation modal asking them to
re-authenticate, since they've just been logged out.
5) User clicks "ok".
6) User is redirected to the login page.
7) User logs in.
8) User is redirected back to the problem page they started out on.
9) User is able to submit an answer
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
self.log_user_out()
with problem_page.handle_alert(confirm=True):
problem_page.click_submit()
login_page = CombinedLoginAndRegisterPage(self.browser)
login_page.wait_for_page()
login_page.login(self.email, self.password)
problem_page.wait_for_page()
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
def test_mathjax_in_hint(self):
"""
Test that MathJax have successfully rendered in problem hint
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, "MATHJAX TEST PROBLEM")
problem_page.verify_mathjax_rendered_in_problem()
# The hint button rotates through multiple hints
problem_page.click_hint(hint_index=0)
self.assertEqual(
["<strong>Hint (1 of 2): </strong>mathjax should work1"],
problem_page.extract_hint_text_from_html
)
problem_page.verify_mathjax_rendered_in_hint()
# Rotate the hint and check the problem hint
problem_page.click_hint(hint_index=1)
self.assertEqual(
[
"<strong>Hint (1 of 2): </strong>mathjax should work1",
"<strong>Hint (2 of 2): </strong>mathjax should work2"
],
problem_page.extract_hint_text_from_html
)
problem_page.verify_mathjax_rendered_in_hint()
def test_reset_correctness_after_changing_answer(self, input_value, correctness, next_input):
"""
Scenario: Test that formula problem can be resetted after changing the answer.
Given I am attempting a formula problem type with randomization:always configuration
When I input an answer
Then the mathjax generated preview should be visible
When I submit the problem, I can see the correctness status
When I only input another answer
Then the correctness status is no longer visible
And I am able to see the reset button
And when I click the reset button
Then input pane contents are cleared
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical(input_value)
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertEqual(problem_page.get_simpleprob_correctness(), correctness)
problem_page.fill_answer_numerical(next_input)
self.assertIsNone(problem_page.get_simpleprob_correctness())
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertEqual(problem_page.get_numerical_input_value, '')
def test_max_attempts(self):
"""
Verifies that the Submit button disables when the max number of attempts is reached.
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
# Submit first answer (correct)
problem_page.click_choice("choice_2")
self.assertFalse(problem_page.is_submit_disabled())
problem_page.click_submit()
problem_page.wait_success_notification()
# Submit second and final answer (incorrect)
problem_page.click_choice("choice_1")
problem_page.click_submit()
problem_page.wait_incorrect_notification()
# Make sure that the Submit button disables.
problem_page.wait_for_submit_disabled()
def test_feedback_notification_hides_after_save(self):
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_0")
problem_page.click_submit()
problem_page.wait_for_feedback_message_visibility()
problem_page.click_choice("choice_1")
problem_page.click_save()
self.assertFalse(
problem_page.is_feedback_message_notification_visible())
def test_status_removed_after_save_before_submit(self):
"""
Scenario: User should see the status removed when saving after submitting an answer and reloading the page.
Given that I have loaded the problem page
And a choice has been selected and submitted
When I change the choice
And Save the problem
And reload the problem page
Then I should see the save notification and I should not see any indication of problem status
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_1")
problem_page.click_submit()
problem_page.wait_incorrect_notification()
problem_page.wait_for_expected_status('label.choicegroup_incorrect',
'incorrect')
problem_page.click_choice("choice_2")
self.assertFalse(
problem_page.is_expected_status_visible(
'label.choicegroup_incorrect'))
problem_page.click_save()
problem_page.wait_for_save_notification()
# Refresh the page and the status should not be added
self.courseware_page.visit()
self.assertFalse(
problem_page.is_expected_status_visible(
'label.choicegroup_incorrect'))
self.assertTrue(problem_page.is_save_notification_visible())
class ProblemStateOnNavigationTest(UniqueCourseTest):
"""
Test courseware with problems in multiple verticals
"""
USERNAME = "******"
EMAIL = "*****@*****.**"
problem1_name = 'MULTIPLE CHOICE TEST PROBLEM 1'
problem2_name = 'MULTIPLE CHOICE TEST PROBLEM 2'
def setUp(self):
super(ProblemStateOnNavigationTest, self).setUp()
self.courseware_page = CoursewarePage(self.browser, self.course_id)
# Install a course with section, tabs and multiple choice problems.
course_fix = CourseFixture(
self.course_info['org'], self.course_info['number'],
self.course_info['run'], self.course_info['display_name']
)
course_fix.add_children(
XBlockFixtureDesc('chapter', 'Test Section 1').add_children(
XBlockFixtureDesc('sequential', 'Test Subsection 1,1').add_children(
self.create_multiple_choice_problem(self.problem1_name),
self.create_multiple_choice_problem(self.problem2_name),
),
),
).install()
# Auto-auth register for the course.
AutoAuthPage(
self.browser, username=self.USERNAME, email=self.EMAIL,
course_id=self.course_id, staff=False
).visit()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def create_multiple_choice_problem(self, problem_name):
"""
Return the Multiple Choice Problem Descriptor, given the name of the problem.
"""
factory = MultipleChoiceResponseXMLFactory()
xml_data = factory.build_xml(
question_text='The correct answer is Choice 2',
choices=[False, False, True, False],
choice_names=['choice_0', 'choice_1', 'choice_2', 'choice_3']
)
return XBlockFixtureDesc(
'problem',
problem_name,
data=xml_data,
metadata={'rerandomize': 'always'}
)
def go_to_tab_and_assert_problem(self, position, problem_name):
"""
Go to sequential tab and assert that we are on problem whose name is given as a parameter.
Args:
position: Position of the sequential tab
problem_name: Name of the problem
"""
self.courseware_page.go_to_sequential_position(position)
self.problem_page.wait_for_element_presence(
self.problem_page.CSS_PROBLEM_HEADER,
'wait for problem header'
)
self.assertEqual(self.problem_page.problem_name, problem_name)
def test_perform_problem_check_and_navigate(self):
"""
Scenario:
I go to sequential position 1
Facing problem1, I select 'choice_1'
Then I click check button
Then I go to sequential position 2
Then I came back to sequential position 1 again
Facing problem1, I observe the problem1 content is not
outdated before and after sequence navigation
"""
# Go to sequential position 1 and assert that we are on problem 1.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
# Update problem 1's content state by clicking check button.
self.problem_page.click_choice('choice_choice_1')
self.problem_page.click_check()
self.problem_page.wait_for_expected_status('label.choicegroup_incorrect', 'incorrect')
# Save problem 1's content state as we're about to switch units in the sequence.
problem1_content_before_switch = self.problem_page.problem_content
# Go to sequential position 2 and assert that we are on problem 2.
self.go_to_tab_and_assert_problem(2, self.problem2_name)
# Come back to our original unit in the sequence and assert that the content hasn't changed.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
problem1_content_after_coming_back = self.problem_page.problem_content
self.assertEqual(problem1_content_before_switch, problem1_content_after_coming_back)
def test_perform_problem_save_and_navigate(self):
"""
Scenario:
I go to sequential position 1
Facing problem1, I select 'choice_1'
Then I click save button
Then I go to sequential position 2
Then I came back to sequential position 1 again
Facing problem1, I observe the problem1 content is not
outdated before and after sequence navigation
"""
# Go to sequential position 1 and assert that we are on problem 1.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
# Update problem 1's content state by clicking save button.
self.problem_page.click_choice('choice_choice_1')
self.problem_page.click_save()
self.problem_page.wait_for_expected_status('div.capa_alert', 'saved')
# Save problem 1's content state as we're about to switch units in the sequence.
problem1_content_before_switch = self.problem_page.problem_content
# Go to sequential position 2 and assert that we are on problem 2.
self.go_to_tab_and_assert_problem(2, self.problem2_name)
# Come back to our original unit in the sequence and assert that the content hasn't changed.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
problem1_content_after_coming_back = self.problem_page.problem_content
self.assertIn(problem1_content_after_coming_back, problem1_content_before_switch)
def test_perform_problem_reset_and_navigate(self):
"""
Scenario:
I go to sequential position 1
Facing problem1, I select 'choice_1'
Then perform the action – check and reset
Then I go to sequential position 2
Then I came back to sequential position 1 again
Facing problem1, I observe the problem1 content is not
outdated before and after sequence navigation
"""
# Go to sequential position 1 and assert that we are on problem 1.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
# Update problem 1's content state – by performing reset operation.
self.problem_page.click_choice('choice_choice_1')
self.problem_page.click_check()
self.problem_page.wait_for_expected_status('label.choicegroup_incorrect', 'incorrect')
self.problem_page.click_reset()
self.problem_page.wait_for_expected_status('span.unanswered', 'unanswered')
# Save problem 1's content state as we're about to switch units in the sequence.
problem1_content_before_switch = self.problem_page.problem_content
# Go to sequential position 2 and assert that we are on problem 2.
self.go_to_tab_and_assert_problem(2, self.problem2_name)
# Come back to our original unit in the sequence and assert that the content hasn't changed.
self.go_to_tab_and_assert_problem(1, self.problem1_name)
problem1_content_after_coming_back = self.problem_page.problem_content
self.assertEqual(problem1_content_before_switch, problem1_content_after_coming_back)
def test_score_reset_after_resetting_problem(self, input_value, correctness, score_before_reset, score_after_reset):
"""
Scenario: Test that score resets after the formula problem is resetted.
Given I am attempting a formula problem type with randomization:always configuration
When I input the answer
Then I should be able to see the MathJax generated preview
When I submit the problem
Then I should be able to view the score that I received
And The reset button should be present and is clickable
When the reset button is clicked
Then the score resets to zero
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical(input_value)
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertEqual(problem_page.get_simpleprob_correctness(), correctness)
self.assertIn(score_before_reset, problem_page.problem_progress_graded_value)
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertIn(score_after_reset, problem_page.problem_progress_graded_value)
def test_feedback_notification_hides_after_save(self):
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_0")
problem_page.click_submit()
problem_page.wait_for_feedback_message_visibility()
problem_page.click_choice("choice_1")
problem_page.click_save()
self.assertFalse(problem_page.is_feedback_message_notification_visible())
def test_logout_after_click_redirect(self):
"""
1) User goes to a problem page.
2) User fills out an answer to the problem.
3) User is logged out because their session id is invalidated or removed.
4) User clicks "check", and sees a confirmation modal asking them to
re-authenticate, since they've just been logged out.
5) User clicks "ok".
6) User is redirected to the login page.
7) User logs in.
8) User is redirected back to the problem page they started out on.
9) User is able to submit an answer
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
self.log_user_out()
with problem_page.handle_alert(confirm=True):
problem_page.click_submit()
login_page = CombinedLoginAndRegisterPage(self.browser)
login_page.wait_for_page()
login_page.login(self.email, self.password)
problem_page.wait_for_page()
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
problem_page.fill_answer_numerical('1')
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
def test_python_execution_in_problem(self):
# Navigate to the problem page
self.course_home_page.visit()
self.course_home_page.outline.go_to_section('Test Section', 'Test Subsection')
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name.upper(), 'PYTHON PROBLEM')
# Does the page have computation results?
self.assertIn("What is the sum of 17 and 3?", problem_page.problem_text)
# Fill in the answer correctly.
problem_page.fill_answer("20")
problem_page.click_submit()
self.assertTrue(problem_page.is_correct())
# Fill in the answer incorrectly.
problem_page.fill_answer("4")
problem_page.click_submit()
self.assertFalse(problem_page.is_correct())
def test_grader_type_displayed(self):
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
self.assertEqual(problem_page.problem_name, 'TEST PROBLEM')
self.assertEqual(problem_page.problem_progress_graded_value,
"1 point possible (ungraded)")
class ProblemTypeTestBase(ProblemsTest, EventsTestMixin):
"""
Base class for testing assesment problem types in bok choy.
This inherits from ProblemsTest, which has capabilities for testing problem
features that are not problem type specific (checking, hinting, etc.).
The following attributes must be explicitly defined when inheriting from
this class:
problem_name (str)
problem_type (str)
factory (ResponseXMLFactory subclass instance)
Additionally, the default values for factory_kwargs and status_indicators
may need to be overridden for some problem types.
"""
__metaclass__ = ProblemTypeTestBaseMeta
problem_name = None
problem_type = None
factory = None
factory_kwargs = {}
status_indicators = {
'correct': ['span.correct'],
'incorrect': ['span.incorrect'],
'unanswered': ['span.unanswered'],
}
def setUp(self):
"""
Visits courseware_page and defines self.problem_page.
"""
super(ProblemTypeTestBase, self).setUp()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def get_problem(self):
"""
Creates a {problem_type} problem
"""
# Generate the problem XML using capa.tests.response_xml_factory
return XBlockFixtureDesc(
'problem',
self.problem_name,
data=self.factory.build_xml(**self.factory_kwargs),
metadata={'rerandomize': 'always', 'show_reset_button': True}
)
def wait_for_status(self, status):
"""
Waits for the expected status indicator.
Args:
status: one of ("correct", "incorrect", "unanswered)
"""
msg = "Wait for status to be {}".format(status)
selector = ', '.join(self.status_indicators[status])
self.problem_page.wait_for_element_visibility(selector, msg)
@abstractmethod
def answer_problem(self, correctness):
"""
Args:
`correct` (bool): Inputs correct answer if True, else inputs
incorrect answer.
"""
raise NotImplementedError()
class ProblemTypeTestBase(ProblemsTest, EventsTestMixin):
"""
Base class for testing assesment problem types in bok choy.
This inherits from ProblemsTest, which has capabilities for testing problem
features that are not problem type specific (checking, hinting, etc.).
The following attributes must be explicitly defined when inheriting from
this class:
problem_name (str)
problem_type (str)
factory (ResponseXMLFactory subclass instance)
Additionally, the default values for factory_kwargs and status_indicators
may need to be overridden for some problem types.
"""
__metaclass__ = ProblemTypeTestBaseMeta
problem_name = None
problem_type = None
problem_points = 1
factory = None
factory_kwargs = {}
status_indicators = {
'correct': ['span.correct'],
'incorrect': ['span.incorrect'],
'unanswered': ['span.unanswered'],
'submitted': ['span.submitted'],
}
def setUp(self):
"""
Visits courseware_page and defines self.problem_page.
"""
super(ProblemTypeTestBase, self).setUp()
self.courseware_page.visit()
self.problem_page = ProblemPage(self.browser)
def get_sequential(self):
""" Allow any class in the inheritance chain to customize subsection metadata."""
return XBlockFixtureDesc('sequential', 'Test Subsection', metadata=getattr(self, 'sequential_metadata', {}))
def get_problem(self):
"""
Creates a {problem_type} problem
"""
# Generate the problem XML using capa.tests.response_xml_factory
return XBlockFixtureDesc(
'problem',
self.problem_name,
data=self.factory.build_xml(**self.factory_kwargs),
metadata={'rerandomize': 'always', 'show_reset_button': True}
)
def wait_for_status(self, status):
"""
Waits for the expected status indicator.
Args:
status: one of ("correct", "incorrect", "unanswered", "submitted")
"""
msg = "Wait for status to be {}".format(status)
selector = ', '.join(self.status_indicators[status])
self.problem_page.wait_for_element_visibility(selector, msg)
@abstractmethod
def answer_problem(self, correctness):
"""
Args:
`correct` (bool): Inputs correct answer if True, else inputs
incorrect answer.
"""
raise NotImplementedError()
def test_reset_problem_after_submission(self, input_value, correctness):
"""
Scenario: Test that reset button works regardless the submission correctness status.
Given I am attempting a formula problem type with randomization:always configuration
When I input the answer
Then I should be able to see the MathJax generated preview
When I submit the problem
Then I should be able to see the reset button
When reset button is clicked
Then the input pane contents should be clear
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical(input_value)
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertEqual(problem_page.get_simpleprob_correctness(), correctness)
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertEqual(problem_page.get_numerical_input_value, '')
def test_notification_updates(self):
"""
Verifies that the notification is removed and not visible when it should be
"""
self.courseware_page.visit()
problem_page = ProblemPage(self.browser)
problem_page.click_choice("choice_2")
self.assertFalse(problem_page.is_success_notification_visible())
problem_page.click_submit()
problem_page.wait_success_notification()
self.assertEqual('Question 1: correct', problem_page.status_sr_text)
# Clicking Save should clear the submit notification
problem_page.click_save()
self.assertFalse(problem_page.is_success_notification_visible())
problem_page.wait_for_save_notification()
# Changing the answer should clear the save notification
problem_page.click_choice("choice_1")
self.assertFalse(problem_page.is_save_notification_visible())
problem_page.click_save()
problem_page.wait_for_save_notification()
# Submitting the problem again should clear the save notification
problem_page.click_submit()
problem_page.wait_incorrect_notification()
self.assertEqual('Question 1: incorrect', problem_page.status_sr_text)
self.assertFalse(problem_page.is_save_notification_visible())
def test_reset_problem_after_changing_correctness(self):
"""
Scenario: Verify that formula problem can be resetted after changing the correctness.
Given I am attempting a formula problem type
When I answer it correctly
Then the correctness status should be visible
And reset button is not rendered
When I change my submission to incorrect
Then the reset button appears and is clickable
"""
problem_page = ProblemPage(self.browser)
problem_page.fill_answer_numerical('R_1*R_2/R_3')
problem_page.verify_mathjax_rendered_in_preview()
problem_page.click_submit()
self.assertTrue(problem_page.simpleprob_is_correct())
self.assertFalse(problem_page.is_reset_button_present())
problem_page.fill_answer_numerical('R_1/R_3')
problem_page.click_submit()
self.assertFalse(problem_page.simpleprob_is_correct())
self.assertTrue(problem_page.is_reset_button_present())
problem_page.click_reset()
self.assertEqual(problem_page.get_numerical_input_value, '')
