def test_errors(): """Tests to ensure that errors are raised appropriately""" # All answers have same length in tuple with raises(ConfigError, match="All possible list answers must have the same length"): grader = ListGrader( answers=(["1", "2", "3"], ["1", "2"]), subgraders=StringGrader() ) # When using grouping, single subgraders must be ListGrader with raises(ConfigError, match="A ListGrader with groupings must have a ListGrader subgrader or a list of subgraders"): grader = ListGrader( answers=["1", "2", "3"], subgraders=StringGrader(), grouping=[1, 1, 2] ) # Must have an answer! with raises(ConfigError, match="Expected at least one answer in answers"): grader = ListGrader( subgraders=StringGrader() ) grader(None, ["Hello"]) # Bad input msg = "Expected answer to have type <type list>, but received <type 'tuple'>" with raises(ConfigError, match=msg): grader = ListGrader( answers=["hello", "there"], subgraders=StringGrader() ) grader(None, ("hello", "there"))
def test_grouping_with_subgraders_list(): """Another test of a nested ListGrader with grouping""" grader = ListGrader( answers=[ [ 'bat', ('ghost', {'expect': 'spectre', 'grade_decimal': 0.5}), 'pumpkin' ], 'Halloween' ], subgraders=[ ListGrader( subgraders=StringGrader() ), StringGrader() ], ordered=True, grouping=[1, 2, 1, 1] ) student_input = ['pumpkin', 'Halloween', 'bird', 'spectre'] expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': 'partial', 'grade_decimal': 0.5, 'msg': ''} ] } assert grader(None, student_input) == expected_result
def test_grouping_errors_subgraderAnd_groups_mismatched_in_size(): """Test that errors are raised when nested ListGraders have size mismatches""" # Too many graders with raises(ConfigError, match="Number of subgraders and number of groups are not equal"): ListGrader( answers=[ ['bat', 'ghost', 'pumpkin'], 'Halloween' ], subgraders=[ ListGrader( subgraders=StringGrader() ), StringGrader() ], ordered=True, grouping=[1, 1, 1, 1] ) # Too few graders with raises(ConfigError, match="Number of subgraders and number of groups are not equal"): ListGrader( answers=[ ['bat', 'ghost', 'pumpkin'], 'Halloween', ], subgraders=[ ListGrader( subgraders=StringGrader() ), StringGrader() ], ordered=True, grouping=[1, 1, 1, 2, 3] )
def test_longer_message_wins_grade_ties(): grader1 = StringGrader(answers=({ 'expect': 'zebra', 'grade_decimal': 1 }, { 'expect': 'horse', 'grade_decimal': 0, 'msg': 'short' }, { 'expect': 'unicorn', 'grade_decimal': 0, 'msg': "longer_msg" })) grader2 = StringGrader(answers=({ 'expect': 'zebra', 'grade_decimal': 1 }, { 'expect': 'unicorn', 'grade_decimal': 0, 'msg': "longer_msg" }, { 'expect': 'horse', 'grade_decimal': 0, 'msg': 'short' })) submission = 'unicorn' expected_result = {'msg': 'longer_msg', 'grade_decimal': 0, 'ok': False} result1 = grader1(None, submission) result2 = grader2(None, submission) assert result1 == result2 == expected_result
def test_infer_expect(): grader = SingleListGrader(subgrader=SingleListGrader( subgrader=StringGrader(), delimiter=','), delimiter=';', debug=True) assert grader.infer_from_expect('a,b;c,d') == [['a', 'b'], ['c', 'd']] assert grader.infer_from_expect('a,b,c;d,e,f;g,h,i') == [['a', 'b', 'c'], ['d', 'e', 'f'], ['g', 'h', 'i']] # Test that the grading process works assert grader('a,b;c,d', 'd,c;b,a')['ok'] # Test that inferred answers show up in the debug log as expected msg = grader('a,b;c,d', 'a')['msg'] assert 'Expect value inferred to be [["a", "b"], ["c", "d"]]' in msg msg = grader('a,b', 'a')['msg'] assert 'Expect value inferred to be [["a", "b"]]' in msg msg = grader('a;b', 'a')['msg'] assert 'Expect value inferred to be [["a"], ["b"]]' in msg # Test heavy nesting grader = SingleListGrader(subgrader=SingleListGrader( subgrader=SingleListGrader(subgrader=StringGrader(), delimiter='-'), delimiter=','), delimiter=';') assert grader.infer_from_expect('a-1-@,b-2;c-3,d-4') == [[['a', '1', '@'], ['b', '2']], [['c', '3'], ['d', '4']]]
def test_errors(): """Tests to ensure that errors are raised appropriately""" # All answers have same length in tuple with raises(ConfigError, match="All possible list answers must have the same length"): SingleListGrader(answers=(["1", "2", "3"], ["1", "2"]), subgrader=StringGrader()) # Answers must not be empty with raises(ConfigError, match="Cannot have an empty list of answers"): SingleListGrader(answers=([], []), subgrader=StringGrader()) # Empty entries raises an error grader = SingleListGrader(answers=['1', '2', '3'], subgrader=StringGrader()) with raises(MissingInput, match="List error: Empty entry detected in position 1"): grader(None, ',1,2,3') with raises(MissingInput, match="List error: Empty entry detected in position 4"): grader(None, '1,2,3,') with raises(MissingInput, match="List error: Empty entry detected in position 2"): grader(None, '1,,2,3') with raises(MissingInput, match="List error: Empty entries detected in positions 1, 3"): grader(None, ',1,,2,3') grader = SingleListGrader(answers=['1', '2', '3'], subgrader=StringGrader(), missing_error=False) grader(None, ',1,2,3')['grade_decimal'] = 0.75 grader(None, '1,2,3,')['grade_decimal'] = 0.75 grader(None, '1,,2,3')['grade_decimal'] = 0.75 grader(None, ',1,2,3,')['grade_decimal'] = 0.6
def test_docs(): """Test the documentation examples""" grader = StringGrader(answers='cat', wrong_msg='Try again!') assert grader(None, 'cat')['ok'] grader = StringGrader(answers={ 'expect': 'zebra', 'ok': True, 'grade_decimal': 1, 'msg': 'Yay!' }, wrong_msg='Try again!') expected_result = {'msg': 'Yay!', 'grade_decimal': 1, 'ok': True} assert grader(None, 'zebra') == expected_result expected_result = {'msg': 'Try again!', 'grade_decimal': 0, 'ok': False} assert grader(None, 'horse') == expected_result grader = StringGrader( answers='cat', # Equivalent to: # answers={'expect': 'cat', 'msg': '', 'grade_decimal': 1, 'ok': True} wrong_msg='Try again!') expected_result = {'msg': '', 'grade_decimal': 1, 'ok': True} assert grader(None, 'cat') == expected_result grader = StringGrader( answers=( # the correct answer 'wolf', # an alternative correct answer 'canis lupus', # a partially correct answer { 'expect': 'dog', 'grade_decimal': 0.5, 'msg': 'No, not dog!' }, # a wrong answer with specific feedback { 'expect': 'unicorn', 'grade_decimal': 0, 'msg': 'No, not unicorn!' }), wrong_msg='Try again!') expected_result = {'msg': '', 'grade_decimal': 1, 'ok': True} assert grader(None, 'wolf') == expected_result assert grader(None, 'canis lupus') == expected_result expected_result = { 'msg': 'No, not dog!', 'grade_decimal': 0.5, 'ok': 'partial' } assert grader(None, 'dog') == expected_result expected_result = { 'msg': 'No, not unicorn!', 'grade_decimal': 0, 'ok': False } assert grader(None, 'unicorn') == expected_result
def test_grouping_errors_group_needs_list_grader(): """Test that anything with grouping needs a ListGrader""" msg = "Grouping index 2 has 3 items, but has a StringGrader subgrader instead of ListGrader" with raises(ConfigError, match=msg): ListGrader( answers=[['bat', 'ghost', 'pumpkin'], 'Halloween'], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 2, 2, 2])
def test_partial_credit_override(): grader0 = SingleListGrader(answers=['moose', 'eagle'], partial_credit=False, subgrader=StringGrader()) grader1 = SingleListGrader(answers=['moose', 'eagle'], subgrader=StringGrader()) submission = "hawk, moose" expected_result0 = {'ok': False, 'grade_decimal': 0, 'msg': ''} expected_result1 = {'ok': 'partial', 'grade_decimal': 0.5, 'msg': ''} assert grader0(None, submission) == expected_result0 assert grader1(None, submission) == expected_result1
def test_errors(): """Tests to ensure that errors are raised appropriately""" # All answers have same length in tuple with raises(ConfigError, match="All possible list answers must have the same length"): grader = SingleListGrader(answers=(["1", "2", "3"], ["1", "2"]), subgrader=StringGrader()) # Answers must not be empty with raises(ConfigError, match="Cannot have an empty list of answers"): grader = SingleListGrader(answers=([], []), subgrader=StringGrader())
def test_wrong_number_of_inputs_with_grouping(): """Test that the right number of inputs is required""" msg = "Grouping indicates 4 inputs are expected, but only 3 inputs exist." with raises(ConfigError, match=msg): grader = ListGrader( answers=[['bat', 'ghost', 'pumpkin'], 'Halloween'], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[2, 1, 1, 1]) grader(None, ['Halloween', 'cat', 'rat'])
def test_grouping_not_contiguous_integers(): """Test that the group numbers are contiguous integers""" msg = "Grouping should be a list of contiguous positive integers starting at 1." with raises(ConfigError, match=msg): ListGrader( answers=[ ['bat', 'ghost', 'pumpkin'], 'Halloween', ], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 1, 1, 3])
def test_empty_entry_in_answers(): msg = ("There is a problem with the author's problem configuration: " "Empty entry detected in answer list. Students receive an error " "when supplying an empty entry. Set 'missing_error' to False in " "order to allow such entries.") # Test base case with raises(ConfigError, match=msg): grader = SingleListGrader(answers=['a', 'b', ' ', 'c'], subgrader=StringGrader()) # Test case with dictionary with raises(ConfigError, match=msg): grader = SingleListGrader(answers=['a', 'b', { 'expect': ' ' }, 'c'], subgrader=StringGrader()) # Test case with unusual expect values with raises(ConfigError, match=msg): grader = SingleListGrader(answers=[ 'a', 'b', { 'comparer': congruence_comparer, 'comparer_params': ['b^2/a', '2*pi'] }, '' ], subgrader=FormulaGrader()) # Test case with really unusual expect values with raises(ConfigError, match=msg): grader = SingleListGrader(answers=[ 'a', 'b', { 'expect': { 'comparer': congruence_comparer, 'comparer_params': ['b^2/a', '2*pi'] }, 'msg': 'Well done!' }, '' ], subgrader=FormulaGrader()) # Test nested case with raises(ConfigError, match=msg): grader = SingleListGrader(answers=[['a', 'b'], ['c', '']], subgrader=SingleListGrader( subgrader=StringGrader(), delimiter=';')) # Test case with no error grader = SingleListGrader(answers=['a', 'b', ' ', 'c'], subgrader=StringGrader(), missing_error=False) grader('a,,b', 'a, b, c')
def test_docs(): """Make sure that the documentation examples work as expected""" grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader()) assert grader(None, "cat, dog")["grade_decimal"] == 1 assert grader(None, "dog, cat")["grade_decimal"] == 1 assert grader(None, "cat, octopus")["grade_decimal"] == 0.5 assert grader(None, "cat")["grade_decimal"] == 0.5 grader = SingleListGrader(answers=( [('cat', 'feline'), 'dog'], ['goat', 'vole'], ), subgrader=StringGrader()) assert grader(None, "cat, dog")["grade_decimal"] == 1 assert grader(None, "feline, dog")["grade_decimal"] == 1 assert grader(None, "goat, vole")["grade_decimal"] == 1 assert grader(None, "cat, vole")["grade_decimal"] == 0.5 assert grader(None, "dog, goat")["grade_decimal"] == 0.5 grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader(), ordered=True) assert grader(None, "cat, dog")["grade_decimal"] == 1 assert grader(None, "cat")["grade_decimal"] == 0.5 assert grader(None, "dog")["grade_decimal"] == 0 grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader(), length_error=True) with raises(MissingInput): grader(None, "cat") with raises(MissingInput): grader(None, "cat, dog, moose") grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader(), delimiter=';') assert grader(None, "cat, dog")["grade_decimal"] == 0 assert grader(None, "dog, cat")["grade_decimal"] == 0 assert grader(None, "cat; dog")["grade_decimal"] == 1 assert grader(None, "dog; cat")["grade_decimal"] == 1 grader = SingleListGrader( answers=[['a', 'b'], ['c', 'd']], subgrader=SingleListGrader(subgrader=StringGrader()), delimiter=';') assert grader(None, "a,b;c,d")["grade_decimal"] == 1 assert grader(None, "b,a;d,c")["grade_decimal"] == 1 assert grader(None, "c,d;a,b")["grade_decimal"] == 1 assert grader(None, "a,c;b,d")["grade_decimal"] == 0.5 grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader(), partial_credit=False) assert grader(None, "cat, octopus")["grade_decimal"] == 0 grader = SingleListGrader(answers=['cat', 'dog'], subgrader=StringGrader(), wrong_msg='Try again!') assert grader(None, "moose, octopus")["msg"] == "Try again!"
def test_multiple_graders_errors(): """Test that exceptions are raised on bad config""" # Wrong number of graders with raises(ConfigError, match='The number of subgraders and answers are different'): ListGrader(answers=['cat', '1'], subgraders=[StringGrader()], ordered=True) # Unordered entry with raises(ConfigError, match='Cannot use unordered lists with multiple graders'): ListGrader(answers=['cat', '1'], subgraders=[StringGrader(), StringGrader()], ordered=False)
def test_siblings_passed_to_subgrader_check_if_ordered_and_subgrader_list(): sg0 = FormulaGrader() sg1 = NumericalGrader() sg2 = StringGrader() grader = ListGrader(answers=['1', '2', '3'], subgraders=[sg0, sg1, sg2], ordered=True) student_input = ['10', '20', '30'] siblings = [{ 'input': '10', 'grader': sg0 }, { 'input': '20', 'grader': sg1 }, { 'input': '30', 'grader': sg2 }] # There must be a better way to spy on multiple things... with mock.patch.object(sg0, 'check', wraps=sg0.check) as check0: with mock.patch.object(sg1, 'check', wraps=sg1.check) as check1: with mock.patch.object(sg2, 'check', wraps=sg2.check) as check2: grader(None, ['10', '20', '30']) # subgrader check has been called three times assert len(check0.call_args_list) == 1 assert len(check1.call_args_list) == 1 assert len(check2.call_args_list) == 1 # subgrader check has been passed the correct siblings for _, kwargs in check0.call_args_list: assert kwargs['siblings'] == siblings for _, kwargs in check1.call_args_list: assert kwargs['siblings'] == siblings for _, kwargs in check2.call_args_list: assert kwargs['siblings'] == siblings
def test_duplicate_items(): grader = ListGrader(answers=['cat', 'dog', 'unicorn', 'cat', 'cat'], subgraders=StringGrader()) submission = ['cat', 'dog', 'dragon', 'dog', 'cat'] expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }] } assert grader(None, submission) == expected_result
def test_too_few_items(): grader = SingleListGrader(answers=[({ 'expect': 'tiger', 'grade_decimal': 1 }, { 'expect': 'lion', 'grade_decimal': 0.5, 'msg': "lion_msg" }), ('skunk'), ({ 'expect': 'zebra', 'grade_decimal': 1 }, { 'expect': 'horse', 'grade_decimal': 0 }, { 'expect': 'unicorn', 'grade_decimal': 0.75, 'msg': "unicorn_msg" })], subgrader=StringGrader(), length_error=False) submission = "skunk, unicorn" expected_result = { 'ok': 'partial', 'msg': "unicorn_msg", 'grade_decimal': approx((1 + 0.75) / 3) } assert grader(None, submission) == expected_result
def test_nesting_with_same_delimiter_raises_config_error(): with raises( ConfigError, match="Nested SingleListGraders must use different delimiters."): # Both delimiters have same default value SingleListGrader(answers=[['a', 'b'], ['c', 'd']], subgrader=SingleListGrader(subgrader=StringGrader()))
def test_single_expect_value_in_config_and_passed_explicitly(): grader = StringGrader( answers='cat' ) submission = 'dog' expected_result = {'msg': '', 'grade_decimal': 0, 'ok': False} assert grader('cat', submission) == expected_result
def test_multiple_list_answers(): """ Check that a SingleListGrader with multiple possible answers is graded correctly """ grader = SingleListGrader(answers=(['cat', 'meow'], ['dog', 'woof']), subgrader=StringGrader()) expected_result = {'ok': True, 'msg': '', 'grade_decimal': 1} result = grader(None, 'cat,meow') assert result == expected_result expected_result = {'ok': 'partial', 'msg': '', 'grade_decimal': 0.5} result = grader(None, 'cat,woof') assert result == expected_result expected_result = {'ok': True, 'msg': '', 'grade_decimal': 1} result = grader(None, 'dog,woof') assert result == expected_result expected_result = {'ok': 'partial', 'msg': '', 'grade_decimal': 0.5} result = grader(None, 'dog,meow') assert result == expected_result expected_result = {'ok': False, 'msg': '', 'grade_decimal': 0} result = grader(None, 'badger,grumble') assert result == expected_result
def test_attempt_based_grading_list(): grader = ListGrader( answers=['cat', 'dog'], subgraders=StringGrader(), attempt_based_credit=LinearCredit(), ) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''} ] } assert grader(None, ['cat', 'dog'], attempt=1) == expected_result expected_result = { 'overall_message': 'Maximum credit for attempt #5 is 20%.', 'input_list': [ {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''}, {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''} ] } assert grader(None, ['cat', 'dog'], attempt=5) == expected_result expected_result = { 'overall_message': 'Maximum credit for attempt #5 is 20%.', 'input_list': [ {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''} ] } assert grader(None, ['cat', 'unicorn'], attempt=5) == expected_result
def test_order_matters(): grader = SingleListGrader(answers=['cat', 'dog', 'fish'], subgrader=StringGrader(), ordered=True) submission = "cat, fish, moose" expected_result = {'ok': 'partial', 'msg': '', 'grade_decimal': 1 / 3} assert grader(None, submission) == expected_result
def test_picking_between_equally_graded_results(): """Check that a listgrader with multiple equally-scoring answers picks the one where high scores occur early""" grader = ListGrader(answers=(['a', 'b', 'c'], ['1', '2', '3']), subgraders=StringGrader()) result = grader(None, ['wrong', 'b', '3']) expected_result = { 'overall_message': '', 'input_list': [{ 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': True, 'grade_decimal': 1.0, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }] } assert result == expected_result
def test_debug_with_input_list(): grader = ListGrader( answers=['cat', 'dog', 'unicorn'], subgraders=StringGrader(), debug=True ) student_response = ["cat", "fish", "dog"] template = ("<pre>" "MITx Grading Library Version {version}\n" "Running on edX using python {python_version}\n" "{debug_content}" "</pre>") debug_content = "Student Responses:\ncat\nfish\ndog" msg = template.format(version=__version__, python_version=platform.python_version(), debug_content=debug_content ).replace("\n", "<br/>\n") expected_result = { 'overall_message': msg, 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''} ] } assert grader(None, student_response) == expected_result
def test_multiple_listAnswers_same_grade(): grader = ListGrader(answers=( [{ 'expect': 'dog', 'msg': 'dog1' }, 'woof'], ['cat', 'woof'], [{ 'expect': 'dog', 'msg': 'dog2' }, 'woof'], ['dolphin', 'squeak'], ), subgraders=StringGrader()) result = grader(None, ['dog', 'woof']) expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1.0, 'msg': 'dog1' }, { 'ok': True, 'grade_decimal': 1.0, 'msg': '' }] } printit(result) assert result == expected_result
def test_two_expect_values_in_config(): grader = StringGrader( answers=('cat', 'horse') ) submission = 'horse' expected_result = {'msg': '', 'grade_decimal': 1, 'ok': True} assert grader(None, submission) == expected_result
def test_partial_credit_assigment(): grader = SingleListGrader(answers=[({ 'expect': 'tiger', 'grade_decimal': 1 }, { 'expect': 'lion', 'grade_decimal': 0.5, 'msg': "lion_msg" }), 'skunk', ({ 'expect': 'zebra', 'grade_decimal': 1 }, { 'expect': 'horse', 'grade_decimal': 0 }, { 'expect': 'unicorn', 'grade_decimal': 0.75, 'msg': "unicorn_msg" })], subgrader=StringGrader()) submission = "skunk, lion, unicorn" expected_result = { 'ok': 'partial', 'msg': ("lion_msg<br/>\n" "unicorn_msg"), 'grade_decimal': approx((1 + 0.5 + 0.75) / 3) } assert grader(None, submission) == expected_result
def test_case(): """Tests that case is working correctly""" grader = StringGrader(answers="cat", case_sensitive=True) assert grader(None, "cat")['ok'] assert not grader(None, "Cat")['ok'] assert not grader(None, "CAT")['ok'] grader = StringGrader(answers="Cat", case_sensitive=True) assert not grader(None, "cat")['ok'] assert grader(None, "Cat")['ok'] assert not grader(None, "CAT")['ok'] grader = StringGrader(answers="CAT", case_sensitive=True) assert not grader(None, "cat")['ok'] assert not grader(None, "Cat")['ok'] assert grader(None, "CAT")['ok']
def test_too_many_items(): grader = SingleListGrader(answers=[ ({ 'expect': 'tiger', 'grade_decimal': 1 }, { 'expect': 'lion', 'grade_decimal': 0.5, 'msg': "lion_msg" }), 'skunk', ({ 'expect': 'zebra', 'grade_decimal': 1 }, { 'expect': 'horse', 'grade_decimal': 0 }, { 'expect': 'unicorn', 'grade_decimal': 0.75, 'msg': "unicorn_msg" }), ], subgrader=StringGrader(), length_error=False) submission = "skunk, fish, lion, unicorn, bear" expected_result = { 'ok': 'partial', 'msg': ("lion_msg<br/>\n" "unicorn_msg"), 'grade_decimal': approx((1 + 0.5 + 0.75) / 3 - 2 * 1 / 3) } assert grader(None, submission) == expected_result