def test_grouping_unordered_inputs(): """ Test nested ListGraders using grouping. Intended to mimick a two-column layout, list mammals in one column birds in another. """ grader = ListGrader( answers=[ ['cat', 'otter', 'bear'], ['eagle', 'sparrow', 'hawk'] ], grouping=[1, 2, 1, 2, 1, 2], subgraders=ListGrader( subgraders=StringGrader() ) ) student_input = [ 'hawk', 'otter', 'falcon', 'bear', 'eagle', 'dog' ] expected_result = { 'overall_message': '', 'input_list': [ # hawk good, otter good {'ok': True, 'msg': '', 'grade_decimal': 1}, {'ok': True, 'msg': '', 'grade_decimal': 1}, # falcon bad, bear good {'ok': False, 'msg': '', 'grade_decimal': 0}, {'ok': True, 'msg': '', 'grade_decimal': 1}, # eagle good, dog bad {'ok': True, 'msg': '', 'grade_decimal': 1}, {'ok': False, 'msg': '', 'grade_decimal': 0}, ] } assert grader(None, student_input) == expected_result
def test_grouping_errors_subgraderAnd_groups_mismatched_in_size(): """Test that errors are raised when nested ListGraders have size mismatches""" # Too many graders with raises( ConfigError, match="Number of subgraders and number of groups are not equal"): ListGrader( answers=[['bat', 'ghost', 'pumpkin'], 'Halloween'], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 1, 1, 1]) # Too few graders with raises( ConfigError, match="Number of subgraders and number of groups are not equal"): ListGrader( answers=[ ['bat', 'ghost', 'pumpkin'], 'Halloween', ], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 1, 1, 2, 3])
def test_errors(): """Tests to ensure that errors are raised appropriately""" # All answers have same length in tuple with raises(ConfigError, match="All possible list answers must have the same length"): grader = ListGrader(answers=(["1", "2", "3"], ["1", "2"]), subgraders=StringGrader()) # When using grouping, single subgraders must be ListGrader with raises( ConfigError, match= "A ListGrader with groupings must have a ListGrader subgrader or a list of subgraders" ): grader = ListGrader(answers=["1", "2", "3"], subgraders=StringGrader(), grouping=[1, 1, 2]) # Must have an answer! with raises(ConfigError, match="Expected at least one answer in answers"): grader = ListGrader(subgraders=StringGrader()) grader(None, ["Hello"]) # Bad input msg = "Expected answer to have type <type list>, but received <type 'tuple'>" with raises(ConfigError, match=msg): grader = ListGrader(answers=["hello", "there"], subgraders=StringGrader()) grader(None, ("hello", "there"))
def test_nested_grouping_ordered(): """Test that ordered nested groupings work appropriately""" grader = ListGrader( answers=[ ['0', '1'], ['2', '3'], ], subgraders=ListGrader( subgraders=FormulaGrader(), ordered=True ), grouping=[1, 1, 2, 2] ) def expect(a, b, c, d): return { 'input_list': [ {'grade_decimal': a, 'msg': '', 'ok': a == 1}, {'grade_decimal': b, 'msg': '', 'ok': b == 1}, {'grade_decimal': c, 'msg': '', 'ok': c == 1}, {'grade_decimal': d, 'msg': '', 'ok': d == 1} ], 'overall_message': '' } assert grader(None, ['0', '1', '2', '3']) == expect(1, 1, 1, 1) assert grader(None, ['1', '0', '3', '2']) == expect(0, 0, 0, 0) assert grader(None, ['2', '3', '0', '1']) == expect(1, 1, 1, 1) assert grader(None, ['3', '2', '1', '0']) == expect(0, 0, 0, 0) assert grader(None, ['1', '3', '2', '0']) == expect(0, 1, 0, 0) assert grader(None, ['0', '2', '3', '1']) == expect(1, 0, 0, 0)
def test_grouping_with_subgraders_list(): """Another test of a nested ListGrader with grouping""" grader = ListGrader( answers=[ [ 'bat', ('ghost', {'expect': 'spectre', 'grade_decimal': 0.5}), 'pumpkin' ], 'Halloween' ], subgraders=[ ListGrader( subgraders=StringGrader() ), StringGrader() ], ordered=True, grouping=[1, 2, 1, 1] ) student_input = ['pumpkin', 'Halloween', 'bird', 'spectre'] expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': 'partial', 'grade_decimal': 0.5, 'msg': ''} ] } assert grader(None, student_input) == expected_result
def test_grouping_errors_group_needs_list_grader(): """Test that anything with grouping needs a ListGrader""" msg = "Grouping index 2 has 3 items, but has a StringGrader subgrader instead of ListGrader" with raises(ConfigError, match=msg): ListGrader( answers=[['bat', 'ghost', 'pumpkin'], 'Halloween'], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 2, 2, 2])
def test_wrong_number_of_inputs_with_grouping(): """Test that the right number of inputs is required""" msg = "Grouping indicates 4 inputs are expected, but only 3 inputs exist." with raises(ConfigError, match=msg): grader = ListGrader( answers=[['bat', 'ghost', 'pumpkin'], 'Halloween'], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[2, 1, 1, 1]) grader(None, ['Halloween', 'cat', 'rat'])
def test_grouping_unordered_different_lengths(): """Test that an error is raised if unordered groupings use different numbers of inputs""" msg = "Groups must all be the same length when unordered" with raises(ConfigError, match=msg): ListGrader(answers=[ ['bat', 'ghost', 'pumpkin'], ['Halloween', 'Easter'], ], subgraders=ListGrader(subgraders=StringGrader()), ordered=False, grouping=[1, 1, 1, 2])
def test_grouping_not_contiguous_integers(): """Test that the group numbers are contiguous integers""" msg = "Grouping should be a list of contiguous positive integers starting at 1." with raises(ConfigError, match=msg): ListGrader( answers=[ ['bat', 'ghost', 'pumpkin'], 'Halloween', ], subgraders=[ListGrader(subgraders=StringGrader()), StringGrader()], ordered=True, grouping=[1, 1, 1, 3])
def test_multiple_nestingAnd_groups(): """ Test of a grouping inside a grouping. Enter the normalized, eigenvalues & eigenvectors of [[1,0],[0,-1]] Of course, in a real problem we wouldn't use StringGrader to grade the numbers TODO Expand this test once the required infrastructure is in place """ grader = ListGrader( answers=[ [ {'expect': '1', 'msg': 'first eigenvalue'}, ( {'expect': '+1, 0', 'msg': 'positive first eigenvector'}, {'expect': '-1, 0', 'msg': 'negative first eigenvector', 'grade_decimal': 0.8} ) ], [ {'expect': '-1', 'msg': 'second eigenvalue'}, ( {'expect': '0, +1', 'msg': 'positive second eigenvector'}, {'expect': '0, -1', 'msg': 'negative second eigenvector', 'grade_decimal': 0.8} ) ], ], subgraders=ListGrader( subgraders=[ FormulaGrader(), StringGrader() ], ordered=True, grouping=[1, 2] ), grouping=[1, 1, 2, 2] ) submission = [ '-1', '0, -1', '1', '+1, 0', ] expected = { 'overall_message': '', 'input_list': [ {'ok': True, 'msg': 'second eigenvalue', 'grade_decimal': 1}, {'ok': 'partial', 'msg': 'negative second eigenvector', 'grade_decimal': 0.8}, {'ok': True, 'msg': 'first eigenvalue', 'grade_decimal': 1}, {'ok': True, 'msg': 'positive first eigenvector', 'grade_decimal': 1} ] } assert grader(None, submission) == expected
def test_multiple_graders_errors(): """Test that exceptions are raised on bad config""" # Wrong number of graders with raises(ConfigError, match='The number of subgraders and answers are different'): ListGrader(answers=['cat', '1'], subgraders=[StringGrader()], ordered=True) # Unordered entry with raises(ConfigError, match='Cannot use unordered lists with multiple graders'): ListGrader(answers=['cat', '1'], subgraders=[StringGrader(), StringGrader()], ordered=False)
def test_readme(): """Tests that the README.md file examples work""" grader = StringGrader(answers='cat') grader = ListGrader(answers=['1', '2'], subgraders=FormulaGrader()) del grader
def test_partial_creditAssigment(): grader = ListGrader( answers=[ ( {'expect': 'tiger', 'grade_decimal': 1}, {'expect': 'lion', 'grade_decimal': 0.5, 'msg': "lion_msg"} ), 'skunk', ( {'expect': 'zebra', 'grade_decimal': 1}, {'expect': 'horse', 'grade_decimal': 0}, {'expect': 'unicorn', 'grade_decimal': 0.75, 'msg': "unicorn_msg"} ) ], subgraders=StringGrader() ) submission = ["skunk", "lion", "unicorn"] expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': 'partial', 'grade_decimal': 0.5, 'msg': 'lion_msg'}, {'ok': 'partial', 'grade_decimal': 0.75, 'msg': 'unicorn_msg'} ] } assert grader(None, submission) == expected_result
def test_insufficientAnswers(): """Check that an error is raised if ListGrader is fed only one answer""" with raises(ConfigError, match='ListGrader does not work with a single answer'): ListGrader( answers=['cat'], subgraders=StringGrader() )
def test_attempt_based_grading_list(): grader = ListGrader( answers=['cat', 'dog'], subgraders=StringGrader(), attempt_based_credit=LinearCredit(), ) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''} ] } assert grader(None, ['cat', 'dog'], attempt=1) == expected_result expected_result = { 'overall_message': 'Maximum credit for attempt #5 is 20%.', 'input_list': [ {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''}, {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''} ] } assert grader(None, ['cat', 'dog'], attempt=5) == expected_result expected_result = { 'overall_message': 'Maximum credit for attempt #5 is 20%.', 'input_list': [ {'ok': 'partial', 'grade_decimal': 0.2, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''} ] } assert grader(None, ['cat', 'unicorn'], attempt=5) == expected_result
def test_fg_evaluates_siblings_appropriately(): grader=ListGrader( answers=['sibling_3 + 1', 'sibling_1^2', 'x'], subgraders=FormulaGrader(variables=['x']), ordered=True ) # All correct! result = grader(None, ['x + 1', 'x^2 + 2*x + 1', 'x']) expected = { 'input_list': [ {'grade_decimal': 1, 'msg': '', 'ok': True}, {'grade_decimal': 1, 'msg': '', 'ok': True}, {'grade_decimal': 1, 'msg': '', 'ok': True}], 'overall_message': '' } assert result == expected # First input wrong, but other two consistent result = grader(None, ['x + 2', 'x^2 + 4*x + 4', 'x']) expected = { 'input_list': [ {'grade_decimal': 0, 'msg': '', 'ok': False}, {'grade_decimal': 1, 'msg': '', 'ok': True}, {'grade_decimal': 1, 'msg': '', 'ok': True}], 'overall_message': '' } assert result == expected # Cannot grade, missing a required input match='Cannot grade answer, a required input is missing.' with raises(MissingInput, match=match): result = grader(None, ['', 'x^2 + 2*x + 1', 'x'])
def test_debug_with_input_list(): grader = ListGrader( answers=['cat', 'dog', 'unicorn'], subgraders=StringGrader(), debug=True ) student_response = ["cat", "fish", "dog"] template = ("<pre>" "MITx Grading Library Version {version}\n" "Running on edX using python {python_version}\n" "{debug_content}" "</pre>") debug_content = "Student Responses:\ncat\nfish\ndog" msg = template.format(version=__version__, python_version=platform.python_version(), debug_content=debug_content ).replace("\n", "<br/>\n") expected_result = { 'overall_message': msg, 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''} ] } assert grader(None, student_response) == expected_result
def test_siblings_passed_to_subgrader_check_if_ordered_and_subgrader_list(): sg0 = FormulaGrader() sg1 = NumericalGrader() sg2 = StringGrader() grader = ListGrader(answers=['1', '2', '3'], subgraders=[sg0, sg1, sg2], ordered=True) student_input = ['10', '20', '30'] siblings = [{ 'input': '10', 'grader': sg0 }, { 'input': '20', 'grader': sg1 }, { 'input': '30', 'grader': sg2 }] # There must be a better way to spy on multiple things... with mock.patch.object(sg0, 'check', wraps=sg0.check) as check0: with mock.patch.object(sg1, 'check', wraps=sg1.check) as check1: with mock.patch.object(sg2, 'check', wraps=sg2.check) as check2: grader(None, ['10', '20', '30']) # subgrader check has been called three times assert len(check0.call_args_list) == 1 assert len(check1.call_args_list) == 1 assert len(check2.call_args_list) == 1 # subgrader check has been passed the correct siblings for _, kwargs in check0.call_args_list: assert kwargs['siblings'] == siblings for _, kwargs in check1.call_args_list: assert kwargs['siblings'] == siblings for _, kwargs in check2.call_args_list: assert kwargs['siblings'] == siblings
def test_multiple_listAnswers_same_grade(): grader = ListGrader(answers=( [{ 'expect': 'dog', 'msg': 'dog1' }, 'woof'], ['cat', 'woof'], [{ 'expect': 'dog', 'msg': 'dog2' }, 'woof'], ['dolphin', 'squeak'], ), subgraders=StringGrader()) result = grader(None, ['dog', 'woof']) expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1.0, 'msg': 'dog1' }, { 'ok': True, 'grade_decimal': 1.0, 'msg': '' }] } printit(result) assert result == expected_result
def test_duplicate_items(): grader = ListGrader(answers=['cat', 'dog', 'unicorn', 'cat', 'cat'], subgraders=StringGrader()) submission = ['cat', 'dog', 'dragon', 'dog', 'cat'] expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }] } assert grader(None, submission) == expected_result
def test_picking_between_equally_graded_results(): """Check that a listgrader with multiple equally-scoring answers picks the one where high scores occur early""" grader = ListGrader(answers=(['a', 'b', 'c'], ['1', '2', '3']), subgraders=StringGrader()) result = grader(None, ['wrong', 'b', '3']) expected_result = { 'overall_message': '', 'input_list': [{ 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': True, 'grade_decimal': 1.0, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }] } assert result == expected_result
def test_nested_debug(): """Ensure that nested debug flags work properly""" grader = ListGrader(answers=["1", "2"], subgraders=FormulaGrader(debug=True), debug=True) result = grader(None, ["1", "2"])['overall_message'] assert result.count("FormulaGrader Debug Info<br/>") == 4
def test_wrong_number_of_inputs(): """Check that an error is raised if number of answers != number of inputs""" expect = r'The number of answers \(2\) and the number of inputs \(1\) are different' with raises(ConfigError, match=expect): grader = ListGrader( answers=['cat', 'dog'], subgraders=StringGrader() ) grader(None, ['cat'])
def test_multiple_listAnswers(): """Check that a listgrader with multiple possible answers is graded correctly""" grader = ListGrader( answers=(['cat', 'meow'], ['dog', 'woof']), subgraders=StringGrader() ) result = grader(None, ['cat', 'meow']) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1.0, 'msg': ''}, {'ok': True, 'grade_decimal': 1.0, 'msg': ''} ] } assert result == expected_result result = grader(None, ['cat', 'woof']) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1.0, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''} ] } assert result == expected_result result = grader(None, ['dog', 'woof']) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1.0, 'msg': ''}, {'ok': True, 'grade_decimal': 1.0, 'msg': ''} ] } assert result == expected_result result = grader(None, ['dog', 'meow']) expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1.0, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''} ] } assert result == expected_result result = grader(None, ['badger', 'grumble']) expected_result = { 'overall_message': '', 'input_list': [ {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''} ] } assert result == expected_result
def test_config_options_errors(): """Tests to ensure that errors are raised appropriately""" # All answers have same length in tuple with raises(ConfigError, match="All possible list answers must have the same length"): grader = ListGrader(answers=(["1", "2", "3"], ["1", "2"]), subgraders=StringGrader()) # When using grouping, single subgraders must be ListGrader with raises( ConfigError, match= "A ListGrader with groupings must have a ListGrader subgrader or a list of subgraders" ): grader = ListGrader(answers=["1", "2", "3"], subgraders=StringGrader(), grouping=[1, 1, 2]) # Must have an answer! with raises(ConfigError, match="Expected at least one answer in answers"): grader = ListGrader(subgraders=StringGrader()) grader(None, ["Hello"])
def test_student_input_errors(): grader = ListGrader(answers=["hello", "there"], subgraders=StringGrader()) # Bad input not_list = "Expected student_input to be a list of text strings, but received {tuple}".format( tuple=tuple) with raises(ConfigError, match=not_list): grader(None, ("hello", "there")) bad_list_entry = ("Expected a list of text strings for student_input, but " "item at position 1 has {int_type}").format(int_type=int) with raises(ConfigError, match=bad_list_entry): grader(None, ["cat", 7, "dog"])
def test_multiple_graders(): """Test multiple graders""" grader = ListGrader(answers=['cat', '1'], subgraders=[StringGrader(), FormulaGrader()], ordered=True) # Test success submission = ['cat', '1'] expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }] } result = grader(None, submission) assert result == expected_result # Test incorrect ordering submission = ['1', 'cat'] with raises( CalcError, match="Invalid Input: 'cat' not permitted in answer as a variable" ): result = grader(None, submission) # Test failure submission = ['dog', '2'] expected_result = { 'overall_message': '', 'input_list': [{ 'ok': False, 'grade_decimal': 0, 'msg': '' }, { 'ok': False, 'grade_decimal': 0, 'msg': '' }] } result = grader(None, submission) assert result == expected_result
def test_shorthandAnswers_specification(): grader = ListGrader( answers=['cat', 'dog', 'unicorn'], subgraders=StringGrader() ) submission = ['cat', 'fish', 'dog'] expected_result = { 'overall_message': '', 'input_list': [ {'ok': True, 'grade_decimal': 1, 'msg': ''}, {'ok': False, 'grade_decimal': 0, 'msg': ''}, {'ok': True, 'grade_decimal': 1, 'msg': ''} ] } assert grader(None, submission) == expected_result
def test_fg_evals_numbered_variables_in_siblings(): subgrader = FormulaGrader(numbered_vars=['x']) grader = ListGrader(answers=['x_{1}', 'x_{2} + sibling_1'], subgraders=subgrader, ordered=True) results = [] side_effect = log_results(results)(subgrader.get_sibling_formulas) with patch.object(subgrader, 'get_sibling_formulas', side_effect=side_effect): grader(None, ['x_{0}+1', 'x_{1} + 1']) # get_sibling_formulas should be called twice, once for each input assert len(results) == 2 # the first call should not provide any sibling formulas assert results[0] == {} # The second call should provide only the first sibling formula assert results[1] == {'sibling_1': 'x_{0}+1'}
def test_nested_listgrader(): """Check that we can use have a SingleListGrader as a subgrader""" grader = ListGrader(answers=[['1', '2'], ['3', '4']], subgraders=SingleListGrader(subgrader=StringGrader())) result = grader(None, ['1,2', '4,3']) expected_result = { 'overall_message': '', 'input_list': [{ 'ok': True, 'grade_decimal': 1, 'msg': '' }, { 'ok': True, 'grade_decimal': 1, 'msg': '' }] } assert result == expected_result