def test_fg_userfunction():
"""Test FormulaGrader with user-defined functions"""
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)",
user_functions={"hello": np.tan})
assert grader(None, "hello(0.4)")['ok']
assert grader(None, "sin(0.4)/cos(0.4)")['ok']
# Test with variable and function names with primes at the end
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)+t''^2",
variables=["t''"],
user_functions={"f'": np.tan})
assert grader(None, "f'(0.4)+t''^2")['ok']
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)",
user_functions={"function2name_2go''''''": np.tan})
assert grader(None, "function2name_2go''''''(0.4)")['ok']
# Primes aren't allowed in the middle
expect = "Invalid Input: Could not parse 'that'sbad\(1\)' as a formula"
with raises(CalcError, match=expect):
grader = FormulaGrader(answers="1",
user_functions={"that'sbad": np.tan})
grader(None, "that'sbad(1)")
expect = "1 is not a valid name for a function \(must be a string\)"
with raises(ConfigError, match=expect):
FormulaGrader(answers="1", user_functions={1: np.tan})
def test_fg_tolerance():
"""Test of FormulaGrader tolerance"""
grader = FormulaGrader(answers="10", tolerance=0.1)
assert not grader(None, '9.85')['ok']
assert grader(None, '9.9')['ok']
assert grader(None, '10')['ok']
assert grader(None, '10.1')['ok']
assert not grader(None, '10.15')['ok']
grader = FormulaGrader(answers="10", tolerance="1%")
assert not grader(None, '9.85')['ok']
assert grader(None, '9.9')['ok']
assert grader(None, '10')['ok']
assert grader(None, '10.1')['ok']
assert not grader(None, '10.15')['ok']
grader = FormulaGrader(answers="10", tolerance=0)
assert not grader(None, '9.999999')['ok']
assert grader(None, '10')['ok']
assert not grader(None, '10.000001')['ok']
expect = "Cannot have a negative percentage for dictionary value @ " + \
"data\['tolerance'\]. Got '-1%'"
with raises(Error, match=expect):
FormulaGrader(answers="10", tolerance="-1%")
def test_fg_whitelist_grading():
grader = FormulaGrader(
answers="sin(0.4)/cos(0.4)",
user_functions={"hello": np.tan},
whitelist=['cos', 'sin']
)
assert grader(None, "hello(0.4)")['ok']
assert grader(None, "sin(0.4)/cos(0.4)")['ok']
# Incorrect answers with forbidden function are marked wrong:
assert not grader(None, "cos(0.4)/sin(0.4)")['ok']
# Correct answers with forbidden function raise error:
expect = r"Invalid Input: function\(s\) 'tan' not permitted in answer"
with raises(InvalidInput, match=expect):
grader(None, "tan(0.4)")
expect = r"Invalid Input: TAN not permitted in answer as a function \(did you mean tan\?\)"
with raises(UndefinedFunction, match=expect):
grader(None, "TAN(0.4)")
grader = FormulaGrader(
answers="1",
whitelist=[None]
)
assert grader(None, "1")['ok']
expect = r"Invalid Input: function\(s\) 'cos' not permitted in answer"
with raises(InvalidInput, match=expect):
grader(None, "cos(0)")
def test_fg_function_sampling():
"""Test random functions in FormulaGrader"""
grader = FormulaGrader(answers="hello(x)",
variables=['x'],
user_functions={'hello': RandomFunction()})
assert grader(None, 'hello(x)')['ok']
assert isinstance(grader.random_funcs['hello'], RandomFunction)
grader = FormulaGrader(answers="hello(x)",
variables=['x'],
user_functions={'hello': [lambda x: x * x]})
assert isinstance(grader.random_funcs['hello'], SpecificFunctions)
assert grader(None, 'hello(x)')['ok']
grader = FormulaGrader(answers="hello(x)",
variables=['x'],
user_functions={'hello': [np.sin, np.cos, np.tan]})
assert isinstance(grader.random_funcs['hello'], SpecificFunctions)
assert grader(None, 'hello(x)')['ok']
grader = FormulaGrader(
answers="hello(x)",
variables=['x'],
user_functions={'hello': SpecificFunctions([np.sin, np.cos, np.tan])})
assert isinstance(grader.random_funcs['hello'], SpecificFunctions)
assert grader(None, 'hello(x)')['ok']
def test_fg_userfunction():
"""Test FormulaGrader with user-defined functions"""
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)",
user_functions={"hello": np.tan})
assert grader(None, "hello(0.4)")['ok']
assert grader(None, "sin(0.4)/cos(0.4)")['ok']
# Test with variable and function names with primes at the end
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)+t''^2",
variables=["t''"],
user_functions={"f'": np.tan})
assert grader(None, "f'(0.4)+t''^2")['ok']
grader = FormulaGrader(answers="sin(0.4)/cos(0.4)",
user_functions={"function2name_2go''''''": np.tan})
assert grader(None, "function2name_2go''''''(0.4)")['ok']
# Primes aren't allowed in the middle
expect = "Invalid Input: Could not parse 'that'sbad\(1\)' as a formula"
with raises(CalcError, match=expect):
grader = FormulaGrader(answers="1",
user_functions={"that'sbad": np.tan})
grader(None, "that'sbad(1)")
expect = ("1 is not a valid key, must be of <type 'str'> for dictionary "
"value @ data\['user_functions'\]. Got {1: <ufunc 'tan'>}")
with raises(Error, match=expect):
FormulaGrader(answers="1", user_functions={1: np.tan})
def test_fg_userconstants():
"""Test FormulaGrader with user-defined constants"""
grader = FormulaGrader(answers="5", user_constants={"hello": 5})
assert grader(None, "hello")['ok']
expect = "1 is not a valid name for a constant \(must be a string\)"
with raises(ConfigError, match=expect):
FormulaGrader(answers="1", user_constants={1: 5})
def test_fg_userconstants():
"""Test FormulaGrader with user-defined constants"""
grader = FormulaGrader(answers="5", user_constants={"hello": 5})
assert grader(None, "hello")['ok']
expect = ("1 is not a valid key, must be of <type 'str'> for dictionary "
"value @ data\['user_constants'\]. Got {1: 5}")
with raises(Error, match=expect):
FormulaGrader(answers="1", user_constants={1: 5})
def test_fg_blacklist_whitelist_config_errors():
with raises(ConfigError,
match="Cannot whitelist and blacklist at the same time"):
FormulaGrader(answers="5", blacklist=['tan'], whitelist=['tan'])
with raises(ConfigError, match="Unknown function in blacklist: test"):
FormulaGrader(answers="5", blacklist=['test'])
with raises(ConfigError, match="Unknown function in whitelist: test"):
FormulaGrader(answers="5", whitelist=['test'])
def test_linear_too_few_comparisons():
FormulaGrader.set_default_comparer(LinearComparer())
grader = FormulaGrader(samples=2)
with raises(
ConfigError,
match='Cannot perform linear comparison with less than 3 samples'):
grader('1.5', '1.5')
# Ensure that NumericalGrader does not use the same default comparer as FormulaGrader
grader = NumericalGrader()
assert grader('1.5', '1.5')['ok']
FormulaGrader.reset_default_comparer()
def test_empty_entry_in_answers():
msg = ("There is a problem with the author's problem configuration: "
"Empty entry detected in answer list. Students receive an error "
"when supplying an empty entry. Set 'missing_error' to False in "
"order to allow such entries.")
# Test base case
with raises(ConfigError, match=msg):
grader = SingleListGrader(answers=['a', 'b', ' ', 'c'],
subgrader=StringGrader())
# Test case with dictionary
with raises(ConfigError, match=msg):
grader = SingleListGrader(answers=['a', 'b', {
'expect': ' '
}, 'c'],
subgrader=StringGrader())
# Test case with unusual expect values
with raises(ConfigError, match=msg):
grader = SingleListGrader(answers=[
'a', 'b', {
'comparer': congruence_comparer,
'comparer_params': ['b^2/a', '2*pi']
}, ''
],
subgrader=FormulaGrader())
# Test case with really unusual expect values
with raises(ConfigError, match=msg):
grader = SingleListGrader(answers=[
'a', 'b', {
'expect': {
'comparer': congruence_comparer,
'comparer_params': ['b^2/a', '2*pi']
},
'msg': 'Well done!'
}, ''
],
subgrader=FormulaGrader())
# Test nested case
with raises(ConfigError, match=msg):
grader = SingleListGrader(answers=[['a', 'b'], ['c', '']],
subgrader=SingleListGrader(
subgrader=StringGrader(), delimiter=';'))
# Test case with no error
grader = SingleListGrader(answers=['a', 'b', ' ', 'c'],
subgrader=StringGrader(),
missing_error=False)
grader('a,,b', 'a, b, c')
def test_infinity():
grader = FormulaGrader(answers="infty", allow_inf=True)
assert grader(None, 'infty')['ok']
assert not grader(None, '-infty')['ok']
assert grader.default_variables['infty'] == float('inf')
grader = FormulaGrader(answers='infty',
user_constants={'infty': float('inf')})
assert 'infty' not in grader.default_variables
with raises(
CalcError,
match=
'Numerical overflow occurred. Does your expression generate very large numbers?'
):
grader(None, 'infty')
def test_overriding_functions():
grader = FormulaGrader(answers='z^2',
variables=['z'],
user_functions={
're': RandomFunction(),
'im': RandomFunction()
},
sample_from={'z': ComplexRectangle()})
learner_input = 're(z)^2 - im(z)^2 + 2*i*re(z)*im(z)'
assert not grader(None, learner_input)['ok']
grader = FormulaGrader(answers='tan(1)',
user_functions={'sin': lambda x: x})
assert grader(None, 'tan(1)')['ok']
assert not grader(None, 'sin(1)/cos(1)')['ok']
def test_nested_grouping_ordered():
"""Test that ordered nested groupings work appropriately"""
grader = ListGrader(
answers=[
['0', '1'],
['2', '3'],
],
subgraders=ListGrader(
subgraders=FormulaGrader(),
ordered=True
),
grouping=[1, 1, 2, 2]
)
def expect(a, b, c, d):
return {
'input_list': [
{'grade_decimal': a, 'msg': '', 'ok': a == 1},
{'grade_decimal': b, 'msg': '', 'ok': b == 1},
{'grade_decimal': c, 'msg': '', 'ok': c == 1},
{'grade_decimal': d, 'msg': '', 'ok': d == 1}
],
'overall_message': ''
}
assert grader(None, ['0', '1', '2', '3']) == expect(1, 1, 1, 1)
assert grader(None, ['1', '0', '3', '2']) == expect(0, 0, 0, 0)
assert grader(None, ['2', '3', '0', '1']) == expect(1, 1, 1, 1)
assert grader(None, ['3', '2', '1', '0']) == expect(0, 0, 0, 0)
assert grader(None, ['1', '3', '2', '0']) == expect(0, 1, 0, 0)
assert grader(None, ['0', '2', '3', '1']) == expect(1, 0, 0, 0)
def test_fg_variables():
"""General test of FormulaGrader using variables"""
grader = FormulaGrader(
answers="1+x^2+y^2",
variables=['x', 'y']
)
assert grader(None, '(x+y+1)^2 - 2*x-2*y-2*x*y')['ok']
def test_fg_custom_comparers():
def is_coterminal_and_large(comparer_params, student_input, utils):
answer = comparer_params[0]
min_value = comparer_params[1]
reduced = student_input % (360)
return utils.within_tolerance(answer, reduced) and student_input > min_value
mock = Mock(side_effect=is_coterminal_and_large,
# The next two kwargs ensure that the Mock behaves nicely for inspect.getargspec
spec=is_coterminal_and_large,
func_code=is_coterminal_and_large.func_code,)
grader = FormulaGrader(
answers={
'comparer': mock,
'comparer_params': ['150 + 50', '360 * 2'],
},
tolerance='1%'
)
assert grader(None, '200 + 3*360') == {'grade_decimal': 1, 'msg': '', 'ok': True}
mock.assert_called_with([200, 720], 1280, grader.comparer_utils)
assert grader(None, '199 + 3*360') == {'grade_decimal': 1, 'msg': '', 'ok': True}
assert grader(None, '197 + 3*360') == {'grade_decimal': 0, 'msg': '', 'ok': False}
def test_fg_evaluates_siblings_appropriately():
grader=ListGrader(
answers=['sibling_3 + 1', 'sibling_1^2', 'x'],
subgraders=FormulaGrader(variables=['x']),
ordered=True
)
# All correct!
result = grader(None, ['x + 1', 'x^2 + 2*x + 1', 'x'])
expected = {
'input_list': [
{'grade_decimal': 1, 'msg': '', 'ok': True},
{'grade_decimal': 1, 'msg': '', 'ok': True},
{'grade_decimal': 1, 'msg': '', 'ok': True}],
'overall_message': ''
}
assert result == expected
# First input wrong, but other two consistent
result = grader(None, ['x + 2', 'x^2 + 4*x + 4', 'x'])
expected = {
'input_list': [
{'grade_decimal': 0, 'msg': '', 'ok': False},
{'grade_decimal': 1, 'msg': '', 'ok': True},
{'grade_decimal': 1, 'msg': '', 'ok': True}],
'overall_message': ''
}
assert result == expected
# Cannot grade, missing a required input
match='Cannot grade answer, a required input is missing.'
with raises(MissingInput, match=match):
result = grader(None, ['', 'x^2 + 2*x + 1', 'x'])
def test_siblings_passed_to_subgrader_check_if_ordered_and_subgrader_list():
sg0 = FormulaGrader()
sg1 = NumericalGrader()
sg2 = StringGrader()
grader = ListGrader(answers=['1', '2', '3'],
subgraders=[sg0, sg1, sg2],
ordered=True)
student_input = ['10', '20', '30']
siblings = [{
'input': '10',
'grader': sg0
}, {
'input': '20',
'grader': sg1
}, {
'input': '30',
'grader': sg2
}]
# There must be a better way to spy on multiple things...
with mock.patch.object(sg0, 'check', wraps=sg0.check) as check0:
with mock.patch.object(sg1, 'check', wraps=sg1.check) as check1:
with mock.patch.object(sg2, 'check', wraps=sg2.check) as check2:
grader(None, ['10', '20', '30'])
# subgrader check has been called three times
assert len(check0.call_args_list) == 1
assert len(check1.call_args_list) == 1
assert len(check2.call_args_list) == 1
# subgrader check has been passed the correct siblings
for _, kwargs in check0.call_args_list:
assert kwargs['siblings'] == siblings
for _, kwargs in check1.call_args_list:
assert kwargs['siblings'] == siblings
for _, kwargs in check2.call_args_list:
assert kwargs['siblings'] == siblings
def test_fg_custom_comparers():
def is_coterminal_and_large(comparer_params, student_input, utils):
answer = comparer_params[0]
min_value = comparer_params[1]
reduced = student_input % (360)
return utils.within_tolerance(answer,
reduced) and student_input > min_value
grader = FormulaGrader(answers={
'comparer': is_coterminal_and_large,
'comparer_params': ['150 + 50', '360 * 2'],
},
tolerance='1%')
assert grader(None, '200 + 3*360') == {
'grade_decimal': 1,
'msg': '',
'ok': True
}
assert grader(None, '199 + 3*360') == {
'grade_decimal': 1,
'msg': '',
'ok': True
}
assert grader(None, '197 + 3*360') == {
'grade_decimal': 0,
'msg': '',
'ok': False
}
def test_linear_comparer_custom_credit_modes():
grader = FormulaGrader(answers={
'comparer_params': ['m*c^2'],
'comparer':
LinearComparer(equals=0.8, proportional=0.6, offset=0.4, linear=0.2)
},
variables=['m', 'c'])
equals_result = {'msg': '', 'grade_decimal': 0.8, 'ok': 'partial'}
proportional_result = {
'msg':
'The submitted answer differs from an expected answer by a constant factor.',
'grade_decimal': 0.6,
'ok': 'partial'
}
offset_result = {'msg': '', 'grade_decimal': 0.4, 'ok': 'partial'}
linear_result = {'msg': '', 'grade_decimal': 0.2, 'ok': 'partial'}
wrong_result = {'msg': '', 'grade_decimal': 0, 'ok': False}
assert grader(None, 'm*c^2') == equals_result
assert grader(None, '3*m*c^2') == proportional_result
assert grader(None, 'm*c^2 + 10') == offset_result
assert grader(None, '-3*m*c^2 + 10') == linear_result
assert grader(None, 'm*c^3') == wrong_result
assert grader(None, '0') == wrong_result
def test_instructor_vars():
"""Ensures that instructor variables are not available to students"""
grader = FormulaGrader(
answers='sin(x)/cos(x)',
variables=['x', 's', 'c'],
numbered_vars=['y'],
sample_from={
'x': [-3.14159, 3.14159],
's': DependentSampler(depends=["x"], formula="sin(x)"),
'c': DependentSampler(depends=["x"], formula="cos(x)")
},
instructor_vars=['x', 'pi', 'y_{0}',
'nothere'] # nothere will be ignored
)
assert grader(None, 's/c')['ok']
assert not grader(None, 'y_{1}')['ok']
with raises(UndefinedVariable,
match="'x' not permitted in answer as a variable"):
grader(None, 'tan(x)')
with raises(UndefinedVariable,
match="'pi' not permitted in answer as a variable"):
grader(None, 'pi')
with raises(UndefinedVariable,
match=r"'y_\{0\}' not permitted in answer as a variable"):
grader(None, 'y_{0}')
def test_readme():
"""Tests that the README.md file examples work"""
grader = StringGrader(answers='cat')
grader = ListGrader(answers=['1', '2'], subgraders=FormulaGrader())
del grader
def test_fg_expressions():
"""General test of FormulaGrader"""
grader = FormulaGrader(answers="1+tan(3/2)", tolerance="0.1%")
assert grader(None, "(cos(3/2) + sin(3/2))/cos(3/2 + 2*pi)")['ok']
# Checking tolerance
assert grader(None, "0.01+(cos(3/2) + sin(3/2))/cos(3/2 + 2*pi)")['ok']
assert not grader(None, "0.02+(cos(3/2) + sin(3/2))/cos(3/2 + 2*pi)")['ok']
def test_whitespace_stripping():
"""Test that formulas work regardless of whitespace"""
grader = FormulaGrader(
variables=['x_{ab}'],
answers='x _ { a b }'
)
assert grader(None, 'x_{a b}')['ok']
def test_fg_invalid_input():
grader = FormulaGrader(answers='2', variables=['m'])
expect = "Invalid Input: 'pi' not permitted in answer as a function " + \
r'\(did you forget to use \* for multiplication\?\)'
with raises(CalcError, match=expect):
grader(None, "pi(3)")
expect = "Invalid Input: 'Im', 'Re' not permitted in answer as a function " + \
r"\(did you mean 'im', 're'\?\)"
with raises(CalcError, match=expect):
grader(None, "Im(3) + Re(2)")
expect = "Invalid Input: 'spin' not permitted in answer as a function"
with raises(CalcError, match=expect):
grader(None, "spin(3)")
expect = "Invalid Input: 'R' not permitted in answer as a variable"
with raises(CalcError, match=expect):
grader(None, "R")
expect = "Invalid Input: 'Q', 'R' not permitted in answer as a variable"
with raises(CalcError, match=expect):
grader(None, "R+Q")
expect = "Invalid Input: 'pp' not permitted directly after a number"
with raises(CalcError, match=expect):
grader(None, "5pp")
expect = "Invalid Input: 'mm', 'pp' not permitted directly after a number"
with raises(CalcError, match=expect):
grader(None, "5pp+6mm")
expect = (r"Invalid Input: 'm' not permitted directly after a number "
r"\(did you forget to use \* for multiplication\?\)")
with raises(CalcError, match=expect):
grader(None, "5m")
expect = (r"There was an error evaluating csc\(...\). "
r"Its input does not seem to be in its domain.")
with raises(CalcError, match=expect):
grader(None, 'csc(0)')
expect = r"There was an error evaluating sinh\(...\). \(Numerical overflow\)."
with raises(CalcError, match=expect):
grader(None, 'sinh(10000)')
expect = (r"There was an error evaluating arccosh\(...\). "
r"Its input does not seem to be in its domain.")
with raises(CalcError, match=expect):
grader(None, 'arccosh(0)')
expect = "Division by zero occurred. Check your input's denominators."
with raises(CalcError, match=expect):
grader(None, '1/0')
expect = "Numerical overflow occurred. Does your input generate very large numbers?"
with raises(CalcError, match=expect):
grader(None, '2^10000')
def test_fg_config_expect():
# If trying to use comparer, a detailed validation error is raised
expect = ("to have 3 arguments, instead it has 2 for dictionary value @ "
"data\['answers'\]\[0\]\['expect'\]\['comparer'\]")
with raises(Error, match=expect):
FormulaGrader(
answers={
'comparer': lambda x, y: x + y,
'comparer_params': ['150 + 50', '360 * 2']
})
# If not, a simpler error is raised:
expect = ("Something's wrong with grader's 'answers' configuration key. "
"Please see documentation for accepted formats.")
with raises(Error, match=expect):
FormulaGrader(answers=5)
def test_fg_userfunc():
"""Test a user function in FormulaGrader"""
grader = FormulaGrader(
answers="hello(2)",
user_functions={"hello": lambda x: x**2-1}
)
assert grader(None, "5+hello(2)-2-3")['ok']
assert not grader(None, "hello(1)")['ok']
def test_overriding_functions():
grader = FormulaGrader(
answers='tan(1)',
user_functions={'sin': lambda x: x},
suppress_warnings=True
)
assert grader(None, 'tan(1)')['ok']
assert not grader(None, 'sin(1)/cos(1)')['ok']
def test_fg_percent():
"""Test a percentage suffix in FormulaGrader"""
grader = FormulaGrader(answers="2%")
assert grader(None, "2%")['ok']
assert grader(None, "0.02")['ok']
with raises(CalcError,
match="Invalid Input: Could not parse '20m' as a formula"):
grader(None, "20m")
def test_fg_required():
"""Test FormulaGrader with required functions in input"""
grader = FormulaGrader(answers="sin(x)/cos(x)",
variables=['x'],
required_functions=['sin', 'cos'])
assert grader(None, 'sin(x)/cos(x)')['ok']
with raises(InvalidInput,
match="Invalid Input: Answer must contain the function sin"):
grader(None, "tan(x)")
def test_fg_percent():
"""Test a percentage suffix in FormulaGrader"""
grader = FormulaGrader(answers="2%")
assert grader(None, "2%")['ok']
assert grader(None, "0.02")['ok']
with raises(
CalcError,
match="Invalid Input: m not permitted directly after a number"):
grader(None, "20m")
