def test_registered_defaults():
# Test that each class has it's own default_variables parameter
AbstractGrader.register_defaults({'test': False})
assert StringGrader.default_values is None
AbstractGrader.clear_registered_defaults()
# Test that registered defaults are used in instantiation
StringGrader.register_defaults({'case_sensitive': False})
assert StringGrader.default_values == {'case_sensitive': False}
grader = StringGrader()
assert grader.config['case_sensitive'] is False
# Test that registered defaults clear correctly
StringGrader.clear_registered_defaults()
assert StringGrader.default_values is None
# Check that registered defaults propagate to subclasses
AbstractGrader.register_defaults({'debug': True})
grader = StringGrader()
assert grader.config['debug']
assert StringGrader.default_values is None
AbstractGrader.clear_registered_defaults()
# Check that registered defaults layer up through a subclass chain
AbstractGrader.register_defaults({'debug': True})
ItemGrader.register_defaults({'wrong_msg': 'haha!'})
StringGrader.register_defaults({'case_sensitive': False})
grader = StringGrader()
assert grader.config['debug']
assert grader.config['wrong_msg'] == 'haha!'
assert not grader.config['case_sensitive']
AbstractGrader.clear_registered_defaults()
ItemGrader.clear_registered_defaults()
StringGrader.clear_registered_defaults()
# Check that registered defaults can be higher level than where they're defined
StringGrader.register_defaults({'debug': True})
assert AbstractGrader.default_values is None
grader = StringGrader()
assert grader.config['debug']
StringGrader.clear_registered_defaults()
# Check that registered defaults are logged in the debug log
StringGrader.register_defaults({'debug': True})
grader = StringGrader()
result = grader('cat', 'cat')
expect = """<pre>MITx Grading Library Version {}<br/>
Running on edX using python {}<br/>
Student Response:<br/>
cat<br/>
Using modified defaults: {{"debug": true}}<br/>
Expect value inferred to be "cat"</pre>""".format(__version__, platform.python_version())
assert result['msg'] == expect
StringGrader.clear_registered_defaults()
def compare_evaluations(self, compare_params_evals, student_evals,
comparer, utils):
"""
Compare the student evaluations to the expected results.
"""
results = []
if isinstance(comparer, CorrelatedComparer):
result = comparer(compare_params_evals, student_evals, utils)
results.append(ItemGrader.standardize_cfn_return(result))
else:
for compare_params_eval, student_eval in zip(
compare_params_evals, student_evals):
result = comparer(compare_params_eval, student_eval, utils)
results.append(ItemGrader.standardize_cfn_return(result))
if self.config['debug']:
self.log_comparison_info(comparer, results)
return results
def consolidate_cfn_return(input_list,
overall_message="",
n_expect=None,
partial_credit=True):
"""
Consolidates a long-form customresponse return dictionary.
Arguments:
input_list (list): a list of customresponse single-answer dictionaries
each has keys 'ok', 'grade_decimal', 'msg'
overall_message (str): an overall message
n_expect: The expected number of answers, defaults to len(input_list).
Used in assigning partial credit.
Usage
=====
>>> input_list = [
... {'ok': True, 'msg': 'msg_0', 'grade_decimal':1},
... {'ok': 'partial', 'msg': 'msg_1', 'grade_decimal':0.5},
... {'ok': False, 'msg': 'msg_2', 'grade_decimal':0},
... {'ok': 'partial', 'msg': 'msg_3', 'grade_decimal':0.1},
... ]
>>> expect = {
... 'ok':'partial',
... 'grade_decimal': (1 + 0.5 + 0 + 0.1)/4,
... 'msg': 'summary\\nmsg_0\\nmsg_1\\nmsg_2\\nmsg_3' # escape newlines in docstring
... }
>>> result = consolidate_cfn_return(input_list,
... overall_message = "summary"
... )
>>> expect == result
True
"""
if n_expect is None:
n_expect = len(input_list)
grade_decimals = [result['grade_decimal'] for result in input_list]
grade_decimal = consolidate_grades(grade_decimals, n_expect)
if not partial_credit:
if grade_decimal < 1:
grade_decimal = 0
ok_status = ItemGrader.grade_decimal_to_ok(grade_decimal)
# TODO Discuss: Do we want the overall_message to go first or last?
messages = [overall_message] + [result['msg'] for result in input_list]
result = {
'grade_decimal': grade_decimal,
'ok': ok_status,
'msg': '\n'.join([message for message in messages if message != ''])
}
return result
def compare_evaluations(self, compare_params_evals, student_evals,
comparer, utils):
"""
Compare the student evaluations to the expected results.
"""
results = []
if isinstance(comparer, CorrelatedComparer):
result = comparer(compare_params_evals, student_evals, utils)
results.append(ItemGrader.standardize_cfn_return(result))
else:
for compare_params_eval, student_eval in zip(
compare_params_evals, student_evals):
result = comparer(compare_params_eval, student_eval, utils)
results.append(ItemGrader.standardize_cfn_return(result))
# TODO: Take out this if statement - should always work.
# However, presently doesn't, because subgraders don't have access to the master debuglog.
if self.config['debug']:
self.log_comparison_info(comparer, results)
return results
def raw_check(self, answer, student_input, **kwargs):
"""Perform the numerical check of student_input vs answer"""
comparer_params = answer['expect']['comparer_params']
siblings = kwargs.get('siblings', None)
required_siblings = self.get_used_vars(comparer_params)
# required_siblings might include some extra variable names, but no matter
sibling_formulas = self.get_sibling_formulas(siblings, required_siblings)
# Generate samples; Include siblings to get numbered_vars from them
expressions = (comparer_params
+ [student_input]
+ [sibling_formulas[key] for key in sibling_formulas])
variables, sample_from_dict = self.generate_variable_list(expressions)
var_samples = gen_symbols_samples(variables,
self.config['samples'],
sample_from_dict,
self.functions,
self.suffixes)
func_samples = gen_symbols_samples(self.random_funcs.keys(),
self.config['samples'],
self.random_funcs,
self.functions,
self.suffixes)
# Make a copy of the functions and variables lists
# We'll add the sampled functions/variables in
funclist = self.functions.copy()
varlist = self.constants.copy()
# Get the comparer function
comparer = answer['expect']['comparer']
num_failures = 0
for i in range(self.config['samples']):
# Update the functions and variables listings with this sample
funclist.update(func_samples[i])
varlist.update(var_samples[i])
def scoped_eval(expression,
variables=varlist,
functions=funclist,
suffixes=self.suffixes,
max_array_dim=self.config['max_array_dim']):
return evaluator(expression, variables, functions, suffixes, max_array_dim)
# Compute the sibling values, and add them to varlist
siblings_eval = {
key: scoped_eval(sibling_formulas[key])[0]
for key in sibling_formulas
}
varlist.update(siblings_eval)
# Compute expressions
comparer_params_eval = self.eval_and_validate_comparer_params(
scoped_eval, comparer_params, siblings_eval)
# Before performing student evaluation, scrub the siblings
# so that students can't use them
for key in siblings_eval:
del varlist[key]
student_eval, used = scoped_eval(student_input)
# Check if expressions agree
comparer_result = comparer(comparer_params_eval, student_eval, self.comparer_utils)
comparer_result = ItemGrader.standardize_cfn_return(comparer_result)
if self.config['debug']:
# Put the siblings back in for the debug output
varlist.update(siblings_eval)
self.log_sample_info(i, varlist, funclist, student_eval,
comparer, comparer_params_eval, comparer_result)
if not comparer_result['ok']:
num_failures += 1
if num_failures > self.config["failable_evals"]:
return comparer_result, used.functions_used
# This response appears to agree with the expected answer
return {
'ok': answer['ok'],
'grade_decimal': answer['grade_decimal'],
'msg': answer['msg']
}, used.functions_used