def test_registered_defaults(): # Test that each class has it's own default_variables parameter AbstractGrader.register_defaults({'test': False}) assert StringGrader.default_values is None AbstractGrader.clear_registered_defaults() # Test that registered defaults are used in instantiation StringGrader.register_defaults({'case_sensitive': False}) assert StringGrader.default_values == {'case_sensitive': False} grader = StringGrader() assert grader.config['case_sensitive'] is False # Test that registered defaults clear correctly StringGrader.clear_registered_defaults() assert StringGrader.default_values is None # Check that registered defaults propagate to subclasses AbstractGrader.register_defaults({'debug': True}) grader = StringGrader() assert grader.config['debug'] assert StringGrader.default_values is None AbstractGrader.clear_registered_defaults() # Check that registered defaults layer up through a subclass chain AbstractGrader.register_defaults({'debug': True}) ItemGrader.register_defaults({'wrong_msg': 'haha!'}) StringGrader.register_defaults({'case_sensitive': False}) grader = StringGrader() assert grader.config['debug'] assert grader.config['wrong_msg'] == 'haha!' assert not grader.config['case_sensitive'] AbstractGrader.clear_registered_defaults() ItemGrader.clear_registered_defaults() StringGrader.clear_registered_defaults() # Check that registered defaults can be higher level than where they're defined StringGrader.register_defaults({'debug': True}) assert AbstractGrader.default_values is None grader = StringGrader() assert grader.config['debug'] StringGrader.clear_registered_defaults() # Check that registered defaults are logged in the debug log StringGrader.register_defaults({'debug': True}) grader = StringGrader() result = grader('cat', 'cat') expect = """<pre>MITx Grading Library Version {}<br/> Running on edX using python {}<br/> Student Response:<br/> cat<br/> Using modified defaults: {{"debug": true}}<br/> Expect value inferred to be "cat"</pre>""".format(__version__, platform.python_version()) assert result['msg'] == expect StringGrader.clear_registered_defaults()
def compare_evaluations(self, compare_params_evals, student_evals, comparer, utils): """ Compare the student evaluations to the expected results. """ results = [] if isinstance(comparer, CorrelatedComparer): result = comparer(compare_params_evals, student_evals, utils) results.append(ItemGrader.standardize_cfn_return(result)) else: for compare_params_eval, student_eval in zip( compare_params_evals, student_evals): result = comparer(compare_params_eval, student_eval, utils) results.append(ItemGrader.standardize_cfn_return(result)) if self.config['debug']: self.log_comparison_info(comparer, results) return results
def consolidate_cfn_return(input_list, overall_message="", n_expect=None, partial_credit=True): """ Consolidates a long-form customresponse return dictionary. Arguments: input_list (list): a list of customresponse single-answer dictionaries each has keys 'ok', 'grade_decimal', 'msg' overall_message (str): an overall message n_expect: The expected number of answers, defaults to len(input_list). Used in assigning partial credit. Usage ===== >>> input_list = [ ... {'ok': True, 'msg': 'msg_0', 'grade_decimal':1}, ... {'ok': 'partial', 'msg': 'msg_1', 'grade_decimal':0.5}, ... {'ok': False, 'msg': 'msg_2', 'grade_decimal':0}, ... {'ok': 'partial', 'msg': 'msg_3', 'grade_decimal':0.1}, ... ] >>> expect = { ... 'ok':'partial', ... 'grade_decimal': (1 + 0.5 + 0 + 0.1)/4, ... 'msg': 'summary\\nmsg_0\\nmsg_1\\nmsg_2\\nmsg_3' # escape newlines in docstring ... } >>> result = consolidate_cfn_return(input_list, ... overall_message = "summary" ... ) >>> expect == result True """ if n_expect is None: n_expect = len(input_list) grade_decimals = [result['grade_decimal'] for result in input_list] grade_decimal = consolidate_grades(grade_decimals, n_expect) if not partial_credit: if grade_decimal < 1: grade_decimal = 0 ok_status = ItemGrader.grade_decimal_to_ok(grade_decimal) # TODO Discuss: Do we want the overall_message to go first or last? messages = [overall_message] + [result['msg'] for result in input_list] result = { 'grade_decimal': grade_decimal, 'ok': ok_status, 'msg': '\n'.join([message for message in messages if message != '']) } return result
def compare_evaluations(self, compare_params_evals, student_evals, comparer, utils): """ Compare the student evaluations to the expected results. """ results = [] if isinstance(comparer, CorrelatedComparer): result = comparer(compare_params_evals, student_evals, utils) results.append(ItemGrader.standardize_cfn_return(result)) else: for compare_params_eval, student_eval in zip( compare_params_evals, student_evals): result = comparer(compare_params_eval, student_eval, utils) results.append(ItemGrader.standardize_cfn_return(result)) # TODO: Take out this if statement - should always work. # However, presently doesn't, because subgraders don't have access to the master debuglog. if self.config['debug']: self.log_comparison_info(comparer, results) return results
def raw_check(self, answer, student_input, **kwargs): """Perform the numerical check of student_input vs answer""" comparer_params = answer['expect']['comparer_params'] siblings = kwargs.get('siblings', None) required_siblings = self.get_used_vars(comparer_params) # required_siblings might include some extra variable names, but no matter sibling_formulas = self.get_sibling_formulas(siblings, required_siblings) # Generate samples; Include siblings to get numbered_vars from them expressions = (comparer_params + [student_input] + [sibling_formulas[key] for key in sibling_formulas]) variables, sample_from_dict = self.generate_variable_list(expressions) var_samples = gen_symbols_samples(variables, self.config['samples'], sample_from_dict, self.functions, self.suffixes) func_samples = gen_symbols_samples(self.random_funcs.keys(), self.config['samples'], self.random_funcs, self.functions, self.suffixes) # Make a copy of the functions and variables lists # We'll add the sampled functions/variables in funclist = self.functions.copy() varlist = self.constants.copy() # Get the comparer function comparer = answer['expect']['comparer'] num_failures = 0 for i in range(self.config['samples']): # Update the functions and variables listings with this sample funclist.update(func_samples[i]) varlist.update(var_samples[i]) def scoped_eval(expression, variables=varlist, functions=funclist, suffixes=self.suffixes, max_array_dim=self.config['max_array_dim']): return evaluator(expression, variables, functions, suffixes, max_array_dim) # Compute the sibling values, and add them to varlist siblings_eval = { key: scoped_eval(sibling_formulas[key])[0] for key in sibling_formulas } varlist.update(siblings_eval) # Compute expressions comparer_params_eval = self.eval_and_validate_comparer_params( scoped_eval, comparer_params, siblings_eval) # Before performing student evaluation, scrub the siblings # so that students can't use them for key in siblings_eval: del varlist[key] student_eval, used = scoped_eval(student_input) # Check if expressions agree comparer_result = comparer(comparer_params_eval, student_eval, self.comparer_utils) comparer_result = ItemGrader.standardize_cfn_return(comparer_result) if self.config['debug']: # Put the siblings back in for the debug output varlist.update(siblings_eval) self.log_sample_info(i, varlist, funclist, student_eval, comparer, comparer_params_eval, comparer_result) if not comparer_result['ok']: num_failures += 1 if num_failures > self.config["failable_evals"]: return comparer_result, used.functions_used # This response appears to agree with the expected answer return { 'ok': answer['ok'], 'grade_decimal': answer['grade_decimal'], 'msg': answer['msg'] }, used.functions_used