def gen_var_and_func_samples(self, *args): """ Generate a list of variable/function sampling dictionaries from the supplied arguments. Arguments may be strings, lists of strings, or dictionaries with string values. Does not flag any bad variables. """ # Make a list of all expressions to check for variables expressions = [] for entry in args: if isinstance(entry, six.text_type): expressions.append(entry) elif isinstance(entry, list): expressions += entry elif isinstance(entry, dict): expressions += [v for k, v in entry.items()] # Generate the variable list variables, sample_from_dict = self.generate_variable_list(expressions) # Generate the samples var_samples = gen_symbols_samples(variables, self.config['samples'], sample_from_dict, self.functions, self.suffixes, self.constants) func_samples = gen_symbols_samples(list(self.random_funcs.keys()), self.config['samples'], self.random_funcs, self.functions, self.suffixes, {}) return var_samples, func_samples
def test_dependent_sampler_infers_dependence(): sampler = DependentSampler(formula="a+1") assert sampler.config['depends'] == ['a'] sampler = DependentSampler(formula="x^2+y^2+sin(c)") assert set(sampler.config['depends']) == set(['x', 'y', 'c']) # Test that 'depends' is now ignored symbols = ["x", "y"] samples = 1 sample_from = { 'x': DependentSampler(depends=["y"], formula="1"), 'y': DependentSampler(depends=["x"], formula="1") } funcs, suffs, consts = {}, {}, {} # This does NOT raise an error; the depends entry is ignored gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts)
def gen_var_and_func_samples(self, answer, student_input, sibling_formulas): """ Generate a list of variable/function sampling dictionaries. """ comparer_params = answer['expect']['comparer_params'] # Generate samples; Include siblings to get numbered_vars from them expressions = (comparer_params + [student_input] + [sibling_formulas[key] for key in sibling_formulas]) variables, sample_from_dict = self.generate_variable_list(expressions) var_samples = gen_symbols_samples(variables, self.config['samples'], sample_from_dict, self.functions, self.suffixes, self.constants) func_samples = gen_symbols_samples(list(self.random_funcs.keys()), self.config['samples'], self.random_funcs, self.functions, self.suffixes, {}) return var_samples, func_samples
def test_overriding_constant_with_dependent_sampling(): symbols = ['a', 'b', 'c'] samples = 1 sample_from = { 'a': RealInterval([10, 10]), 'b': DependentSampler(depends=["a"], formula="a+1"), 'c': DependentSampler(depends=["b"], formula="b+1") } funcs, suffs = {}, {} consts = {'unity': 1, 'b': 3.14} result = gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts)[0] a, b, c = [result[sym] for sym in 'abc'] assert a == 10 assert b == 11 assert c == 12
def test_dependent_sampler(): """Tests the DependentSampler class""" # Test basic usage and multiple samples result = gen_symbols_samples( ["a", "b"], 2, { 'a': IntegerRange([1, 1]), 'b': DependentSampler(depends=["a"], formula="a+1") }) assert result == [{"a": 1, "b": 2.0}, {"a": 1, "b": 2.0}] result = gen_symbols_samples( ["a", "b", "c", "d"], 1, { 'a': RealInterval([1, 1]), 'd': DependentSampler(depends=["c"], formula="c+1"), 'c': DependentSampler(depends=["b"], formula="b+1"), 'b': DependentSampler(depends=["a"], formula="a+1") })[0] assert result["b"] == 2 and result["c"] == 3 and result["d"] == 4 result = gen_symbols_samples( ["x", "y", "z", "r"], 1, { 'x': RealInterval([-5, 5]), 'y': RealInterval([-5, 5]), 'z': RealInterval([-5, 5]), 'r': DependentSampler(depends=["x", "y", "z"], formula="sqrt(x^2+y^2+z^2)") })[0] assert result["x"]**2 + result["y"]**2 + result["z"]**2 == approx( result["r"]**2) with raises(ConfigError, match="Circularly dependent DependentSamplers detected: x, y"): gen_symbols_samples( ["x", "y"], 1, { 'x': DependentSampler(depends=["y"], formula="1"), 'y': DependentSampler(depends=["x"], formula="1") }) with raises(ConfigError, match=r"Formula error in dependent sampling formula: 1\+\(2"): gen_symbols_samples( ["x"], 1, {'x': DependentSampler(depends=[], formula="1+(2")}) with raises(Exception, match="DependentSampler must be invoked with compute_sample."): DependentSampler(depends=[], formula="1").gen_sample()
def raw_check(self, answer, cleaned_input): """Perform the numerical check of student_input vs answer""" var_samples = gen_symbols_samples(self.config['variables'], self.config['samples'], self.config['sample_from']) func_samples = gen_symbols_samples(self.random_funcs.keys(), self.config['samples'], self.random_funcs) # Make a copy of the functions and variables lists # We'll add the sampled functions/variables in funcscope = self.functions.copy() varscope = self.constants.copy() num_failures = 0 for i in range(self.config['samples']): # Update the functions and variables listings with this sample funcscope.update(func_samples[i]) varscope.update(var_samples[i]) # Evaluate integrals. Error handling here is in two parts because # 1. custom error messages we've added # 2. scipy's warnings re-raised as error messages try: expected_re, expected_im = self.evaluate_int( answer['integrand'], answer['lower'], answer['upper'], answer['integration_variable'], varscope=varscope, funcscope=funcscope) except IntegrationError as e: msg = "Integration Error with author's stored answer: {}" raise ConfigError(msg.format(e.message)) student_re, student_im = self.evaluate_int( cleaned_input['integrand'], cleaned_input['lower'], cleaned_input['upper'], cleaned_input['integration_variable'], varscope=varscope, funcscope=funcscope) # scipy raises integration warnings when things go wrong, # except they aren't really warnings, they're just printed to stdout # so we use quad's full_output option to catch the messages, and then raise errors. # The 4th component only exists when its warning message is non-empty if len(student_re) == 4: raise IntegrationError(student_re[3]) if len(expected_re) == 4: raise ConfigError(expected_re[3]) if len(student_im) == 4: raise IntegrationError(student_im[3]) if len(expected_im) == 4: raise ConfigError(expected_im[3]) self.log( self.debug_appendix_template.format( samplenum=i, variables=varscope, student_re_eval=student_re[0], student_re_error=student_re[1], student_re_neval=student_re[2]['neval'], student_im_eval=student_im[0], student_im_error=student_im[1], student_im_neval=student_im[2]['neval'], author_re_eval=expected_re[0], author_re_error=expected_re[1], author_re_neval=expected_re[2]['neval'], author_im_eval=expected_im[0], author_im_error=expected_im[1], author_im_neval=expected_im[2]['neval'], )) # Check if expressions agree expected = expected_re[0] + (expected_im[0] or 0) * 1j student = student_re[0] + (student_im[0] or 0) * 1j if not within_tolerance(expected, student, self.config['tolerance']): num_failures += 1 if num_failures > self.config["failable_evals"]: return {'ok': False, 'grade_decimal': 0, 'msg': ''} # This response appears to agree with the expected answer return {'ok': True, 'grade_decimal': 1, 'msg': ''}
def raw_check(self, answer, student_input, **kwargs): """Perform the numerical check of student_input vs answer""" comparer_params = answer['expect']['comparer_params'] siblings = kwargs.get('siblings', None) required_siblings = self.get_used_vars(comparer_params) # required_siblings might include some extra variable names, but no matter sibling_formulas = self.get_sibling_formulas(siblings, required_siblings) # Generate samples; Include siblings to get numbered_vars from them expressions = (comparer_params + [student_input] + [sibling_formulas[key] for key in sibling_formulas]) variables, sample_from_dict = self.generate_variable_list(expressions) var_samples = gen_symbols_samples(variables, self.config['samples'], sample_from_dict, self.functions, self.suffixes) func_samples = gen_symbols_samples(self.random_funcs.keys(), self.config['samples'], self.random_funcs, self.functions, self.suffixes) # Make a copy of the functions and variables lists # We'll add the sampled functions/variables in funclist = self.functions.copy() varlist = self.constants.copy() # Get the comparer function comparer = answer['expect']['comparer'] num_failures = 0 for i in range(self.config['samples']): # Update the functions and variables listings with this sample funclist.update(func_samples[i]) varlist.update(var_samples[i]) def scoped_eval(expression, variables=varlist, functions=funclist, suffixes=self.suffixes, max_array_dim=self.config['max_array_dim']): return evaluator(expression, variables, functions, suffixes, max_array_dim) # Compute the sibling values, and add them to varlist siblings_eval = { key: scoped_eval(sibling_formulas[key])[0] for key in sibling_formulas } varlist.update(siblings_eval) # Compute expressions comparer_params_eval = self.eval_and_validate_comparer_params( scoped_eval, comparer_params, siblings_eval) # Before performing student evaluation, scrub the siblings # so that students can't use them for key in siblings_eval: del varlist[key] student_eval, used = scoped_eval(student_input) # Check if expressions agree comparer_result = comparer(comparer_params_eval, student_eval, self.comparer_utils) comparer_result = ItemGrader.standardize_cfn_return(comparer_result) if self.config['debug']: # Put the siblings back in for the debug output varlist.update(siblings_eval) self.log_sample_info(i, varlist, funclist, student_eval, comparer, comparer_params_eval, comparer_result) if not comparer_result['ok']: num_failures += 1 if num_failures > self.config["failable_evals"]: return comparer_result, used.functions_used # This response appears to agree with the expected answer return { 'ok': answer['ok'], 'grade_decimal': answer['grade_decimal'], 'msg': answer['msg'] }, used.functions_used
def test_dependent_sampler(): """Tests the DependentSampler class""" # Test basic usage and multiple samples symbols = ["a", "b"] samples = 2 sample_from = { 'a': IntegerRange([1, 1]), 'b': DependentSampler(depends=["a"], formula="a+1") } funcs, suffs, consts = {}, {}, {} result = gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts) assert result == [{"a": 1, "b": 2.0}, {"a": 1, "b": 2.0}] symbols = ["a", "b", "c", "d"] samples = 1 sample_from = { 'a': RealInterval([1, 1]), 'd': DependentSampler(depends=["c"], formula="c+1"), 'c': DependentSampler(depends=["b"], formula="b+1"), 'b': DependentSampler(depends=["a"], formula="a+1") } funcs, suffs, consts = {}, {}, {} result = gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts)[0] assert result["b"] == 2 and result["c"] == 3 and result["d"] == 4 symbols = ["x", "y", "z", "r"] samples = 1 sample_from = { 'x': RealInterval([-5, 5]), 'y': RealInterval([-5, 5]), 'z': RealInterval([-5, 5]), 'r': DependentSampler(depends=["x", "y", "z"], formula="sqrt(x^2+y^2+z^2 + unity)") } funcs = {'sqrt': lambda x: x**0.5} consts = {'unity': 1} suffs = {} result = gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts)[0] assert result["x"]**2 + result["y"]**2 + result["z"]**2 + 1 == approx(result["r"]**2) symbols = ["x", "y"] samples = 1 sample_from = { 'x': DependentSampler(depends=["y"], formula="y"), 'y': DependentSampler(depends=["x"], formula="x") } funcs, suffs, consts = {}, {}, {} with raises(ConfigError, match="Circularly dependent DependentSamplers detected: x, y"): gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts) with raises(ConfigError, match=r"Formula error in dependent sampling formula: 1\+\(2"): DependentSampler(formula="1+(2") symbols = ["x"] samples = 1 sample_from = {'x': DependentSampler(formula="min(j, i)")} with raises(ConfigError, match=r"Formula error in dependent sampling formula: min\(j, i\)"): gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, {'i': 1j, 'j': 1j}) with raises(ConfigError, match=r"DependentSamplers depend on undefined quantities: i, j"): gen_symbols_samples(symbols, samples, sample_from, funcs, suffs, consts) with raises(Exception, match="DependentSampler must be invoked with compute_sample."): DependentSampler(depends=[], formula="1").gen_sample()