def set_information(self, key, value): self.information[key] = utils.shorten_str(str(value))
def has_expr( state, incorrect_msg=None, error_msg=None, undefined_msg=None, append=None, extra_env=None, context_vals=None, pre_code=None, expr_code=None, name=None, copy=True, func=None, override=None, test=None, # todo: default or arg before state ): if ( append is None ): # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: if name: incorrect_msg = DEFAULT_INCORRECT_NAME_MSG elif expr_code: incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG else: incorrect_msg = DEFAULT_INCORRECT_MSG if undefined_msg is None: undefined_msg = DEFAULT_UNDEFINED_NAME_MSG if error_msg is None: if test == "error": error_msg = DEFAULT_ERROR_MSG_INV else: error_msg = DEFAULT_ERROR_MSG if state.solution_code is not None and isinstance(expr_code, str): expr_code = expr_code.replace("__focus__", state.solution_code) get_func = partial( evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, name=name, copy=copy, ) if override is not None: # don't bother with running expression and fetching output/value # eval_sol, str_sol = eval eval_sol, str_sol = override, str(override) else: eval_sol, str_sol = get_func( tree=state.solution_ast, process=state.solution_process, context=state.solution_context, env=state.solution_env, ) if (test == "error") ^ isinstance(eval_sol, Exception): raise InstructorError.from_message( "Evaluating expression raised error in solution process (or didn't raise if testing for one). " "Error: {} - {}".format(type(eval_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise InstructorError.from_message( "Couldn't extract the value for the highlighted expression from the solution process: " + eval_sol.info) eval_stu, str_stu = get_func( tree=state.student_ast, process=state.student_process, context=state.student_context, env=state.student_env, ) # kwargs --- fmt_kwargs = { "stu_part": state.student_parts, "sol_part": state.solution_parts, "name": name, "test": test, "test_desc": "" if test == "value" else "the %s " % test, "expr_code": expr_code, } fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu)) fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol)) if incorrect_msg == DEFAULT_INCORRECT_MSG and ( fmt_kwargs["stu_eval"] is None or fmt_kwargs["sol_eval"] is None or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]): incorrect_msg = "Expected something different." # tests --- # error in process if (test == "error") ^ isinstance(eval_stu, Exception): fmt_kwargs["stu_str"] = str_stu state.report(error_msg, fmt_kwargs, append=append) # name is undefined after running expression if isinstance(eval_stu, UndefinedValue): state.report(undefined_msg, fmt_kwargs, append=append) # test equality of results state.do_test( EqualTest( eval_stu, eval_sol, FeedbackComponent(incorrect_msg, fmt_kwargs, append=append), func, )) return state
def has_expr(incorrect_msg=DEFAULT_INCORRECT_MSG, error_msg=DEFAULT_ERROR_MSG, undefined_msg=DEFAULT_UNDEFINED_MSG, extra_env=None, context_vals=None, pre_code=None, expr_code=None, name=None, copy=True, func=None, state=None, test=None): rep = Reporter.active_reporter get_func = partial(evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, name=name, copy=copy, do_exec=True if test == 'output' else False) eval_sol, str_sol = get_func(tree=state.solution_tree, process=state.solution_process, context=state.solution_context, env=state.solution_env) if (test == 'error') ^ isinstance(str_sol, Exception): raise ValueError( "Evaluating expression raised error in solution process (or not an error if testing for one). " "Error: {} - {}".format(type(str_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise ValueError( "Couldn't figure out the value of a default argument: " + eval_sol.info) eval_stu, str_stu = get_func(tree=state.student_tree, process=state.student_process, context=state.student_context, env=state.student_env) # kwargs --- fmt_kwargs = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'extra_env': str(extra_env) if extra_env else "", 'context_vals': context_vals } fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) # tests --- # error in process if (test == 'error') ^ isinstance(str_stu, Exception): _msg = state.build_message(error_msg, fmt_kwargs) feedback = Feedback(_msg, state) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(str_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, state))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func)) return state
def has_expr(incorrect_msg=None, error_msg=None, undefined_msg=None, append=None, extra_env=None, context_vals=None, pre_code=None, expr_code=None, name=None, copy=True, func=None, override=None, state=None, test=None): if append is None: # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: if name: incorrect_msg = DEFAULT_INCORRECT_NAME_MSG elif expr_code: incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG else: incorrect_msg = DEFAULT_INCORRECT_MSG if undefined_msg is None: undefined_msg = DEFAULT_UNDEFINED_NAME_MSG if error_msg is None: if test == 'error': error_msg = DEFAULT_ERROR_MSG_INV else: error_msg = DEFAULT_ERROR_MSG rep = Reporter.active_reporter get_func = partial(evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, name=name, copy=copy) if override is not None: # don't bother with running expression and fetching output/value # eval_sol, str_sol = eval eval_sol, str_sol = override, str(override) else: eval_sol, str_sol = get_func(tree=state.solution_tree, process=state.solution_process, context=state.solution_context, env=state.solution_env) if (test == 'error') ^ isinstance(eval_sol, Exception): raise InstructorError( "Evaluating expression raised error in solution process (or not an error if testing for one). " "Error: {} - {}".format(type(eval_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise InstructorError( "Couldn't extract the value for the highlighted expression from the solution process: " + eval_sol.info) eval_stu, str_stu = get_func(tree=state.student_tree, process=state.student_process, context=state.student_context, env=state.student_env) # kwargs --- fmt_kwargs = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'test_desc': '' if test == 'value' else 'the %s ' % test, 'expr_code': expr_code } fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) if incorrect_msg == DEFAULT_INCORRECT_MSG and \ ( fmt_kwargs['stu_eval'] is None or fmt_kwargs['sol_eval'] is None or fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ): incorrect_msg = "Expected something different." # tests --- # error in process if (test == 'error') ^ isinstance(eval_stu, Exception): fmt_kwargs['stu_str'] = str_stu _msg = state.build_message(error_msg, fmt_kwargs, append=append) feedback = Feedback(_msg, state) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(eval_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs, append=append) rep.do_test(Test(Feedback(_msg, state))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func)) return state
def test_operator(index=1, eq_condition="equal", used=None, do_eval=True, not_found_msg=None, incorrect_op_msg=None, incorrect_result_msg=None, state=None): """THIS FUNCTION IS DEPRECATED Test if operator groups match. This function compares an operator group in the student's code with the corresponding one in the solution code. It will cause the reporter to fail if the corresponding operators do not match. The fail message that is returned will depend on the sort of fail. We say that one operator group correpsonds to a group of operators that is evaluated to one value (e.g. 3 + 5 * (1/3)). Args: index (int): Index of the operator group to be checked. Defaults to 1. eq_condition (str): how results of operators are compared. Currently, only "equal" is supported, meaning that the result in student and solution process should have exactly the same value. used(List[str]): A list of operators that have to be in the group. Valid operators are: "+", "-", "*", "/", "%", "**", "<<", ">>", "|", "^", "&" and "//". If the list is None, operators that are in the group in the solution have to be in the student code. Defaults to None. do_eval (bool): Boolean representing whether the group should be evaluated and compared or not. Defaults to True. not_found_msg (str): Feedback message if not enough operators groups are found in the student's code. incorrect_op_msg (str): Feedback message if the wrong operators are used in the student's code. incorrect_result_msg (str): Feedback message if the operator group evaluates to the wrong result in the student's code. :Example: Student code:: 1 + 5 * (3+5) 1 + 1 * 238 Solution code:: 3.1415 + 5 1 + 238 SCT:: test_operator(index = 2, used = ["+"]) # pass test_operator(index = 2) # fail test_operator(index = 1, incorrect_op_msg = "Use the correct operators") # fail test_operator(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail """ rep = Reporter.active_reporter rep.set_tag("fun", "test_operator") # Indexing starts at 1 for the pythonwhat user. index = index - 1 eq_map = {"equal": EqualTest} if eq_condition not in eq_map: raise NameError("%r not a valid equality condition " % eq_condition) # Parses through code and extracts a data structure holding all operator groups. student_ops = state.student_operators solution_ops = state.solution_operators # Check if number of student operations is greater than index (which is decreased with 1 by now) _msg = state.build_message(not_found_msg or "You didn't define enough operations in your code.") rep.do_test( BiggerTest( len(student_ops), index, _msg)) expr_student, used_student = student_ops[index] # Throw error if solution code is invalid with SCT if index > len(solution_ops) + 1: raise IndexError("index not found in solution: %d" % index) expr_solution, used_solution = solution_ops[index] build_incorrect_msg = "The highlighted operation" used_student = set(used_student) used_solution = set(used_solution) if used is not None: used = set(used) else: used = used_solution for op in used: if incorrect_op_msg is None: incorrect_op_msg = build_incorrect_msg + (" is missing a `%s` operation." % op) _msg = state.build_message(incorrect_op_msg) rep.do_test(DefinedCollTest(op, used_student, Feedback(_msg, expr_student))) if (do_eval): eval_solution, str_solution = getResultInProcess(process = state.solution_process, tree = expr_solution) if str_solution is None: raise ValueError("Running the operation in the solution environment raised an error") if isinstance(eval_solution, ReprFail): raise ValueError("Couldn't find the result of the operation in the solution process: " + eval_solution.info) eval_student, str_student = getResultInProcess(process = state.student_process, tree = expr_student) # Compare the evaluated operation groups if incorrect_result_msg is None: if isinstance(str_student, Exception): stud_patt = "an error" else: stud_patt = "`%s`" % utils.shorten_str(str_student) incorrect_result_msg = build_incorrect_msg + (" evaluates to %s, should be `%s`." % (stud_patt, utils.shorten_str(str_solution))) _msg = state.build_message(incorrect_result_msg) rep.do_test(eq_map[eq_condition]( eval_student, eval_solution, Feedback(_msg, expr_student)))
def has_expr( incorrect_msg="__JINJA__:Unexpected expression {{test}}: expected `{{sol_eval}}`, got `{{stu_eval}}`{{' with values ' + extra_env if extra_env}}.", error_msg="Running an expression in the student process caused an issue.", undefined_msg="FMT:Have you defined `{name}` without errors?", extra_env=None, context_vals=None, expr_code=None, pre_code=None, keep_objs_in_env=None, name=None, highlight=None, copy=True, func=None, state=None, test=None): """Run student and solution code, compare returned value, printed output, or errors. Args: incorrect_msg (str): feedback message if the output of the expression in the solution doesn't match the one of the student. This feedback message will be expanded if it is used in the context of another test function, like test_if_else. error_msg (str): feedback message if there was an error when running the student code. Note that when testing for an error, this message is displayed when none is raised. undefined_msg (str): feedback message if the name argument is defined, but a variable with that name doesn't exist after running the student code. extra_env (dict): set variables to the extra environment. They will update the student and solution environment in the active state before the student/solution code in the active state is ran. This argument should contain a dictionary with the keys the names of the variables you want to set, and the values are the values of these variables. context_vals (list): set variables which are bound in a for loop to certain values. This argument is only useful if you use the function in a test_for_loop. It contains a list with the values of the bound variables. expr_code (str): if this variable is not None, the expression in the student/solution code will not be ran. Instead, the given piece of code will be ran in the student as well as the solution environment and the result will be compared. pre_code (str): the code in string form that should be executed before the expression is executed. This is the ideal place to set a random seed, for example. keep_obj_in_env (list()): a list of variable names that should be hold in the copied environment where the expression is evaluated. All primitive types are copied automatically, other objects have to be passed explicitely. name (str): the name of a variable, or expression, whose value will be tested after running the student and solution code. This could be thought of as post code. copy (bool): whether to try to deep copy objects in the environment, such as lists, that could accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues. func: custom binary function of form f(stu_result, sol_result), for equality testing. """ rep = Reporter.active_reporter # run function to highlight a block of code if callable(highlight): try: highlight = highlight(state=state).student_tree except: pass highlight = highlight or state.highlight get_func = partial(evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, keep_objs_in_env=keep_objs_in_env, name=name, copy=copy, do_exec=True if test == 'output' else False) eval_sol, str_sol = get_func(tree=state.solution_tree, process=state.solution_process, context=state.solution_context) if (test == 'error') ^ isinstance(str_sol, Exception): raise ValueError( "evaluating expression raised error in solution process (or not an error if testing for one). " "Error: %s - %s" % (type(str_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise ValueError( "Couldn't figure out the value of a default argument: " + eval_sol.info) eval_stu, str_stu = get_func(tree=state.student_tree, process=state.student_process, context=state.student_context) # kwargs --- fmt_kwargs = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'extra_env': str(extra_env) if extra_env else "", 'context_vals': context_vals } fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) # tests --- # error in process if (test == 'error') ^ isinstance(str_stu, Exception): _msg = state.build_message(error_msg, fmt_kwargs) feedback = Feedback(_msg, highlight) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(str_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, highlight))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight), func)) return state
def has_expr( incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.", error_msg="Running an expression in the student process caused an issue.", undefined_msg="FMT:Have you defined `{name}` without errors?", extra_env=None, context_vals=None, expr_code=None, pre_code=None, keep_objs_in_env=None, name=None, highlight=None, state=None, test=None): rep = Reporter.active_reporter # run function to highlight a block of code if callable(highlight): try: highlight = highlight(state=state).student_tree except: pass highlight = highlight or state.highlight get_func = partial(evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, keep_objs_in_env=keep_objs_in_env, name=name, do_exec=True if test == 'output' else False) eval_sol, str_sol = get_func(tree=state.solution_tree, process=state.solution_process, context=state.solution_context) if (test == 'error') ^ isinstance(str_sol, Exception): raise ValueError( "evaluating expression raised error in solution process (or not an error if testing for one). " "Error: %s - %s" % (type(str_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise ValueError( "Couldn't figure out the value of a default argument: " + eval_sol.info) eval_stu, str_stu = get_func(tree=state.student_tree, process=state.student_process, context=state.student_context) # kwargs --- fmt_kwargs = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'extra_env': " " + str(extra_env or ""), 'context_vals': context_vals } fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) # tests --- # error in process if (test == 'error') ^ isinstance(str_stu, Exception): _msg = state.build_message(error_msg, fmt_kwargs) feedback = Feedback(_msg, highlight) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(str_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, highlight))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight))) return state
def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.", error_msg="Running an expression in the student process caused an issue.", undefined_msg="FMT:Have you defined `{name}` without errors?", extra_env=None, context_vals=None, expr_code=None, pre_code=None, keep_objs_in_env=None, name=None, highlight=None, state=None, test=None): rep = Reporter.active_reporter # run function to highlight a block of code if callable(highlight): try: highlight = highlight(state=state).student_tree except: pass highlight = highlight or state.highlight get_func = partial(evalCalls[test], extra_env = extra_env, context_vals = context_vals, pre_code = pre_code, expr_code = expr_code, keep_objs_in_env = keep_objs_in_env, name=name, do_exec = True if test == 'output' else False) eval_sol, str_sol = get_func(tree = state.solution_tree, process = state.solution_process, context = state.solution_context) if (test == 'error') ^ isinstance(str_sol, Exception): raise ValueError("evaluating expression raised error in solution process (or not an error if testing for one). " "Error: %s - %s"%(type(str_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise ValueError("Couldn't figure out the value of a default argument: " + eval_sol.info) eval_stu, str_stu = get_func(tree = state.student_tree, process = state.student_process, context = state.student_context) # kwargs --- fmt_kwargs = {'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'extra_env': " "+str(extra_env or ""), 'context_vals': context_vals} fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) # tests --- # error in process if (test == 'error') ^ isinstance(str_stu, Exception): _msg = state.build_message(error_msg, fmt_kwargs) feedback = Feedback(_msg, highlight) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(str_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, highlight))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight))) return state