def set_information(self, key, value):
self.information[key] = utils.shorten_str(str(value))
def has_expr(
state,
incorrect_msg=None,
error_msg=None,
undefined_msg=None,
append=None,
extra_env=None,
context_vals=None,
pre_code=None,
expr_code=None,
name=None,
copy=True,
func=None,
override=None,
test=None, # todo: default or arg before state
):
if (
append is None
): # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
if name:
incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
elif expr_code:
incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
else:
incorrect_msg = DEFAULT_INCORRECT_MSG
if undefined_msg is None:
undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
if error_msg is None:
if test == "error":
error_msg = DEFAULT_ERROR_MSG_INV
else:
error_msg = DEFAULT_ERROR_MSG
if state.solution_code is not None and isinstance(expr_code, str):
expr_code = expr_code.replace("__focus__", state.solution_code)
get_func = partial(
evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
name=name,
copy=copy,
)
if override is not None:
# don't bother with running expression and fetching output/value
# eval_sol, str_sol = eval
eval_sol, str_sol = override, str(override)
else:
eval_sol, str_sol = get_func(
tree=state.solution_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
)
if (test == "error") ^ isinstance(eval_sol, Exception):
raise InstructorError.from_message(
"Evaluating expression raised error in solution process (or didn't raise if testing for one). "
"Error: {} - {}".format(type(eval_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise InstructorError.from_message(
"Couldn't extract the value for the highlighted expression from the solution process: "
+ eval_sol.info)
eval_stu, str_stu = get_func(
tree=state.student_ast,
process=state.student_process,
context=state.student_context,
env=state.student_env,
)
# kwargs ---
fmt_kwargs = {
"stu_part": state.student_parts,
"sol_part": state.solution_parts,
"name": name,
"test": test,
"test_desc": "" if test == "value" else "the %s " % test,
"expr_code": expr_code,
}
fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu))
fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol))
if incorrect_msg == DEFAULT_INCORRECT_MSG and (
fmt_kwargs["stu_eval"] is None or fmt_kwargs["sol_eval"] is None
or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]):
incorrect_msg = "Expected something different."
# tests ---
# error in process
if (test == "error") ^ isinstance(eval_stu, Exception):
fmt_kwargs["stu_str"] = str_stu
state.report(error_msg, fmt_kwargs, append=append)
# name is undefined after running expression
if isinstance(eval_stu, UndefinedValue):
state.report(undefined_msg, fmt_kwargs, append=append)
# test equality of results
state.do_test(
EqualTest(
eval_stu,
eval_sol,
FeedbackComponent(incorrect_msg, fmt_kwargs, append=append),
func,
))
return state
def has_expr(incorrect_msg=DEFAULT_INCORRECT_MSG,
error_msg=DEFAULT_ERROR_MSG,
undefined_msg=DEFAULT_UNDEFINED_MSG,
extra_env=None,
context_vals=None,
pre_code=None,
expr_code=None,
name=None,
copy=True,
func=None,
state=None,
test=None):
rep = Reporter.active_reporter
get_func = partial(evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
name=name,
copy=copy,
do_exec=True if test == 'output' else False)
eval_sol, str_sol = get_func(tree=state.solution_tree,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env)
if (test == 'error') ^ isinstance(str_sol, Exception):
raise ValueError(
"Evaluating expression raised error in solution process (or not an error if testing for one). "
"Error: {} - {}".format(type(str_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise ValueError(
"Couldn't figure out the value of a default argument: " +
eval_sol.info)
eval_stu, str_stu = get_func(tree=state.student_tree,
process=state.student_process,
context=state.student_context,
env=state.student_env)
# kwargs ---
fmt_kwargs = {
'stu_part': state.student_parts,
'sol_part': state.solution_parts,
'name': name,
'test': test,
'extra_env': str(extra_env) if extra_env else "",
'context_vals': context_vals
}
fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
# tests ---
# error in process
if (test == 'error') ^ isinstance(str_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
feedback = Feedback(_msg, state)
rep.do_test(Test(feedback))
# name is undefined after running expression
if isinstance(str_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs)
rep.do_test(Test(Feedback(_msg, state)))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs)
rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))
return state
def has_expr(incorrect_msg=None,
error_msg=None,
undefined_msg=None,
append=None,
extra_env=None,
context_vals=None,
pre_code=None,
expr_code=None,
name=None,
copy=True,
func=None,
override=None,
state=None,
test=None):
if append is None: # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
if name:
incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
elif expr_code:
incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
else:
incorrect_msg = DEFAULT_INCORRECT_MSG
if undefined_msg is None:
undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
if error_msg is None:
if test == 'error':
error_msg = DEFAULT_ERROR_MSG_INV
else:
error_msg = DEFAULT_ERROR_MSG
rep = Reporter.active_reporter
get_func = partial(evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
name=name,
copy=copy)
if override is not None:
# don't bother with running expression and fetching output/value
# eval_sol, str_sol = eval
eval_sol, str_sol = override, str(override)
else:
eval_sol, str_sol = get_func(tree=state.solution_tree,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env)
if (test == 'error') ^ isinstance(eval_sol, Exception):
raise InstructorError(
"Evaluating expression raised error in solution process (or not an error if testing for one). "
"Error: {} - {}".format(type(eval_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise InstructorError(
"Couldn't extract the value for the highlighted expression from the solution process: "
+ eval_sol.info)
eval_stu, str_stu = get_func(tree=state.student_tree,
process=state.student_process,
context=state.student_context,
env=state.student_env)
# kwargs ---
fmt_kwargs = {
'stu_part': state.student_parts,
'sol_part': state.solution_parts,
'name': name,
'test': test,
'test_desc': '' if test == 'value' else 'the %s ' % test,
'expr_code': expr_code
}
fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
if incorrect_msg == DEFAULT_INCORRECT_MSG and \
( fmt_kwargs['stu_eval'] is None or
fmt_kwargs['sol_eval'] is None or
fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ):
incorrect_msg = "Expected something different."
# tests ---
# error in process
if (test == 'error') ^ isinstance(eval_stu, Exception):
fmt_kwargs['stu_str'] = str_stu
_msg = state.build_message(error_msg, fmt_kwargs, append=append)
feedback = Feedback(_msg, state)
rep.do_test(Test(feedback))
# name is undefined after running expression
if isinstance(eval_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs, append=append)
rep.do_test(Test(Feedback(_msg, state)))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))
return state
def test_operator(index=1,
eq_condition="equal",
used=None,
do_eval=True,
not_found_msg=None,
incorrect_op_msg=None,
incorrect_result_msg=None,
state=None):
"""THIS FUNCTION IS DEPRECATED
Test if operator groups match.
This function compares an operator group in the student's code with the corresponding one in the solution
code. It will cause the reporter to fail if the corresponding operators do not match. The fail message
that is returned will depend on the sort of fail. We say that one operator group correpsonds to a group of
operators that is evaluated to one value (e.g. 3 + 5 * (1/3)).
Args:
index (int): Index of the operator group to be checked. Defaults to 1.
eq_condition (str): how results of operators are compared. Currently, only "equal" is supported,
meaning that the result in student and solution process should have exactly the same value.
used(List[str]): A list of operators that have to be in the group. Valid operators are: "+", "-",
"*", "/", "%", "**", "<<", ">>", "|", "^", "&" and "//". If the list is None, operators that are
in the group in the solution have to be in the student code. Defaults to None.
do_eval (bool): Boolean representing whether the group should be evaluated and compared or not.
Defaults to True.
not_found_msg (str): Feedback message if not enough operators groups are found in the student's code.
incorrect_op_msg (str): Feedback message if the wrong operators are used in the student's code.
incorrect_result_msg (str): Feedback message if the operator group evaluates to the wrong result in
the student's code.
:Example:
Student code::
1 + 5 * (3+5)
1 + 1 * 238
Solution code::
3.1415 + 5
1 + 238
SCT::
test_operator(index = 2, used = ["+"]) # pass
test_operator(index = 2) # fail
test_operator(index = 1, incorrect_op_msg = "Use the correct operators") # fail
test_operator(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail
"""
rep = Reporter.active_reporter
rep.set_tag("fun", "test_operator")
# Indexing starts at 1 for the pythonwhat user.
index = index - 1
eq_map = {"equal": EqualTest}
if eq_condition not in eq_map:
raise NameError("%r not a valid equality condition " % eq_condition)
# Parses through code and extracts a data structure holding all operator groups.
student_ops = state.student_operators
solution_ops = state.solution_operators
# Check if number of student operations is greater than index (which is decreased with 1 by now)
_msg = state.build_message(not_found_msg or "You didn't define enough operations in your code.")
rep.do_test(
BiggerTest(
len(student_ops),
index,
_msg))
expr_student, used_student = student_ops[index]
# Throw error if solution code is invalid with SCT
if index > len(solution_ops) + 1:
raise IndexError("index not found in solution: %d" % index)
expr_solution, used_solution = solution_ops[index]
build_incorrect_msg = "The highlighted operation"
used_student = set(used_student)
used_solution = set(used_solution)
if used is not None:
used = set(used)
else:
used = used_solution
for op in used:
if incorrect_op_msg is None:
incorrect_op_msg = build_incorrect_msg + (" is missing a `%s` operation." % op)
_msg = state.build_message(incorrect_op_msg)
rep.do_test(DefinedCollTest(op, used_student,
Feedback(_msg, expr_student)))
if (do_eval):
eval_solution, str_solution = getResultInProcess(process = state.solution_process, tree = expr_solution)
if str_solution is None:
raise ValueError("Running the operation in the solution environment raised an error")
if isinstance(eval_solution, ReprFail):
raise ValueError("Couldn't find the result of the operation in the solution process: " + eval_solution.info)
eval_student, str_student = getResultInProcess(process = state.student_process, tree = expr_student)
# Compare the evaluated operation groups
if incorrect_result_msg is None:
if isinstance(str_student, Exception):
stud_patt = "an error"
else:
stud_patt = "`%s`" % utils.shorten_str(str_student)
incorrect_result_msg = build_incorrect_msg + (" evaluates to %s, should be `%s`." % (stud_patt, utils.shorten_str(str_solution)))
_msg = state.build_message(incorrect_result_msg)
rep.do_test(eq_map[eq_condition](
eval_student, eval_solution, Feedback(_msg, expr_student)))
def set_information(self, key, value):
self.information[key] = utils.shorten_str(str(value))
def has_expr(
incorrect_msg="__JINJA__:Unexpected expression {{test}}: expected `{{sol_eval}}`, got `{{stu_eval}}`{{' with values ' + extra_env if extra_env}}.",
error_msg="Running an expression in the student process caused an issue.",
undefined_msg="FMT:Have you defined `{name}` without errors?",
extra_env=None,
context_vals=None,
expr_code=None,
pre_code=None,
keep_objs_in_env=None,
name=None,
highlight=None,
copy=True,
func=None,
state=None,
test=None):
"""Run student and solution code, compare returned value, printed output, or errors.
Args:
incorrect_msg (str): feedback message if the output of the expression in the solution doesn't match
the one of the student. This feedback message will be expanded if it is used in the context of
another test function, like test_if_else.
error_msg (str): feedback message if there was an error when running the student code.
Note that when testing for an error, this message is displayed when none is raised.
undefined_msg (str): feedback message if the name argument is defined, but a variable
with that name doesn't exist after running the student code.
extra_env (dict): set variables to the extra environment. They will update the student
and solution environment in the active state before the student/solution code in the active
state is ran. This argument should contain a dictionary with the keys the names of
the variables you want to set, and the values are the values of these variables.
context_vals (list): set variables which are bound in a for loop to certain values. This argument is
only useful if you use the function in a test_for_loop. It contains a list with the values
of the bound variables.
expr_code (str): if this variable is not None, the expression in the student/solution code will not
be ran. Instead, the given piece of code will be ran in the student as well as the solution environment
and the result will be compared.
pre_code (str): the code in string form that should be executed before the expression is executed.
This is the ideal place to set a random seed, for example.
keep_obj_in_env (list()): a list of variable names that should be hold in the copied environment where
the expression is evaluated. All primitive types are copied automatically, other objects have to
be passed explicitely.
name (str): the name of a variable, or expression, whose value will be tested after running the
student and solution code. This could be thought of as post code.
copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues.
func: custom binary function of form f(stu_result, sol_result), for equality testing.
"""
rep = Reporter.active_reporter
# run function to highlight a block of code
if callable(highlight):
try:
highlight = highlight(state=state).student_tree
except:
pass
highlight = highlight or state.highlight
get_func = partial(evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
keep_objs_in_env=keep_objs_in_env,
name=name,
copy=copy,
do_exec=True if test == 'output' else False)
eval_sol, str_sol = get_func(tree=state.solution_tree,
process=state.solution_process,
context=state.solution_context)
if (test == 'error') ^ isinstance(str_sol, Exception):
raise ValueError(
"evaluating expression raised error in solution process (or not an error if testing for one). "
"Error: %s - %s" % (type(str_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise ValueError(
"Couldn't figure out the value of a default argument: " +
eval_sol.info)
eval_stu, str_stu = get_func(tree=state.student_tree,
process=state.student_process,
context=state.student_context)
# kwargs ---
fmt_kwargs = {
'stu_part': state.student_parts,
'sol_part': state.solution_parts,
'name': name,
'test': test,
'extra_env': str(extra_env) if extra_env else "",
'context_vals': context_vals
}
fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
# tests ---
# error in process
if (test == 'error') ^ isinstance(str_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
feedback = Feedback(_msg, highlight)
rep.do_test(Test(feedback))
# name is undefined after running expression
if isinstance(str_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs)
rep.do_test(Test(Feedback(_msg, highlight)))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs)
rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight), func))
return state
def has_expr(
incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.",
error_msg="Running an expression in the student process caused an issue.",
undefined_msg="FMT:Have you defined `{name}` without errors?",
extra_env=None,
context_vals=None,
expr_code=None,
pre_code=None,
keep_objs_in_env=None,
name=None,
highlight=None,
state=None,
test=None):
rep = Reporter.active_reporter
# run function to highlight a block of code
if callable(highlight):
try:
highlight = highlight(state=state).student_tree
except:
pass
highlight = highlight or state.highlight
get_func = partial(evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
keep_objs_in_env=keep_objs_in_env,
name=name,
do_exec=True if test == 'output' else False)
eval_sol, str_sol = get_func(tree=state.solution_tree,
process=state.solution_process,
context=state.solution_context)
if (test == 'error') ^ isinstance(str_sol, Exception):
raise ValueError(
"evaluating expression raised error in solution process (or not an error if testing for one). "
"Error: %s - %s" % (type(str_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise ValueError(
"Couldn't figure out the value of a default argument: " +
eval_sol.info)
eval_stu, str_stu = get_func(tree=state.student_tree,
process=state.student_process,
context=state.student_context)
# kwargs ---
fmt_kwargs = {
'stu_part': state.student_parts,
'sol_part': state.solution_parts,
'name': name,
'test': test,
'extra_env': " " + str(extra_env or ""),
'context_vals': context_vals
}
fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
# tests ---
# error in process
if (test == 'error') ^ isinstance(str_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
feedback = Feedback(_msg, highlight)
rep.do_test(Test(feedback))
# name is undefined after running expression
if isinstance(str_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs)
rep.do_test(Test(Feedback(_msg, highlight)))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs)
rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight)))
return state
def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.",
error_msg="Running an expression in the student process caused an issue.",
undefined_msg="FMT:Have you defined `{name}` without errors?",
extra_env=None,
context_vals=None,
expr_code=None,
pre_code=None,
keep_objs_in_env=None,
name=None,
highlight=None,
state=None,
test=None):
rep = Reporter.active_reporter
# run function to highlight a block of code
if callable(highlight):
try: highlight = highlight(state=state).student_tree
except: pass
highlight = highlight or state.highlight
get_func = partial(evalCalls[test],
extra_env = extra_env,
context_vals = context_vals,
pre_code = pre_code,
expr_code = expr_code,
keep_objs_in_env = keep_objs_in_env,
name=name,
do_exec = True if test == 'output' else False)
eval_sol, str_sol = get_func(tree = state.solution_tree,
process = state.solution_process,
context = state.solution_context)
if (test == 'error') ^ isinstance(str_sol, Exception):
raise ValueError("evaluating expression raised error in solution process (or not an error if testing for one). "
"Error: %s - %s"%(type(str_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise ValueError("Couldn't figure out the value of a default argument: " + eval_sol.info)
eval_stu, str_stu = get_func(tree = state.student_tree,
process = state.student_process,
context = state.student_context)
# kwargs ---
fmt_kwargs = {'stu_part': state.student_parts, 'sol_part': state.solution_parts,
'name': name, 'test': test,
'extra_env': " "+str(extra_env or ""), 'context_vals': context_vals}
fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
# tests ---
# error in process
if (test == 'error') ^ isinstance(str_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
feedback = Feedback(_msg, highlight)
rep.do_test(Test(feedback))
# name is undefined after running expression
if isinstance(str_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs)
rep.do_test(Test(Feedback(_msg, highlight)))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs)
rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight)))
return state