def array_element_tests(test, student_data, solution_data, feedback, *args, **kwargs):
tests = []
if not isinstance(student_data, list) or not isinstance(solution_data, list):
# If student data is None; Feedback will be provided by a different test.
return tests
for i, (element_student_data, element_solution_data) in enumerate(
zip(student_data, solution_data)
):
if isinstance(feedback, str):
item_feedback = Feedback(feedback)
else:
item_feedback = copy.deepcopy(feedback)
item_feedback.message = item_feedback.message.format(
ordinal=selectors.get_ord(i + 1),
expected=element_solution_data,
actual=element_student_data,
)
tests.append(
test(
element_student_data,
element_solution_data,
item_feedback,
*args,
**kwargs,
)
)
return tests
def _test(state, incorrect_msg, exact_names, tv_name, highlight_name):
# get parts for testing from state
# TODO: this could be rewritten to use check_part_index -> has_equal_part, etc..
stu_vars = state.student_parts[tv_name]
sol_vars = state.solution_parts[tv_name]
child_state = state.to_child(
student_ast=state.student_parts.get(highlight_name),
solution_ast=state.solution_parts.get(highlight_name),
)
# variables exposed to messages
d = {"stu_vars": stu_vars, "sol_vars": sol_vars, "num_vars": len(sol_vars)}
if exact_names:
# message for wrong iter var names
_msg = state.build_message(incorrect_msg, d)
# test
state.do_test(
EqualTest(stu_vars, sol_vars, Feedback(_msg, child_state)))
else:
# message for wrong number of iter vars
_msg = state.build_message(incorrect_msg, d)
# test
state.do_test(
EqualTest(len(stu_vars), len(sol_vars),
Feedback(_msg, child_state)))
return state
def report(self, feedback: str):
test_feedback = Feedback(feedback, self)
if test_feedback.highlight is None and self is not getattr(
self, "root_state", None
):
test_feedback.highlight = self.student_ast
test = Fail(test_feedback)
return self.do_test(test)
def test_highlighting_path_no_position():
r = Reporter()
f = Feedback(FeedbackComponent("msg"), path=Path("test.py"))
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", "path": "test.py"}
assert payload == expected_payload
def test_highlighting_offset_proxy():
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight_range_1),
highlight_offset=highlight_range_2,
)
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **highlight_combined}
assert payload == expected_payload
def has_equal_part(state, name, msg):
d = {
"stu_part": state.student_parts,
"sol_part": state.solution_parts,
"name": name,
}
_msg = state.build_message(msg, d)
state.do_test(
EqualTest(d["stu_part"][name], d["sol_part"][name],
Feedback(_msg, state)))
return state
def test_highlighting_offset(offset, highlight, payload_highlight_info):
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight),
highlight_offset=offset,
)
payload = r.build_failed_payload(f)
expected_payload = {"correct": False, "message": "msg", **payload_highlight_info}
assert payload == expected_payload
def __init__(self, feedback):
"""
Initialize the standard test.
Args:
feedback: string or Feedback object
"""
if issubclass(type(feedback), Feedback):
self.feedback = feedback
elif issubclass(type(feedback), str):
self.feedback = Feedback(feedback)
else:
raise TypeError(
"When creating a test, specify either a string or a Feedback object"
)
self.result = None
def is_instance(state, inst, not_instance_msg=None):
"""Check whether an object is an instance of a certain class.
``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is
used to 'zoom in' on the object of interest.
Args:
inst (class): The class that the object should have.
not_instance_msg (str): When specified, this overrides the automatically generated message in case
the object does not have the expected class.
state (State): The state that is passed in through the SCT chain (don't specify this).
:Example:
Student code and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
SCT::
# Verify the class of arr
import numpy
Ex().check_object('arr').is_instance(numpy.ndarray)
"""
state.assert_is(["object_assignments"], "is_instance", ["check_object"])
sol_name = state.solution_parts.get("name")
stu_name = state.student_parts.get("name")
if not_instance_msg is None:
not_instance_msg = "Is it a {{inst.__name__}}?"
if not isInstanceInProcess(sol_name, inst, state.solution_process):
raise InstructorError(
"`is_instance()` noticed that `%s` is not a `%s` in the solution process."
% (sol_name, inst.__name__))
_msg = state.build_message(not_instance_msg, {"inst": inst})
feedback = Feedback(_msg, state)
state.do_test(
InstanceProcessTest(stu_name, inst, state.student_process, feedback))
return state
def test_highlighting_path():
r = Reporter()
f = Feedback(
FeedbackComponent("msg"),
highlight=Highlight(highlight_range_1),
path=Path("test.py"),
)
payload = r.build_failed_payload(f)
expected_payload = {
"correct": False,
"message": "msg",
"path": "test.py",
**highlight_payload_1,
}
assert payload == expected_payload
def has_code(state, text, pattern=True, not_typed_msg=None):
"""Test the student code.
Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``,
as it is more robust to small syntactical differences that don't change the code's behavior.
Args:
text (str): the text that is searched for
pattern (bool): if True (the default), the text is treated as a pattern. If False, it is treated as plain text.
not_typed_msg (str): feedback message to be displayed if the student did not type the text.
:Example:
Student code and solution code::
y = 1 + 2 + 3
SCT::
# Verify that student code contains pattern (not robust!!):
Ex().has_code(r"1\\s*\\+2\\s*\\+3")
"""
if not not_typed_msg:
if pattern:
not_typed_msg = "Could not find the correct pattern in your code."
else:
not_typed_msg = "Could not find the following text in your code: %r" % text
student_code = state.student_code
_msg = state.build_message(not_typed_msg)
state.do_test(
StringContainsTest(student_code, text, pattern, Feedback(_msg, state)))
return state
def build_failed_payload(self, feedback: Feedback):
highlight = Counter()
code_highlight = feedback.get_highlight_data()
path = code_highlight.get("path", None)
if path is not None:
del code_highlight["path"]
if code_highlight:
highlight.update(self.highlight_offset)
if "line_start" in highlight and "line_end" not in highlight:
highlight["line_end"] = highlight["line_start"]
highlight.update(code_highlight)
highlight.update(self.ast_highlight_offset)
if path is not None:
highlight["path"] = str(path)
return {
"correct": False,
"message": Reporter.to_html(feedback.message),
**highlight,
}
def from_message(cls, message: str) -> "Failure":
return cls(Feedback(FeedbackComponent(message)), [])
def has_expr(
state,
incorrect_msg=None,
error_msg=None,
undefined_msg=None,
append=None,
extra_env=None,
context_vals=None,
pre_code=None,
expr_code=None,
name=None,
copy=True,
func=None,
override=None,
test=None, # todo: default or arg before state
):
if (
append is None
): # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
if name:
incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
elif expr_code:
incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
else:
incorrect_msg = DEFAULT_INCORRECT_MSG
if undefined_msg is None:
undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
if error_msg is None:
if test == "error":
error_msg = DEFAULT_ERROR_MSG_INV
else:
error_msg = DEFAULT_ERROR_MSG
if state.solution_code is not None and isinstance(expr_code, str):
expr_code = expr_code.replace("__focus__", state.solution_code)
get_func = partial(
evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
name=name,
copy=copy,
)
if override is not None:
# don't bother with running expression and fetching output/value
# eval_sol, str_sol = eval
eval_sol, str_sol = override, str(override)
else:
eval_sol, str_sol = get_func(
tree=state.solution_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
)
if (test == "error") ^ isinstance(eval_sol, Exception):
raise InstructorError(
"Evaluating expression raised error in solution process (or didn't raise if testing for one). "
"Error: {} - {}".format(type(eval_sol), str_sol))
if isinstance(eval_sol, ReprFail):
raise InstructorError(
"Couldn't extract the value for the highlighted expression from the solution process: "
+ eval_sol.info)
eval_stu, str_stu = get_func(
tree=state.student_ast,
process=state.student_process,
context=state.student_context,
env=state.student_env,
)
# kwargs ---
fmt_kwargs = {
"stu_part": state.student_parts,
"sol_part": state.solution_parts,
"name": name,
"test": test,
"test_desc": "" if test == "value" else "the %s " % test,
"expr_code": expr_code,
}
fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu))
fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol))
if incorrect_msg == DEFAULT_INCORRECT_MSG and (
fmt_kwargs["stu_eval"] is None or fmt_kwargs["sol_eval"] is None
or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]):
incorrect_msg = "Expected something different."
# tests ---
# error in process
if (test == "error") ^ isinstance(eval_stu, Exception):
fmt_kwargs["stu_str"] = str_stu
_msg = state.build_message(error_msg, fmt_kwargs, append=append)
state.report(_msg)
# name is undefined after running expression
if isinstance(eval_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs, append=append)
state.report(_msg)
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
state.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))
return state
def has_equal_ast(state,
incorrect_msg=None,
code=None,
exact=True,
append=None):
"""Test whether abstract syntax trees match between the student and solution code.
``has_equal_ast()`` can be used in two ways:
* As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission.
But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST.
Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``.
* As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments.
Args:
incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this.
code: optional code to use instead of the solution AST.
exact: whether the representations must match exactly. If false, the solution AST
only needs to be contained within the student AST (similar to using test student typed).
Defaults to ``True``, unless the ``code`` argument has been specified.
:Example:
Student and Solution Code::
dict(a = 'value').keys()
SCT::
# all pass
Ex().has_equal_ast()
Ex().has_equal_ast(code = "dict(a = 'value').keys()")
Ex().has_equal_ast(code = "dict(a = 'value')", exact = False)
Student and Solution Code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Check underlying value of arugment a of np.mean:
Ex().check_function('numpy.mean').check_args('a').has_equal_ast()
# Only check AST equality of expression used to specify argument a:
Ex().check_function('numpy.mean').check_args('a').has_equal_ast()
"""
if utils.v2_only():
state.assert_is_not(["object_assignments"], "has_equal_ast",
["check_object"])
state.assert_is_not(["function_calls"], "has_equal_ast",
["check_function"])
if code and incorrect_msg is None:
raise InstructorError(
"If you manually specify the code to match inside has_equal_ast(), "
"you have to explicitly set the `incorrect_msg` argument.")
if (
append is None
): # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
incorrect_msg = "Expected `{{sol_str}}`, but got `{{stu_str}}`."
def parse_tree(tree):
# get contents of module.body if only 1 element
crnt = (tree.body[0] if isinstance(tree, ast.Module)
and len(tree.body) == 1 else tree)
# remove Expr if it exists
return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt)
stu_rep = parse_tree(state.student_ast)
sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code))
fmt_kwargs = {
"sol_str": state.solution_code if not code else code,
"stu_str": state.student_code,
}
_msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
if exact and not code:
state.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state)))
elif not sol_rep in stu_rep:
state.report(_msg)
return state
def call(state,
args,
test="value",
incorrect_msg=None,
error_msg=None,
argstr=None,
func=None,
**kwargs):
"""Use ``check_call()`` in combination with ``has_equal_x()`` instead.
"""
if incorrect_msg is None:
incorrect_msg = MSG_CALL_INCORRECT
if error_msg is None:
error_msg = MSG_CALL_ERROR_INV if test == "error" else MSG_CALL_ERROR
assert test in ("value", "output", "error")
get_func = evalCalls[test]
# Run for Solution --------------------------------------------------------
eval_sol, str_sol = run_call(args, state.solution_parts["node"],
state.solution_process, get_func, **kwargs)
if (test == "error") ^ isinstance(eval_sol, Exception):
_msg = (state.build_message(
"Calling {{argstr}} resulted in an error (or not an error if testing for one). Error message: {{type_err}} {{str_sol}}",
dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr),
), )
raise InstructorError(_msg)
if isinstance(eval_sol, ReprFail):
_msg = state.build_message(
"Can't get the result of calling {{argstr}}: {{eval_sol.info}}",
dict(argstr=argstr, eval_sol=eval_sol),
)
raise InstructorError(_msg)
# Run for Submission ------------------------------------------------------
eval_stu, str_stu = run_call(args, state.student_parts["node"],
state.student_process, get_func, **kwargs)
action_strs = {
"value": "return",
"output": "print out",
"error": "error out with the message",
}
fmt_kwargs = {
"part": argstr,
"argstr": argstr,
"str_sol": str_sol,
"str_stu": str_stu,
"action": action_strs[test],
}
# either error test and no error, or vice-versa
stu_node = state.student_parts["node"]
stu_state = state.to_child(highlight=stu_node)
if (test == "error") ^ isinstance(eval_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
stu_state.report(_msg)
# incorrect result
_msg = state.build_message(incorrect_msg, fmt_kwargs)
state.do_test(
EqualTest(eval_sol, eval_stu, Feedback(_msg, stu_state), func))
return state
def build_failed_payload(self, feedback: Feedback):
return {
"correct": False,
"message": Reporter.to_html(feedback.get_message()),
**feedback.get_highlight(),
}
def check_keys(state, key, missing_msg=None, expand_msg=None):
"""Check whether an object (dict, DataFrame, etc) has a key.
``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is
used to 'zoom in' on the object of interest.
Args:
key (str): Name of the key that the object should have.
missing_msg (str): When specified, this overrides the automatically generated
message in case the key does not exist.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
state (State): The state that is passed in through the SCT chain (don't specify this).
:Example:
Student code and solution code::
x = {'a': 2}
SCT::
# Verify that x contains a key a
Ex().check_object('x').check_keys('a')
# Verify that x contains a key a and a is correct.
Ex().check_object('x').check_keys('a').has_equal_value()
"""
state.assert_is(["object_assignments"], "is_instance",
["check_object", "check_df"])
if missing_msg is None:
missing_msg = "There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`."
if expand_msg is None:
expand_msg = "Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? "
sol_name = state.solution_parts.get("name")
stu_name = state.student_parts.get("name")
if not isDefinedCollInProcess(sol_name, key, state.solution_process):
raise InstructorError(
"`check_keys()` couldn't find key `%s` in object `%s` in the solution process."
% (key, sol_name))
# check if key available
_msg = state.build_message(missing_msg, {"key": key})
state.do_test(
DefinedCollProcessTest(stu_name, key, state.student_process,
Feedback(_msg, state)))
def get_part(name, key, highlight):
if isinstance(key, str):
slice_val = ast.Str(s=key)
else:
slice_val = ast.parse(str(key)).body[0].value
expr = ast.Subscript(
value=ast.Name(id=name, ctx=ast.Load()),
slice=ast.Index(value=slice_val),
ctx=ast.Load(),
)
ast.fix_missing_locations(expr)
return {"node": expr, "highlight": highlight}
stu_part = get_part(stu_name, key, state.student_parts.get("highlight"))
sol_part = get_part(sol_name, key, state.solution_parts.get("highlight"))
append_message = {"msg": expand_msg, "kwargs": {"key": key}}
child = part_to_child(stu_part, sol_part, append_message, state)
return child
def report(self, feedback: str):
return self.do_test(Fail(Feedback(feedback)))
def check_object(state,
index,
missing_msg=None,
expand_msg=None,
typestr="variable"):
"""Check object existence (and equality)
Check whether an object is defined in the student's process, and zoom in on its value in both
student and solution process to inspect quality (with has_equal_value().
In ``pythonbackend``, both the student's submission as well as the solution code are executed, in separate processes.
``check_object()`` looks at these processes and checks if the referenced object is available in the student process.
Next, you can use ``has_equal_value()`` to check whether the objects in the student and solution process correspond.
Args:
index (str): the name of the object which value has to be checked.
missing_msg (str): feedback message when the object is not defined in the student process.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
:Example:
Suppose you want the student to create a variable ``x``, equal to 15: ::
x = 15
The following SCT will verify this: ::
Ex().check_object("x").has_equal_value()
- ``check_object()`` will check if the variable ``x`` is defined in the student process.
- ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process.
Note that ``has_equal_value()`` only looks at **end result** of a variable in the student process.
In the example, how the object ``x`` came about in the student's submission, does not matter.
This means that all of the following submission will also pass the above SCT: ::
x = 15
x = 12 + 3
x = 3; x += 12
:Example:
As the previous example mentioned, ``has_equal_value()`` only looks at the **end result**. If your exercise is
first initializing and object and further down the script is updating the object, you can only look at the final value!
Suppose you want the student to initialize and populate a list `my_list` as follows: ::
my_list = []
for i in range(20):
if i % 3 == 0:
my_list.append(i)
There is no robust way to verify whether `my_list = [0]` was coded correctly in a separate way.
The best SCT would look something like this: ::
msg = "Have you correctly initialized `my_list`?"
Ex().check_correct(
check_object('my_list').has_equal_value(),
multi(
# check initialization: [] or list()
check_or(
has_equal_ast(code = "[]", incorrect_msg = msg),
check_function('list')
),
check_for_loop().multi(
check_iter().has_equal_value(),
check_body().check_if_else().multi(
check_test().multi(
set_context(2).has_equal_value(),
set_context(3).has_equal_value()
),
check_body().set_context(3).\\
set_env(my_list = [0]).\\
has_equal_value(name = 'my_list')
)
)
)
)
- ``check_correct()`` is used to robustly check whether ``my_list`` was built correctly.
- If ``my_list`` is not correct, **both** the initialization and the population code are checked.
:Example:
Because checking object correctness incorrectly is such a common misconception, we're adding another example: ::
import pandas as pd
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
df['c'] = [7, 8, 9]
The following SCT would be **wrong**, as it does not factor in the possibility that the 'add column ``c``' step could've been wrong: ::
Ex().check_correct(
check_object('df').has_equal_value(),
check_function('pandas.DataFrame').check_args(0).has_equal_value()
)
The following SCT would be better, as it is specific to the steps: ::
# verify the df = pd.DataFrame(...) step
Ex().check_correct(
check_df('df').multi(
check_keys('a').has_equal_value(),
check_keys('b').has_equal_value()
),
check_function('pandas.DataFrame').check_args(0).has_equal_value()
)
# verify the df['c'] = [...] step
Ex().check_df('df').check_keys('c').has_equal_value()
:Example:
pythonwhat compares the objects in the student and solution process with the ``==`` operator.
For basic objects, this ``==`` is operator is properly implemented, so that the objects can be effectively compared.
For more complex objects that are produced by third-party packages, however, it's possible that this equality operator is not implemented in a way you'd expect.
Often, for these object types the ``==`` will compare the actual object instances: ::
# pre exercise code
class Number():
def __init__(self, n):
self.n = n
# solution
x = Number(1)
# sct that won't work
Ex().check_object().has_equal_value()
# sct
Ex().check_object().has_equal_value(expr_code = 'x.n')
# submissions that will pass this sct
x = Number(1)
x = Number(2 - 1)
The basic SCT like in the previous example will notwork here.
Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking.
Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n``
and seeing if the result of running this expression in both student and solution process match.
"""
# Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1'
if v2_only():
extra_msg = "If you want to check the value of an object in e.g. a for loop, use `has_equal_value(name = 'my_obj')` instead."
state.assert_execution_root("check_object", extra_msg=extra_msg)
if missing_msg is None:
missing_msg = "Did you define the {{typestr}} `{{index}}` without errors?"
if expand_msg is None:
expand_msg = "Did you correctly define the {{typestr}} `{{index}}`? "
if (not isDefinedInProcess(index, state.solution_process)
and state.has_different_processes()):
raise InstructorError(
"`check_object()` couldn't find object `%s` in the solution process."
% index)
append_message = {
"msg": expand_msg,
"kwargs": {
"index": index,
"typestr": typestr
}
}
# create child state, using either parser output, or create part from name
fallback = lambda: ObjectAssignmentParser.get_part(index)
stu_part = state.ast_dispatcher.find("object_assignments",
state.student_ast).get(
index, fallback())
sol_part = state.ast_dispatcher.find("object_assignments",
state.solution_ast).get(
index, fallback())
# test object exists
_msg = state.build_message(missing_msg, append_message["kwargs"])
state.do_test(
DefinedProcessTest(index, state.student_process, Feedback(_msg)))
child = part_to_child(stu_part,
sol_part,
append_message,
state,
node_name="object_assignments")
return child