def with_context(state, *args, child=None):
# set up context in processes
solution_res = setUpNewEnvInProcess(
process=state.solution_process, context=state.solution_parts["with_items"]
)
if isinstance(solution_res, Exception):
raise InstructorError(
"error in the solution, running test_with(): %s" % str(solution_res)
)
student_res = setUpNewEnvInProcess(
process=state.student_process, context=state.student_parts["with_items"]
)
if isinstance(student_res, AttributeError):
state.report(
Feedback(
"In your `with` statement, you're not using a correct context manager.",
child.highlight, # TODO
)
)
if isinstance(student_res, (AssertionError, ValueError, TypeError)):
state.report(
Feedback(
"In your `with` statement, the number of values in your context manager "
"doesn't correspond to the number of variables you're trying to assign it to.",
child.highlight,
)
)
# run subtests
try:
multi(state, *args)
finally:
# exit context
close_solution_context = breakDownNewEnvInProcess(
process=state.solution_process
)
if isinstance(close_solution_context, Exception):
raise InstructorError(
"error in the solution, closing the `with` fails with: %s"
% close_solution_context
)
close_student_context = breakDownNewEnvInProcess(process=state.student_process)
if isinstance(close_student_context, Exception):
state.report(
Feedback(
"Your `with` statement can not be closed off correctly, you're "
+ "not using the context manager correctly.",
state,
)
)
return state
def has_part(state, name, msg, fmt_kwargs=None, index=None):
d = {
"sol_part": state.solution_parts,
"stu_part": state.student_parts,
**fmt_kwargs,
}
def verify(part, index):
if index is not None:
if isinstance(index, list):
for ind in index:
part = part[ind]
else:
part = part[index]
if part is None:
raise KeyError
# Chceck if it's there in the solution
_msg = state.build_message(msg, d)
_err_msg = "SCT fails on solution: " + _msg
try:
verify(state.solution_parts[name], index)
except (KeyError, IndexError):
raise InstructorError(_err_msg)
try:
verify(state.student_parts[name], index)
except (KeyError, IndexError):
state.report(Feedback(_msg, state))
return state
def parse_internal(self, code):
try:
return self.ast_dispatcher.parse(code)
except Exception as e:
raise InstructorError(
"Something went wrong when parsing the solution code: %s" % str(e)
)
def get_dispatcher(self):
try:
return Dispatcher(self.pre_exercise_code)
except Exception as e:
raise InstructorError(
"Something went wrong when parsing the PEC: %s" % str(e)
)
def run_call(args, node, process, get_func, **kwargs):
# Get function expression
if isinstance(node, ast.FunctionDef): # function name
func_expr = ast.Name(id=node.name, ctx=ast.Load())
elif isinstance(node, ast.Lambda): # lambda body expr
func_expr = node
else:
raise InstructorError("Only function definition or lambda may be called")
ast.fix_missing_locations(func_expr)
return get_func(process=process, tree=func_expr, call=args, **kwargs)
def get_bash_history_info():
try:
old_history_length = int(
Path(os.environ[BASH_HISTORY_INFO_PATH_ENV]).read_text()
)
except FileNotFoundError:
raise InstructorError(
"`update_bash_history_info` wasn't called"
)
return old_history_length
def has_chosen(state, correct, msgs):
"""Test multiple choice exercise.
Test for a MultipleChoiceExercise. The correct answer (as an integer) and feedback messages
are passed to this function.
Args:
correct (int): the index of the correct answer (should be an instruction). Starts at 1.
msgs (list(str)): a list containing all feedback messages belonging to each choice of the
student. The list should have the same length as the number of options.
"""
if not issubclass(type(correct), int):
raise InstructorError(
"Inside `has_chosen()`, the argument `correct` should be an integer."
)
student_process = state.student_process
if not isDefinedInProcess(MC_VAR_NAME, student_process):
raise InstructorError("Option not available in the student process")
else:
selected_option = getOptionFromProcess(student_process, MC_VAR_NAME)
if not issubclass(type(selected_option), int):
raise InstructorError("selected_option should be an integer")
if selected_option < 1 or correct < 1:
raise InstructorError(
"selected_option and correct should be greater than zero"
)
if selected_option > len(msgs) or correct > len(msgs):
raise InstructorError("there are not enough feedback messages defined")
feedback_msg = msgs[selected_option - 1]
state.reporter.success_msg = msgs[correct - 1]
state.do_test(EqualTest(selected_option, correct, feedback_msg))
def parse(self, text, test=True):
result = None
if self.ast_dispatcher:
try:
result = self.ast_dispatcher.parse(text)
except self.ast_dispatcher.ParseError as e:
if test:
self.report(e.message)
else:
raise InstructorError(
"Something went wrong when parsing PEC or solution code: %s"
% str(e)
)
return result
def is_instance(state, inst, not_instance_msg=None):
"""Check whether an object is an instance of a certain class.
``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is
used to 'zoom in' on the object of interest.
Args:
inst (class): The class that the object should have.
not_instance_msg (str): When specified, this overrides the automatically generated message in case
the object does not have the expected class.
state (State): The state that is passed in through the SCT chain (don't specify this).
:Example:
Student code and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
SCT::
# Verify the class of arr
import numpy
Ex().check_object('arr').is_instance(numpy.ndarray)
"""
state.assert_is(["object_assignments"], "is_instance", ["check_object"])
sol_name = state.solution_parts.get("name")
stu_name = state.student_parts.get("name")
if not_instance_msg is None:
not_instance_msg = "Is it a {{inst.__name__}}?"
if not isInstanceInProcess(sol_name, inst, state.solution_process):
raise InstructorError(
"`is_instance()` noticed that `%s` is not a `%s` in the solution process."
% (sol_name, inst.__name__))
_msg = state.build_message(not_instance_msg, {"inst": inst})
feedback = Feedback(_msg, state)
state.do_test(
InstanceProcessTest(stu_name, inst, state.student_process, feedback))
return state
def assert_ast(state, element, fmt_kwargs):
patt = (
"You are zooming in on the {{part}}, but it is not an AST, so it can't be re-run."
+ " If this error occurred because of ``check_args()``," +
"you may have to refer to your argument differently, e.g. `['args', 0]` or `['kwargs', 'a']`. "
+
"Read https://pythonwhat.readthedocs.io/en/latest/articles/checking_function_calls.html#signatures for more info."
)
_err_msg = "SCT fails on solution: "
_err_msg += state.build_message(patt, fmt_kwargs)
# element can also be { 'node': AST }
if isinstance(element, dict):
element = element["node"]
if isinstance(element, ast.AST):
return
if isinstance(element, list) and all(
[isinstance(el, ast.AST) for el in element]):
return
raise InstructorError(_err_msg)
def get_signature(name, mapped_name, signature, manual_sigs, env):
if isinstance(signature, str):
if signature in manual_sigs:
signature = inspect.Signature(manual_sigs[signature])
else:
raise InstructorError(
"signature error - specified signature not found")
if signature is None:
# establish function
try:
fun = eval(mapped_name, env)
except:
raise InstructorError("%s() was not found." % mapped_name)
# first go through manual sigs
# try to get signature
try:
if name in manual_sigs:
signature = inspect.Signature(manual_sigs[name])
else:
# it might be a method, and we have to find the general method name
if "." in mapped_name:
els = name.split(".")
try:
els[0] = type(eval(els[0], env)).__name__
generic_name = ".".join(els[:])
except:
raise InstructorError(
"signature error - cannot convert call")
if generic_name in manual_sigs:
signature = inspect.Signature(
manual_sigs[generic_name])
else:
raise InstructorError(
"signature error - %s not in builtins" %
generic_name)
else:
raise InstructorError("manual signature not found")
except Exception as e:
try:
signature = inspect.signature(fun)
except:
raise InstructorError(e.args[0] +
" and cannot determine signature")
return signature
def check_object(state,
index,
missing_msg=None,
expand_msg=None,
typestr="variable"):
"""Check object existence (and equality)
Check whether an object is defined in the student's process, and zoom in on its value in both
student and solution process to inspect quality (with has_equal_value().
In ``pythonbackend``, both the student's submission as well as the solution code are executed, in separate processes.
``check_object()`` looks at these processes and checks if the referenced object is available in the student process.
Next, you can use ``has_equal_value()`` to check whether the objects in the student and solution process correspond.
Args:
index (str): the name of the object which value has to be checked.
missing_msg (str): feedback message when the object is not defined in the student process.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
:Example:
Suppose you want the student to create a variable ``x``, equal to 15: ::
x = 15
The following SCT will verify this: ::
Ex().check_object("x").has_equal_value()
- ``check_object()`` will check if the variable ``x`` is defined in the student process.
- ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process.
Note that ``has_equal_value()`` only looks at **end result** of a variable in the student process.
In the example, how the object ``x`` came about in the student's submission, does not matter.
This means that all of the following submission will also pass the above SCT: ::
x = 15
x = 12 + 3
x = 3; x += 12
:Example:
As the previous example mentioned, ``has_equal_value()`` only looks at the **end result**. If your exercise is
first initializing and object and further down the script is updating the object, you can only look at the final value!
Suppose you want the student to initialize and populate a list `my_list` as follows: ::
my_list = []
for i in range(20):
if i % 3 == 0:
my_list.append(i)
There is no robust way to verify whether `my_list = [0]` was coded correctly in a separate way.
The best SCT would look something like this: ::
msg = "Have you correctly initialized `my_list`?"
Ex().check_correct(
check_object('my_list').has_equal_value(),
multi(
# check initialization: [] or list()
check_or(
has_equal_ast(code = "[]", incorrect_msg = msg),
check_function('list')
),
check_for_loop().multi(
check_iter().has_equal_value(),
check_body().check_if_else().multi(
check_test().multi(
set_context(2).has_equal_value(),
set_context(3).has_equal_value()
),
check_body().set_context(3).\\
set_env(my_list = [0]).\\
has_equal_value(name = 'my_list')
)
)
)
)
- ``check_correct()`` is used to robustly check whether ``my_list`` was built correctly.
- If ``my_list`` is not correct, **both** the initialization and the population code are checked.
:Example:
Because checking object correctness incorrectly is such a common misconception, we're adding another example: ::
import pandas as pd
df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
df['c'] = [7, 8, 9]
The following SCT would be **wrong**, as it does not factor in the possibility that the 'add column ``c``' step could've been wrong: ::
Ex().check_correct(
check_object('df').has_equal_value(),
check_function('pandas.DataFrame').check_args(0).has_equal_value()
)
The following SCT would be better, as it is specific to the steps: ::
# verify the df = pd.DataFrame(...) step
Ex().check_correct(
check_df('df').multi(
check_keys('a').has_equal_value(),
check_keys('b').has_equal_value()
),
check_function('pandas.DataFrame').check_args(0).has_equal_value()
)
# verify the df['c'] = [...] step
Ex().check_df('df').check_keys('c').has_equal_value()
:Example:
pythonwhat compares the objects in the student and solution process with the ``==`` operator.
For basic objects, this ``==`` is operator is properly implemented, so that the objects can be effectively compared.
For more complex objects that are produced by third-party packages, however, it's possible that this equality operator is not implemented in a way you'd expect.
Often, for these object types the ``==`` will compare the actual object instances: ::
# pre exercise code
class Number():
def __init__(self, n):
self.n = n
# solution
x = Number(1)
# sct that won't work
Ex().check_object().has_equal_value()
# sct
Ex().check_object().has_equal_value(expr_code = 'x.n')
# submissions that will pass this sct
x = Number(1)
x = Number(2 - 1)
The basic SCT like in the previous example will notwork here.
Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking.
Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n``
and seeing if the result of running this expression in both student and solution process match.
"""
# Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1'
if v2_only():
extra_msg = "If you want to check the value of an object in e.g. a for loop, use `has_equal_value(name = 'my_obj')` instead."
state.assert_execution_root("check_object", extra_msg=extra_msg)
if missing_msg is None:
missing_msg = "Did you define the {{typestr}} `{{index}}` without errors?"
if expand_msg is None:
expand_msg = "Did you correctly define the {{typestr}} `{{index}}`? "
if (not isDefinedInProcess(index, state.solution_process)
and state.has_different_processes()):
raise InstructorError(
"`check_object()` couldn't find object `%s` in the solution process."
% index)
append_message = {
"msg": expand_msg,
"kwargs": {
"index": index,
"typestr": typestr
}
}
# create child state, using either parser output, or create part from name
fallback = lambda: ObjectAssignmentParser.get_part(index)
stu_part = state.ast_dispatcher.find("object_assignments",
state.student_ast).get(
index, fallback())
sol_part = state.ast_dispatcher.find("object_assignments",
state.solution_ast).get(
index, fallback())
# test object exists
_msg = state.build_message(missing_msg, append_message["kwargs"])
state.do_test(
DefinedProcessTest(index, state.student_process, Feedback(_msg)))
child = part_to_child(stu_part,
sol_part,
append_message,
state,
node_name="object_assignments")
return child
def has_printout(
state, index, not_printed_msg=None, pre_code=None, name=None, copy=False
):
"""Check if the right printouts happened.
``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in
the solution process, capture its output, and verify whether the output is present in the output of the student.
This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
printouts as they want, as long as they do the correct one somewhere.
Args:
index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
not_printed_msg (str): if specified, this overrides the default message that is generated when the output
is not found in the student output.
pre_code (str): Python code as a string that is executed before running the targeted student call.
This is the ideal place to set a random seed, for example.
copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
accidentally be mutated. Disabled by default, which speeds up SCTs.
state (State): state as passed by the SCT chain. Don't specify this explicitly.
:Example:
Suppose you want somebody to print out 4: ::
print(1, 2, 3, 4)
The following SCT would check that: ::
Ex().has_printout(0)
All of the following SCTs would pass: ::
print(1, 2, 3, 4)
print('1 2 3 4')
print(1, 2, '3 4')
print("random"); print(1, 2, 3, 4)
:Example:
Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed.
If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work.
Suppose you have the following solution: ::
x = 4
print(x)
x = 6
The following SCT will not work: ::
Ex().has_printout(0)
Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated.
In cases like these, ``has_printout()`` cannot be used.
:Example:
Inside a for loop ``has_printout()``
Suppose you have the following solution: ::
for i in range(5):
print(i)
The following SCT will not work: ::
Ex().check_for_loop().check_body().has_printout(0)
The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``.
If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: ::
Ex().check_for_loop().check_body().\\
set_context(0).check_function('print').\\
check_args(0).has_equal_value()
"""
extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead."
state.assert_root("has_printout", extra_msg=extra_msg)
if not_printed_msg is None:
not_printed_msg = (
"Have you used `{{sol_call}}` to do the appropriate printouts?"
)
try:
sol_call_ast = state.ast_dispatcher("function_calls", state.solution_ast)[
"print"
][index]["node"]
except (KeyError, IndexError):
raise InstructorError(
"`has_printout({})` couldn't find the {} print call in your solution.".format(
index, utils.get_ord(index + 1)
)
)
out_sol, str_sol = getOutputInProcess(
tree=sol_call_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
pre_code=pre_code,
copy=copy,
)
sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast)
if isinstance(str_sol, Exception):
raise InstructorError(
"Evaluating the solution expression {} raised error in solution process."
"Error: {} - {}".format(sol_call_str, type(out_sol), str_sol)
)
_msg = state.build_message(not_printed_msg, {"sol_call": sol_call_str})
has_output(state, out_sol.strip(), pattern=False, no_output_msg=_msg)
return state
def _has_context(state, incorrect_msg, exact_names):
raise InstructorError(
"_has_context的第一个参数必须是状态实例或子类（State instance or subclass）")
def assert_execution_root(self, fun, extra_msg=""):
if not (self.is_root or self.is_creator_type("run")):
raise InstructorError(
"`%s()` should only be called focusing on a full script, following `Ex()` or `run()`. %s"
% (fun, extra_msg)
)
def has_command(state, pattern, msg, fixed=False, commands=None):
"""Test whether the bash history has a command matching the pattern
Args:
state: State instance describing student and solution code. Can be omitted if used with Ex().
pattern: text that command must contain (can be a regex pattern or a simple string)
msg: feedback message if no matching command is found
fixed: whether to match text exactly, rather than using regular expressions
commands: the bash history commands to check against.
By default this will be all commands since the last bash history info update.
Otherwise pass a list of commands to search through, created by calling the helper function
``get_bash_history()``.
Note:
The helper function ``update_bash_history_info(bash_history_path=None)``
needs to be called in the pre-exercise code in exercise types that don't have
built-in support for bash history features.
Note:
If the bash history info is updated every time code is submitted
(by using ``update_bash_history_info()`` in the pre-exercise code),
it's advised to only use this function as the second part of a ``check_correct()``
to help students debug the command they haven't correctly run yet.
Look at the examples to see what could go wrong.
If bash history info is only updated at the start of an exercise,
this can be used everywhere as the (cumulative) commands from all submissions are known.
:Example:
The goal of an exercise is to use ``man``.
If the exercise doesn't have built-in support for bash history SCTs,
update the bash history info in the pre-exercise code::
update_bash_history_info()
In the SCT, check whether a command with ``man`` was used::
Ex().has_command("$man\s", "Your command should start with ``man ...``.")
:Example:
The goal of an exercise is to use ``touch`` to create two files.
In the pre-exercise code, put::
update_bash_history_info()
This SCT can cause problems::
Ex().has_command("touch.*file1", "Use `touch` to create `file1`")
Ex().has_command("touch.*file2", "Use `touch` to create `file2`")
If a student submits after running ``touch file0 && touch file1`` in the console,
they will get feedback to create ``file2``.
If they submit again after running ``touch file2`` in the console,
they will get feedback to create ``file1``, since the SCT only has access
to commands after the last bash history info update (only the second command in this case).
Only if they execute all required commands in a single submission the SCT will pass.
A better SCT in this situation checks the outcome first
and checks the command to help the student achieve it::
Ex().check_correct(
check_file('file1', parse=False),
has_command("touch.*file1", "Use `touch` to create `file1`")
)
Ex().check_correct(
check_file('file2', parse=False),
has_command("touch.*file2", "Use `touch` to create `file2`")
)
"""
if commands is None:
commands = get_bash_history()
if not commands:
state.report("Looking for an executed shell command, we didn't find any.")
if not state.is_root:
raise InstructorError(
"`has_command()` should only be called from the root state, `Ex()`."
)
correct = False
for command in commands:
# similar to has_code
if pattern in command if fixed else re.search(pattern, command):
correct = True
break
if not correct:
state.report(msg)
return state
def call(state,
args,
test="value",
incorrect_msg=None,
error_msg=None,
argstr=None,
func=None,
**kwargs):
"""Use ``check_call()`` in combination with ``has_equal_x()`` instead.
"""
if incorrect_msg is None:
incorrect_msg = MSG_CALL_INCORRECT
if error_msg is None:
error_msg = MSG_CALL_ERROR_INV if test == "error" else MSG_CALL_ERROR
assert test in ("value", "output", "error")
get_func = evalCalls[test]
# Run for Solution --------------------------------------------------------
eval_sol, str_sol = run_call(args, state.solution_parts["node"],
state.solution_process, get_func, **kwargs)
if (test == "error") ^ isinstance(eval_sol, Exception):
_msg = (state.build_message(
"Calling {{argstr}} resulted in an error (or not an error if testing for one). Error message: {{type_err}} {{str_sol}}",
dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr),
), )
raise InstructorError(_msg)
if isinstance(eval_sol, ReprFail):
_msg = state.build_message(
"Can't get the result of calling {{argstr}}: {{eval_sol.info}}",
dict(argstr=argstr, eval_sol=eval_sol),
)
raise InstructorError(_msg)
# Run for Submission ------------------------------------------------------
eval_stu, str_stu = run_call(args, state.student_parts["node"],
state.student_process, get_func, **kwargs)
action_strs = {
"value": "return",
"output": "print out",
"error": "error out with the message",
}
fmt_kwargs = {
"part": argstr,
"argstr": argstr,
"str_sol": str_sol,
"str_stu": str_stu,
"action": action_strs[test],
}
# either error test and no error, or vice-versa
stu_node = state.student_parts["node"]
stu_state = state.to_child(highlight=stu_node)
if (test == "error") ^ isinstance(eval_stu, Exception):
_msg = state.build_message(error_msg, fmt_kwargs)
stu_state.report(_msg)
# incorrect result
_msg = state.build_message(incorrect_msg, fmt_kwargs)
state.do_test(
EqualTest(eval_sol, eval_stu, Feedback(_msg, stu_state), func))
return state
def set_context(state, *args, **kwargs):
"""Update context values for student and solution environments.
When ``has_equal_x()`` is used after this, the context values (in ``for`` loops and function definitions, for example)
will have the values specified through his function. It is the function equivalent of the ``context_vals`` argument of
the ``has_equal_x()`` functions.
- Note 1: excess args and unmatched kwargs will be unused in the student environment.
- Note 2: When you try to set context values that don't match any target variables in the solution code,
``set_context()`` raises an exception that lists the ones available.
- Note 3: positional arguments are more robust to the student using different names for context values.
- Note 4: You have to specify arguments either by position, either by name. A combination is not possible.
:Example:
Solution code::
total = 0
for i in range(10):
print(i ** 2)
Student submission that will pass (different iterator, different calculation)::
total = 0
for j in range(10):
print(j * j)
SCT::
# set_context is robust against different names of context values.
Ex().check_for_loop().check_body().multi(
set_context(1).has_equal_output(),
set_context(2).has_equal_output(),
set_context(3).has_equal_output()
)
# equivalent SCT, by setting context_vals in has_equal_output()
Ex().check_for_loop().check_body().\\
multi([s.has_equal_output(context_vals=[i]) for i in range(1, 4)])
"""
stu_crnt = state.student_context.context
sol_crnt = state.solution_context.context
# for now, you can't specify both
if len(args) > 0 and len(kwargs) > 0:
raise InstructorError(
"In `set_context()`, specify arguments either by position, either by name."
)
# set args specified by pos -----------------------------------------------
if args:
# stop if too many pos args for solution
if len(args) > len(sol_crnt):
raise InstructorError(
"Too many positional args. There are {} context vals, but tried to set {}"
.format(len(sol_crnt), len(args)))
# set pos args
upd_sol = sol_crnt.update(dict(zip(sol_crnt.keys(), args)))
upd_stu = stu_crnt.update(dict(zip(stu_crnt.keys(), args)))
else:
upd_sol = sol_crnt
upd_stu = stu_crnt
# set args specified by keyword -------------------------------------------
if kwargs:
# stop if keywords don't match with solution
if set(kwargs) - set(upd_sol):
raise InstructorError(
"`set_context()` failed: context val names are {}, but you tried to set {}."
.format(upd_sol or "missing", sorted(list(kwargs.keys()))))
out_sol = upd_sol.update(kwargs)
# need to match keys in kwargs with corresponding keys in stu context
# in case they used, e.g., different loop variable names
match_keys = dict(zip(sol_crnt.keys(), stu_crnt.keys()))
out_stu = upd_stu.update(
{match_keys[k]: v
for k, v in kwargs.items() if k in match_keys})
else:
out_sol = upd_sol
out_stu = upd_stu
return state.to_child(student_context=out_stu,
solution_context=out_sol,
highlight=state.highlight)
def check_function(
state,
name,
index=0,
missing_msg=None,
params_not_matched_msg=None,
expand_msg=None,
signature=True,
):
"""Check whether a particular function is called.
``check_function()`` is typically followed by:
- ``check_args()`` to check whether the arguments were specified.
In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()``
to assert that the arguments were correctly specified.
- ``has_equal_value()`` to check whether rerunning the function call coded by the student
gives the same result as calling the function call as in the solution.
Checking function calls is a tricky topic. Please visit the
`dedicated article <articles/checking_function_calls.html>`_ for more explanation,
edge cases and best practices.
Args:
name (str): the name of the function to be tested. When checking functions in packages, always
use the 'full path' of the function.
index (int): index of the function call to be checked. Defaults to 0.
missing_msg (str): If specified, this overrides an automatically generated feedback message in case
the student did not call the function correctly.
params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
in case the function parameters were not successfully matched.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
signature (Signature): Normally, check_function() can figure out what the function signature is,
but it might be necessary to use ``sig_from_params()`` to manually build a signature and pass this along.
state (State): State object that is passed from the SCT Chain (don't specify this).
:Examples:
Student code and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Verify whether arr was correctly set in np.mean
Ex().check_function('numpy.mean').check_args('a').has_equal_value()
# Verify whether np.mean(arr) produced the same result
Ex().check_function('numpy.mean').has_equal_value()
"""
append_missing = missing_msg is None
append_params_not_matched = params_not_matched_msg is None
if missing_msg is None:
missing_msg = MISSING_MSG
if expand_msg is None:
expand_msg = PREPEND_MSG
if params_not_matched_msg is None:
params_not_matched_msg = SIG_ISSUE_MSG
stu_out = state.ast_dispatcher.find("function_calls", state.student_ast)
sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast)
student_mappings = state.ast_dispatcher.find("mappings", state.student_ast)
fmt_kwargs = {
"times": get_times(index + 1),
"ord": get_ord(index + 1),
"index": index,
"mapped_name": get_mapped_name(name, student_mappings),
}
# Get Parts ----
# Copy, otherwise signature binding overwrites sol_out[name][index]['args']
try:
sol_parts = {**sol_out[name][index]}
except KeyError:
raise InstructorError(
"`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!"
% name)
except IndexError:
raise InstructorError(
"`check_function()` couldn't find %s calls of `%s()` in your solution code."
% (index + 1, name))
try:
# Copy, otherwise signature binding overwrites stu_out[name][index]['args']
stu_parts = {**stu_out[name][index]}
except (KeyError, IndexError):
_msg = state.build_message(missing_msg,
fmt_kwargs,
append=append_missing)
state.report(_msg)
# Signatures -----
if signature:
signature = None if isinstance(signature, bool) else signature
get_sig = partial(
getSignatureInProcess,
name=name,
signature=signature,
manual_sigs=state.get_manual_sigs(),
)
try:
sol_sig = get_sig(mapped_name=sol_parts["name"],
process=state.solution_process)
sol_parts["args"] = bind_args(sol_sig, sol_parts["args"])
except Exception as e:
raise InstructorError(
"`check_function()` couldn't match the %s call of `%s` to its signature:\n%s "
% (get_ord(index + 1), name, e))
try:
stu_sig = get_sig(mapped_name=stu_parts["name"],
process=state.student_process)
stu_parts["args"] = bind_args(stu_sig, stu_parts["args"])
except Exception:
_msg = state.build_message(params_not_matched_msg,
fmt_kwargs,
append=append_params_not_matched)
state.to_child(highlight=stu_parts["node"]).report(_msg)
# three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
append_message = {"msg": expand_msg, "kwargs": fmt_kwargs}
child = part_to_child(stu_parts,
sol_parts,
append_message,
state,
node_name="function_calls")
return child
def _has_context(state, incorrect_msg, exact_names):
raise InstructorError(
"first argument to _has_context must be a State instance or subclass")
def check_keys(state, key, missing_msg=None, expand_msg=None):
"""Check whether an object (dict, DataFrame, etc) has a key.
``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is
used to 'zoom in' on the object of interest.
Args:
key (str): Name of the key that the object should have.
missing_msg (str): When specified, this overrides the automatically generated
message in case the key does not exist.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
state (State): The state that is passed in through the SCT chain (don't specify this).
:Example:
Student code and solution code::
x = {'a': 2}
SCT::
# Verify that x contains a key a
Ex().check_object('x').check_keys('a')
# Verify that x contains a key a and a is correct.
Ex().check_object('x').check_keys('a').has_equal_value()
"""
state.assert_is(["object_assignments"], "is_instance",
["check_object", "check_df"])
if missing_msg is None:
missing_msg = "There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`."
if expand_msg is None:
expand_msg = "Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? "
sol_name = state.solution_parts.get("name")
stu_name = state.student_parts.get("name")
if not isDefinedCollInProcess(sol_name, key, state.solution_process):
raise InstructorError(
"`check_keys()` couldn't find key `%s` in object `%s` in the solution process."
% (key, sol_name))
# check if key available
_msg = state.build_message(missing_msg, {"key": key})
state.do_test(
DefinedCollProcessTest(stu_name, key, state.student_process,
Feedback(_msg, state)))
def get_part(name, key, highlight):
if isinstance(key, str):
slice_val = ast.Str(s=key)
else:
slice_val = ast.parse(str(key)).body[0].value
expr = ast.Subscript(
value=ast.Name(id=name, ctx=ast.Load()),
slice=ast.Index(value=slice_val),
ctx=ast.Load(),
)
ast.fix_missing_locations(expr)
return {"node": expr, "highlight": highlight}
stu_part = get_part(stu_name, key, state.student_parts.get("highlight"))
sol_part = get_part(sol_name, key, state.solution_parts.get("highlight"))
append_message = {"msg": expand_msg, "kwargs": {"key": key}}
child = part_to_child(stu_part, sol_part, append_message, state)
return child
def has_expr(
state,
incorrect_msg=None,
error_msg=None,
undefined_msg=None,
append=None,
extra_env=None,
context_vals=None,
pre_code=None,
expr_code=None,
name=None,
copy=True,
func=None,
override=None,
test=None, # todo: default or arg before state
):
if (
append is None
): # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
if name:
incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
elif expr_code:
incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
else:
incorrect_msg = DEFAULT_INCORRECT_MSG
if undefined_msg is None:
undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
if error_msg is None:
if test == "error":
error_msg = DEFAULT_ERROR_MSG_INV
else:
error_msg = DEFAULT_ERROR_MSG
get_func = partial(
evalCalls[test],
extra_env=extra_env,
context_vals=context_vals,
pre_code=pre_code,
expr_code=expr_code,
name=name,
copy=copy,
)
if override is not None:
# don't bother with running expression and fetching output/value
# eval_sol, str_sol = eval
eval_sol, str_sol = override, str(override)
else:
eval_sol, str_sol = get_func(
tree=state.solution_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
)
if (test == "error") ^ isinstance(eval_sol, Exception):
raise InstructorError(
"Evaluating expression raised error in solution process (or didn't raise if testing for one). "
"Error: {} - {}".format(type(eval_sol), str_sol)
)
if isinstance(eval_sol, ReprFail):
raise InstructorError(
"Couldn't extract the value for the highlighted expression from the solution process: "
+ eval_sol.info
)
eval_stu, str_stu = get_func(
tree=state.student_ast,
process=state.student_process,
context=state.student_context,
env=state.student_env,
)
# kwargs ---
fmt_kwargs = {
"stu_part": state.student_parts,
"sol_part": state.solution_parts,
"name": name,
"test": test,
"test_desc": "" if test == "value" else "the %s " % test,
"expr_code": expr_code,
}
fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu))
fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol))
if incorrect_msg == DEFAULT_INCORRECT_MSG and (
fmt_kwargs["stu_eval"] is None
or fmt_kwargs["sol_eval"] is None
or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]
):
incorrect_msg = "Expected something different."
# tests ---
# error in process
if (test == "error") ^ isinstance(eval_stu, Exception):
fmt_kwargs["stu_str"] = str_stu
_msg = state.build_message(error_msg, fmt_kwargs, append=append)
feedback = Feedback(_msg, state)
state.report(feedback)
# name is undefined after running expression
if isinstance(eval_stu, UndefinedValue):
_msg = state.build_message(undefined_msg, fmt_kwargs, append=append)
state.report(Feedback(_msg, state))
# test equality of results
_msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
state.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))
return state
def test_function_v2(state,
name,
index=1,
params=[],
signature=True,
eq_condition="equal",
do_eval=True,
not_called_msg=None,
params_not_matched_msg=None,
params_not_specified_msg=None,
incorrect_msg=None,
add_more=False,
**kwargs):
index = index - 1
if not isinstance(params, list):
raise InstructorError(
"Inside test_function_v2, make sure to specify a LIST of params.")
if isinstance(do_eval, bool) or do_eval is None:
do_eval = [do_eval] * len(params)
if len(params) != len(do_eval):
raise InstructorError(
"Inside test_function_v2, make sure that do_eval has the same length as params."
)
# if params_not_specified_msg is a str or None, convert into list
if isinstance(params_not_specified_msg,
str) or params_not_specified_msg is None:
params_not_specified_msg = [params_not_specified_msg] * len(params)
if len(params) != len(params_not_specified_msg):
raise InstructorError(
"Inside test_function_v2, make sure that params_not_specified_msg has the same length as params."
)
# if incorrect_msg is a str or None, convert into list
if isinstance(incorrect_msg, str) or incorrect_msg is None:
incorrect_msg = [incorrect_msg] * len(params)
if len(params) != len(incorrect_msg):
raise InstructorError(
"Inside test_function_v2, make sure that incorrect_msg has the same length as params."
)
# if root-level (not in compound statement) calls that can be evaluated: use has_printout
eligible = do_eval[0] if isinstance(do_eval,
list) and len(do_eval) > 0 else do_eval
if name == "print" and state.parent_state is None and eligible:
try:
return has_printout(state,
index=index,
not_printed_msg=incorrect_msg[0])
except TestFail:
# The test didn't pass; just continue with the more strict check_function test.
pass
if len(params) == 0:
signature = False
fun_state = check_function(
state,
name=name,
index=index,
missing_msg=not_called_msg,
params_not_matched_msg=params_not_matched_msg,
signature=signature,
)
for i in range(len(params)):
arg_test(
fun_state,
name=params[i],
do_eval=do_eval[i],
missing_msg=params_not_specified_msg[i],
incorrect_msg=incorrect_msg[i],
)
return state
def assert_is_not(self, klasses, fun, prev_fun):
if self.__class__.__name__ in klasses:
raise InstructorError(
"`%s()` should not be called on %s."
% (fun, " or ".join(["`%s()`" % pf for pf in prev_fun]))
)
def has_import(
state,
name,
same_as=False,
not_imported_msg="Did you import `{{pkg}}`?",
incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?",
):
"""Checks whether student imported a package or function correctly.
Python features many ways to import packages.
All of these different methods revolve around the ``import``, ``from`` and ``as`` keywords.
``has_import()`` provides a robust way to check whether a student correctly imported a certain package.
By default, ``has_import()`` allows for different ways of aliasing the imported package or function.
If you want to make sure the correct alias was used to refer to the package or function that was imported,
set ``same_as=True``.
Args:
name (str): the name of the package that has to be checked.
same_as (bool): if True, the alias of the package or function has to be the same. Defaults to False.
not_imported_msg (str): feedback message when the package is not imported.
incorrect_as_msg (str): feedback message if the alias is wrong.
:Example:
Example 1, where aliases don't matter (defaut): ::
# solution
import matplotlib.pyplot as plt
# sct
Ex().has_import("matplotlib.pyplot")
# passing submissions
import matplotlib.pyplot as plt
from matplotlib import pyplot as plt
import matplotlib.pyplot as pltttt
# failing submissions
import matplotlib as mpl
Example 2, where the SCT is coded so aliases do matter: ::
# solution
import matplotlib.pyplot as plt
# sct
Ex().has_import("matplotlib.pyplot", same_as=True)
# passing submissions
import matplotlib.pyplot as plt
from matplotlib import pyplot as plt
# failing submissions
import matplotlib.pyplot as pltttt
"""
student_imports = state.ast_dispatcher("imports", state.student_ast)
solution_imports = state.ast_dispatcher("imports", state.solution_ast)
if name not in solution_imports:
raise InstructorError(
"`has_import()` couldn't find an import of the package %s in your solution code."
% name
)
fmt_kwargs = {"pkg": name, "alias": solution_imports[name]}
_msg = state.build_message(not_imported_msg, fmt_kwargs)
state.do_test(DefinedCollTest(name, student_imports, _msg))
if same_as:
_msg = state.build_message(incorrect_as_msg, fmt_kwargs)
state.do_test(EqualTest(solution_imports[name], student_imports[name], _msg))
return state
def has_equal_ast(state, incorrect_msg=None, code=None, exact=True, append=None):
"""Test whether abstract syntax trees match between the student and solution code.
``has_equal_ast()`` can be used in two ways:
* As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission.
But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST.
Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``.
* As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments.
Args:
incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this.
code: optional code to use instead of the solution AST.
exact: whether the representations must match exactly. If false, the solution AST
only needs to be contained within the student AST (similar to using test student typed).
Defaults to ``True``, unless the ``code`` argument has been specified.
:Example:
Student and Solution Code::
dict(a = 'value').keys()
SCT::
# all pass
Ex().has_equal_ast()
Ex().has_equal_ast(code = "dict(a = 'value').keys()")
Ex().has_equal_ast(code = "dict(a = 'value')", exact = False)
Student and Solution Code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Check underlying value of arugment a of np.mean:
Ex().check_function('numpy.mean').check_args('a').has_equal_ast()
# Only check AST equality of expression used to specify argument a:
Ex().check_function('numpy.mean').check_args('a').has_equal_ast()
"""
if utils.v2_only():
state.assert_is_not(["object_assignments"], "has_equal_ast", ["check_object"])
state.assert_is_not(["function_calls"], "has_equal_ast", ["check_function"])
if code and incorrect_msg is None:
raise InstructorError(
"If you manually specify the code to match inside has_equal_ast(), "
"you have to explicitly set the `incorrect_msg` argument."
)
if (
append is None
): # if not specified, set to False if incorrect_msg was manually specified
append = incorrect_msg is None
if incorrect_msg is None:
incorrect_msg = "Expected `{{sol_str}}`, but got `{{stu_str}}`."
def parse_tree(tree):
# get contents of module.body if only 1 element
crnt = (
tree.body[0]
if isinstance(tree, ast.Module) and len(tree.body) == 1
else tree
)
# remove Expr if it exists
return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt)
stu_rep = parse_tree(state.student_ast)
sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code))
fmt_kwargs = {
"sol_str": state.solution_code if not code else code,
"stu_str": state.student_code,
}
_msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
if exact and not code:
state.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state)))
elif not sol_rep in stu_rep:
state.report(Feedback(_msg, state))
return state
def assert_root(self, fun, extra_msg=""):
if self.parent_state is not None:
raise InstructorError(
"`%s()` should only be called from the root state, `Ex()`. %s"
% (fun, extra_msg))
