def with_context(state, *args, child=None):
# set up context in processes
solution_res = setUpNewEnvInProcess(
process=state.solution_process, context=state.solution_parts["with_items"]
)
if isinstance(solution_res, Exception):
with debugger(state):
state.report(
"error in the solution, running test_with(): %s" % str(solution_res)
)
student_res = setUpNewEnvInProcess(
process=state.student_process, context=state.student_parts["with_items"]
)
if isinstance(student_res, AttributeError):
child.report(
"In your `with` statement, you're not using a correct context manager."
)
if isinstance(student_res, (AssertionError, ValueError, TypeError)):
child.report(
"In your `with` statement, the number of values in your context manager "
"doesn't correspond to the number of variables you're trying to assign it to."
)
# run subtests
try:
multi(state, *args)
finally:
# exit context
close_solution_context = breakDownNewEnvInProcess(
process=state.solution_process
)
if isinstance(close_solution_context, Exception):
with debugger(state):
state.report(
"error in the solution, closing the `with` fails with: %s"
% close_solution_context
)
close_student_context = breakDownNewEnvInProcess(process=state.student_process)
if isinstance(close_student_context, Exception):
state.report(
"Your `with` statement can not be closed off correctly, you're "
"not using the context manager correctly."
)
return state
def has_part(state, name, msg, fmt_kwargs=None, index=None):
d = {
"sol_part": state.solution_parts,
"stu_part": state.student_parts,
**fmt_kwargs,
}
def verify(part, index):
if index is not None:
if isinstance(index, list):
for ind in index:
part = part[ind]
else:
part = part[index]
if part is None:
raise KeyError
# TODO: instructor error if msg is not str
# Check if it's there in the solution
try:
verify(state.solution_parts[name], index)
except (KeyError, IndexError):
with debugger(state):
err_msg = "SCT fails on solution: {}".format(msg)
state.report(err_msg, d)
try:
verify(state.student_parts[name], index)
except (KeyError, IndexError):
state.report(msg, d)
return state
def get_dispatcher(self):
try:
return Dispatcher(self.pre_exercise_code)
except Exception as e:
with debugger(self):
self.report("Something went wrong when parsing the PEC: %s" %
str(e))
def check_file(
state: State,
path,
missing_msg="Did you create the file `{}`?",
is_dir_msg="Want to check the file `{}`, but found a directory.",
parse=True,
solution_code=None,
):
"""Test whether file exists, and make its contents the student code.
Args:
state: State instance describing student and solution code. Can be omitted if used with Ex().
path: expected location of the file
missing_msg: feedback message if no file is found in the expected location
is_dir_msg: feedback message if the path is a directory instead of a file
parse: If ``True`` (the default) the content of the file is interpreted as code in the main exercise technology.
This enables more checks on the content of the file.
solution_code: this argument can be used to pass the expected code for the file
so it can be used by subsequent checks.
Note:
This SCT fails if the file is a directory.
:Example:
To check if a user created the file ``my_output.txt`` in the subdirectory ``resources``
of the directory where the exercise is run, use this SCT::
Ex().check_file("resources/my_output.txt", parse=False)
"""
path_obj = Path(path)
if not path_obj.exists():
state.report(missing_msg.format(path)) # test file exists
if path_obj.is_dir():
state.report(is_dir_msg.format(path)) # test its not a dir
code = get_file_content(path_obj)
sol_kwargs = {"solution_code": solution_code, "solution_ast": None}
if solution_code:
solution_ast = False
if parse:
with debugger(state):
solution_ast = state.parse(solution_code)
sol_kwargs["solution_ast"] = solution_ast
child_state = state.to_child(
append_message="We checked the file `{}`. ".format(path),
student_code=code,
student_ast=state.parse(code) if parse else False,
**sol_kwargs)
child_state.path = path_obj # .parent + .name
return child_state
def __init__(
self,
student_code,
solution_code,
pre_exercise_code,
student_process,
solution_process,
raw_student_output,
# solution output
reporter,
force_diagnose=False,
highlight=None,
highlight_offset=None,
highlighting_disabled=None,
feedback_context=None,
creator=None,
student_ast=None,
solution_ast=None,
student_ast_tokens=None,
solution_ast_tokens=None,
student_parts=None,
solution_parts=None,
student_context=Context(),
solution_context=Context(),
student_env=Context(),
solution_env=Context(),
):
args = locals().copy()
self.debug = False
for k, v in args.items():
if k != "self":
setattr(self, k, v)
self.ast_dispatcher = self.get_dispatcher()
# Parse solution and student code
# if possible, not done yet and wanted (ast arguments not False)
if isinstance(self.student_code, str) and student_ast is None:
self.student_ast = self.parse(student_code)
if isinstance(self.solution_code, str) and solution_ast is None:
with debugger(self):
self.solution_ast = self.parse(solution_code)
if highlight is None: # todo: check parent_state? (move check to reporting?)
self.highlight = self.student_ast
self.converters = get_manual_converters(
) # accessed only from root state
self.manual_sigs = None
def assert_ast(state, element, fmt_kwargs):
err_msg = (
"SCT fails on solution: "
"You are zooming in on the {{part}}, but it is not an AST, so it can't be re-run."
" If this error occurred because of ``check_args()``,"
"you may have to refer to your argument differently, e.g. `['args', 0]` or `['kwargs', 'a']`. "
"Read https://pythonwhat.readthedocs.io/en/latest/articles/checking_function_calls.html#signatures for more info."
)
# element can also be { 'node': AST }
if isinstance(element, dict):
element = element["node"]
if isinstance(element, ast.AST):
return
if isinstance(element, list) and all([isinstance(el, ast.AST) for el in element]):
return
with debugger(state):
state.report(err_msg, fmt_kwargs)
def __init__(
self,
student_code,
solution_code,
pre_exercise_code,
student_conn,
solution_conn,
student_result,
solution_result,
reporter,
force_diagnose=False,
highlight_offset=None,
highlighting_disabled=False,
feedback_context=None,
creator=None,
solution_ast=None,
student_ast=None,
ast_dispatcher=None,
):
args = locals().copy()
self.debug = False
for k, v in args.items():
if k != "self":
setattr(self, k, v)
if ast_dispatcher is None:
self.ast_dispatcher = self.get_dispatcher()
# Parse solution and student code
# if possible, not done yet and wanted (ast arguments not False)
if isinstance(self.solution_code, str) and self.solution_ast is None:
with debugger(self):
self.solution_ast = self.parse(self.solution_code)
if isinstance(self.student_code, str) and self.student_ast is None:
self.student_ast = self.parse(self.student_code)
def has_printout(state,
index,
not_printed_msg=None,
pre_code=None,
name=None,
copy=False):
"""Check if the right printouts happened.
``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in
the solution process, capture its output, and verify whether the output is present in the output of the student.
This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
printouts as they want, as long as they do the correct one somewhere.
Args:
index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
not_printed_msg (str): if specified, this overrides the default message that is generated when the output
is not found in the student output.
pre_code (str): Python code as a string that is executed before running the targeted student call.
This is the ideal place to set a random seed, for example.
copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
accidentally be mutated. Disabled by default, which speeds up SCTs.
state (State): state as passed by the SCT chain. Don't specify this explicitly.
:Example:
Suppose you want somebody to print out 4: ::
print(1, 2, 3, 4)
The following SCT would check that: ::
Ex().has_printout(0)
All of the following SCTs would pass: ::
print(1, 2, 3, 4)
print('1 2 3 4')
print(1, 2, '3 4')
print("random"); print(1, 2, 3, 4)
:Example:
Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed.
If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work.
Suppose you have the following solution: ::
x = 4
print(x)
x = 6
The following SCT will not work: ::
Ex().has_printout(0)
Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated.
In cases like these, ``has_printout()`` cannot be used.
:Example:
Inside a for loop ``has_printout()``
Suppose you have the following solution: ::
for i in range(5):
print(i)
The following SCT will not work: ::
Ex().check_for_loop().check_body().has_printout(0)
The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``.
If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: ::
Ex().check_for_loop().check_body().\\
set_context(0).check_function('print').\\
check_args(0).has_equal_value()
"""
extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead."
state.assert_execution_root("has_printout", extra_msg=extra_msg)
if not_printed_msg is None:
not_printed_msg = (
"Have you used `{{sol_call}}` to do the appropriate printouts?")
try:
sol_call_ast = state.ast_dispatcher.find(
"function_calls", state.solution_ast)["print"][index]["node"]
except (KeyError, IndexError):
raise InstructorError.from_message(
"`has_printout({})` couldn't find the {} print call in your solution."
.format(index, get_ord(index + 1)))
out_sol, str_sol = getOutputInProcess(
tree=sol_call_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
pre_code=pre_code,
copy=copy,
)
sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast)
if isinstance(str_sol, Exception):
with debugger(state):
state.report(
"Evaluating the solution expression {} raised error in solution process."
"Error: {} - {}".format(sol_call_str, type(out_sol), str_sol))
has_output(
state,
out_sol.strip(),
pattern=False,
no_output_msg=FeedbackComponent(not_printed_msg,
{"sol_call": sol_call_str}),
)
return state
def check_function(
state,
name,
index=0,
missing_msg=None,
params_not_matched_msg=None,
expand_msg=None,
signature=True,
):
"""Check whether a particular function is called.
``check_function()`` is typically followed by:
- ``check_args()`` to check whether the arguments were specified.
In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()``
to assert that the arguments were correctly specified.
- ``has_equal_value()`` to check whether rerunning the function call coded by the student
gives the same result as calling the function call as in the solution.
Checking function calls is a tricky topic. Please visit the
`dedicated article <articles/checking_function_calls.html>`_ for more explanation,
edge cases and best practices.
Args:
name (str): the name of the function to be tested. When checking functions in packages, always
use the 'full path' of the function.
index (int): index of the function call to be checked. Defaults to 0.
missing_msg (str): If specified, this overrides an automatically generated feedback message in case
the student did not call the function correctly.
params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
in case the function parameters were not successfully matched.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
signature (Signature): Normally, check_function() can figure out what the function signature is,
but it might be necessary to use ``sig_from_params()`` to manually build a signature and pass this along.
state (State): State object that is passed from the SCT Chain (don't specify this).
:Examples:
Student code and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Verify whether arr was correctly set in np.mean
Ex().check_function('numpy.mean').check_args('a').has_equal_value()
# Verify whether np.mean(arr) produced the same result
Ex().check_function('numpy.mean').has_equal_value()
"""
append_missing = missing_msg is None
append_params_not_matched = params_not_matched_msg is None
if missing_msg is None:
missing_msg = MISSING_MSG
if expand_msg is None:
expand_msg = PREPEND_MSG
if params_not_matched_msg is None:
params_not_matched_msg = SIG_ISSUE_MSG
stu_out = state.ast_dispatcher.find("function_calls", state.student_ast)
sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast)
student_mappings = state.ast_dispatcher.find("mappings", state.student_ast)
fmt_kwargs = {
"times": get_times(index + 1),
"ord": get_ord(index + 1),
"index": index,
"mapped_name": get_mapped_name(name, student_mappings),
}
# Get Parts ----
# Copy, otherwise signature binding overwrites sol_out[name][index]['args']
with debugger(state):
try:
sol_parts = {**sol_out[name][index]}
except KeyError:
state.report(
"`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!"
% name)
except IndexError:
state.report(
"`check_function()` couldn't find %s calls of `%s()` in your solution code."
% (index + 1, name))
try:
# Copy, otherwise signature binding overwrites stu_out[name][index]['args']
stu_parts = {**stu_out[name][index]}
except (KeyError, IndexError):
state.report(missing_msg, fmt_kwargs, append=append_missing)
# Signatures -----
if signature:
signature = None if isinstance(signature, bool) else signature
get_sig = partial(
getSignatureInProcess,
name=name,
signature=signature,
manual_sigs=state.get_manual_sigs(),
)
try:
sol_sig = get_sig(mapped_name=sol_parts["name"],
process=state.solution_process)
sol_parts["args"] = bind_args(sol_sig, sol_parts["args"])
except Exception as e:
with debugger(state):
state.report(
"`check_function()` couldn't match the %s call of `%s` to its signature:\n%s "
% (get_ord(index + 1), name, e))
try:
stu_sig = get_sig(mapped_name=stu_parts["name"],
process=state.student_process)
stu_parts["args"] = bind_args(stu_sig, stu_parts["args"])
except Exception:
state.to_child(highlight=stu_parts["node"]).report(
params_not_matched_msg,
fmt_kwargs,
append=append_params_not_matched)
# three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
append_message = FeedbackComponent(expand_msg, fmt_kwargs)
child = part_to_child(stu_parts,
sol_parts,
append_message,
state,
node_name="function_calls")
return child
def call(state,
args,
test="value",
incorrect_msg=None,
error_msg=None,
argstr=None,
func=None,
**kwargs):
"""Use ``check_call()`` in combination with ``has_equal_x()`` instead.
"""
if incorrect_msg is None:
incorrect_msg = MSG_CALL_INCORRECT
if error_msg is None:
error_msg = MSG_CALL_ERROR_INV if test == "error" else MSG_CALL_ERROR
assert test in ("value", "output", "error")
get_func = evalCalls[test]
# Run for Solution --------------------------------------------------------
eval_sol, str_sol = run_call(args, state.solution_parts["node"],
state.solution_process, get_func, **kwargs)
if (test == "error") ^ isinstance(eval_sol, Exception):
with debugger(state):
state.report(
"Calling {{argstr}} resulted in an error (or not an error if testing for one). Error message: {{type_err}} {{str_sol}}",
dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr),
)
if isinstance(eval_sol, ReprFail):
with debugger(state):
state.report(
"Can't get the result of calling {{argstr}}: {{eval_sol.info}}",
dict(argstr=argstr, eval_sol=eval_sol),
)
# Run for Submission ------------------------------------------------------
eval_stu, str_stu = run_call(args, state.student_parts["node"],
state.student_process, get_func, **kwargs)
action_strs = {
"value": "return",
"output": "print out",
"error": "error out with the message",
}
fmt_kwargs = {
"part": argstr,
"argstr": argstr,
"str_sol": str_sol,
"str_stu": str_stu,
"action": action_strs[test],
}
# either error test and no error, or vice-versa
stu_node = state.student_parts["node"]
stu_state = state.to_child(highlight=stu_node)
if (test == "error") ^ isinstance(eval_stu, Exception):
stu_state.report(error_msg, fmt_kwargs)
# incorrect result
stu_state.do_test(
EqualTest(eval_sol, eval_stu,
FeedbackComponent(incorrect_msg, fmt_kwargs), func))
return state
def assert_is_not(self, klasses, fun, prev_fun):
if self.__class__.__name__ in klasses:
with debugger(self):
self.report(
"`%s()` should not be called on %s." %
(fun, " or ".join(["`%s()`" % pf for pf in prev_fun])))
def assert_execution_root(self, fun, extra_msg=""):
if not (self.is_root or self.is_creator_type("run")):
with debugger(self):
self.report(
"`%s()` should only be called focusing on a full script, following `Ex()` or `run()`. %s"
% (fun, extra_msg))
def has_command(state, pattern, msg, fixed=False, commands=None):
"""Test whether the bash history has a command matching the pattern
Args:
state: State instance describing student and solution code. Can be omitted if used with Ex().
pattern: text that command must contain (can be a regex pattern or a simple string)
msg: feedback message if no matching command is found
fixed: whether to match text exactly, rather than using regular expressions
commands: the bash history commands to check against.
By default this will be all commands since the last bash history info update.
Otherwise pass a list of commands to search through, created by calling the helper function
``get_bash_history()``.
Note:
The helper function ``update_bash_history_info(bash_history_path=None)``
needs to be called in the pre-exercise code in exercise types that don't have
built-in support for bash history features.
Note:
If the bash history info is updated every time code is submitted
(by using ``update_bash_history_info()`` in the pre-exercise code),
it's advised to only use this function as the second part of a ``check_correct()``
to help students debug the command they haven't correctly run yet.
Look at the examples to see what could go wrong.
If bash history info is only updated at the start of an exercise,
this can be used everywhere as the (cumulative) commands from all submissions are known.
:Example:
The goal of an exercise is to use ``man``.
If the exercise doesn't have built-in support for bash history SCTs,
update the bash history info in the pre-exercise code::
update_bash_history_info()
In the SCT, check whether a command with ``man`` was used::
Ex().has_command("$man\s", "Your command should start with ``man ...``.")
:Example:
The goal of an exercise is to use ``touch`` to create two files.
In the pre-exercise code, put::
update_bash_history_info()
This SCT can cause problems::
Ex().has_command("touch.*file1", "Use `touch` to create `file1`")
Ex().has_command("touch.*file2", "Use `touch` to create `file2`")
If a student submits after running ``touch file0 && touch file1`` in the console,
they will get feedback to create ``file2``.
If they submit again after running ``touch file2`` in the console,
they will get feedback to create ``file1``, since the SCT only has access
to commands after the last bash history info update (only the second command in this case).
Only if they execute all required commands in a single submission the SCT will pass.
A better SCT in this situation checks the outcome first
and checks the command to help the student achieve it::
Ex().check_correct(
check_file('file1', parse=False),
has_command("touch.*file1", "Use `touch` to create `file1`")
)
Ex().check_correct(
check_file('file2', parse=False),
has_command("touch.*file2", "Use `touch` to create `file2`")
)
"""
if commands is None:
commands = get_bash_history()
if not commands:
state.report(
"Looking for an executed shell command, we didn't find any.")
if not state.is_root:
with debugger(state):
state.report(
"`has_command()` should only be called from the root state, `Ex()`."
)
correct = False
for command in commands:
# similar to has_code
if pattern in command if fixed else re.search(pattern, command):
correct = True
break
if not correct:
state.report(msg)
return state
def _has_context(state, incorrect_msg, exact_names):
with debugger(state):
state.report(
"first argument to _has_context must be a State instance or subclass"
)