def check_node(name, index=0, typestr='{ordinal} node',
missing_msg=None,
expand_msg=None,
state=None):
if missing_msg is None: missing_msg = "__JINJA__:The system wants to check the {{typestr}} but hasn't found it."
if expand_msg is None: expand_msg = "__JINJA__:Check the {{typestr}}. "
rep = Reporter.active_reporter
stu_out = getattr(state, 'student_'+name)
sol_out = getattr(state, 'solution_'+name)
# check if there are enough nodes for index
fmt_kwargs = {'ordinal': get_ord(index+1) if isinstance(index, int) else "",
'index': index,
'name': name}
fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs)
# test if node can be indexed succesfully
try: stu_out[index]
except (KeyError, IndexError): # TODO comment errors
_msg = state.build_message(missing_msg, fmt_kwargs)
rep.do_test(Test(Feedback(_msg, state)))
# get node at index
stu_part = stu_out[index]
sol_part = sol_out[index]
append_message = {
'msg': expand_msg,
'kwargs': fmt_kwargs
}
return part_to_child(stu_part, sol_part, append_message, state, node_name=name)
def check_part_index(name, index, part_msg,
missing_msg=None,
expand_msg=None,
state=None):
"""Return child state with indexed name part as its ast tree.
``index`` can be:
- an integer, in which case the student/solution_parts are indexed by position.
- a string, in which case the student/solution_parts are expected to be a dictionary.
- a list of indices (which can be integer or string), in which case the student parts are indexed step by step.
"""
if missing_msg is None: missing_msg = "__JINJA__:Are you sure you defined the {{part}}? "
if expand_msg is None: expand_msg = "__JINJA__:Did you correctly specify the {{part}}? "
# create message
ordinal = get_ord(index+1) if isinstance(index, int) else ""
fmt_kwargs = {
'index': index,
'ordinal': ordinal
}
fmt_kwargs.update(part = part_msg.format(**fmt_kwargs))
append_message = {
'msg': expand_msg,
'kwargs': fmt_kwargs
}
# check there are enough parts for index
has_part(name, missing_msg, state, fmt_kwargs, index)
# get part at index
stu_part = state.student_parts[name]
sol_part = state.solution_parts[name]
if isinstance(index, list):
for ind in index:
stu_part = stu_part[ind]
sol_part = sol_part[ind]
else:
stu_part = stu_part[index]
sol_part = sol_part[index]
assert_ast(state, sol_part, fmt_kwargs)
# return child state from part
return part_to_child(stu_part, sol_part, append_message, state)
def has_printout(index,
not_printed_msg=None,
pre_code=None,
name=None,
copy=False,
state=None):
"""Check if the output of print() statement in the solution is in the output the student generated.
This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
printouts as they want, as long as they do the correct one somewhere.
.. note::
When zooming in on parts of the student submission (with e.g. ``check_for_loop()``), we are not
zooming in on the piece of the student output that is related to that piece of the student code.
In other words, ``has_printout()`` always considers the entire student output.
Args:
index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
not_printed_msg (str): if specified, this overrides the default message that is generated when the output
is not found in the student output.
pre_code (str): Python code as a string that is executed before running the targeted student call.
This is the ideal place to set a random seed, for example.
copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
accidentally be mutated. Disabled by default, which speeds up SCTs.
state (State): state as passed by the SCT chain. Don't specify this explicitly.
:Example:
Solution::
print(1, 2, 3, 4)
SCT::
Ex().has_printout(0)
Each of these submissions will pass::
print(1, 2, 3, 4)
print('1 2 3 4')
print(1, 2, '3 4')
print("random"); print(1, 2, 3, 4)
"""
state.assert_root('has_printout')
if not_printed_msg is None:
not_printed_msg = "__JINJA__:Have you used `{{sol_call}}` to do the appropriate printouts?"
try:
sol_call_ast = state.solution_function_calls['print'][index]['node']
except (KeyError, IndexError):
raise InstructorError(
"`has_printout({})` couldn't find the {} print call in your solution."
.format(index, utils.get_ord(index + 1)))
out_sol, str_sol = getOutputInProcess(tree=sol_call_ast,
process=state.solution_process,
context=state.solution_context,
env=state.solution_env,
pre_code=pre_code,
copy=copy)
sol_call_str = state.solution_tree_tokens.get_text(sol_call_ast)
if isinstance(str_sol, Exception):
raise InstructorError(
"Evaluating the solution expression {} raised error in solution process."
"Error: {} - {}".format(sol_call_str, type(out_sol), str_sol))
_msg = state.build_message(not_printed_msg, {'sol_call': sol_call_str})
has_output(out_sol.strip(), pattern=False, no_output_msg=_msg, state=state)
return state
def test_get_ord(self):
self.assertEqual(utils.get_ord(1), "first")
self.assertEqual(utils.get_ord(2), "second")
self.assertEqual(utils.get_ord(3), "third")
self.assertEqual(utils.get_ord(11), "11th")
def check_args(name, missing_msg=None, state=None):
"""Check whether a function argument is specified.
This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified.
If you want to go on and check whether the argument was correctly specified, you can can continue chaining with
``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check)
This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been
specified.
Args:
name (str): the name of the argument for which you want to check it is specified. This can also be
a number, in which case it refers to the positional arguments. Named argumetns take precedence.
missing_msg (str): If specified, this overrides an automatically generated feedback message in case
the student did specify the argument.
state (State): State object that is passed from the SCT Chain (don't specify this).
:Examples:
Student and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Verify whether arr was correctly set in np.mean
# has_equal_value() checks the value of arr, used to set argument a
Ex().check_function('numpy.mean').check_args('a').has_equal_value()
# Verify whether arr was correctly set in np.mean
# has_equal_ast() checks the expression used to set argument a
Ex().check_function('numpy.mean').check_args('a').has_equal_ast()
Student and solution code::
def my_power(x):
print("calculating sqrt...")
return(x * x)
SCT::
Ex().check_function_def('my_power').multi(
check_args('x') # will fail if student used y as arg
check_args(0) # will still pass if student used y as arg
)
"""
if missing_msg is None:
missing_msg = '__JINJA__:Did you specify the {{part}}?'
if name in ['*args', '**kwargs']: # for check_function_def
return check_part(name, name, state=state, missing_msg = missing_msg)
else:
if isinstance(name, list): # dealing with args or kwargs
if name[0] == 'args':
arg_str = "%s argument passed as a variable length argument"%get_ord(name[1]+1)
else:
arg_str = "argument `%s`"%name[1]
else:
arg_str = "%s argument" % get_ord(name+1) if isinstance(name, int) else "argument `%s`" % name
return check_part_index('args', name, arg_str, missing_msg = missing_msg, state=state)
def test_comp(typestr,
comptype,
index,
iter_vars_names,
not_called_msg,
insufficient_ifs_msg,
incorrect_iter_vars_msg,
comp_iter,
ifs,
key=None,
body=None,
value=None,
expand_message=True,
rep=None,
state=None):
MSG_NOT_CALLED = "FMT:The system wants to check the {typestr} but hasn't found it."
MSG_PREPEND = "FMT:Check the {typestr}. "
MSG_INCORRECT_ITER_VARS = "FMT:Have you used the correct iterator variables in the {parent[typestr]}? Be sure to use the correct names."
MSG_INCORRECT_NUM_ITER_VARS = "FMT:Have you used {num_vars} iterator variables in the {parent[typestr]}?"
MSG_INSUFFICIENT_IFS = "FMT:Have you used {sol_len} ifs inside the {parent[typestr]}?"
# if true, set expand_message to default (for backwards compatibility)
expand_message = MSG_PREPEND if expand_message is True else (expand_message
or "")
# make sure other messages are set to default if None
if insufficient_ifs_msg is None:
insufficient_ifs_msg = MSG_INSUFFICIENT_IFS
if not_called_msg is None: not_called_msg = MSG_NOT_CALLED
# TODO MSG: function was not consistent with prepending, so use state w/o expand_message
quiet_state = check_node(comptype,
index - 1,
typestr,
not_called_msg,
expand_msg="",
state=state)
# get comprehension
state = check_node(comptype,
index - 1,
typestr,
not_called_msg,
expand_msg=None if expand_message else "",
state=state)
# test comprehension iter and its variable names (or number of variables)
if comp_iter:
multi(comp_iter,
state=check_part("iter", "iterable part", state=state))
# test iterator variables
default_msg = MSG_INCORRECT_ITER_VARS if iter_vars_names else MSG_INCORRECT_NUM_ITER_VARS
has_context(incorrect_iter_vars_msg or default_msg,
iter_vars_names,
state=quiet_state)
# test the main expressions.
if body:
multi(body,
state=check_part("body",
"body",
expand_msg=None if expand_message else "",
state=state)) # list and gen comp
if key:
multi(key,
state=check_part("key",
"key part",
expand_msg=None if expand_message else "",
state=state)) # dict comp
if value:
multi(value,
state=check_part("value",
"value part",
expand_msg=None if expand_message else "",
state=state)) # ""
# test a list of ifs. each entry corresponds to a filter in the comprehension.
for i, if_test in enumerate(ifs or []):
# test that ifs are same length
has_equal_part_len('ifs', insufficient_ifs_msg, state=quiet_state)
# test individual ifs
multi(if_test,
state=check_part_index("ifs",
i,
utils.get_ord(i + 1) + " if",
state=state))
def test_with(
index,
context_vals=False, # whether to check number of context vals
context_tests=None, # check on context expressions
body=None,
undefined_msg=None,
context_vals_len_msg=None,
context_vals_msg=None,
expand_message=True,
state=None):
"""Test a with statement.
with open_file('...') as bla:
[ open_file('...').__enter__() ]
with open_file('...') as file:
[ ]
"""
MSG_MISSING = "Define more `with` statements."
MSG_PREPEND = "FMT:Check the {typestr}. "
MSG_NUM_CTXT = "Make sure to use the correct number of context variables. It seems you defined too many."
MSG_NUM_CTXT2 = "Make sure to use the correct number of context variables. It seems you defined too little."
MSG_CTXT_NAMES = "FMT:Make sure to use the correct context variable names. Was expecting `{sol_vars}` but got `{stu_vars}`."
check_with = partial(check_node,
'withs',
index - 1,
"{ordinal} `with` statement",
MSG_MISSING,
state=state)
child = check_with(MSG_PREPEND if expand_message else "")
child2 = check_with(MSG_PREPEND if expand_message else "")
if context_vals:
# test context var names ----
has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES,
exact_names=True,
state=child)
# test num context vars ----
has_equal_part_len('context', MSG_NUM_CTXT, state=child)
# Context sub tests ----
if context_tests and not isinstance(context_tests, list):
context_tests = [context_tests]
expand_msg = None if expand_message else ""
for i, context_test in enumerate(context_tests or []):
# partial the substate check, because the function uses two prepended messages
check_context = partial(check_part_index,
'context',
i,
"%s context" % utils.get_ord(i + 1),
missing_msg=MSG_NUM_CTXT2,
expand_msg=expand_msg)
check_context(state=child) # test exist
ctxt_state = check_context(state=child2) # sub tests
multi(context_test, state=ctxt_state)
# Body sub tests ----
if body is not None:
body_state = check_part('body',
'body',
expand_msg=expand_msg,
state=child2)
with_context(body, state=body_state)
def check_function(name,
index=0,
missing_msg=None,
params_not_matched_msg=None,
expand_msg=None,
signature=True,
state=None):
"""Check whether a particular function is called.
This function is typically followed by ``check_args()`` to check whether the arguments were
specified correctly.
Args:
name (str): the name of the function to be tested. When checking functions in packages, always
use the 'full path' of the function.
index (int): index of the function call to be checked. Defaults to 0.
missing_msg (str): If specified, this overrides an automatically generated feedback message in case
the student did not call the function correctly.
params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
in case the function parameters were not successfully matched.
expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
signature (Signature): Normally, check_function() can figure out what the function signature is,
but it might be necessary to use build_sig to manually build a signature and pass this along.
state (State): State object that is passed from the SCT Chain (don't specify this).
:Examples:
Student code and solution code::
import numpy as np
arr = np.array([1, 2, 3, 4, 5])
np.mean(arr)
SCT::
# Verify whether arr was correctly set in np.mean
Ex().check_function('numpy.mean').check_args('a').has_equal_value()
# Verify whether np.mean(arr) produced the same result
Ex().check_function('numpy.mean').has_equal_value()
"""
append_missing = missing_msg is None
append_params_not_matched = params_not_matched_msg is None
if missing_msg is None:
missing_msg = MISSING_MSG
if expand_msg is None:
expand_msg = PREPEND_MSG
if params_not_matched_msg is None:
params_not_matched_msg = SIG_ISSUE_MSG
rep = Reporter.active_reporter
stu_out = state.student_function_calls
sol_out = state.solution_function_calls
student_mappings = state.student_mappings
fmt_kwargs = {
'times': get_times(index + 1),
'ord': get_ord(index + 1),
'index': index,
'mapped_name': get_mapped_name(name, student_mappings)
}
# Get Parts ----
# Copy, otherwise signature binding overwrites sol_out[name][index]['args']
try:
sol_parts = {**sol_out[name][index]}
except KeyError:
raise InstructorError(
"`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!"
% name)
except IndexError:
raise InstructorError(
"`check_function()` couldn't find %s calls of `%s()` in your solution code."
% (index + 1, name))
try:
# Copy, otherwise signature binding overwrites stu_out[name][index]['args']
stu_parts = {**stu_out[name][index]}
except (KeyError, IndexError):
_msg = state.build_message(missing_msg,
fmt_kwargs,
append=append_missing)
rep.do_test(Test(Feedback(_msg, state)))
# Signatures -----
if signature:
signature = None if isinstance(signature, bool) else signature
get_sig = partial(getSignatureInProcess,
name=name,
signature=signature,
manual_sigs=state.get_manual_sigs())
try:
sol_sig = get_sig(mapped_name=sol_parts['name'],
process=state.solution_process)
sol_parts['args'] = bind_args(sol_sig, sol_parts['args'])
except:
raise InstructorError(
"`check_function()` couldn't match the %s call of `%s` to its signature. "
% (get_ord(index + 1), name))
try:
stu_sig = get_sig(mapped_name=stu_parts['name'],
process=state.student_process)
stu_parts['args'] = bind_args(stu_sig, stu_parts['args'])
except Exception:
_msg = state.build_message(params_not_matched_msg,
fmt_kwargs,
append=append_params_not_matched)
rep.do_test(
Test(
Feedback(
_msg,
StubState(stu_parts['node'],
state.highlighting_disabled))))
# three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}
child = part_to_child(stu_parts,
sol_parts,
append_message,
state,
node_name='function_calls')
return child