def test_call(name, call_ind, signature, params, do_eval, solution_args,
eq_fun, add_more, index,
params_not_specified_msg, params_not_matched_msg, incorrect_msg,
keywords, # pulled from solution process
state, highlight):
#stud_name = get_mapped_name(name, state.student_mappings)
rep = Reporter.active_reporter
# Parse Signature for Submission. TODO: more info
try:
student_call, arguments, keywords, stud_name = state.student_function_calls[name][call_ind]['_spec1']
student_sig = getSignatureInProcess(name = name, mapped_name = stud_name,
signature=signature,
manual_sigs = state.get_manual_sigs(),
process=state.student_process)
student_args, student_params = bind_args(signature = student_sig, arguments=arguments, keyws=keywords)
except:
# -prep feedback-
if not params_not_matched_msg:
params_not_matched_msg = ("Something went wrong in figuring out how you specified the " + \
"arguments for `%s()`; have another look at your code and its output.") % stud_name
_msg = state.build_message(params_not_matched_msg)
feedback = Feedback(_msg, student_call if highlight else None)
# run subtest
rep.do_test(Test(feedback)) # TODO: sub_call
# Fail if student didn't use all params ---------------------------
setdiff = list(set(params) - set(student_args.keys()))
if setdiff:
# -prep feedback-
first_missing = setdiff[0] # TODO: sets are not ordered! equiv to set.pop()?
param_ind = params.index(first_missing)
if params_not_specified_msg[param_ind] is None:
msg = "Have you specified all required arguments inside `%s()`?" % stud_name
# only if value can be supplied as keyword argument, give more info:
if student_params[first_missing].kind in [1, 3, 4]:
msg += " You didn't specify `%s`." % first_missing
else:
msg = params_not_specified_msg[param_ind]
_msg = state.build_message(msg)
feedback = Feedback(_msg, student_call if highlight else None)
# run subtest
rep.do_test(Test(feedback)) # TODO: sub_call
# TEST EACH PARAM
for ind, param in enumerate(params):
if do_eval[ind] is None: continue
arg_student = student_args[param]
arg_solution = solution_args[param]
param_kind = student_params[param].kind
test_arg(param, do_eval[ind],
arg_student, arg_solution, param_kind, stud_name,
eq_fun, add_more,
incorrect_msg[ind], state=state, highlight = arg_student if highlight else None)
# If all is still good, we have a winner!
state.set_used(name, call_ind, index)
def test_call(
name,
call_ind,
signature,
params,
do_eval,
solution_args,
eq_fun,
add_more,
index,
params_not_specified_msg,
params_not_matched_msg,
incorrect_msg,
keywords, # pulled from solution process
state,
highlight):
#stud_name = get_mapped_name(name, state.student_mappings)
rep = Reporter.active_reporter
# Parse Signature for Submission. TODO: more info
try:
student_call, arguments, keywords, stud_name = state.student_function_calls[
name][call_ind]['_spec1']
student_sig = getSignatureInProcess(
name=name,
mapped_name=stud_name,
signature=signature,
manual_sigs=state.get_manual_sigs(),
process=state.student_process)
student_args, student_params = bind_args(signature=student_sig,
arguments=arguments,
keyws=keywords)
except:
# -prep feedback-
if not params_not_matched_msg:
params_not_matched_msg = ("Something went wrong in figuring out how you specified the " + \
"arguments for `%s()`; have another look at your code and its output.") % stud_name
_msg = state.build_message(params_not_matched_msg)
feedback = Feedback(_msg, student_call if highlight else None)
# run subtest
rep.do_test(Test(feedback)) # TODO: sub_call
# Fail if student didn't use all params ---------------------------
setdiff = list(set(params) - set(student_args.keys()))
if setdiff:
# -prep feedback-
first_missing = setdiff[
0] # TODO: sets are not ordered! equiv to set.pop()?
param_ind = params.index(first_missing)
if params_not_specified_msg[param_ind] is None:
msg = "Have you specified all required arguments inside `%s()`?" % stud_name
# only if value can be supplied as keyword argument, give more info:
if student_params[first_missing].kind in [1, 3, 4]:
msg += " You didn't specify `%s`." % first_missing
else:
msg = params_not_specified_msg[param_ind]
_msg = state.build_message(msg)
feedback = Feedback(_msg, student_call if highlight else None)
# run subtest
rep.do_test(Test(feedback)) # TODO: sub_call
# TEST EACH PARAM
for ind, param in enumerate(params):
if do_eval[ind] is None: continue
arg_student = student_args[param]
arg_solution = solution_args[param]
param_kind = student_params[param].kind
test_arg(param,
do_eval[ind],
arg_student,
arg_solution,
param_kind,
stud_name,
eq_fun,
add_more,
incorrect_msg[ind],
state=state,
highlight=arg_student if highlight else None)
# If all is still good, we have a winner!
state.set_used(name, call_ind, index)
def test_function_v2(name,
index=1,
params=[],
signature=None,
eq_condition="equal",
do_eval=True,
not_called_msg=None,
params_not_matched_msg=None,
params_not_specified_msg=None,
incorrect_msg=None,
add_more=False,
highlight=True,
state=None):
"""Test if function calls match (v2).
This function compares a function call in the student's code with the corresponding one in the solution
code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
that is returned will depend on the sort of fail.
Args:
name (str): the name of the function to be tested.
index (int): index of the function call to be checked. Defaults to 1.
params (list(str)): the parameter names of the function call that you want to check.
signature (Signature): Normally, test_function() can figure out what the function signature is,
but it might be necessary to use build_sig to manually build a signature and pass this along.
eq_condition (str): how parameters are compared. Currently, only "equal" is supported,
meaning that the arguments in student and solution process should have exactly the same value.
do_eval (list(bool)): Boolean or list of booleans (parameter-specific) that specify whether or
not arguments should be evaluated.
True: arguments are evaluated and compared.
False: arguments are not evaluated but 'string-matched'.
None: arguments are not evaluated; it is only checked if they are specified.
not_called_msg (str): custom feedback message if the function is not called.
params_not_matched_message (str): custom feedback message if the function parameters were not successfully matched.
params_not_specified_msg (str): string or list of strings (parameter-specific). Custom feedback message if not all
parameters listed in params are specified by the student.
incorrect_msg (list(str)): string or list of strings (parameter-specific). Custom feedback messages if the arguments
don't correspond between student and solution code.
"""
rep = Reporter.active_reporter
rep.set_tag("fun", "test_function")
index = index - 1
eq_map = {"equal": EqualTest}
# ARG CHECKS --------------------------------------------------------------
if eq_condition not in eq_map:
raise NameError("%r not a valid equality condition " % eq_condition)
eq_fun = eq_map[eq_condition]
if not isinstance(params, list):
raise NameError(
"Inside test_function_v2, make sure to specify a LIST of params.")
if isinstance(do_eval, bool) or do_eval is None:
do_eval = [do_eval] * len(params)
if len(params) != len(do_eval):
raise NameError(
"Inside test_function_v2, make sure that do_eval has the same length as params."
)
# if params_not_specified_msg is a str or None, convert into list
if isinstance(params_not_specified_msg,
str) or params_not_specified_msg is None:
params_not_specified_msg = [params_not_specified_msg] * len(params)
if len(params) != len(params_not_specified_msg):
raise NameError(
"Inside test_function_v2, make sure that params_not_specified_msg has the same length as params."
)
# if incorrect_msg is a str or None, convert into list
if isinstance(incorrect_msg, str) or incorrect_msg is None:
incorrect_msg = [incorrect_msg] * len(params)
if len(params) != len(incorrect_msg):
raise NameError(
"Inside test_function_v2, make sure that incorrect_msg has the same length as params."
)
# STATE STUFF -------------------------------------------------------------
student_process, solution_process = state.student_process, state.solution_process
solution_calls = state.solution_function_calls
student_calls = state.student_function_calls
student_mappings = state.student_mappings
solution_mappings = state.solution_mappings
stud_name = get_mapped_name(name, student_mappings)
#sol_name = get_mapped_name(name, solution_mappings)
if not_called_msg is None:
if index == 0:
not_called_msg = "Have you called `%s()`?" % stud_name
else:
not_called_msg = (
"The system wants to check the %s call of `%s()`, " +
"but hasn't found it; have another look at your code.") % (
get_ord(index + 1), stud_name)
if name not in solution_calls or len(solution_calls[name]) <= index:
raise NameError("%r not in solution environment (often enough)" % name)
# TODO: test if function name in dict of calls
_msg = state.build_message(not_called_msg)
rep.do_test(DefinedCollTest(name, student_calls, _msg))
# TODO: test if number of specific function calls is less than index
rep.do_test(BiggerTest(len(student_calls[name]), index, _msg)) # TODO
# TODO pull into own function
if len(params) > 0:
# Parse Signature -----------------------------------------------------
try:
sol_call, arguments, keywords, sol_name = solution_calls[name][
index]['_spec1']
sol_sig = getSignatureInProcess(
name=name,
mapped_name=sol_name,
signature=signature,
manual_sigs=state.get_manual_sigs(),
process=solution_process)
solution_args, _ = bind_args(signature=sol_sig,
arguments=arguments,
keyws=keywords)
except:
raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
" You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))
# Check if params are in signature
if set(params) - set(solution_args.keys()):
raise ValueError(
"When testing %s(), the solution call doesn't specify the listed parameters."
% name)
# Get all options (some function calls may be blacklisted)
call_indices = state.get_options(name,
list(range(len(student_calls[name]))),
index)
feedback = None
# Test all calls ------------------------------------------------------
from functools import partial
sub_tests = [
partial(test_call,
name,
call_ind,
signature,
params,
do_eval,
solution_args,
eq_fun,
add_more,
index,
params_not_specified_msg,
params_not_matched_msg,
incorrect_msg,
keywords,
state=state,
highlight=highlight) for call_ind in call_indices
]
test_or(*sub_tests, state=state)
def test_function_v2(name,
index=1,
params=[],
signature=None,
eq_condition="equal",
do_eval=True,
not_called_msg=None,
params_not_matched_msg=None,
params_not_specified_msg=None,
incorrect_msg=None,
add_more=False,
highlight=True,
state=None):
"""Test if function calls match (v2).
This function compares a function call in the student's code with the corresponding one in the solution
code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
that is returned will depend on the sort of fail.
Args:
name (str): the name of the function to be tested.
index (int): index of the function call to be checked. Defaults to 1.
params (list(str)): the parameter names of the function call that you want to check.
signature (Signature): Normally, test_function() can figure out what the function signature is,
but it might be necessary to use build_sig to manually build a signature and pass this along.
eq_condition (str): how parameters are compared. Currently, only "equal" is supported,
meaning that the arguments in student and solution process should have exactly the same value.
do_eval (list(bool)): Boolean or list of booleans (parameter-specific) that specify whether or
not arguments should be evaluated.
True: arguments are evaluated and compared.
False: arguments are not evaluated but 'string-matched'.
None: arguments are not evaluated; it is only checked if they are specified.
not_called_msg (str): custom feedback message if the function is not called.
params_not_matched_message (str): custom feedback message if the function parameters were not successfully matched.
params_not_specified_msg (str): string or list of strings (parameter-specific). Custom feedback message if not all
parameters listed in params are specified by the student.
incorrect_msg (list(str)): string or list of strings (parameter-specific). Custom feedback messages if the arguments
don't correspond between student and solution code.
"""
rep = Reporter.active_reporter
rep.set_tag("fun", "test_function")
index = index - 1
eq_map = {"equal": EqualTest}
# ARG CHECKS --------------------------------------------------------------
if eq_condition not in eq_map:
raise NameError("%r not a valid equality condition " % eq_condition)
eq_fun = eq_map[eq_condition]
if not isinstance(params, list):
raise NameError("Inside test_function_v2, make sure to specify a LIST of params.")
if isinstance(do_eval, bool) or do_eval is None:
do_eval = [do_eval] * len(params)
if len(params) != len(do_eval):
raise NameError("Inside test_function_v2, make sure that do_eval has the same length as params.")
# if params_not_specified_msg is a str or None, convert into list
if isinstance(params_not_specified_msg, str) or params_not_specified_msg is None:
params_not_specified_msg = [params_not_specified_msg] * len(params)
if len(params) != len(params_not_specified_msg):
raise NameError("Inside test_function_v2, make sure that params_not_specified_msg has the same length as params.")
# if incorrect_msg is a str or None, convert into list
if isinstance(incorrect_msg, str) or incorrect_msg is None:
incorrect_msg = [incorrect_msg] * len(params)
if len(params) != len(incorrect_msg):
raise NameError("Inside test_function_v2, make sure that incorrect_msg has the same length as params.")
# STATE STUFF -------------------------------------------------------------
student_process, solution_process = state.student_process, state.solution_process
solution_calls = state.solution_function_calls
student_calls = state.student_function_calls
student_mappings = state.student_mappings
solution_mappings = state.solution_mappings
stud_name = get_mapped_name(name, student_mappings)
#sol_name = get_mapped_name(name, solution_mappings)
if not_called_msg is None:
if index == 0:
not_called_msg = "Have you called `%s()`?" % stud_name
else:
not_called_msg = ("The system wants to check the %s call of `%s()`, " +
"but hasn't found it; have another look at your code.") % (get_ord(index + 1), stud_name)
if name not in solution_calls or len(solution_calls[name]) <= index:
raise NameError("%r not in solution environment (often enough)" % name)
# TODO: test if function name in dict of calls
_msg = state.build_message(not_called_msg)
rep.do_test(DefinedCollTest(name, student_calls, _msg))
# TODO: test if number of specific function calls is less than index
rep.do_test(BiggerTest(len(student_calls[name]), index, _msg)) # TODO
# TODO pull into own function
if len(params) > 0:
# Parse Signature -----------------------------------------------------
try:
sol_call, arguments, keywords, sol_name = solution_calls[name][index]['_spec1']
sol_sig = getSignatureInProcess(name=name, mapped_name=sol_name,
signature=signature,
manual_sigs = state.get_manual_sigs(),
process=solution_process)
solution_args, _ = bind_args(signature = sol_sig, arguments=arguments, keyws=keywords)
except:
raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
" You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))
# Check if params are in signature
if set(params) - set(solution_args.keys()):
raise ValueError("When testing %s(), the solution call doesn't specify the listed parameters." % name)
# Get all options (some function calls may be blacklisted)
call_indices = state.get_options(name, list(range(len(student_calls[name]))), index)
feedback = None
# Test all calls ------------------------------------------------------
from functools import partial
sub_tests = [partial(test_call, name, call_ind, signature, params, do_eval, solution_args,
eq_fun, add_more, index,
params_not_specified_msg, params_not_matched_msg, incorrect_msg,
keywords, state=state, highlight = highlight)
for call_ind in call_indices]
test_or(*sub_tests, state=state)
def test_function_v2(name,
index=1,
params=[],
signature=None,
eq_condition="equal",
do_eval=True,
not_called_msg=None,
params_not_matched_msg=None,
params_not_specified_msg=None,
incorrect_msg=None,
add_more=False,
state=None,
**kwargs):
rep = Reporter.active_reporter
index = index - 1
eq_map = {"equal": EqualTest}
do_highlight = index == 0
# ARG CHECKS --------------------------------------------------------------
if eq_condition not in eq_map:
raise NameError("%r not a valid equality condition " % eq_condition)
eq_fun = eq_map[eq_condition]
if not isinstance(params, list):
raise NameError(
"Inside test_function_v2, make sure to specify a LIST of params.")
if isinstance(do_eval, bool) or do_eval is None:
do_eval = [do_eval] * len(params)
if len(params) != len(do_eval):
raise NameError(
"Inside test_function_v2, make sure that do_eval has the same length as params."
)
# if params_not_specified_msg is a str or None, convert into list
if isinstance(params_not_specified_msg,
str) or params_not_specified_msg is None:
params_not_specified_msg = [params_not_specified_msg] * len(params)
if len(params) != len(params_not_specified_msg):
raise NameError(
"Inside test_function_v2, make sure that params_not_specified_msg has the same length as params."
)
# if incorrect_msg is a str or None, convert into list
if isinstance(incorrect_msg, str) or incorrect_msg is None:
incorrect_msg = [incorrect_msg] * len(params)
if len(params) != len(incorrect_msg):
raise NameError(
"Inside test_function_v2, make sure that incorrect_msg has the same length as params."
)
# STATE STUFF -------------------------------------------------------------
solution_process = state.solution_process
solution_calls = state.solution_function_calls
student_calls = state.student_function_calls
student_mappings = state.student_mappings
stud_name = get_mapped_name(name, student_mappings)
if not_called_msg is None:
if index == 0:
not_called_msg = "Have you called `%s()`?" % stud_name
else:
not_called_msg = (
"The system wants to check the %s call of `%s()`, " +
"but hasn't found it; have another look at your code.") % (
get_ord(index + 1), stud_name)
if name not in solution_calls or len(solution_calls[name]) <= index:
raise NameError("%r not in solution environment (often enough)" % name)
# TODO: test if function name in dict of calls
_msg = state.build_message(not_called_msg)
rep.do_test(DefinedCollTest(name, student_calls, Feedback(_msg, state)))
# TODO: test if number of specific function calls is less than index
rep.do_test(
BiggerTest(len(student_calls[name]), index, Feedback(_msg,
state))) # TODO
# TODO pull into own function
if len(params) > 0:
# Parse Signature -----------------------------------------------------
try:
_, arguments, keywords, sol_name = solution_calls[name][index][
'_spec1']
sol_sig = getSignatureInProcess(
name=name,
mapped_name=sol_name,
signature=signature,
manual_sigs=state.get_manual_sigs(),
process=solution_process)
solution_args, _ = bind_args(signature=sol_sig,
arguments=arguments,
keyws=keywords)
except:
raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
" You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))
# Check if params are in signature
if set(params) - set(solution_args.keys()):
raise ValueError(
"When testing %s(), the solution call doesn't specify the listed parameters."
% name)
# Get all options (some function calls may be blacklisted)
call_indices = state.get_options(name,
list(range(len(student_calls[name]))),
index)
# Test all calls ------------------------------------------------------
from functools import partial
sub_tests = [
partial(test_call,
name,
call_ind,
signature,
params,
do_eval,
solution_args,
eq_fun,
add_more,
index,
params_not_specified_msg,
params_not_matched_msg,
incorrect_msg,
keywords,
do_highlight=do_highlight,
state=state,
**kwargs) for call_ind in call_indices
]
test_or(*sub_tests, state=state)
