def with_context(*args, state=None): rep = Reporter.active_reporter # set up context in processes solution_res = setUpNewEnvInProcess(process = state.solution_process, context = state.solution_parts['with_items']) if isinstance(solution_res, Exception): raise InstructorError("error in the solution, running test_with(): %s" % str(solution_res)) student_res = setUpNewEnvInProcess(process = state.student_process, context = state.student_parts['with_items']) if isinstance(student_res, AttributeError): rep.do_test(Test(Feedback("In your `with` statement, you're not using a correct context manager.", child.highlight))) if isinstance(student_res, (AssertionError, ValueError, TypeError)): rep.do_test(Test(Feedback("In your `with` statement, the number of values in your context manager " "doesn't correspond to the number of variables you're trying to assign it to.", child.highlight))) # run subtests try: multi(*args, state=state) finally: # exit context if breakDownNewEnvInProcess(process = state.solution_process): raise InstructorError("error in the solution, closing the `with` fails with: %s" % (close_solution_context)) if breakDownNewEnvInProcess(process = state.student_process): rep.do_test(Test(Feedback("Your `with` statement can not be closed off correctly, you're " + \ "not using the context manager correctly.", state))) return state
def _test(state, incorrect_msg, exact_names, tv_name, highlight_name): rep = Reporter.active_reporter # get parts for testing from state # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc.. stu_vars = state.student_parts[tv_name] sol_vars = state.solution_parts[tv_name] child_state = state.to_child_state( student_subtree=state.student_parts.get(highlight_name), solution_subtree=state.solution_parts.get(highlight_name)) # variables exposed to messages d = {'stu_vars': stu_vars, 'sol_vars': sol_vars, 'num_vars': len(sol_vars)} if exact_names: # message for wrong iter var names _msg = state.build_message(incorrect_msg, d) # test rep.do_test(EqualTest(stu_vars, sol_vars, Feedback(_msg, child_state))) else: # message for wrong number of iter vars _msg = state.build_message(incorrect_msg, d) # test rep.do_test( EqualTest(len(stu_vars), len(sol_vars), Feedback(_msg, child_state))) return state
def parse_external(x): rep = Reporter.active_reporter res = (None, None) try: res = asttokens.ASTTokens(x, parse=True) return (res, res._tree) except IndentationError as e: e.filename = "script.py" # no line info for now rep.do_test( Test( Feedback( "Your code could not be parsed due to an error in the indentation:<br>`%s.`" % str(e)))) except SyntaxError as e: e.filename = "script.py" # no line info for now rep.do_test( Test( Feedback( "Your code can not be executed due to a syntax error:<br>`%s.`" % str(e)))) # Can happen, can't catch this earlier because we can't differentiate between # TypeError in parsing or TypeError within code (at runtime). except: rep.do_test( Test( Feedback("Something went wrong while parsing your code."))) return (res)
def call(args, test='value', incorrect_msg=None, error_msg=None, argstr=None, func=None, state=None, **kwargs): """Use ``check_call()`` in combination with ``has_equal_x()`` instead. """ if incorrect_msg is None: incorrect_msg = MSG_CALL_INCORRECT if error_msg is None: error_msg = MSG_CALL_ERROR_INV if test == 'error' else MSG_CALL_ERROR if argstr is None: bracks = stringify(fix_format(args)) if hasattr(state.student_parts['node'], 'name'): # Lambda function doesn't have name argstr = '`{}{}`'.format(state.student_parts['node'].name, bracks) else: argstr = 'it with the arguments `{}`'.format(bracks) rep = Reporter.active_reporter assert test in ('value', 'output', 'error') get_func = evalCalls[test] # Run for Solution -------------------------------------------------------- eval_sol, str_sol = run_call(args, state.solution_parts['node'], state.solution_process, get_func, **kwargs) if (test == 'error') ^ isinstance(eval_sol, Exception): _msg = state.build_message("FMT:Calling {argstr} resulted in an error (or not an error if testing for one). Error message: {type_err} {str_sol}", dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr)), raise InstructorError(_msg) if isinstance(eval_sol, ReprFail): _msg = state.build_message("FMT:Can't get the result of calling {argstr}: {eval_sol.info}", dict(argstr = argstr, eval_sol=eval_sol)) raise InstructorError(_msg) # Run for Submission ------------------------------------------------------ eval_stu, str_stu = run_call(args, state.student_parts['node'], state.student_process, get_func, **kwargs) action_strs = {'value': 'return', 'output': 'print out', 'error': 'error out with the message'} fmt_kwargs = {'part': argstr, 'argstr': argstr, 'str_sol': str_sol, 'str_stu': str_stu, 'action': action_strs[test]} # either error test and no error, or vice-versa stu_node = state.student_parts['node'] stu_state = StubState(stu_node, state.highlighting_disabled) if (test == 'error') ^ isinstance(eval_stu, Exception): _msg = state.build_message(error_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, stu_state))) # incorrect result _msg = state.build_message(incorrect_msg, fmt_kwargs) rep.do_test(EqualTest(eval_sol, eval_stu, Feedback(_msg, stu_state), func)) return state
def check_node(name, index=0, typestr='{ordinal} node', missing_msg=None, expand_msg=None, state=None): if missing_msg is None: missing_msg = "__JINJA__:The system wants to check the {{typestr}} but hasn't found it." if expand_msg is None: expand_msg = "__JINJA__:Check the {{typestr}}. " rep = Reporter.active_reporter stu_out = getattr(state, 'student_'+name) sol_out = getattr(state, 'solution_'+name) # check if there are enough nodes for index fmt_kwargs = {'ordinal': get_ord(index+1) if isinstance(index, int) else "", 'index': index, 'name': name} fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs) # test if node can be indexed succesfully try: stu_out[index] except (KeyError, IndexError): # TODO comment errors _msg = state.build_message(missing_msg, fmt_kwargs) rep.do_test(Test(Feedback(_msg, state))) # get node at index stu_part = stu_out[index] sol_part = sol_out[index] append_message = { 'msg': expand_msg, 'kwargs': fmt_kwargs } return part_to_child(stu_part, sol_part, append_message, state, node_name=name)
def fail(msg="", state=None): """Fail test with message""" rep = Reporter.active_reporter _msg = state.build_message(msg) rep.do_test(Test(Feedback(_msg, state))) return state
def has_equal_part_len(name, unequal_msg, state=None): """Verify that a part that is zoomed in on has equal length. Typically used in the context of ``check_function_def()`` Arguments: name (str): name of the part for which to check the length to the corresponding part in the solution. unequal_msg (str): Message in case the lengths do not match. state (State): state as passed by the SCT chain. Don't specify this explicitly. :Examples: Student and solution code:: def shout(word): return word + '!!!' SCT that checks number of arguments:: Ex().check_function_def('shout').has_equal_part_len('args', 'not enough args!') """ rep = Reporter.active_reporter d = dict(stu_len=len(state.student_parts[name]), sol_len=len(state.solution_parts[name])) if d['stu_len'] != d['sol_len']: _msg = state.build_message(unequal_msg, d) rep.do_test(Test(Feedback(_msg, state))) return state
def has_part(name, msg, state=None, fmt_kwargs=None, index=None): rep = Reporter.active_reporter d = { 'sol_part': state.solution_parts, 'stu_part': state.student_parts, **fmt_kwargs } def verify(part, index): if index is not None: if isinstance(index, list): for ind in index: part = part[ind] else: part = part[index] if part is None: raise KeyError # Chceck if it's there in the solution _msg = state.build_message(msg, d) _err_msg = "SCT fails on solution: " + _msg try: verify(state.solution_parts[name], index) except (KeyError, IndexError): raise InstructorError(_err_msg) try: verify(state.student_parts[name], index) except (KeyError, IndexError): rep.do_test(Test(Feedback(_msg, state))) return state
def has_equal_part(name, msg, state): rep = Reporter.active_reporter d = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name } _msg = state.build_message(msg, d) rep.do_test( EqualTest(d['stu_part'][name], d['sol_part'][name], Feedback(_msg, state))) return state
def is_instance(inst, not_instance_msg=None, state=None): """Check whether an object is an instance of a certain class. ``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is used to 'zoom in' on the object of interest. Args: inst (class): The class that the object should have. not_instance_msg (str): When specified, this overrides the automatically generated message in case the object does not have the expected class. state (State): The state that is passed in through the SCT chain (don't specify this). :Example: Student code and solution code:: import numpy as np arr = np.array([1, 2, 3, 4, 5]) SCT:: # Verify the class of arr import numpy Ex().check_object('arr').is_instance(numpy.ndarray) """ state.assert_is(['object_assignments'], 'is_instance', ['check_object']) rep = Reporter.active_reporter sol_name = state.solution_parts.get('name') stu_name = state.student_parts.get('name') if not_instance_msg is None: not_instance_msg = "__JINJA__:Is it a {{inst.__name__}}?" if not isInstanceInProcess(sol_name, inst, state.solution_process): raise InstructorError( "`is_instance()` noticed that `%s` is not a `%s` in the solution process." % (sol_name, inst.__name__)) _msg = state.build_message(not_instance_msg, {'inst': inst}) feedback = Feedback(_msg, state) rep.do_test( InstanceProcessTest(stu_name, inst, state.student_process, feedback)) return state
def __init__(self, feedback): """ Initialize the standard test. Args: feedback: string or Feedback object """ if (issubclass(type(feedback), Feedback)): self.feedback = feedback elif (issubclass(type(feedback), str)): self.feedback = Feedback(feedback) else: raise TypeError( "When creating a test, specify either a string or a Feedback object" ) self.result = None
def has_code(text, pattern=True, not_typed_msg=None, state=None): """Test the student code. Tests if the student typed a (pattern of) text. It is advised to use ``has_equal_ast()`` instead of ``has_code()``, as it is more robust to small syntactical differences that don't change the code's behavior. Args: text (str): the text that is searched for pattern (bool): if True (the default), the text is treated as a pattern. If False, it is treated as plain text. not_typed_msg (str): feedback message to be displayed if the student did not type the text. :Example: Student code and solution code:: y = 1 + 2 + 3 SCT:: # Verify that student code contains pattern (not robust!!): Ex().has_code(r"1\\s*\\+2\\s*\\+3") """ rep = Reporter.active_reporter if not not_typed_msg: if pattern: not_typed_msg = "Could not find the correct pattern in your code." else: not_typed_msg = "Could not find the following text in your code: %r" % text student_code = state.student_code _msg = state.build_message(not_typed_msg) rep.do_test( StringContainsTest(student_code, text, pattern, Feedback(_msg, state))) return state
def check_object(index, missing_msg=None, expand_msg=None, state=None, typestr="variable"): """Check object existence (and equality) Check whether an object is defined in the student's process, and zoom in on its value in both student and solution process to inspect quality (with has_equal_value(). Args: index (str): the name of the object which value has to be checked. missing_msg (str): feedback message when the object is not defined in the student process. expand_msg (str): prepending message to put in front. :Example: Student code:: b = 1 c = 3 Solution code:: a = 1 b = 2 c = 3 SCT:: Ex().check_object("a") # fail Ex().check_object("b") # pass Ex().check_object("b").has_equal_value() # fail Ex().check_object("c").has_equal_value() # pass """ # Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1' if v2_only(): state.assert_root('check_object') if missing_msg is None: missing_msg = "__JINJA__:Did you define the {{typestr}} `{{index}}` without errors?" if expand_msg is None: expand_msg = "__JINJA__:Did you correctly define the {{typestr}} `{{index}}`? " rep = Reporter.active_reporter if not isDefinedInProcess( index, state.solution_process) and state.has_different_processes(): raise InstructorError( "`check_object()` couldn't find object `%s` in the solution process." % index) append_message = { 'msg': expand_msg, 'kwargs': { 'index': index, 'typestr': typestr } } # create child state, using either parser output, or create part from name fallback = lambda: ObjectAssignmentParser.get_part(index) stu_part = state.student_object_assignments.get(index, fallback()) sol_part = state.solution_object_assignments.get(index, fallback()) # test object exists _msg = state.build_message(missing_msg, append_message['kwargs']) rep.do_test( DefinedProcessTest(index, state.student_process, Feedback(_msg))) child = part_to_child(stu_part, sol_part, append_message, state, node_name='object_assignments') return child
def has_expr(incorrect_msg=None, error_msg=None, undefined_msg=None, append=None, extra_env=None, context_vals=None, pre_code=None, expr_code=None, name=None, copy=True, func=None, override=None, state=None, test=None): if append is None: # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: if name: incorrect_msg = DEFAULT_INCORRECT_NAME_MSG elif expr_code: incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG else: incorrect_msg = DEFAULT_INCORRECT_MSG if undefined_msg is None: undefined_msg = DEFAULT_UNDEFINED_NAME_MSG if error_msg is None: if test == 'error': error_msg = DEFAULT_ERROR_MSG_INV else: error_msg = DEFAULT_ERROR_MSG rep = Reporter.active_reporter get_func = partial(evalCalls[test], extra_env=extra_env, context_vals=context_vals, pre_code=pre_code, expr_code=expr_code, name=name, copy=copy) if override is not None: # don't bother with running expression and fetching output/value # eval_sol, str_sol = eval eval_sol, str_sol = override, str(override) else: eval_sol, str_sol = get_func(tree=state.solution_tree, process=state.solution_process, context=state.solution_context, env=state.solution_env) if (test == 'error') ^ isinstance(eval_sol, Exception): raise InstructorError( "Evaluating expression raised error in solution process (or not an error if testing for one). " "Error: {} - {}".format(type(eval_sol), str_sol)) if isinstance(eval_sol, ReprFail): raise InstructorError( "Couldn't extract the value for the highlighted expression from the solution process: " + eval_sol.info) eval_stu, str_stu = get_func(tree=state.student_tree, process=state.student_process, context=state.student_context, env=state.student_env) # kwargs --- fmt_kwargs = { 'stu_part': state.student_parts, 'sol_part': state.solution_parts, 'name': name, 'test': test, 'test_desc': '' if test == 'value' else 'the %s ' % test, 'expr_code': expr_code } fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu)) fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol)) if incorrect_msg == DEFAULT_INCORRECT_MSG and \ ( fmt_kwargs['stu_eval'] is None or fmt_kwargs['sol_eval'] is None or fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ): incorrect_msg = "Expected something different." # tests --- # error in process if (test == 'error') ^ isinstance(eval_stu, Exception): fmt_kwargs['stu_str'] = str_stu _msg = state.build_message(error_msg, fmt_kwargs, append=append) feedback = Feedback(_msg, state) rep.do_test(Test(feedback)) # name is undefined after running expression if isinstance(eval_stu, UndefinedValue): _msg = state.build_message(undefined_msg, fmt_kwargs, append=append) rep.do_test(Test(Feedback(_msg, state))) # test equality of results _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func)) return state
def has_equal_ast(incorrect_msg=None, code=None, exact=True, append=None, state=None): """Test whether abstract syntax trees match between the student and solution code. ``has_equal_ast()`` can be used in two ways: * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission. * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments. Args: incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this. code: optional code to use instead of the solution AST. exact: whether the representations must match exactly. If false, the solution AST only needs to be contained within the student AST (similar to using test student typed). Defaults to ``True``, unless the ``code`` argument has been specified. :Example: Student and Solution Code:: dict(a = 'value').keys() SCT:: # all pass Ex().has_equal_ast() Ex().has_equal_ast(code = "dict(a = 'value').keys()") Ex().has_equal_ast(code = "dict(a = 'value')", exact = False) Student and Solution Code:: import numpy as np arr = np.array([1, 2, 3, 4, 5]) np.mean(arr) SCT:: # Check underlying value of arugment a of np.mean: Ex().check_function('numpy.mean').check_args('a').has_equal_ast() # Only check AST equality of expression used to specify argument a: Ex().check_function('numpy.mean').check_args('a').has_equal_ast() """ rep = Reporter.active_reporter if utils.v2_only(): state.assert_is_not(['object_assignments'], 'has_equal_ast', ['check_object']) state.assert_is_not(['function_calls'], 'has_equal_ast', ['check_function']) if code and incorrect_msg is None: raise InstructorError( "If you manually specify the code to match inside has_equal_ast(), " "you have to explicitly set the `incorrect_msg` argument.") if append is None: # if not specified, set to False if incorrect_msg was manually specified append = incorrect_msg is None if incorrect_msg is None: incorrect_msg = "__JINJA__:Expected `{{sol_str}}`, but got `{{stu_str}}`." def parse_tree(tree): # get contents of module.body if only 1 element crnt = tree.body[0] if isinstance(tree, ast.Module) and len( tree.body) == 1 else tree # remove Expr if it exists return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt) stu_rep = parse_tree(state.student_tree) sol_rep = parse_tree(state.solution_tree if not code else ast.parse(code)) fmt_kwargs = { 'sol_str': state.solution_code if not code else code, 'stu_str': state.student_code } _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append) if exact and not code: rep.do_test(EqualTest(stu_rep, sol_rep, Feedback(_msg, state))) elif not sol_rep in stu_rep: rep.do_test(Test(Feedback(_msg, state))) return state
def check_keys(key, missing_msg=None, expand_msg=None, state=None): """Check whether an object (dict, DataFrame, etc) has a key. ``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is used to 'zoom in' on the object of interest. Args: key (str): Name of the key that the object should have. missing_msg (str): When specified, this overrides the automatically generated message in case the key does not exist. state (State): The state that is passed in through the SCT chain (don't specify this). :Example: Student code and solution code:: x = {'a': 2} SCT:: # Verify that x contains a key a Ex().check_object('x').check_keys('a') # Verify that x contains a key a and a is correct. Ex().check_object('x').check_keys('a').has_equal_value() """ state.assert_is(['object_assignments'], 'is_instance', ['check_object', 'check_df']) if missing_msg is None: missing_msg = "__JINJA__:There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`." if expand_msg is None: expand_msg = "__JINJA__:Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? " rep = Reporter.active_reporter sol_name = state.solution_parts.get('name') stu_name = state.student_parts.get('name') if not isDefinedCollInProcess(sol_name, key, state.solution_process): raise InstructorError( "`check_keys()` couldn't find key `%s` in object `%s` in the solution process." % (key, sol_name)) # check if key available _msg = state.build_message(missing_msg, {'key': key}) rep.do_test( DefinedCollProcessTest(stu_name, key, state.student_process, Feedback(_msg, state))) def get_part(name, key, highlight): if isinstance(key, str): slice_val = ast.Str(s=key) else: slice_val = ast.parse('{}'.format(key)).body[0].value expr = ast.Subscript(value=ast.Name(id=name, ctx=ast.Load()), slice=ast.Index(value=slice_val), ctx=ast.Load()) ast.fix_missing_locations(expr) return {'node': expr, 'highlight': highlight} stu_part = get_part(stu_name, key, state.student_parts.get('highlight')) sol_part = get_part(sol_name, key, state.solution_parts.get('highlight')) append_message = {'msg': expand_msg, 'kwargs': {'key': key}} child = part_to_child(stu_part, sol_part, append_message, state) return child
def check_function(name, index=0, missing_msg=None, params_not_matched_msg=None, expand_msg=None, signature=True, state=None): """Check whether a particular function is called. This function is typically followed by ``check_args()`` to check whether the arguments were specified correctly. Args: name (str): the name of the function to be tested. When checking functions in packages, always use the 'full path' of the function. index (int): index of the function call to be checked. Defaults to 0. missing_msg (str): If specified, this overrides an automatically generated feedback message in case the student did not call the function correctly. params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message in case the function parameters were not successfully matched. expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains. signature (Signature): Normally, check_function() can figure out what the function signature is, but it might be necessary to use build_sig to manually build a signature and pass this along. state (State): State object that is passed from the SCT Chain (don't specify this). :Examples: Student code and solution code:: import numpy as np arr = np.array([1, 2, 3, 4, 5]) np.mean(arr) SCT:: # Verify whether arr was correctly set in np.mean Ex().check_function('numpy.mean').check_args('a').has_equal_value() # Verify whether np.mean(arr) produced the same result Ex().check_function('numpy.mean').has_equal_value() """ append_missing = missing_msg is None append_params_not_matched = params_not_matched_msg is None if missing_msg is None: missing_msg = MISSING_MSG if expand_msg is None: expand_msg = PREPEND_MSG if params_not_matched_msg is None: params_not_matched_msg = SIG_ISSUE_MSG rep = Reporter.active_reporter stu_out = state.student_function_calls sol_out = state.solution_function_calls student_mappings = state.student_mappings fmt_kwargs = { 'times': get_times(index + 1), 'ord': get_ord(index + 1), 'index': index, 'mapped_name': get_mapped_name(name, student_mappings) } # Get Parts ---- # Copy, otherwise signature binding overwrites sol_out[name][index]['args'] try: sol_parts = {**sol_out[name][index]} except KeyError: raise InstructorError( "`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!" % name) except IndexError: raise InstructorError( "`check_function()` couldn't find %s calls of `%s()` in your solution code." % (index + 1, name)) try: # Copy, otherwise signature binding overwrites stu_out[name][index]['args'] stu_parts = {**stu_out[name][index]} except (KeyError, IndexError): _msg = state.build_message(missing_msg, fmt_kwargs, append=append_missing) rep.do_test(Test(Feedback(_msg, state))) # Signatures ----- if signature: signature = None if isinstance(signature, bool) else signature get_sig = partial(getSignatureInProcess, name=name, signature=signature, manual_sigs=state.get_manual_sigs()) try: sol_sig = get_sig(mapped_name=sol_parts['name'], process=state.solution_process) sol_parts['args'] = bind_args(sol_sig, sol_parts['args']) except: raise InstructorError( "`check_function()` couldn't match the %s call of `%s` to its signature. " % (get_ord(index + 1), name)) try: stu_sig = get_sig(mapped_name=stu_parts['name'], process=state.student_process) stu_parts['args'] = bind_args(stu_sig, stu_parts['args']) except Exception: _msg = state.build_message(params_not_matched_msg, fmt_kwargs, append=append_params_not_matched) rep.do_test( Test( Feedback( _msg, StubState(stu_parts['node'], state.highlighting_disabled)))) # three types of parts: pos_args, keywords, args (e.g. these are bound to sig) append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs} child = part_to_child(stu_parts, sol_parts, append_message, state, node_name='function_calls') return child