Example #1
0
def with_context(*args, state=None):
    # set up context in processes
    solution_res = setUpNewEnvInProcess(process = state.solution_process,
                                        context = state.solution_parts['with_items'])
    if isinstance(solution_res, Exception):
        raise Exception("error in the solution, running test_with() on with %d: %s" % (index - 1, str(solution_res)))

    student_res = setUpNewEnvInProcess(process = state.student_process,
                                       context = state.student_parts['with_items'])
    if isinstance(student_res, AttributeError):
        rep.do_test(Test(Feedback("In your %s `with` statement, you're not using a correct context manager." % (get_ord(index)), child.highlight)))

    if isinstance(student_res, (AssertionError, ValueError, TypeError)):
        rep.do_test(Test(Feedback("In your %s `with` statement, the number of values in your context manager " + \
            "doesn't correspond to the number of variables you're trying to assign it to." % (get_ord(index)), child.highlight)))

    # run subtests
    try:
        multi(*args, state=state)
    finally:
        # exit context
        if breakDownNewEnvInProcess(process = state.solution_process):
            raise Exception("error in the solution, closing the %s with fails with: %s" %
                (get_ord(index), close_solution_context))

        if breakDownNewEnvInProcess(process = state.student_process):

            rep.do_test(Test(Feedback("Your %s `with` statement can not be closed off correctly, you're " + \
                            "not using the context manager correctly." % (get_ord(index)), state.highlight)),
                        fallback_ast = state.highlight)
    return state
Example #2
0
def check_node(
    state, name, index=0, typestr="{{ordinal}} node", missing_msg=None, expand_msg=None
):

    if missing_msg is None:
        missing_msg = "系统想要检查 {{typestr}} 但没有找到它."
    if expand_msg is None:
        expand_msg = "检查 {{typestr}}. "

    stu_out = state.ast_dispatcher(name, state.student_ast)
    sol_out = state.ast_dispatcher(name, state.solution_ast)

    # check if there are enough nodes for index
    fmt_kwargs = {
        "ordinal": get_ord(index + 1) if isinstance(index, int) else "",
        "index": index,
        "name": name,
    }
    fmt_kwargs["typestr"] = render(typestr, fmt_kwargs)

    # test if node can be indexed succesfully
    try:
        stu_out[index]
    except (KeyError, IndexError):  # TODO comment errors
        _msg = state.build_message(missing_msg, fmt_kwargs)
        state.report(Feedback(_msg, state))

    # get node at index
    stu_part = stu_out[index]
    sol_part = sol_out[index]

    append_message = {"msg": expand_msg, "kwargs": fmt_kwargs}

    return part_to_child(stu_part, sol_part, append_message, state, node_name=name)
Example #3
0
def test_comp(typestr, comptype, index, iter_vars_names,
              not_called_msg, insufficient_ifs_msg, incorrect_iter_vars_msg,
              comp_iter, ifs, key=None, body=None, value=None,
              expand_message = True,
              rep=None, state=None):

    # if true, set expand_message to default (for backwards compatibility)
    expand_message = MSG_PREPEND if expand_message is True else (expand_message or "")
    # make sure other messages are set to default if None
    if insufficient_ifs_msg is None: insufficient_ifs_msg = MSG_INSUFFICIENT_IFS
    if not_called_msg is None: not_called_msg = MSG_NOT_CALLED

    # TODO MSG: function was not consistent with prepending, so use state w/o expand_message
    quiet_state = check_node(comptype, index-1, typestr, not_called_msg, "", state)

    # get comprehension
    state = check_node(comptype, index-1, typestr, not_called_msg, expand_message, state)

    # test comprehension iter and its variable names (or number of variables)
    if comp_iter: multi(comp_iter, state=check_part("iter", "iterable part", state))
    has_iter_vars(incorrect_iter_vars_msg, iter_vars_names, state=quiet_state)

    # test the main expressions.
    if body:   multi(body,  state=check_part("body", "body", state))        # list and gen comp
    if key:    multi(key,   state=check_part("key", "key part", state))     # dict comp
    if value:  multi(value, state=check_part("value", "value part", state)) # ""

    # test a list of ifs. each entry corresponds to a filter in the comprehension.
    for i, if_test in enumerate(ifs or []):
        # test that ifs are same length
        has_equal_part_len('ifs', insufficient_ifs_msg, state=quiet_state)
        # test individual ifs
        multi(if_test, state=check_part_index("ifs", i, get_ord(i+1) + " if", state=state))
def test_comp(typestr, comptype, index, iter_vars_names,
              not_called_msg, insufficient_ifs_msg, incorrect_iter_vars_msg,
              comp_iter, ifs, key=None, body=None, value=None,
              rep=None, state=None):

    MSG_INCORRECT_ITER_VARS = "Have you used the correct iterator variables?"
    MSG_INCORRECT_NUM_ITER_VARS = "Have you used {{num_vars}} iterator variables?"
    MSG_INSUFFICIENT_IFS = "Have you used {{sol_len}} ifs?"

    # make sure other messages are set to default if None
    if insufficient_ifs_msg is None:
        insufficient_ifs_msg = MSG_INSUFFICIENT_IFS

    # get comprehension
    child = check_node(comptype, index-1, typestr, missing_msg=not_called_msg, state=state)

    # test comprehension iter and its variable names (or number of variables)
    if comp_iter: multi(comp_iter, state=check_part("iter", "iterable part", state=child))

    # test iterator variables
    default_msg = MSG_INCORRECT_ITER_VARS if iter_vars_names else MSG_INCORRECT_NUM_ITER_VARS
    has_context(incorrect_iter_vars_msg or default_msg, iter_vars_names, state=child)

    # test the main expressions.
    if body:   multi(body,  state=check_part("body", "body", state=child))        # list and gen comp
    if key:    multi(key,   state=check_part("key", "key part",  state=child))    # dict comp
    if value:  multi(value, state=check_part("value", "value part", state=child)) # ""

    # test a list of ifs. each entry corresponds to a filter in the comprehension.
    for i, if_test in enumerate(ifs or []):
        # test that ifs are same length
        has_equal_part_len('ifs', insufficient_ifs_msg, state=child)
        # test individual ifs
        multi(if_test, state=check_part_index("ifs", i, utils.get_ord(i+1) + " if", state=child))
Example #5
0
def check_part_index(name,
                     index,
                     part_msg,
                     missing_msg="FMT:Are you sure it is defined?",
                     state=None,
                     expand_msg=""):
    """Return child state with indexed name part as its ast tree"""

    rep = Reporter.active_reporter

    # create message
    ordinal = "" if isinstance(index, str) else get_ord(index + 1)
    fmt_kwargs = {'index': index, 'ordinal': ordinal}
    fmt_kwargs['part'] = part_msg.format(**fmt_kwargs)

    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}

    # check there are enough parts for index
    has_part(name, missing_msg, state, append_message['kwargs'], index)

    # get part at index
    stu_part = state.student_parts[name][index]
    sol_part = state.solution_parts[name][index]

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
Example #6
0
def check_part_index(name,
                     index,
                     part_msg,
                     missing_msg="FMT:Are you sure it is defined?",
                     state=None,
                     expand_msg=""):
    """Return child state with indexed name part as its ast tree"""

    rep = Reporter.active_reporter

    # create message
    ordinal = "" if isinstance(index, str) else get_ord(index + 1)
    fmt_kwargs = {'index': index, 'ordinal': ordinal}
    fmt_kwargs['part'] = part_msg.format(**fmt_kwargs)

    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}

    # check there are enough parts for index
    stu_parts = state.student_parts[name]
    try:
        stu_parts[index]
    except (KeyError, IndexError):
        _msg = state.build_message(missing_msg, append_message['kwargs'])
        rep.do_test(Test(Feedback(_msg, state.highlight)))

    # get part at index
    stu_part = state.student_parts[name][index]
    sol_part = state.solution_parts[name][index]

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
Example #7
0
def check_node(name, index, typestr, missing_msg=MSG_MISSING, expand_msg=MSG_PREPEND, state=None):
    rep = Reporter.active_reporter
    stu_out = getattr(state, 'student_'+name)
    sol_out = getattr(state, 'solution_'+name)

    # check if there are enough nodes for index
    fmt_kwargs = {'ordinal': get_ord(index+1) if isinstance(index, int) else "",
                  'index': index}
    fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs)

    # test if node can be indexed succesfully
    try: stu_out[index]
    except (KeyError, IndexError):                  # TODO comment errors
        _msg = state.build_message(missing_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, state.highlight)))

    # get node at index
    stu_part = stu_out[index]
    sol_part = sol_out[index]

    append_message = {'msg': expand_msg,
                      'kwargs': fmt_kwargs
                      }

    return part_to_child(stu_part, sol_part, append_message, state, node_name=name)
Example #8
0
def check_part_index(name, index, part_msg,
                     missing_msg="FMT:Are you sure it is defined?",
                     state=None, expand_msg=""):
    """Return child state with indexed name part as its ast tree"""

    rep = Reporter.active_reporter

    # create message
    ordinal = "" if isinstance(index, str) else get_ord(index+1)
    fmt_kwargs = {'index': index, 'ordinal': ordinal}
    fmt_kwargs['part'] = part_msg.format(**fmt_kwargs)

    append_message = {'msg': expand_msg,
                      'kwargs': fmt_kwargs}

    # check there are enough parts for index
    stu_parts = state.student_parts[name]
    try: stu_parts[index]
    except (KeyError, IndexError):
        _msg = state.build_message(missing_msg, append_message['kwargs'])
        rep.do_test(Test(Feedback(_msg, state.highlight)))

    # get part at index
    stu_part = state.student_parts[name][index]
    sol_part = state.solution_parts[name][index]

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
def check_function(
        name,
        index,
        missing_msg="FMT:Did you define the {typestr}?",
        params_not_matched_msg="FMT:Something went wrong in figuring out how you specified the "
    "arguments for `{name}`; have another look at your code and its output.",
        expand_msg=MSG_PREPEND,
        signature=True,
        typestr="{ordinal} function call `{name}()`",
        state=None):
    rep = Reporter.active_reporter
    stu_out = state.student_function_calls
    sol_out = state.solution_function_calls

    fmt_kwargs = {'ordinal': get_ord(index + 1), 'index': index, 'name': name}
    fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs)

    # Get Parts ----
    try:
        stu_parts = stu_out[name][index]
    except (KeyError, IndexError):
        _msg = state.build_message(missing_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, state.highlight)))

    sol_parts = sol_out[name][index]

    # Signatures -----
    if signature:
        signature = None if isinstance(signature, bool) else signature
        get_sig = partial(getSignatureInProcess,
                          name=name,
                          signature=signature,
                          manual_sigs=state.get_manual_sigs())

        try:
            sol_sig = get_sig(mapped_name=sol_parts['name'],
                              process=state.solution_process)
            sol_parts['args'], _ = bind_args(sol_sig, sol_parts['args'])
        except:
            raise ValueError(
                "Something went wrong in matching call index {index} of {name} to its signature. "
                "You might have to manually specify or correct the signature.".
                format(index=index, name=name))

        try:
            stu_sig = get_sig(mapped_name=stu_parts['name'],
                              process=state.student_process)
            stu_parts['args'], _ = bind_args(stu_sig, stu_parts['args'])
        except Exception as e:
            _msg = state.build_message(params_not_matched_msg, fmt_kwargs)
            rep.do_test(Test(Feedback(_msg, stu_parts['node'])))

    # three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}
    child = part_to_child(stu_parts,
                          sol_parts,
                          append_message,
                          state,
                          node_name='function_calls')
    return child
Example #10
0
def test_with(
        index,
        context_vals=False,  # whether to check number of context vals
        context_tests=None,  # check on context expressions
        body=None,
        undefined_msg=None,
        context_vals_len_msg=None,
        context_vals_msg=None,
        expand_message=True,
        state=None):
    """Test a with statement.
with open_file('...') as bla:

    [ open_file('...').__enter__() ]


with open_file('...') as file:
    [ ]

    """

    check_with = partial(check_node,
                         'withs',
                         index - 1,
                         "`with` statement",
                         MSG_MISSING,
                         state=state)

    child = check_with(MSG_PREPEND if expand_message else "")
    child2 = check_with(MSG_PREPEND2 if expand_message else "")

    if context_vals:
        # test context var names ----
        has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES,
                    exact_names=True,
                    state=child)

        # test num context vars ----
        has_equal_part_len('context', MSG_NUM_CTXT, state=child)

    # Context sub tests ----
    if context_tests and not isinstance(context_tests, list):
        context_tests = [context_tests]

    for i, context_test in enumerate(context_tests or []):
        # partial the substate check, because the function uses two prepended messages
        check_context = partial(check_part_index, 'context', i,
                                "%s context" % utils.get_ord(i + 1),
                                MSG_NUM_CTXT2)

        check_context(state=child)  # test exist

        ctxt_state = check_context(state=child2)  # sub tests
        multi(context_test, state=ctxt_state)

    # Body sub tests ----
    if body is not None:
        body_state = check_part('body', 'body', state=child2)

        with_context(body, state=body_state)
Example #11
0
def test_comp(typestr,
              comptype,
              index,
              iter_vars_names,
              not_called_msg,
              insufficient_ifs_msg,
              incorrect_iter_vars_msg,
              comp_iter,
              ifs,
              key=None,
              body=None,
              value=None,
              expand_message=True,
              rep=None,
              state=None):

    # if true, set expand_message to default (for backwards compatibility)
    expand_message = MSG_PREPEND if expand_message is True else (expand_message
                                                                 or "")
    # make sure other messages are set to default if None
    if insufficient_ifs_msg is None:
        insufficient_ifs_msg = MSG_INSUFFICIENT_IFS
    if not_called_msg is None: not_called_msg = MSG_NOT_CALLED

    # TODO MSG: function was not consistent with prepending, so use state w/o expand_message
    quiet_state = check_node(comptype, index - 1, typestr, not_called_msg, "",
                             state)

    # get comprehension
    state = check_node(comptype, index - 1, typestr, not_called_msg,
                       expand_message, state)

    # test comprehension iter and its variable names (or number of variables)
    if comp_iter:
        multi(comp_iter, state=check_part("iter", "iterable part", state))

    # test iterator variables
    default_msg = MSG_INCORRECT_ITER_VARS if iter_vars_names else MSG_INCORRECT_NUM_ITER_VARS
    has_context(incorrect_iter_vars_msg or default_msg,
                iter_vars_names,
                state=quiet_state)

    # test the main expressions.
    if body:
        multi(body, state=check_part("body", "body",
                                     state))  # list and gen comp
    if key: multi(key, state=check_part("key", "key part", state))  # dict comp
    if value:
        multi(value, state=check_part("value", "value part", state))  # ""

    # test a list of ifs. each entry corresponds to a filter in the comprehension.
    for i, if_test in enumerate(ifs or []):
        # test that ifs are same length
        has_equal_part_len('ifs', insufficient_ifs_msg, state=quiet_state)
        # test individual ifs
        multi(if_test,
              state=check_part_index("ifs",
                                     i,
                                     get_ord(i + 1) + " if",
                                     state=state))
Example #12
0
 def check_context(state):
     return check_part_index(
         state,
         "context",
         i,
         "%s context" % utils.get_ord(i + 1),
         missing_msg=MSG_NUM_CTXT2,
     )
Example #13
0
def test_with(index,
              context_vals=False, # whether to check number of context vals
              context_tests=None, # check on context expressions
              body=None,
              undefined_msg=None,
              context_vals_len_msg=None,
              context_vals_msg=None,
              expand_message=True,
              state=None):
    """Test a with statement.
with open_file('...') as bla:

    [ open_file('...').__enter__() ]


with open_file('...') as file:
    [ ]

    """
    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_with")

    check_with = partial(check_node, 'withs', index-1, "`with` statement", MSG_MISSING, state=state)

    child =  check_with(MSG_PREPEND  if expand_message else "")
    child2 = check_with(MSG_PREPEND2 if expand_message else "")
    quiet_child = quiet(1, child)

    if context_vals:
        # test num context vars ----
        too_many = len(child.student_parts['context']) > len(child.solution_parts['context'])
        if too_many:
            _msg = child.build_message(MSG_NUM_CTXT)
            rep.do_test(Test(Feedback(_msg, child.student_tree)))

        # test context var names ----
        for i in range(len(child.solution_parts['context'])):
            ctxt_state = check_part_index('context', i, "", state=child)
            has_equal_part('target_vars', MSG_CTXT_NAMES, state=ctxt_state)

    
    # Context sub tests ----
    if context_tests and not isinstance(context_tests, list): context_tests = [context_tests]

    for i, context_test in enumerate(context_tests or []):
        # partial the substate check, because the function uses two prepended messages
        check_context = partial(check_part_index, 'context', i, "%s context"%utils.get_ord(i+1), MSG_NUM_CTXT2)

        check_context(state=child)                   # test exist

        ctxt_state = check_context(state=child2)     # sub tests
        multi(context_test, state=ctxt_state)
    
    # Body sub tests ----
    if body is not None:
        body_state = check_part('body', 'body', state=child2)

        with_context(body, state=body_state)
def test_with(index,
              context_vals=False, # whether to check number of context vals
              context_tests=None, # check on context expressions
              body=None,
              undefined_msg=None,
              context_vals_len_msg=None,
              context_vals_msg=None,
              state=None):
    """Test a with statement.
with open_file('...') as bla:

    [ open_file('...').__enter__() ]


with open_file('...') as file:
    [ ]

    """

    MSG_NUM_CTXT = "Make sure to use the correct number of context variables. It seems you defined too many."
    MSG_NUM_CTXT2 = "Make sure to use the correct number of context variables. It seems you defined too little."
    MSG_CTXT_NAMES = "Make sure to use the correct context variable names. Was expecting `{{sol_vars}}` but got `{{stu_vars}}`."

    check_with = partial(check_node, 'withs', index-1, "{{ordinal}} `with` statement", state=state)

    child =  check_with()
    child2 = check_with()

    if context_vals:
        # test context var names ----
        has_context(incorrect_msg=context_vals_msg or MSG_CTXT_NAMES, exact_names = True, state=child)

        # test num context vars ----
        has_equal_part_len('context', MSG_NUM_CTXT, state=child)
        
    
    # Context sub tests ----
    if context_tests and not isinstance(context_tests, list): context_tests = [context_tests]

    for i, context_test in enumerate(context_tests or []):
        # partial the substate check, because the function uses two prepended messages
        check_context = partial(check_part_index, 'context', i, "%s context"%utils.get_ord(i+1), missing_msg=MSG_NUM_CTXT2)

        check_context(state=child)                   # test exist

        ctxt_state = check_context(state=child2)     # sub tests
        multi(context_test, state=ctxt_state)
    
    # Body sub tests ----
    if body is not None:
        body_state = check_part('body', 'body', state=child2)

        with_context(body, state=body_state)
Example #15
0
def with_context(*args, state=None):
    # set up context in processes
    solution_res = setUpNewEnvInProcess(
        process=state.solution_process,
        context=state.solution_parts['with_items'])
    if isinstance(solution_res, Exception):
        raise Exception(
            "error in the solution, running test_with() on with %d: %s" %
            (index - 1, str(solution_res)))

    student_res = setUpNewEnvInProcess(
        process=state.student_process,
        context=state.student_parts['with_items'])
    if isinstance(student_res, AttributeError):
        rep.do_test(
            Test(
                Feedback(
                    "In your %s `with` statement, you're not using a correct context manager."
                    % (get_ord(index)), child.highlight)))

    if isinstance(student_res, (AssertionError, ValueError, TypeError)):
        rep.do_test(Test(Feedback("In your %s `with` statement, the number of values in your context manager " + \
            "doesn't correspond to the number of variables you're trying to assign it to." % (get_ord(index)), child.highlight)))

    # run subtests
    try:
        multi(*args, state=state)
    finally:
        # exit context
        if breakDownNewEnvInProcess(process=state.solution_process):
            raise Exception(
                "error in the solution, closing the %s with fails with: %s" %
                (get_ord(index), close_solution_context))

        if breakDownNewEnvInProcess(process=state.student_process):

            rep.do_test(Test(Feedback("Your %s `with` statement can not be closed off correctly, you're " + \
                            "not using the context manager correctly." % (get_ord(index)), state.highlight)),
                        fallback_ast = state.highlight)
    return state
Example #16
0
def check_args(name,
               missing_msg='FMT:Are you sure it is defined?',
               state=None):
    if name in ['*args', '**kwargs']:
        return check_part(name, name, state=state, missing_msg=missing_msg)
    else:
        arg_str = "%s argument" % get_ord(name + 1) if isinstance(
            name, int) else "argument `%s`" % name
        return check_part_index('args',
                                name,
                                arg_str,
                                state=state,
                                missing_msg=missing_msg)
Example #17
0
def check_part_index(state,
                     name,
                     index,
                     part_msg,
                     missing_msg=None,
                     expand_msg=None):
    """Return child state with indexed name part as its ast tree.

    ``index`` can be:

    - an integer, in which case the student/solution_parts are indexed by position.
    - a string, in which case the student/solution_parts are expected to be a dictionary.
    - a list of indices (which can be integer or string), in which case the student parts are indexed step by step.
    """

    if missing_msg is None:
        missing_msg = "Are you sure you defined the {{part}}? "
    if expand_msg is None:
        expand_msg = "Did you correctly specify the {{part}}? "

    # create message
    ordinal = get_ord(index + 1) if isinstance(index, int) else ""
    fmt_kwargs = {"index": index, "ordinal": ordinal}
    fmt_kwargs.update(part=render(part_msg, fmt_kwargs))

    append_message = {"msg": expand_msg, "kwargs": fmt_kwargs}

    # check there are enough parts for index
    has_part(state, name, missing_msg, fmt_kwargs, index)

    # get part at index
    stu_part = state.student_parts[name]
    sol_part = state.solution_parts[name]

    if isinstance(index, list):
        for ind in index:
            stu_part = stu_part[ind]
            sol_part = sol_part[ind]
    else:
        stu_part = stu_part[index]
        sol_part = sol_part[index]

    assert_ast(state, sol_part, fmt_kwargs)

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
Example #18
0
def check_part_index(name,
                     index,
                     part_msg,
                     missing_msg=None,
                     expand_msg=None,
                     state=None):
    """Return child state with indexed name part as its ast tree.

    ``index`` can be:

    - an integer, in which case the student/solution_parts are indexed by position.
    - a string, in which case the student/solution_parts are expected to be a dictionary.
    - a list of indices (which can be integer or string), in which case the student parts are indexed step by step.
    """

    if missing_msg is None:
        missing_msg = "__JINJA__:Are you sure you defined the {{part}}? "
    if expand_msg is None:
        expand_msg = "__JINJA__:Did you correctly specify the {{part}}? "

    # create message
    ordinal = get_ord(index + 1) if isinstance(index, int) else ""
    fmt_kwargs = {'index': index, 'ordinal': ordinal}
    fmt_kwargs['part'] = part_msg.format(**fmt_kwargs)

    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}

    # check there are enough parts for index
    has_part(name, missing_msg, state, append_message['kwargs'], index)

    # get part at index
    stu_part = state.student_parts[name]
    sol_part = state.solution_parts[name]

    if isinstance(index, list):
        for ind in index:
            stu_part = stu_part[ind]
            sol_part = sol_part[ind]
    else:
        stu_part = stu_part[index]
        sol_part = sol_part[index]

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
Example #19
0
def check_node(name,
               index=0,
               typestr='{ordinal} node',
               missing_msg=None,
               expand_msg=None,
               state=None):

    if missing_msg is None:
        missing_msg = NODE_MISSING_MSG
    if expand_msg is None:
        expand_msg = NODE_PREPEND_MSG

    rep = Reporter.active_reporter
    stu_out = getattr(state, 'student_' + name)
    sol_out = getattr(state, 'solution_' + name)

    # check if there are enough nodes for index
    fmt_kwargs = {
        'ordinal': get_ord(index + 1) if isinstance(index, int) else "",
        'index': index,
        'name': name
    }
    fmt_kwargs['typestr'] = typestr.format(**fmt_kwargs)

    # test if node can be indexed succesfully
    try:
        stu_out[index]
    except (KeyError, IndexError):  # TODO comment errors
        _msg = state.build_message(missing_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, state)))

    # get node at index
    stu_part = stu_out[index]
    sol_part = sol_out[index]

    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}

    return part_to_child(stu_part,
                         sol_part,
                         append_message,
                         state,
                         node_name=name)
Example #20
0
def check_args(state, name, missing_msg=None):
    """Check whether a function argument is specified.

    This function can follow ``check_function()`` in an SCT chain and verifies whether an argument is specified.
    If you want to go on and check whether the argument was correctly specified, you can can continue chaining with
    ``has_equal_value()`` (value-based check) or ``has_equal_ast()`` (AST-based check)

    This function can also follow ``check_function_def()`` or ``check_lambda_function()`` to see if arguments have been
    specified.

    Args:
        name (str): the name of the argument for which you want to check it is specified. This can also be
            a number, in which case it refers to the positional arguments. Named argumetns take precedence.
        missing_msg (str): If specified, this overrides an automatically generated feedback message in case
            the student did specify the argument.
        state (State): State object that is passed from the SCT Chain (don't specify this).

    :Examples:

        Student and solution code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Verify whether arr was correctly set in np.mean
            # has_equal_value() checks the value of arr, used to set argument a
            Ex().check_function('numpy.mean').check_args('a').has_equal_value()

            # Verify whether arr was correctly set in np.mean
            # has_equal_ast() checks the expression used to set argument a
            Ex().check_function('numpy.mean').check_args('a').has_equal_ast()

        Student and solution code::

            def my_power(x):
                print("calculating sqrt...")
                return(x * x)

        SCT::

            Ex().check_function_def('my_power').multi(
                check_args('x') # will fail if student used y as arg
                check_args(0)   # will still pass if student used y as arg
            )

    """
    if missing_msg is None:
        missing_msg = "Did you specify the {{part}}?"

    if name in ["*args", "**kwargs"]:  # for check_function_def
        return check_part(state, name, name, missing_msg=missing_msg)
    else:
        if isinstance(name, list):  # dealing with args or kwargs
            if name[0] == "args":
                arg_str = "{} argument passed as a variable length argument".format(
                    get_ord(name[1] + 1))
            else:
                arg_str = "argument `{}`".format(name[1])
        else:
            arg_str = ("{} argument".format(get_ord(name + 1)) if isinstance(
                name, int) else "argument `{}`".format(name))
        return check_part_index(state,
                                "args",
                                name,
                                arg_str,
                                missing_msg=missing_msg)
Example #21
0
def check_function(name, index=0,
                   missing_msg=None,
                   params_not_matched_msg=None,
                   expand_msg=None,
                   signature=True,
                   typestr="__JINJA__:`{name} function call{} of `{name}()`",
                   state=None):
    """Check whether a particular function is called.

    This function is typically followed by ``check_args()`` to check whether the arguments were
    specified correctly.

    Args:
        name (str): the name of the function to be tested. When checking functions in packages, always
            use the 'full path' of the function.
        index (int): index of the function call to be checked. Defaults to 0.
        missing_msg (str): If specified, this overrides an automatically generated feedback message in case
            the student did not call the function correctly.
        params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
            in case the function parameters were not successfully matched.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
        signature (Signature): Normally, check_function() can figure out what the function signature is,
            but it might be necessary to use build_sig to manually build a signature and pass this along.
        typestr (formatted string): If specified, this overrides how the function call is automatically referred to.
        state (State): State object that is passed from the SCT Chain (don't specify this).

    :Examples:

        Student code and solution code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Verify whether arr was correctly set in np.mean
            Ex().check_function('numpy.mean').check_args('a').has_equal_value()

            # Verify whether np.mean(arr) produced the same result
            Ex().check_function('numpy.mean').has_equal_value()
    """

    append_missing = missing_msg is None
    append_params_not_matched = params_not_matched_msg is None
    if missing_msg is None:
        missing_msg = MISSING_MSG
    if expand_msg is None:
        expand_msg = PREPEND_MSG
    if params_not_matched_msg is None:
        params_not_matched_msg = SIG_ISSUE_MSG

    rep = Reporter.active_reporter
    stu_out = state.student_function_calls
    sol_out = state.solution_function_calls

    student_mappings = state.student_mappings

    fmt_kwargs = {'times': get_times(index+1),
                  'ord': get_ord(index+1),
                  'index': index,
                  'name': get_mapped_name(name, student_mappings)}

    # Get Parts ----
    # Copy, otherwise signature binding overwrites sol_out[name][index]['args']
    sol_parts = {**sol_out[name][index]}

    try:
        # Copy, otherwise signature binding overwrites stu_out[name][index]['args']
        stu_parts = {**stu_out[name][index]}
    except (KeyError, IndexError):
        _msg = state.build_message(missing_msg, fmt_kwargs, append=append_missing)
        rep.do_test(Test(Feedback(_msg, state)))

    # Signatures -----
    if signature:
        signature = None if isinstance(signature, bool) else signature
        get_sig = partial(getSignatureInProcess, name=name, signature=signature,
                                                manual_sigs = state.get_manual_sigs())

        try:
            sol_sig = get_sig(mapped_name=sol_parts['name'], process=state.solution_process)
            sol_parts['args'] = bind_args(sol_sig, sol_parts['args'])
        except:
            raise ValueError("Something went wrong in matching call index {index} of {name} to its signature. "
                             "You might have to manually specify or correct the signature."
                             .format(index=index, name=name))

        try:
            stu_sig = get_sig(mapped_name=stu_parts['name'], process=state.student_process)
            stu_parts['args'] = bind_args(stu_sig, stu_parts['args'])
        except Exception:
            _msg = state.build_message(params_not_matched_msg, fmt_kwargs, append=append_params_not_matched)
            rep.do_test(Test(Feedback(_msg, StubState(stu_parts['node'], state.highlighting_disabled))))

    # three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
    append_message = {'msg': expand_msg, 'kwargs': fmt_kwargs}
    child = part_to_child(stu_parts, sol_parts, append_message, state, node_name='function_calls')
    return child
def test_function_v2(name,
                     index=1,
                     params=[],
                     signature=None,
                     eq_condition="equal",
                     do_eval=True,
                     not_called_msg=None,
                     params_not_matched_msg=None,
                     params_not_specified_msg=None,
                     incorrect_msg=None,
                     add_more=False,
                     state=None,
                     **kwargs):
    rep = Reporter.active_reporter

    index = index - 1
    eq_map = {"equal": EqualTest}

    do_highlight = index == 0

    # ARG CHECKS --------------------------------------------------------------
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    if not isinstance(params, list):
        raise NameError(
            "Inside test_function_v2, make sure to specify a LIST of params.")

    if isinstance(do_eval, bool) or do_eval is None:
        do_eval = [do_eval] * len(params)

    if len(params) != len(do_eval):
        raise NameError(
            "Inside test_function_v2, make sure that do_eval has the same length as params."
        )

    # if params_not_specified_msg is a str or None, convert into list
    if isinstance(params_not_specified_msg,
                  str) or params_not_specified_msg is None:
        params_not_specified_msg = [params_not_specified_msg] * len(params)

    if len(params) != len(params_not_specified_msg):
        raise NameError(
            "Inside test_function_v2, make sure that params_not_specified_msg has the same length as params."
        )

    # if incorrect_msg is a str or None, convert into list
    if isinstance(incorrect_msg, str) or incorrect_msg is None:
        incorrect_msg = [incorrect_msg] * len(params)

    if len(params) != len(incorrect_msg):
        raise NameError(
            "Inside test_function_v2, make sure that incorrect_msg has the same length as params."
        )

    # STATE STUFF -------------------------------------------------------------
    solution_process = state.solution_process

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings

    stud_name = get_mapped_name(name, student_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = (
                "The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (
                    get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)
    # TODO: test if function name in dict of calls
    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, Feedback(_msg, state)))

    # TODO: test if number of specific function calls is less than index
    rep.do_test(
        BiggerTest(len(student_calls[name]), index, Feedback(_msg,
                                                             state)))  # TODO

    # TODO pull into own function
    if len(params) > 0:

        # Parse Signature -----------------------------------------------------
        try:
            _, arguments, keywords, sol_name = solution_calls[name][index][
                '_spec1']
            sol_sig = getSignatureInProcess(
                name=name,
                mapped_name=sol_name,
                signature=signature,
                manual_sigs=state.get_manual_sigs(),
                process=solution_process)
            solution_args, _ = bind_args(signature=sol_sig,
                                         arguments=arguments,
                                         keyws=keywords)
        except:
            raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
                " You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))

        # Check if params are in signature
        if set(params) - set(solution_args.keys()):
            raise ValueError(
                "When testing %s(), the solution call doesn't specify the listed parameters."
                % name)

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name,
                                         list(range(len(student_calls[name]))),
                                         index)

        # Test all calls ------------------------------------------------------
        from functools import partial
        sub_tests = [
            partial(test_call,
                    name,
                    call_ind,
                    signature,
                    params,
                    do_eval,
                    solution_args,
                    eq_fun,
                    add_more,
                    index,
                    params_not_specified_msg,
                    params_not_matched_msg,
                    incorrect_msg,
                    keywords,
                    do_highlight=do_highlight,
                    state=state,
                    **kwargs) for call_ind in call_indices
        ]
        test_or(*sub_tests, state=state)
def test_lambda_function(index,
                         arg_names=True,
                         arg_defaults=True,
                         body=None,
                         results=[],
                         errors=[],
                         not_called_msg=None,
                         nb_args_msg=None,
                         arg_names_msg=None,
                         arg_defaults_msg=None,
                         wrong_result_msg=None,
                         no_error_msg=None,
                         expand_message=True,
                         state=None):
    """Test a lambda function definition.

    This function helps you test a lambda function definition. Generally four things can be tested:
        1) The argument names of the function (including if the correct defaults are used)
        2) The body of the functions (does it output correctly, are the correct functions used)
        3) The return value with a certain input
        4) Whether certain inputs generate an error

    Custom feedback messages can be set for all these parts, default messages are generated
    automatically if none are set.

    Args:
        index (int): the number of the lambda function you want to test.
        arg_names (bool): if True, the argument names will be tested, if False they won't be tested. Defaults
            to True.
        arg_defaults (bool): if True, the default values of the arguments will be tested, if False they won't
            be tested. Defaults to True.
        body: this arguments holds the part of the code that will be ran to check the body of the function
            definition. It should be passed as a lambda expression or a function. The functions that are
            ran should be other pythonwhat test functions, and they will be tested specifically on only the
            body of the for loop. Defaults to None.
        results (list(str)): a list of strings representing function calls to the lam function. The lam
            function will be replaced by the actual lambda function from the student and solution code. The result
            of calling the lambda function will be compared between student and solution.
        errors (list(str)): a list of strings representing function calls to the lam function. The lam
            function will be replaced by the actual lambda function from the student and solution code. It will be
            checked if an error is generated appropriately for the specified inputs.
        not_called_msg (str): message if the function is not defined.
        nb_args_msg (str): message if the number of arguments do not matched.
        arg_names_msg (str): message if the argument names do not match.
        arg_defaults_msg (str): message if the argument default values do not match.
        wrong_result_msg (str): message if one of the tested function calls' result did not match.
        no_error_msg (str): message if one of the tested function calls' result did not generate an error.
        expand_message (bool): only relevant if there is a body test. If True, feedback messages defined in the
            body test will be preceded by 'In your definition of ___, '. If False, :code:`test_function_definition()`
            will generate no extra feedback if the body test fails. Defaults to True.
    """

    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_lambda_function")

    # what the lambda will be referred to as
    typestr = "the {} lambda function".format(get_ord(index))
    get_func_child = partial(check_node,
                             'lambda_functions',
                             index - 1,
                             typestr,
                             not_called_msg or MSG_MISSING,
                             state=state)
    child = get_func_child(expand_msg=MSG_PREPEND if expand_message else "")

    # make a temporary child states, to reflect that there were two types of
    # messages prepended in the original function
    quiet_child = get_func_child(expand_msg="")
    prep_child2 = get_func_child(
        expand_msg=MSG_PREPEND_ARG if expand_message else "")

    test_args(arg_names, arg_defaults, nb_args_msg, arg_names_msg,
              arg_defaults_msg, prep_child2, quiet_child)

    multi(body, state=check_part('body', "", child))

    # Test function calls -----------------------------------------------------

    student_fun = state.student_lambda_functions[index - 1]['node']
    solution_fun = state.solution_lambda_functions[index - 1]['node']

    for el in results:
        argstr = el.replace('lam', '')
        call(el,
             incorrect_msg=wrong_result_msg or MSG_RES_INCORRECT,
             error_msg=wrong_result_msg or MSG_RES_ERROR,
             argstr=argstr,
             state=child)

    for el in errors:
        argstr = el.replace('lam', '')
        call(el,
             'error',
             incorrect_msg=no_error_msg or MSG_ERR_WRONG,
             error_msg=no_error_msg or MSG_ERR_WRONG,
             argstr=argstr,
             state=child)
Example #24
0
def test_get_ord(input, output):
    assert utils.get_ord(input) == output
def test_lambda_function(index,
                         arg_names=True,
                         arg_defaults=True,
                         body=None,
                         results=[],
                         errors=[],
                         not_called_msg=None,
                         nb_args_msg=None,
                         arg_names_msg=None,
                         arg_defaults_msg=None,
                         wrong_result_msg=None,
                         no_error_msg=None,
                         expand_message=True,
                         state=None):
    """Test a lambda function definition.

    This function helps you test a lambda function definition. Generally four things can be tested:
        1) The argument names of the function (including if the correct defaults are used)
        2) The body of the functions (does it output correctly, are the correct functions used)
        3) The return value with a certain input
        4) Whether certain inputs generate an error

    Custom feedback messages can be set for all these parts, default messages are generated
    automatically if none are set.

    Args:
        index (int): the number of the lambda function you want to test.
        arg_names (bool): if True, the argument names will be tested, if False they won't be tested. Defaults
            to True.
        arg_defaults (bool): if True, the default values of the arguments will be tested, if False they won't
            be tested. Defaults to True.
        body: this arguments holds the part of the code that will be ran to check the body of the function
            definition. It should be passed as a lambda expression or a function. The functions that are
            ran should be other pythonwhat test functions, and they will be tested specifically on only the
            body of the for loop. Defaults to None.
        results (list(str)): a list of strings representing function calls to the lam function. The lam
            function will be replaced by the actual lambda function from the student and solution code. The result
            of calling the lambda function will be compared between student and solution.
        errors (list(str)): a list of strings representing function calls to the lam function. The lam
            function will be replaced by the actual lambda function from the student and solution code. It will be
            checked if an error is generated appropriately for the specified inputs.
        not_called_msg (str): message if the function is not defined.
        nb_args_msg (str): message if the number of arguments do not matched.
        arg_names_msg (str): message if the argument names do not match.
        arg_defaults_msg (str): message if the argument default values do not match.
        wrong_result_msg (str): message if one of the tested function calls' result did not match.
        no_error_msg (str): message if one of the tested function calls' result did not generate an error.
        expand_message (bool): only relevant if there is a body test. If True, feedback messages defined in the
            body test will be preceded by 'In your definition of ___, '. If False, :code:`test_function_definition()`
            will generate no extra feedback if the body test fails. Defaults to True.
    """

    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_lambda_function")

    # what the lambda will be referred to as
    typestr = "the {} lambda function".format(get_ord(index))
    get_func_child = partial(check_node, 'lambda_functions', index-1, typestr, not_called_msg or MSG_MISSING, state=state)
    child = get_func_child(expand_msg = MSG_PREPEND if expand_message else "")

    # make a temporary child states, to reflect that there were two types of 
    # messages prepended in the original function
    quiet_child = get_func_child(expand_msg = "")
    prep_child2 = get_func_child(expand_msg =  MSG_PREPEND_ARG if expand_message else "")

    test_args(arg_names, arg_defaults, 
              nb_args_msg, arg_names_msg, arg_defaults_msg, 
              prep_child2, quiet_child)

    multi(body, state=check_part('body', "", child))

    # Test function calls -----------------------------------------------------

    student_fun  = state.student_lambda_functions[index-1]['node']
    solution_fun = state.solution_lambda_functions[index-1]['node']

    for el in results:
        argstr = el.replace('lam', '')
        call(el, 
             incorrect_msg = wrong_result_msg or MSG_RES_INCORRECT, 
             error_msg = wrong_result_msg or MSG_RES_ERROR,
             argstr = argstr,
             state = child)

    for el in errors:
        argstr = el.replace('lam', '')
        call(el, 'error',
             incorrect_msg = no_error_msg or MSG_ERR_WRONG, 
             error_msg = no_error_msg or MSG_ERR_WRONG,
             argstr = argstr,
             state = child)
Example #26
0
def test_with(
        index,
        context_vals=False,  # whether to check number of context vals
        context_tests=None,  # check on context expressions
        body=None,
        undefined_msg=None,
        context_vals_len_msg=None,
        context_vals_msg=None,
        expand_message=True,
        state=None):
    """Test a with statement.
with open_file('...') as bla:

    [ open_file('...').__enter__() ]


with open_file('...') as file:
    [ ]

    """
    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_with")

    check_with = partial(check_node,
                         'withs',
                         index - 1,
                         "`with` statement",
                         MSG_MISSING,
                         state=state)

    child = check_with(MSG_PREPEND if expand_message else "")
    child2 = check_with(MSG_PREPEND2 if expand_message else "")
    quiet_child = quiet(1, child)

    if context_vals:
        # test num context vars ----
        too_many = len(child.student_parts['context']) > len(
            child.solution_parts['context'])
        if too_many:
            _msg = child.build_message(MSG_NUM_CTXT)
            rep.do_test(Test(Feedback(_msg, child.student_tree)))

        # test context var names ----
        for i in range(len(child.solution_parts['context'])):
            ctxt_state = check_part_index('context', i, "", state=child)
            has_equal_part('target_vars', MSG_CTXT_NAMES, state=ctxt_state)

    # Context sub tests ----
    if context_tests and not isinstance(context_tests, list):
        context_tests = [context_tests]

    for i, context_test in enumerate(context_tests or []):
        # partial the substate check, because the function uses two prepended messages
        check_context = partial(check_part_index, 'context', i,
                                "%s context" % utils.get_ord(i + 1),
                                MSG_NUM_CTXT2)

        check_context(state=child)  # test exist

        ctxt_state = check_context(state=child2)  # sub tests
        multi(context_test, state=ctxt_state)

    # Body sub tests ----
    if body is not None:
        body_state = check_part('body', 'body', state=child2)

        with_context(body, state=body_state)
Example #27
0
def has_printout(
    state, index, not_printed_msg=None, pre_code=None, name=None, copy=False
):
    """Check if the right printouts happened.

    ``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in
    the solution process, capture its output, and verify whether the output is present in the output of the student.

    This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
    printouts as they want, as long as they do the correct one somewhere.

    Args:
        index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
        not_printed_msg (str): if specified, this overrides the default message that is generated when the output
          is not found in the student output.
        pre_code (str): Python code as a string that is executed before running the targeted student call.
          This is the ideal place to set a random seed, for example.
        copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
          accidentally be mutated. Disabled by default, which speeds up SCTs.
        state (State): state as passed by the SCT chain. Don't specify this explicitly.

    :Example:

        Suppose you want somebody to print out 4: ::

            print(1, 2, 3, 4)

        The following SCT would check that: ::

            Ex().has_printout(0)

        All of the following SCTs would pass: ::

            print(1, 2, 3, 4)
            print('1 2 3 4')
            print(1, 2, '3 4')
            print("random"); print(1, 2, 3, 4)

    :Example:

        Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed.
        If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work.

        Suppose you have the following solution: ::

            x = 4
            print(x)
            x = 6

        The following SCT will not work: ::

            Ex().has_printout(0)

        Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated.
        In cases like these, ``has_printout()`` cannot be used.

    :Example:

        Inside a for loop ``has_printout()``

        Suppose you have the following solution: ::

            for i in range(5):
                print(i)

        The following SCT will not work: ::

            Ex().check_for_loop().check_body().has_printout(0)

        The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``.
        If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: ::

            Ex().check_for_loop().check_body().\\
                set_context(0).check_function('print').\\
                check_args(0).has_equal_value()

    """

    extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead."
    state.assert_root("has_printout", extra_msg=extra_msg)

    if not_printed_msg is None:
        not_printed_msg = (
            "Have you used `{{sol_call}}` to do the appropriate printouts?"
        )

    try:
        sol_call_ast = state.ast_dispatcher("function_calls", state.solution_ast)[
            "print"
        ][index]["node"]
    except (KeyError, IndexError):
        raise InstructorError(
            "`has_printout({})` couldn't find the {} print call in your solution.".format(
                index, utils.get_ord(index + 1)
            )
        )

    out_sol, str_sol = getOutputInProcess(
        tree=sol_call_ast,
        process=state.solution_process,
        context=state.solution_context,
        env=state.solution_env,
        pre_code=pre_code,
        copy=copy,
    )

    sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast)

    if isinstance(str_sol, Exception):
        raise InstructorError(
            "Evaluating the solution expression {} raised error in solution process."
            "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol)
        )

    _msg = state.build_message(not_printed_msg, {"sol_call": sol_call_str})

    has_output(state, out_sol.strip(), pattern=False, no_output_msg=_msg)

    return state
Example #28
0
def check_function(
    state,
    name,
    index=0,
    missing_msg=None,
    params_not_matched_msg=None,
    expand_msg=None,
    signature=True,
):
    """Check whether a particular function is called.

    ``check_function()`` is typically followed by:

    - ``check_args()`` to check whether the arguments were specified.
      In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()``
      to assert that the arguments were correctly specified.
    - ``has_equal_value()`` to check whether rerunning the function call coded by the student
      gives the same result as calling the function call as in the solution.

    Checking function calls is a tricky topic. Please visit the
    `dedicated article <articles/checking_function_calls.html>`_ for more explanation,
    edge cases and best practices.

    Args:
        name (str): the name of the function to be tested. When checking functions in packages, always
            use the 'full path' of the function.
        index (int): index of the function call to be checked. Defaults to 0.
        missing_msg (str): If specified, this overrides an automatically generated feedback message in case
            the student did not call the function correctly.
        params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
            in case the function parameters were not successfully matched.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
        signature (Signature): Normally, check_function() can figure out what the function signature is,
            but it might be necessary to use ``sig_from_params()`` to manually build a signature and pass this along.
        state (State): State object that is passed from the SCT Chain (don't specify this).

    :Examples:

        Student code and solution code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Verify whether arr was correctly set in np.mean
            Ex().check_function('numpy.mean').check_args('a').has_equal_value()

            # Verify whether np.mean(arr) produced the same result
            Ex().check_function('numpy.mean').has_equal_value()
    """

    append_missing = missing_msg is None
    append_params_not_matched = params_not_matched_msg is None
    if missing_msg is None:
        missing_msg = MISSING_MSG
    if expand_msg is None:
        expand_msg = PREPEND_MSG
    if params_not_matched_msg is None:
        params_not_matched_msg = SIG_ISSUE_MSG

    stu_out = state.ast_dispatcher.find("function_calls", state.student_ast)
    sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast)

    student_mappings = state.ast_dispatcher.find("mappings", state.student_ast)

    fmt_kwargs = {
        "times": get_times(index + 1),
        "ord": get_ord(index + 1),
        "index": index,
        "mapped_name": get_mapped_name(name, student_mappings),
    }

    # Get Parts ----
    # Copy, otherwise signature binding overwrites sol_out[name][index]['args']
    try:
        sol_parts = {**sol_out[name][index]}
    except KeyError:
        raise InstructorError(
            "`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!"
            % name)
    except IndexError:
        raise InstructorError(
            "`check_function()` couldn't find %s calls of `%s()` in your solution code."
            % (index + 1, name))

    try:
        # Copy, otherwise signature binding overwrites stu_out[name][index]['args']
        stu_parts = {**stu_out[name][index]}
    except (KeyError, IndexError):
        _msg = state.build_message(missing_msg,
                                   fmt_kwargs,
                                   append=append_missing)
        state.report(_msg)

    # Signatures -----
    if signature:
        signature = None if isinstance(signature, bool) else signature
        get_sig = partial(
            getSignatureInProcess,
            name=name,
            signature=signature,
            manual_sigs=state.get_manual_sigs(),
        )

        try:
            sol_sig = get_sig(mapped_name=sol_parts["name"],
                              process=state.solution_process)
            sol_parts["args"] = bind_args(sol_sig, sol_parts["args"])
        except Exception as e:
            raise InstructorError(
                "`check_function()` couldn't match the %s call of `%s` to its signature:\n%s "
                % (get_ord(index + 1), name, e))

        try:
            stu_sig = get_sig(mapped_name=stu_parts["name"],
                              process=state.student_process)
            stu_parts["args"] = bind_args(stu_sig, stu_parts["args"])
        except Exception:
            _msg = state.build_message(params_not_matched_msg,
                                       fmt_kwargs,
                                       append=append_params_not_matched)
            state.to_child(highlight=stu_parts["node"]).report(_msg)

    # three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
    append_message = {"msg": expand_msg, "kwargs": fmt_kwargs}
    child = part_to_child(stu_parts,
                          sol_parts,
                          append_message,
                          state,
                          node_name="function_calls")
    return child
def test_function(name,
                  index=1,
                  args=None,
                  keywords=None,
                  eq_condition="equal",
                  do_eval=True,
                  not_called_msg=None,
                  args_not_specified_msg=None,
                  incorrect_msg=None,
                  add_more=False,
                  state=None,
                  **kwargs):
    rep = Reporter.active_reporter

    index = index - 1

    do_highlight = index == 0

    eq_map = {"equal": EqualTest}
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings

    # for messaging purposes: replace with original alias or import again.
    stud_name = get_mapped_name(name, student_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = (
                "The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (
                    get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)

    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, Feedback(_msg, state)))

    rep.do_test(
        BiggerTest(len(student_calls[name]), index, Feedback(_msg, state)))

    _, args_solution, keyw_solution, _ = solution_calls[name][index]['_spec1']
    keyw_solution = {keyword.arg: keyword.value for keyword in keyw_solution}

    if args is None:
        args = list(range(len(args_solution)))

    if keywords is None:
        keywords = list(keyw_solution.keys())

    if len(args) > 0 or len(keywords) > 0:

        success = None

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name,
                                         list(range(len(student_calls[name]))),
                                         index)

        feedback = None

        for call_ind in call_indices:
            student_call, args_student, keyw_student, stud_name = student_calls[
                name][call_ind]['_spec1']
            keyw_student = {
                keyword.arg: keyword.value
                for keyword in keyw_student
            }

            success = True
            dflt = "Have you specified all required arguments inside `%s()`?" % stud_name

            setdiff = list(set(keywords) - set(keyw_student.keys()))
            if (len(args) > 0 and
                (max(args) >= len(args_student))) or len(setdiff) > 0:
                if feedback is None:
                    if not args_not_specified_msg:
                        args_not_specified_msg = dflt
                    _st = StubState(
                        student_call,
                        state.highlighting_disabled if do_highlight else True)
                    feedback = Feedback(args_not_specified_msg, _st)
                success = False
                continue

            if do_eval is None:
                # don't have to go further: set used and break from the for loop
                state.set_used(name, call_ind, index)
                break

            feedback_msg = "Did you call `%s()` with the correct arguments?" % stud_name
            for arg in args:
                arg_student = args_student[arg]
                arg_solution = args_solution[arg]
                if incorrect_msg is None:
                    msg = feedback_msg + (
                        " The %s argument seems to be incorrect." %
                        get_ord(arg + 1))
                else:
                    msg = incorrect_msg

                test = build_test(arg_student,
                                  arg_solution,
                                  state,
                                  do_eval,
                                  eq_fun,
                                  msg,
                                  add_more=add_more,
                                  do_highlight=do_highlight,
                                  **kwargs)
                test.test()

                if not test.result:
                    if feedback is None:
                        feedback = test.get_feedback()
                    success = False
                    break

            if success:
                for key in keywords:
                    key_student = keyw_student[key]
                    key_solution = keyw_solution[key]
                    if incorrect_msg is None:
                        msg = feedback_msg + (
                            " Keyword `%s` seems to be incorrect." % key)
                        add_more = True
                    else:
                        msg = incorrect_msg
                        add_more = False

                    test = build_test(key_student,
                                      key_solution,
                                      state,
                                      do_eval,
                                      eq_fun,
                                      msg,
                                      add_more=add_more,
                                      do_highlight=do_highlight,
                                      **kwargs)
                    test.test()

                    if not test.result:
                        if feedback is None:
                            feedback = test.get_feedback()
                        success = False
                        break

            if success:
                # we have a winner that passes all argument and keyword checks
                state.set_used(name, call_ind, index)
                break

        if not success:
            if feedback is None:
                _msg = state.build_message(
                    "You haven't used enough appropriate calls of `%s()`" %
                    stud_name)
                feedback = Feedback(_msg, state)
            rep.do_test(Test(feedback))
Example #30
0
def test_function(name,
                  index=1,
                  args=None,
                  keywords=None,
                  eq_condition="equal",
                  do_eval=True,
                  not_called_msg=None,
                  args_not_specified_msg=None,
                  incorrect_msg=None,
                  add_more=False,
                  highlight=True,
                  state=None):
    """Test if function calls match.

    This function compares a function call in the student's code with the corresponding one in the solution
    code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
    that is returned will depend on the sort of fail.

    Args:
        name (str): the name of the function to be tested.
        index (int): index of the function call to be checked. Defaults to 1.
        args (list(int)): the indices of the positional arguments that have to be checked. If it is set to
          None, all positional arguments which are in the solution will be checked.
        keywords (list(str)): the indices of the keyword arguments that have to be checked. If it is set to
          None, all keyword arguments which are in the solution will be checked.
        eq_condition (str): how arguments/keywords are compared. Currently, only "equal" is supported,
          meaning that the result in student and solution process should have exactly the same value.
        do_eval (bool): True: arguments are evaluated and compared. False: arguments are not evaluated but
            'string-matched'. None: arguments are not evaluated; it is only checked if they are specified.
        not_called_msg (str): feedback message if the function is not called.
        args_not_specified_msg (str): feedback message if the function is called but not all required arguments are specified
        incorrect_msg (str): feedback message if the arguments of the function in the solution doesn't match
          the one of the student.

    :Example:

        Student code::

            import numpy as np
            np.mean([1,2,3])
            np.std([2,3,4])

        Solution code::

            import numpy
            numpy.mean([1,2,3], axis = 0)
            numpy.std([4,5,6])

        SCT::

            test_function("numpy.mean", index = 1, keywords = []) # pass
            test_function("numpy.mean", index = 1)                # fail
            test_function(index = 1, incorrect_op_msg = "Use the correct operators")       # fail
            test_function(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail
    """
    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_function")

    index = index - 1

    eq_map = {"equal": EqualTest}
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    student_process, solution_process = state.student_process, state.solution_process

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings

    # for messaging purposes: replace with original alias or import again.
    stud_name = get_mapped_name(name, student_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = ("The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)

    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, _msg))

    rep.do_test(BiggerTest(len(student_calls[name]), index, _msg))

    solution_call, args_solution, keyw_solution, sol_name = solution_calls[name][index]['_spec1']
    keyw_solution = {keyword.arg: keyword.value for keyword in keyw_solution}


    if args is None:
        args = list(range(len(args_solution)))

    if keywords is None:
        keywords = list(keyw_solution.keys())

    if len(args) > 0 or len(keywords) > 0:

        success = None

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name, list(range(len(student_calls[name]))), index)

        feedback = None

        for call_ind in call_indices:
            student_call, args_student, keyw_student, stud_name = student_calls[name][call_ind]['_spec1']
            keyw_student = {keyword.arg: keyword.value for keyword in keyw_student}

            success = True
            dflt = "Have you specified all required arguments inside `%s()`?" % stud_name

            setdiff = list(set(keywords) - set(keyw_student.keys()))
            if (len(args) > 0 and (max(args) >= len(args_student))) or len(setdiff) > 0:
                if feedback is None:
                    if not args_not_specified_msg:
                        args_not_specified_msg = dflt
                    feedback = Feedback(args_not_specified_msg, student_call if highlight else None)
                success = False
                continue

            if do_eval is None:
                # don't have to go further: set used and break from the for loop
                state.set_used(name, call_ind, index)
                break

            feedback_msg = "Did you call `%s()` with the correct arguments?" % stud_name
            for arg in args:
                arg_student = args_student[arg]
                arg_solution = args_solution[arg]
                if incorrect_msg is None:
                    msg = feedback_msg + (" The %s argument seems to be incorrect." % get_ord(arg + 1))
                else:
                    msg = incorrect_msg

                test = build_test(arg_student, arg_solution,
                                  state,
                                  do_eval, eq_fun, msg, add_more=add_more,
                                  highlight=arg_student if highlight else None)
                test.test()

                if not test.result:
                    if feedback is None:
                        feedback = test.get_feedback()
                    success = False
                    break

            if success:
                for key in keywords:
                    key_student = keyw_student[key]
                    key_solution = keyw_solution[key]
                    if incorrect_msg is None:
                        msg = feedback_msg + (" Keyword `%s` seems to be incorrect." % key)
                        add_more = True
                    else:
                        msg = incorrect_msg
                        add_more = False

                    test = build_test(key_student, key_solution,
                                      state,
                                      do_eval, eq_fun, msg, add_more=add_more,
                                      highlight=key_student if highlight else None)
                    test.test()

                    if not test.result:
                        if feedback is None:
                            feedback = test.get_feedback()
                        success = False
                        break

            if success:
                # we have a winner that passes all argument and keyword checks
                state.set_used(name, call_ind, index)
                break

        if not success:
            if feedback is None:
                _msg = state.build_message("You haven't used enough appropriate calls of `%s()`" % stud_name)
                feedback = Feedback(_msg)
            rep.do_test(Test(feedback))
Example #31
0
def test_function_v2(name,
                     index=1,
                     params=[],
                     signature=None,
                     eq_condition="equal",
                     do_eval=True,
                     not_called_msg=None,
                     params_not_matched_msg=None,
                     params_not_specified_msg=None,
                     incorrect_msg=None,
                     add_more=False,
                     highlight=True,
                     state=None):
    """Test if function calls match (v2).

    This function compares a function call in the student's code with the corresponding one in the solution
    code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
    that is returned will depend on the sort of fail.

    Args:
        name (str): the name of the function to be tested.
        index (int): index of the function call to be checked. Defaults to 1.
        params (list(str)): the parameter names of the function call that you want to check.
        signature (Signature): Normally, test_function() can figure out what the function signature is,
            but it might be necessary to use build_sig to manually build a signature and pass this along.
        eq_condition (str): how parameters are compared. Currently, only "equal" is supported,
            meaning that the arguments in student and solution process should have exactly the same value.
        do_eval (list(bool)): Boolean or list of booleans (parameter-specific) that specify whether or
            not arguments should be evaluated.
            True: arguments are evaluated and compared.
            False: arguments are not evaluated but 'string-matched'.
            None: arguments are not evaluated; it is only checked if they are specified.
        not_called_msg (str): custom feedback message if the function is not called.
        params_not_matched_message (str): custom feedback message if the function parameters were not successfully matched.
        params_not_specified_msg (str): string or list of strings (parameter-specific). Custom feedback message if not all
            parameters listed in params are specified by the student.
        incorrect_msg (list(str)): string or list of strings (parameter-specific). Custom feedback messages if the arguments
            don't correspond between student and solution code.
    """

    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_function")

    index = index - 1
    eq_map = {"equal": EqualTest}

    # ARG CHECKS --------------------------------------------------------------
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    if not isinstance(params, list):
        raise NameError("Inside test_function_v2, make sure to specify a LIST of params.")

    if isinstance(do_eval, bool) or do_eval is None:
        do_eval = [do_eval] * len(params)

    if len(params) != len(do_eval):
        raise NameError("Inside test_function_v2, make sure that do_eval has the same length as params.")

    # if params_not_specified_msg is a str or None, convert into list
    if isinstance(params_not_specified_msg, str) or params_not_specified_msg is None:
        params_not_specified_msg = [params_not_specified_msg] * len(params)

    if len(params) != len(params_not_specified_msg):
        raise NameError("Inside test_function_v2, make sure that params_not_specified_msg has the same length as params.")

    # if incorrect_msg is a str or None, convert into list
    if isinstance(incorrect_msg, str) or incorrect_msg is None:
        incorrect_msg = [incorrect_msg] * len(params)

    if len(params) != len(incorrect_msg):
        raise NameError("Inside test_function_v2, make sure that incorrect_msg has the same length as params.")

    # STATE STUFF -------------------------------------------------------------
    student_process, solution_process = state.student_process, state.solution_process

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings
    solution_mappings = state.solution_mappings

    stud_name = get_mapped_name(name, student_mappings)
    #sol_name = get_mapped_name(name, solution_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = ("The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)
    # TODO: test if function name in dict of calls
    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, _msg))

    # TODO: test if number of specific function calls is less than index
    rep.do_test(BiggerTest(len(student_calls[name]), index, _msg))  # TODO

    # TODO pull into own function
    if len(params) > 0:

        # Parse Signature -----------------------------------------------------
        try:
            sol_call, arguments, keywords, sol_name = solution_calls[name][index]['_spec1']
            sol_sig = getSignatureInProcess(name=name, mapped_name=sol_name,
                                            signature=signature,
                                            manual_sigs = state.get_manual_sigs(),
                                            process=solution_process)
            solution_args, _ = bind_args(signature = sol_sig, arguments=arguments, keyws=keywords)
        except:
            raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
                " You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))

        # Check if params are in signature
        if set(params) - set(solution_args.keys()):
            raise ValueError("When testing %s(), the solution call doesn't specify the listed parameters." % name)

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name, list(range(len(student_calls[name]))), index)

        feedback = None

        # Test all calls ------------------------------------------------------
        from functools import partial
        sub_tests = [partial(test_call, name, call_ind, signature, params, do_eval, solution_args, 
                             eq_fun, add_more, index,
                             params_not_specified_msg, params_not_matched_msg, incorrect_msg, 
                             keywords, state=state, highlight = highlight)
                     for call_ind in call_indices]
        test_or(*sub_tests, state=state)
Example #32
0
	def test_get_ord(self):
		self.assertEqual(utils.get_ord(1), "first")
		self.assertEqual(utils.get_ord(2), "second")
		self.assertEqual(utils.get_ord(3), "third")
		self.assertEqual(utils.get_ord(11), "11th")
Example #33
0
def test_function_v2(name,
                     index=1,
                     params=[],
                     signature=None,
                     eq_condition="equal",
                     do_eval=True,
                     not_called_msg=None,
                     params_not_matched_msg=None,
                     params_not_specified_msg=None,
                     incorrect_msg=None,
                     add_more=False,
                     highlight=True,
                     state=None):
    """Test if function calls match (v2).

    This function compares a function call in the student's code with the corresponding one in the solution
    code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
    that is returned will depend on the sort of fail.

    Args:
        name (str): the name of the function to be tested.
        index (int): index of the function call to be checked. Defaults to 1.
        params (list(str)): the parameter names of the function call that you want to check.
        signature (Signature): Normally, test_function() can figure out what the function signature is,
            but it might be necessary to use build_sig to manually build a signature and pass this along.
        eq_condition (str): how parameters are compared. Currently, only "equal" is supported,
            meaning that the arguments in student and solution process should have exactly the same value.
        do_eval (list(bool)): Boolean or list of booleans (parameter-specific) that specify whether or
            not arguments should be evaluated.
            True: arguments are evaluated and compared.
            False: arguments are not evaluated but 'string-matched'.
            None: arguments are not evaluated; it is only checked if they are specified.
        not_called_msg (str): custom feedback message if the function is not called.
        params_not_matched_message (str): custom feedback message if the function parameters were not successfully matched.
        params_not_specified_msg (str): string or list of strings (parameter-specific). Custom feedback message if not all
            parameters listed in params are specified by the student.
        incorrect_msg (list(str)): string or list of strings (parameter-specific). Custom feedback messages if the arguments
            don't correspond between student and solution code.
    """

    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_function")

    index = index - 1
    eq_map = {"equal": EqualTest}

    # ARG CHECKS --------------------------------------------------------------
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    if not isinstance(params, list):
        raise NameError(
            "Inside test_function_v2, make sure to specify a LIST of params.")

    if isinstance(do_eval, bool) or do_eval is None:
        do_eval = [do_eval] * len(params)

    if len(params) != len(do_eval):
        raise NameError(
            "Inside test_function_v2, make sure that do_eval has the same length as params."
        )

    # if params_not_specified_msg is a str or None, convert into list
    if isinstance(params_not_specified_msg,
                  str) or params_not_specified_msg is None:
        params_not_specified_msg = [params_not_specified_msg] * len(params)

    if len(params) != len(params_not_specified_msg):
        raise NameError(
            "Inside test_function_v2, make sure that params_not_specified_msg has the same length as params."
        )

    # if incorrect_msg is a str or None, convert into list
    if isinstance(incorrect_msg, str) or incorrect_msg is None:
        incorrect_msg = [incorrect_msg] * len(params)

    if len(params) != len(incorrect_msg):
        raise NameError(
            "Inside test_function_v2, make sure that incorrect_msg has the same length as params."
        )

    # STATE STUFF -------------------------------------------------------------
    student_process, solution_process = state.student_process, state.solution_process

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings
    solution_mappings = state.solution_mappings

    stud_name = get_mapped_name(name, student_mappings)
    #sol_name = get_mapped_name(name, solution_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = (
                "The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (
                    get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)
    # TODO: test if function name in dict of calls
    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, _msg))

    # TODO: test if number of specific function calls is less than index
    rep.do_test(BiggerTest(len(student_calls[name]), index, _msg))  # TODO

    # TODO pull into own function
    if len(params) > 0:

        # Parse Signature -----------------------------------------------------
        try:
            sol_call, arguments, keywords, sol_name = solution_calls[name][
                index]['_spec1']
            sol_sig = getSignatureInProcess(
                name=name,
                mapped_name=sol_name,
                signature=signature,
                manual_sigs=state.get_manual_sigs(),
                process=solution_process)
            solution_args, _ = bind_args(signature=sol_sig,
                                         arguments=arguments,
                                         keyws=keywords)
        except:
            raise ValueError(("Something went wrong in matching the %s call of %s to its signature." + \
                " You might have to manually specify or correct the function signature.") % (get_ord(index + 1), sol_name))

        # Check if params are in signature
        if set(params) - set(solution_args.keys()):
            raise ValueError(
                "When testing %s(), the solution call doesn't specify the listed parameters."
                % name)

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name,
                                         list(range(len(student_calls[name]))),
                                         index)

        feedback = None

        # Test all calls ------------------------------------------------------
        from functools import partial
        sub_tests = [
            partial(test_call,
                    name,
                    call_ind,
                    signature,
                    params,
                    do_eval,
                    solution_args,
                    eq_fun,
                    add_more,
                    index,
                    params_not_specified_msg,
                    params_not_matched_msg,
                    incorrect_msg,
                    keywords,
                    state=state,
                    highlight=highlight) for call_ind in call_indices
        ]
        test_or(*sub_tests, state=state)
Example #34
0
def test_function(name,
                  index=1,
                  args=None,
                  keywords=None,
                  eq_condition="equal",
                  do_eval=True,
                  not_called_msg=None,
                  args_not_specified_msg=None,
                  incorrect_msg=None,
                  add_more=False,
                  highlight=True,
                  state=None):
    """Test if function calls match.

    This function compares a function call in the student's code with the corresponding one in the solution
    code. It will cause the reporter to fail if the corresponding calls do not match. The fail message
    that is returned will depend on the sort of fail.

    Args:
        name (str): the name of the function to be tested.
        index (int): index of the function call to be checked. Defaults to 1.
        args (list(int)): the indices of the positional arguments that have to be checked. If it is set to
          None, all positional arguments which are in the solution will be checked.
        keywords (list(str)): the indices of the keyword arguments that have to be checked. If it is set to
          None, all keyword arguments which are in the solution will be checked.
        eq_condition (str): how arguments/keywords are compared. Currently, only "equal" is supported,
          meaning that the result in student and solution process should have exactly the same value.
        do_eval (bool): True: arguments are evaluated and compared. False: arguments are not evaluated but
            'string-matched'. None: arguments are not evaluated; it is only checked if they are specified.
        not_called_msg (str): feedback message if the function is not called.
        args_not_specified_msg (str): feedback message if the function is called but not all required arguments are specified
        incorrect_msg (str): feedback message if the arguments of the function in the solution doesn't match
          the one of the student.

    :Example:

        Student code::

            import numpy as np
            np.mean([1,2,3])
            np.std([2,3,4])

        Solution code::

            import numpy
            numpy.mean([1,2,3], axis = 0)
            numpy.std([4,5,6])

        SCT::

            test_function("numpy.mean", index = 1, keywords = []) # pass
            test_function("numpy.mean", index = 1)                # fail
            test_function(index = 1, incorrect_op_msg = "Use the correct operators")       # fail
            test_function(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail
    """
    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_function")

    index = index - 1

    eq_map = {"equal": EqualTest}
    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)
    eq_fun = eq_map[eq_condition]

    student_process, solution_process = state.student_process, state.solution_process

    solution_calls = state.solution_function_calls
    student_calls = state.student_function_calls
    student_mappings = state.student_mappings

    # for messaging purposes: replace with original alias or import again.
    stud_name = get_mapped_name(name, student_mappings)

    if not_called_msg is None:
        if index == 0:
            not_called_msg = "Have you called `%s()`?" % stud_name
        else:
            not_called_msg = (
                "The system wants to check the %s call of `%s()`, " +
                "but hasn't found it; have another look at your code.") % (
                    get_ord(index + 1), stud_name)

    if name not in solution_calls or len(solution_calls[name]) <= index:
        raise NameError("%r not in solution environment (often enough)" % name)

    _msg = state.build_message(not_called_msg)
    rep.do_test(DefinedCollTest(name, student_calls, _msg))

    rep.do_test(BiggerTest(len(student_calls[name]), index, _msg))

    solution_call, args_solution, keyw_solution, sol_name = solution_calls[
        name][index]['_spec1']
    keyw_solution = {keyword.arg: keyword.value for keyword in keyw_solution}

    if args is None:
        args = list(range(len(args_solution)))

    if keywords is None:
        keywords = list(keyw_solution.keys())

    if len(args) > 0 or len(keywords) > 0:

        success = None

        # Get all options (some function calls may be blacklisted)
        call_indices = state.get_options(name,
                                         list(range(len(student_calls[name]))),
                                         index)

        feedback = None

        for call_ind in call_indices:
            student_call, args_student, keyw_student, stud_name = student_calls[
                name][call_ind]['_spec1']
            keyw_student = {
                keyword.arg: keyword.value
                for keyword in keyw_student
            }

            success = True
            dflt = "Have you specified all required arguments inside `%s()`?" % stud_name

            setdiff = list(set(keywords) - set(keyw_student.keys()))
            if (len(args) > 0 and
                (max(args) >= len(args_student))) or len(setdiff) > 0:
                if feedback is None:
                    if not args_not_specified_msg:
                        args_not_specified_msg = dflt
                    feedback = Feedback(args_not_specified_msg,
                                        student_call if highlight else None)
                success = False
                continue

            if do_eval is None:
                # don't have to go further: set used and break from the for loop
                state.set_used(name, call_ind, index)
                break

            feedback_msg = "Did you call `%s()` with the correct arguments?" % stud_name
            for arg in args:
                arg_student = args_student[arg]
                arg_solution = args_solution[arg]
                if incorrect_msg is None:
                    msg = feedback_msg + (
                        " The %s argument seems to be incorrect." %
                        get_ord(arg + 1))
                else:
                    msg = incorrect_msg

                test = build_test(arg_student,
                                  arg_solution,
                                  state,
                                  do_eval,
                                  eq_fun,
                                  msg,
                                  add_more=add_more,
                                  highlight=arg_student if highlight else None)
                test.test()

                if not test.result:
                    if feedback is None:
                        feedback = test.get_feedback()
                    success = False
                    break

            if success:
                for key in keywords:
                    key_student = keyw_student[key]
                    key_solution = keyw_solution[key]
                    if incorrect_msg is None:
                        msg = feedback_msg + (
                            " Keyword `%s` seems to be incorrect." % key)
                        add_more = True
                    else:
                        msg = incorrect_msg
                        add_more = False

                    test = build_test(
                        key_student,
                        key_solution,
                        state,
                        do_eval,
                        eq_fun,
                        msg,
                        add_more=add_more,
                        highlight=key_student if highlight else None)
                    test.test()

                    if not test.result:
                        if feedback is None:
                            feedback = test.get_feedback()
                        success = False
                        break

            if success:
                # we have a winner that passes all argument and keyword checks
                state.set_used(name, call_ind, index)
                break

        if not success:
            if feedback is None:
                _msg = state.build_message(
                    "You haven't used enough appropriate calls of `%s()`" %
                    stud_name)
                feedback = Feedback(_msg)
            rep.do_test(Test(feedback))
Example #35
0
 def test_get_ord(self):
     self.assertEqual(utils.get_ord(1), "first")
     self.assertEqual(utils.get_ord(2), "second")
     self.assertEqual(utils.get_ord(3), "third")
     self.assertEqual(utils.get_ord(11), "11th")
Example #36
0
 def build_arg_str(name):
     return "%s argument" % get_ord(name + 1) if isinstance(
         name, int) else "argument `%s`" % name