コード例 #1
0
def _test(state, incorrect_msg, exact_names, tv_name, highlight_name):
    # get parts for testing from state
    # TODO: this could be rewritten to use check_part_index -> has_equal_part, etc..
    stu_vars = state.student_parts[tv_name]
    sol_vars = state.solution_parts[tv_name]

    child_state = state.to_child(
        student_ast=state.student_parts.get(highlight_name),
        solution_ast=state.solution_parts.get(highlight_name),
    )

    # variables exposed to messages
    d = {"stu_vars": stu_vars, "sol_vars": sol_vars, "num_vars": len(sol_vars)}

    if exact_names:
        # feedback for wrong iter var names
        child_state.do_test(
            EqualTest(stu_vars, sol_vars, FeedbackComponent(incorrect_msg, d)))
    else:
        # feedback for wrong number of iter vars
        child_state.do_test(
            EqualTest(len(stu_vars), len(sol_vars),
                      FeedbackComponent(incorrect_msg, d)))

    return state
コード例 #2
0
ファイル: check_funcs.py プロジェクト: datacamp/pythonwhat
def check_node(
    state, name, index=0, typestr="{{ordinal}} node", missing_msg=None, expand_msg=None
):

    if missing_msg is None:
        missing_msg = "The system wants to check the {{typestr}} but hasn't found it."
    if expand_msg is None:
        expand_msg = "Check the {{typestr}}. "

    stu_out = state.ast_dispatcher.find(name, state.student_ast)
    sol_out = state.ast_dispatcher.find(name, state.solution_ast)

    # check if there are enough nodes for index
    fmt_kwargs = {
        "ordinal": get_ord(index + 1) if isinstance(index, int) else "",
        "index": index,
        "name": name,
    }
    fmt_kwargs["typestr"] = render(typestr, fmt_kwargs)

    # test if node can be indexed succesfully
    try:
        stu_out[index]
    except (KeyError, IndexError):  # TODO comment errors
        state.report(missing_msg, fmt_kwargs)

    # get node at index
    stu_part = stu_out[index]
    sol_part = sol_out[index]

    append_message = FeedbackComponent(expand_msg, fmt_kwargs)

    return part_to_child(stu_part, sol_part, append_message, state, node_name=name)
コード例 #3
0
    def to_child(self,
                 append_message: Union[str, FeedbackComponent] = None,
                 **kwargs):
        """Basic implementation of returning a child state"""

        bad_parameters = set(kwargs) - set(self.parameters)
        if bad_parameters:
            raise ValueError("Invalid init parameters for State: %s" %
                             ", ".join(bad_parameters))

        if append_message and not isinstance(append_message,
                                             FeedbackComponent):
            if isinstance(append_message, str):
                append_message = FeedbackComponent(append_message)
            else:
                raise ValueError(
                    "append_message should be a FeedbackComponent or a string")
        kwargs["feedback_context"] = append_message
        kwargs["creator"] = {"type": "to_child", "args": {"state": self}}

        child = copy(self)
        for k, v in kwargs.items():
            setattr(child, k, v)

        return child
コード例 #4
0
def _debug(state: "State", msg="", on_error=False, force=True):
    """
    This SCT function makes the SCT fail with a message containing debugging information
    and highlights the focus of the SCT at that point.

    To make the interruption behave like a student failure, use ``force=False``.
    """
    checks = check_history(state.state_history)

    feedback = ""
    if msg:
        feedback += msg + "\n"

    if checks:
        feedback += "SCT function state history: `{}`".format(" > ".join(checks))

    if state.reporter.tests:
        feedback += "\nLast test: `{}`".format(repr(state.reporter.tests[-1]))

    if on_error:
        # debug on next failure
        state.debug = True
        # or at the end (to prevent debug mode in production)
        state.reporter.fail = True
    else:
        # latest highlight added automatically
        failure_type = InstructorError if force else TestFail
        raise failure_type(state.get_feedback(FeedbackComponent(feedback)), state.state_history)

    return state
コード例 #5
0
ファイル: test_reporter.py プロジェクト: alexyarosh/protowhat
def test_highlighting_path_no_position():
    r = Reporter()
    f = Feedback(FeedbackComponent("msg"), path=Path("test.py"))

    payload = r.build_failed_payload(f)

    expected_payload = {"correct": False, "message": "msg", "path": "test.py"}

    assert payload == expected_payload
コード例 #6
0
def test_object_accessed(state, name, times=1, not_accessed_msg=None):
    """Test if object accessed

    Checks whether an object, or the attribute of an object, are accessed

    Args:
        name (str): the name of the object that should be accessed; can contain dots (for attributes)
        times (int): how often the object specified in name should be accessed.
        not_accessed_msg (str): custom feedback message when the object was not accessed.

    Examples:


        Student code

        | ``import numpy as np``
        | ``arr = np.array([1, 2, 3])``
        | ``x = arr.shape``

        Solution code

        | ``import numpy as np``
        | ``arr = np.array([1, 2, 3])``
        | ``x = arr.shape``
        | ``t = arr.dtype``

        SCT

        | ``test_object_accessed("arr")``: pass.
        | ``test_object_accessed("arr.shape")``: pass.
        | ``test_object_accessed("arr.dtype")``: fail.
    """
    student_object_accesses = state.ast_dispatcher.find(
        "object_accesses", state.student_ast
    )
    student_mappings = state.ast_dispatcher.find("oa_mappings", state.student_ast)

    if not not_accessed_msg:
        stud_name = name
        if "." in stud_name:
            for orig, full_name in student_mappings.items():
                if name.startswith(full_name):
                    stud_name = name.replace(full_name, orig)

        add = " at least %s" % get_times(times) if times > 1 else ""
        not_accessed_msg = "Have you accessed `%s`%s?" % (stud_name, add)

    # name should be contained inside the student_object_accesses.
    # hack: add a dot and do a match on the name with the dot,
    # to make sure you're not matching substrings
    student_hits = [c for c in student_object_accesses if name + "." in c + "."]
    state.do_test(
        BiggerTest(len(student_hits) + 1, times, FeedbackComponent(not_accessed_msg))
    )
コード例 #7
0
def has_equal_part(state, name, msg):
    d = {
        "stu_part": state.student_parts,
        "sol_part": state.solution_parts,
        "name": name,
    }

    state.do_test(
        EqualTest(d["stu_part"][name], d["sol_part"][name],
                  FeedbackComponent(msg, d)))

    return state
コード例 #8
0
ファイル: test_reporter.py プロジェクト: alexyarosh/protowhat
def test_highlighting_offset(offset, highlight, payload_highlight_info):
    r = Reporter()
    f = Feedback(
        FeedbackComponent("msg"),
        highlight=Highlight(highlight),
        highlight_offset=offset,
    )

    payload = r.build_failed_payload(f)

    expected_payload = {"correct": False, "message": "msg", **payload_highlight_info}

    assert payload == expected_payload
コード例 #9
0
ファイル: test_reporter.py プロジェクト: alexyarosh/protowhat
def test_highlighting_offset_proxy():
    r = Reporter()
    f = Feedback(
        FeedbackComponent("msg"),
        highlight=Highlight(highlight_range_1),
        highlight_offset=highlight_range_2,
    )

    payload = r.build_failed_payload(f)

    expected_payload = {"correct": False, "message": "msg", **highlight_combined}

    assert payload == expected_payload
コード例 #10
0
    def __init__(self, feedback: Union[str, FeedbackComponent]):
        """
        Initialize the standard test.

        Args:
            feedback: string or Feedback object
        """
        if issubclass(type(feedback), FeedbackComponent):
            self.feedback = feedback
        elif issubclass(type(feedback), str):
            self.feedback = FeedbackComponent(feedback)
        else:
            raise TypeError(
                "When creating a test, specify either a string or a FeedbackComponent object"
            )

        self.result = None
コード例 #11
0
ファイル: test_reporter.py プロジェクト: alexyarosh/protowhat
def test_highlighting_path():
    r = Reporter()
    f = Feedback(
        FeedbackComponent("msg"),
        highlight=Highlight(highlight_range_1),
        path=Path("test.py"),
    )

    payload = r.build_failed_payload(f)

    expected_payload = {
        "correct": False,
        "message": "msg",
        "path": "test.py",
        **highlight_payload_1,
    }

    assert payload == expected_payload
コード例 #12
0
def is_instance(state, inst, not_instance_msg=None):
    """Check whether an object is an instance of a certain class.

    ``is_instance()`` can currently only be used when chained from ``check_object()``, the function that is
    used to 'zoom in' on the object of interest.

    Args:
        inst (class): The class that the object should have.
        not_instance_msg (str): When specified, this overrides the automatically generated message in case
            the object does not have the expected class.
        state (State): The state that is passed in through the SCT chain (don't specify this).

    :Example:

        Student code and solution code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])

        SCT::

            # Verify the class of arr
            import numpy
            Ex().check_object('arr').is_instance(numpy.ndarray)
    """

    state.assert_is(["object_assignments"], "is_instance", ["check_object"])

    sol_name = state.solution_parts.get("name")
    stu_name = state.student_parts.get("name")

    if not_instance_msg is None:
        not_instance_msg = "Is it a {{inst.__name__}}?"

    if not isInstanceInProcess(sol_name, inst, state.solution_process):
        raise InstructorError.from_message(
            "`is_instance()` noticed that `%s` is not a `%s` in the solution process."
            % (sol_name, inst.__name__))

    feedback = FeedbackComponent(not_instance_msg, {"inst": inst})
    state.do_test(
        InstanceProcessTest(stu_name, inst, state.student_process, feedback))

    return state
コード例 #13
0
ファイル: check_funcs.py プロジェクト: datacamp/pythonwhat
def check_call(state, callstr, argstr=None, expand_msg=None):
    """When checking a function definition of lambda function,
    prepare has_equal_x for checking the call of a user-defined function.

    Args:
        callstr (str): call string that specifies how the function should be called, e.g. `f(1, a = 2)`.
           ``check_call()`` will replace ``f`` with the function/lambda you're targeting.
        argstr (str): If specified, this overrides the way the function call is refered to in the expand message.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
        state (State): state object that is chained from.

    :Example:

        Student and solution code::

            def my_power(x):
                print("calculating sqrt...")
                return(x * x)

        SCT::

            Ex().check_function_def('my_power').multi(
                check_call("f(3)").has_equal_value()
                check_call("f(3)").has_equal_output()
            )
    """

    state.assert_is(
        ["function_defs", "lambda_functions"],
        "check_call",
        ["check_function_def", "check_lambda_function"],
    )

    if expand_msg is None:
        expand_msg = "To verify it, we reran {{argstr}}. "

    stu_part, _argstr = build_call(callstr, state.student_parts["node"])
    sol_part, _ = build_call(callstr, state.solution_parts["node"])

    append_message = FeedbackComponent(expand_msg, {"argstr": argstr or _argstr})
    child = part_to_child(stu_part, sol_part, append_message, state)

    return child
コード例 #14
0
ファイル: check_funcs.py プロジェクト: datacamp/pythonwhat
def check_part(state, name, part_msg, missing_msg=None, expand_msg=None):
    """Return child state with name part as its ast tree"""

    if missing_msg is None:
        missing_msg = "Are you sure you defined the {{part}}? "
    if expand_msg is None:
        expand_msg = "Did you correctly specify the {{part}}? "

    if not part_msg:
        part_msg = name
    append_message = FeedbackComponent(expand_msg, {"part": part_msg})

    has_part(state, name, missing_msg, append_message.kwargs)

    stu_part = state.student_parts[name]
    sol_part = state.solution_parts[name]

    assert_ast(state, sol_part, append_message.kwargs)

    return part_to_child(stu_part, sol_part, append_message, state)
コード例 #15
0
ファイル: check_funcs.py プロジェクト: datacamp/pythonwhat
def check_part_index(state, name, index, part_msg, missing_msg=None, expand_msg=None):
    """Return child state with indexed name part as its ast tree.

    ``index`` can be:

    - an integer, in which case the student/solution_parts are indexed by position.
    - a string, in which case the student/solution_parts are expected to be a dictionary.
    - a list of indices (which can be integer or string), in which case the student parts are indexed step by step.
    """

    if missing_msg is None:
        missing_msg = "Are you sure you defined the {{part}}? "
    if expand_msg is None:
        expand_msg = "Did you correctly specify the {{part}}? "

    # create message
    ordinal = get_ord(index + 1) if isinstance(index, int) else ""
    fmt_kwargs = {"index": index, "ordinal": ordinal}
    fmt_kwargs.update(part=render(part_msg, fmt_kwargs))

    append_message = FeedbackComponent(expand_msg, fmt_kwargs)

    # check there are enough parts for index
    has_part(state, name, missing_msg, fmt_kwargs, index)

    # get part at index
    stu_part = state.student_parts[name]
    sol_part = state.solution_parts[name]

    if isinstance(index, list):
        for ind in index:
            stu_part = stu_part[ind]
            sol_part = sol_part[ind]
    else:
        stu_part = stu_part[index]
        sol_part = sol_part[index]

    assert_ast(state, sol_part, fmt_kwargs)

    # return child state from part
    return part_to_child(stu_part, sol_part, append_message, state)
コード例 #16
0
ファイル: check_logic.py プロジェクト: datacamp/pythonwhat
def override(state, solution):
    """Override the solution code with something arbitrary.

    There might be cases in which you want to temporarily override the solution code
    so you can allow for alternative ways of solving an exercise.
    When you use ``override()`` in an SCT chain, the remainder of that SCT chain will
    run as if the solution code you specified is the only code that was in the solution.

    Check the glossary for an example (pandas plotting)

    Args:
        solution: solution code as a string that overrides the original solution code.
        state: State instance describing student and solution code. Can be omitted if used with Ex().
    """

    # the old ast may be a number of node types, but generally either a
    # (1) ast.Module, or for single expressions...
    # (2) whatever was grabbed using module.body[0]
    # (3) module.body[0].value, when module.body[0] is an Expr node
    old_ast = state.solution_ast
    new_ast = ast.parse(solution)
    if not isinstance(old_ast, ast.Module) and len(new_ast.body) == 1:
        expr = new_ast.body[0]
        candidates = [expr, expr.value] if isinstance(expr,
                                                      ast.Expr) else [expr]
        for node in candidates:
            if isinstance(node, old_ast.__class__):
                new_ast = node
                break

    kwargs = state.feedback_context.kwargs if state.feedback_context else {}
    child = state.to_child(
        solution_ast=new_ast,
        student_ast=state.student_ast,
        highlight=state.highlight,
        append_message=FeedbackComponent("", kwargs),
    )

    return child
コード例 #17
0
ファイル: utils.py プロジェクト: datacamp/pythonwhat
def call(state,
         args,
         test="value",
         incorrect_msg=None,
         error_msg=None,
         argstr=None,
         func=None,
         **kwargs):
    """Use ``check_call()`` in combination with ``has_equal_x()`` instead.
    """

    if incorrect_msg is None:
        incorrect_msg = MSG_CALL_INCORRECT
    if error_msg is None:
        error_msg = MSG_CALL_ERROR_INV if test == "error" else MSG_CALL_ERROR

    assert test in ("value", "output", "error")

    get_func = evalCalls[test]

    # Run for Solution --------------------------------------------------------
    eval_sol, str_sol = run_call(args, state.solution_parts["node"],
                                 state.solution_process, get_func, **kwargs)

    if (test == "error") ^ isinstance(eval_sol, Exception):
        with debugger(state):
            state.report(
                "Calling {{argstr}} resulted in an error (or not an error if testing for one). Error message: {{type_err}} {{str_sol}}",
                dict(type_err=type(eval_sol), str_sol=str_sol, argstr=argstr),
            )

    if isinstance(eval_sol, ReprFail):
        with debugger(state):
            state.report(
                "Can't get the result of calling {{argstr}}: {{eval_sol.info}}",
                dict(argstr=argstr, eval_sol=eval_sol),
            )

    # Run for Submission ------------------------------------------------------
    eval_stu, str_stu = run_call(args, state.student_parts["node"],
                                 state.student_process, get_func, **kwargs)
    action_strs = {
        "value": "return",
        "output": "print out",
        "error": "error out with the message",
    }
    fmt_kwargs = {
        "part": argstr,
        "argstr": argstr,
        "str_sol": str_sol,
        "str_stu": str_stu,
        "action": action_strs[test],
    }

    # either error test and no error, or vice-versa
    stu_node = state.student_parts["node"]
    stu_state = state.to_child(highlight=stu_node)
    if (test == "error") ^ isinstance(eval_stu, Exception):
        stu_state.report(error_msg, fmt_kwargs)

    # incorrect result
    stu_state.do_test(
        EqualTest(eval_sol, eval_stu,
                  FeedbackComponent(incorrect_msg, fmt_kwargs), func))

    return state
コード例 #18
0
 def from_message(cls, message: str) -> "Failure":
     return cls(Feedback(FeedbackComponent(message)), [])
コード例 #19
0
def has_import(
    state,
    name,
    same_as=False,
    not_imported_msg="Did you import `{{pkg}}`?",
    incorrect_as_msg="Did you import `{{pkg}}` as `{{alias}}`?",
):
    """Checks whether student imported a package or function correctly.

    Python features many ways to import packages.
    All of these different methods revolve around the ``import``, ``from`` and ``as`` keywords.
    ``has_import()`` provides a robust way to check whether a student correctly imported a certain package.

    By default, ``has_import()`` allows for different ways of aliasing the imported package or function.
    If you want to make sure the correct alias was used to refer to the package or function that was imported,
    set ``same_as=True``.

    Args:
        name (str): the name of the package that has to be checked.
        same_as (bool): if True, the alias of the package or function has to be the same. Defaults to False.
        not_imported_msg (str): feedback message when the package is not imported.
        incorrect_as_msg (str): feedback message if the alias is wrong.

    :Example:

        Example 1, where aliases don't matter (defaut): ::

            # solution
            import matplotlib.pyplot as plt

            # sct
            Ex().has_import("matplotlib.pyplot")

            # passing submissions
            import matplotlib.pyplot as plt
            from matplotlib import pyplot as plt
            import matplotlib.pyplot as pltttt

            # failing submissions
            import matplotlib as mpl

        Example 2, where the SCT is coded so aliases do matter: ::

            # solution
            import matplotlib.pyplot as plt

            # sct
            Ex().has_import("matplotlib.pyplot", same_as=True)

            # passing submissions
            import matplotlib.pyplot as plt
            from matplotlib import pyplot as plt

            # failing submissions
            import matplotlib.pyplot as pltttt

    """
    student_imports = state.ast_dispatcher.find("imports", state.student_ast)
    solution_imports = state.ast_dispatcher.find("imports", state.solution_ast)

    if name not in solution_imports:
        raise InstructorError.from_message(
            "`has_import()` couldn't find an import of the package %s in your solution code."
            % name)

    fmt_kwargs = {"pkg": name, "alias": solution_imports[name]}

    state.do_test(
        DefinedCollTest(name, student_imports,
                        FeedbackComponent(not_imported_msg, fmt_kwargs)))

    if same_as:
        state.do_test(
            EqualTest(
                solution_imports[name],
                student_imports[name],
                FeedbackComponent(incorrect_as_msg, fmt_kwargs),
            ))

    return state
コード例 #20
0
def has_printout(state,
                 index,
                 not_printed_msg=None,
                 pre_code=None,
                 name=None,
                 copy=False):
    """Check if the right printouts happened.

    ``has_printout()`` will look for the printout in the solution code that you specified with ``index`` (0 in this case), rerun the ``print()`` call in
    the solution process, capture its output, and verify whether the output is present in the output of the student.

    This is more robust as ``Ex().check_function('print')`` initiated chains as students can use as many
    printouts as they want, as long as they do the correct one somewhere.

    Args:
        index (int): index of the ``print()`` call in the solution whose output you want to search for in the student output.
        not_printed_msg (str): if specified, this overrides the default message that is generated when the output
          is not found in the student output.
        pre_code (str): Python code as a string that is executed before running the targeted student call.
          This is the ideal place to set a random seed, for example.
        copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
          accidentally be mutated. Disabled by default, which speeds up SCTs.
        state (State): state as passed by the SCT chain. Don't specify this explicitly.

    :Example:

        Suppose you want somebody to print out 4: ::

            print(1, 2, 3, 4)

        The following SCT would check that: ::

            Ex().has_printout(0)

        All of the following SCTs would pass: ::

            print(1, 2, 3, 4)
            print('1 2 3 4')
            print(1, 2, '3 4')
            print("random"); print(1, 2, 3, 4)

    :Example:

        Watch out: ``has_printout()`` will effectively **rerun** the ``print()`` call in the solution process after the entire solution script was executed.
        If your solution script updates the value of `x` after executing it, ``has_printout()`` will not work.

        Suppose you have the following solution: ::

            x = 4
            print(x)
            x = 6

        The following SCT will not work: ::

            Ex().has_printout(0)

        Why? When the ``print(x)`` call is executed, the value of ``x`` will be 6, and pythonwhat will look for the output `'6`' in the output the student generated.
        In cases like these, ``has_printout()`` cannot be used.

    :Example:

        Inside a for loop ``has_printout()``

        Suppose you have the following solution: ::

            for i in range(5):
                print(i)

        The following SCT will not work: ::

            Ex().check_for_loop().check_body().has_printout(0)

        The reason is that ``has_printout()`` can only be called from the root state. ``Ex()``.
        If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead: ::

            Ex().check_for_loop().check_body().\\
                set_context(0).check_function('print').\\
                check_args(0).has_equal_value()

    """

    extra_msg = "If you want to check printouts done in e.g. a for loop, you have to use a `check_function('print')` chain instead."
    state.assert_execution_root("has_printout", extra_msg=extra_msg)

    if not_printed_msg is None:
        not_printed_msg = (
            "Have you used `{{sol_call}}` to do the appropriate printouts?")

    try:
        sol_call_ast = state.ast_dispatcher.find(
            "function_calls", state.solution_ast)["print"][index]["node"]
    except (KeyError, IndexError):
        raise InstructorError.from_message(
            "`has_printout({})` couldn't find the {} print call in your solution."
            .format(index, get_ord(index + 1)))

    out_sol, str_sol = getOutputInProcess(
        tree=sol_call_ast,
        process=state.solution_process,
        context=state.solution_context,
        env=state.solution_env,
        pre_code=pre_code,
        copy=copy,
    )

    sol_call_str = state.solution_ast_tokens.get_text(sol_call_ast)

    if isinstance(str_sol, Exception):
        with debugger(state):
            state.report(
                "Evaluating the solution expression {} raised error in solution process."
                "Error: {} - {}".format(sol_call_str, type(out_sol), str_sol))

    has_output(
        state,
        out_sol.strip(),
        pattern=False,
        no_output_msg=FeedbackComponent(not_printed_msg,
                                        {"sol_call": sol_call_str}),
    )

    return state
コード例 #21
0
def has_expr(
        state,
        incorrect_msg=None,
        error_msg=None,
        undefined_msg=None,
        append=None,
        extra_env=None,
        context_vals=None,
        pre_code=None,
        expr_code=None,
        name=None,
        copy=True,
        func=None,
        override=None,
        test=None,  # todo: default or arg before state
):

    if (
            append is None
    ):  # if not specified, set to False if incorrect_msg was manually specified
        append = incorrect_msg is None
    if incorrect_msg is None:
        if name:
            incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
        elif expr_code:
            incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
        else:
            incorrect_msg = DEFAULT_INCORRECT_MSG
    if undefined_msg is None:
        undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
    if error_msg is None:
        if test == "error":
            error_msg = DEFAULT_ERROR_MSG_INV
        else:
            error_msg = DEFAULT_ERROR_MSG

    if state.solution_code is not None and isinstance(expr_code, str):
        expr_code = expr_code.replace("__focus__", state.solution_code)

    get_func = partial(
        evalCalls[test],
        extra_env=extra_env,
        context_vals=context_vals,
        pre_code=pre_code,
        expr_code=expr_code,
        name=name,
        copy=copy,
    )

    if override is not None:
        # don't bother with running expression and fetching output/value
        # eval_sol, str_sol = eval
        eval_sol, str_sol = override, str(override)
    else:
        eval_sol, str_sol = get_func(
            tree=state.solution_ast,
            process=state.solution_process,
            context=state.solution_context,
            env=state.solution_env,
        )

        if (test == "error") ^ isinstance(eval_sol, Exception):
            raise InstructorError.from_message(
                "Evaluating expression raised error in solution process (or didn't raise if testing for one). "
                "Error: {} - {}".format(type(eval_sol), str_sol))
        if isinstance(eval_sol, ReprFail):
            raise InstructorError.from_message(
                "Couldn't extract the value for the highlighted expression from the solution process: "
                + eval_sol.info)

    eval_stu, str_stu = get_func(
        tree=state.student_ast,
        process=state.student_process,
        context=state.student_context,
        env=state.student_env,
    )

    # kwargs ---
    fmt_kwargs = {
        "stu_part": state.student_parts,
        "sol_part": state.solution_parts,
        "name": name,
        "test": test,
        "test_desc": "" if test == "value" else "the %s " % test,
        "expr_code": expr_code,
    }

    fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu))
    fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol))
    if incorrect_msg == DEFAULT_INCORRECT_MSG and (
            fmt_kwargs["stu_eval"] is None or fmt_kwargs["sol_eval"] is None
            or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]):
        incorrect_msg = "Expected something different."

    # tests ---
    # error in process
    if (test == "error") ^ isinstance(eval_stu, Exception):
        fmt_kwargs["stu_str"] = str_stu
        state.report(error_msg, fmt_kwargs, append=append)

    # name is undefined after running expression
    if isinstance(eval_stu, UndefinedValue):
        state.report(undefined_msg, fmt_kwargs, append=append)

    # test equality of results
    state.do_test(
        EqualTest(
            eval_stu,
            eval_sol,
            FeedbackComponent(incorrect_msg, fmt_kwargs, append=append),
            func,
        ))

    return state
コード例 #22
0
def has_equal_ast(state,
                  incorrect_msg=None,
                  code=None,
                  exact=True,
                  append=None):
    """Test whether abstract syntax trees match between the student and solution code.

    ``has_equal_ast()`` can be used in two ways:

    * As a robust version of ``has_code()``. By setting ``code``, you can look for the AST representation of ``code`` in the student's submission.
      But be aware that ``a`` and ``a = 1`` won't match, as reading and assigning are not the same in an AST.
      Use ``ast.dump(ast.parse(code))`` to see an AST representation of ``code``.
    * As an expression-based check when using more advanced SCT chain, e.g. to compare the equality of expressions to set function arguments.

    Args:
        incorrect_msg: message displayed when ASTs mismatch. When you specify ``code`` yourself, you have to specify this.
        code: optional code to use instead of the solution AST.
        exact: whether the representations must match exactly. If false, the solution AST
               only needs to be contained within the student AST (similar to using test student typed).
               Defaults to ``True``, unless the ``code`` argument has been specified.

    :Example:

        Student and Solution Code::

            dict(a = 'value').keys()

        SCT::

            # all pass
            Ex().has_equal_ast()
            Ex().has_equal_ast(code = "dict(a = 'value').keys()")
            Ex().has_equal_ast(code = "dict(a = 'value')", exact = False)

        Student and Solution Code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Check underlying value of arugment a of np.mean:
            Ex().check_function('numpy.mean').check_args('a').has_equal_ast()

            # Only check AST equality of expression used to specify argument a:
            Ex().check_function('numpy.mean').check_args('a').has_equal_ast()

    """
    if utils.v2_only():
        state.assert_is_not(["object_assignments"], "has_equal_ast",
                            ["check_object"])
        state.assert_is_not(["function_calls"], "has_equal_ast",
                            ["check_function"])

    if code and incorrect_msg is None:
        raise InstructorError.from_message(
            "If you manually specify the code to match inside has_equal_ast(), "
            "you have to explicitly set the `incorrect_msg` argument.")

    if (
            append is None
    ):  # if not specified, set to False if incorrect_msg was manually specified
        append = incorrect_msg is None
    if incorrect_msg is None:
        incorrect_msg = "Expected `{{sol_str}}`, but got `{{stu_str}}`."

    def parse_tree(tree):
        # get contents of module.body if only 1 element
        crnt = (tree.body[0] if isinstance(tree, ast.Module)
                and len(tree.body) == 1 else tree)

        # remove Expr if it exists
        return ast.dump(crnt.value if isinstance(crnt, ast.Expr) else crnt)

    stu_rep = parse_tree(state.student_ast)
    sol_rep = parse_tree(state.solution_ast if not code else ast.parse(code))

    fmt_kwargs = {
        "sol_str": state.solution_code if not code else code,
        "stu_str": state.student_code,
    }

    if exact and not code:
        state.do_test(
            EqualTest(
                stu_rep,
                sol_rep,
                FeedbackComponent(incorrect_msg, fmt_kwargs, append=append),
            ))
    elif sol_rep not in stu_rep:
        state.report(incorrect_msg, fmt_kwargs, append=append)

    return state
コード例 #23
0
    def report(self, feedback: str, kwargs=None, append=True):
        test_feedback = FeedbackComponent(feedback, kwargs, append)
        test = Fail(test_feedback)

        return self.do_test(test)
コード例 #24
0
def check_keys(state, key, missing_msg=None, expand_msg=None):
    """Check whether an object (dict, DataFrame, etc) has a key.

    ``check_keys()`` can currently only be used when chained from ``check_object()``, the function that is
    used to 'zoom in' on the object of interest.

    Args:
        key (str): Name of the key that the object should have.
        missing_msg (str): When specified, this overrides the automatically generated
            message in case the key does not exist.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
        state (State): The state that is passed in through the SCT chain (don't specify this).

    :Example:

        Student code and solution code::

            x = {'a': 2}

        SCT::

            # Verify that x contains a key a
            Ex().check_object('x').check_keys('a')

            # Verify that x contains a key a and a is correct.
            Ex().check_object('x').check_keys('a').has_equal_value()

    """

    state.assert_is(["object_assignments"], "is_instance",
                    ["check_object", "check_df"])

    if missing_msg is None:
        missing_msg = "There is no {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`."
    if expand_msg is None:
        expand_msg = "Did you correctly set the {{ 'column' if 'DataFrame' in parent.typestr else 'key' }} `'{{key}}'`? "

    sol_name = state.solution_parts.get("name")
    stu_name = state.student_parts.get("name")

    if not isDefinedCollInProcess(sol_name, key, state.solution_process):
        raise InstructorError.from_message(
            "`check_keys()` couldn't find key `%s` in object `%s` in the solution process."
            % (key, sol_name))

    # check if key available
    state.do_test(
        DefinedCollProcessTest(stu_name, key, state.student_process,
                               FeedbackComponent(missing_msg, {"key": key})))

    def get_part(name, key, highlight):
        if isinstance(key, str):
            slice_val = ast.Str(s=key)
        else:
            slice_val = ast.parse(str(key)).body[0].value
        expr = ast.Subscript(
            value=ast.Name(id=name, ctx=ast.Load()),
            slice=ast.Index(value=slice_val),
            ctx=ast.Load(),
        )
        ast.fix_missing_locations(expr)
        return {"node": expr, "highlight": highlight}

    stu_part = get_part(stu_name, key, state.student_parts.get("highlight"))
    sol_part = get_part(sol_name, key, state.solution_parts.get("highlight"))
    append_message = FeedbackComponent(expand_msg, {"key": key})
    child = part_to_child(stu_part, sol_part, append_message, state)
    return child
コード例 #25
0
    def to_child(self, append_message=None, node_name="", **kwargs):
        """Dive into nested tree.

        Set the current state as a state with a subtree of this syntax tree as
        student tree and solution tree. This is necessary when testing if statements or
        for loops for example.
        """
        bad_parameters = set(kwargs) - set(self.parameters)
        if bad_parameters:
            raise ValueError("Invalid init parameters for State: %s" %
                             ", ".join(bad_parameters))

        base_kwargs = {
            attr: getattr(self, attr)
            for attr in self.parameters if hasattr(self, attr)
            and attr not in ["ast_dispatcher", "highlight"]
        }

        if append_message and not isinstance(append_message,
                                             FeedbackComponent):
            append_message = FeedbackComponent(append_message)
        kwargs["feedback_context"] = append_message
        kwargs["creator"] = {"type": "to_child", "args": {"state": self}}

        def update_kwarg(name, func):
            kwargs[name] = func(kwargs[name])

        def update_context(name):
            update_kwarg(name, getattr(self, name).update_ctx)

        for ast_arg in ["student_ast", "solution_ast"]:
            if isinstance(kwargs.get(ast_arg), list):
                update_kwarg(ast_arg, wrap_in_module)

        if kwargs.get("student_ast") and kwargs.get("student_code") is None:
            kwargs["student_code"] = self.student_ast_tokens.get_text(
                kwargs["student_ast"])
        if kwargs.get("solution_ast") and kwargs.get("solution_code") is None:
            kwargs["solution_code"] = self.solution_ast_tokens.get_text(
                kwargs["solution_ast"])

        for context in [
                "student_context",
                "solution_context",
                "student_env",
                "solution_env",
        ]:
            if context in kwargs:
                if kwargs[context] is not None:
                    update_context(context)
                else:
                    kwargs.pop(context)

        klass = self.SUBCLASSES[node_name] if node_name else State
        init_kwargs = {**base_kwargs, **kwargs}
        child = klass(**init_kwargs)

        extra_attrs = set(vars(self)) - set(self.parameters)
        for attr in extra_attrs:
            # don't copy attrs set on new instances in init
            # the cached manual_sigs is passed
            if attr not in {"ast_dispatcher", "converters"}:
                setattr(child, attr, getattr(self, attr))

        return child
コード例 #26
0
def check_object(state,
                 index,
                 missing_msg=None,
                 expand_msg=None,
                 typestr="variable"):
    """Check object existence (and equality)

    Check whether an object is defined in the student's process, and zoom in on its value in both
    student and solution process to inspect quality (with has_equal_value().

    In ``pythonbackend``, both the student's submission as well as the solution code are executed, in separate processes.
    ``check_object()`` looks at these processes and checks if the referenced object is available in the student process.
    Next, you can use ``has_equal_value()`` to check whether the objects in the student and solution process correspond.

    Args:
        index (str): the name of the object which value has to be checked.
        missing_msg (str): feedback message when the object is not defined in the student process.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.

    :Example:

        Suppose you want the student to create a variable ``x``, equal to 15: ::

            x = 15

        The following SCT will verify this: ::

            Ex().check_object("x").has_equal_value()

        - ``check_object()`` will check if the variable ``x`` is defined in the student process.
        - ``has_equal_value()`` will check whether the value of ``x`` in the solution process is the same as in the student process.

        Note that ``has_equal_value()`` only looks at **end result** of a variable in the student process.
        In the example, how the object ``x`` came about in the student's submission, does not matter.
        This means that all of the following submission will also pass the above SCT: ::

            x = 15
            x = 12 + 3
            x = 3; x += 12

    :Example:

        As the previous example mentioned, ``has_equal_value()`` only looks at the **end result**. If your exercise is
        first initializing and object and further down the script is updating the object, you can only look at the final value!

        Suppose you want the student to initialize and populate a list `my_list` as follows: ::

            my_list = []
            for i in range(20):
                if i % 3 == 0:
                    my_list.append(i)

        There is no robust way to verify whether `my_list = [0]` was coded correctly in a separate way.
        The best SCT would look something like this: ::

            msg = "Have you correctly initialized `my_list`?"
            Ex().check_correct(
                check_object('my_list').has_equal_value(),
                multi(
                    # check initialization: [] or list()
                    check_or(
                        has_equal_ast(code = "[]", incorrect_msg = msg),
                        check_function('list')
                    ),
                    check_for_loop().multi(
                        check_iter().has_equal_value(),
                        check_body().check_if_else().multi(
                            check_test().multi(
                                set_context(2).has_equal_value(),
                                set_context(3).has_equal_value()
                            ),
                            check_body().set_context(3).\\
                                set_env(my_list = [0]).\\
                                has_equal_value(name = 'my_list')
                        )
                    )
                )
            )

        - ``check_correct()`` is used to robustly check whether ``my_list`` was built correctly.
        - If ``my_list`` is not correct, **both** the initialization and the population code are checked.

    :Example:

        Because checking object correctness incorrectly is such a common misconception, we're adding another example: ::

            import pandas as pd
            df = pd.DataFrame({'a': [1, 2, 3], 'b': [4, 5, 6]})
            df['c'] = [7, 8, 9]

        The following SCT would be **wrong**, as it does not factor in the possibility that the 'add column ``c``' step could've been wrong: ::

            Ex().check_correct(
                check_object('df').has_equal_value(),
                check_function('pandas.DataFrame').check_args(0).has_equal_value()
            )

        The following SCT would be better, as it is specific to the steps: ::

            # verify the df = pd.DataFrame(...) step
            Ex().check_correct(
                check_df('df').multi(
                    check_keys('a').has_equal_value(),
                    check_keys('b').has_equal_value()
                ),
                check_function('pandas.DataFrame').check_args(0).has_equal_value()
            )

            # verify the df['c'] = [...] step
            Ex().check_df('df').check_keys('c').has_equal_value()

    :Example:

        pythonwhat compares the objects in the student and solution process with the ``==`` operator.
        For basic objects, this ``==`` is operator is properly implemented, so that the objects can be effectively compared.
        For more complex objects that are produced by third-party packages, however, it's possible that this equality operator is not implemented in a way you'd expect.
        Often, for these object types the ``==`` will compare the actual object instances: ::

            # pre exercise code
            class Number():
                def __init__(self, n):
                    self.n = n

            # solution
            x = Number(1)

            # sct that won't work
            Ex().check_object().has_equal_value()

            # sct
            Ex().check_object().has_equal_value(expr_code = 'x.n')

            # submissions that will pass this sct
            x = Number(1)
            x = Number(2 - 1)

        The basic SCT like in the previous example will notwork here.
        Notice how we used the ``expr_code`` argument to _override_ which value `has_equal_value()` is checking.
        Instead of checking whether `x` corresponds between student and solution process, it's now executing the expression ``x.n``
        and seeing if the result of running this expression in both student and solution process match.

    """

    # Only do the assertion if PYTHONWHAT_V2_ONLY is set to '1'
    if v2_only():
        extra_msg = "If you want to check the value of an object in e.g. a for loop, use `has_equal_value(name = 'my_obj')` instead."
        state.assert_execution_root("check_object", extra_msg=extra_msg)

    if missing_msg is None:
        missing_msg = "Did you define the {{typestr}} `{{index}}` without errors?"

    if expand_msg is None:
        expand_msg = "Did you correctly define the {{typestr}} `{{index}}`? "

    if (not isDefinedInProcess(index, state.solution_process)
            and state.has_different_processes()):
        raise InstructorError.from_message(
            "`check_object()` couldn't find object `%s` in the solution process."
            % index)

    append_message = FeedbackComponent(expand_msg, {
        "index": index,
        "typestr": typestr
    })

    # create child state, using either parser output, or create part from name
    fallback = lambda: ObjectAssignmentParser.get_part(index)
    stu_part = state.ast_dispatcher.find("object_assignments",
                                         state.student_ast).get(
                                             index, fallback())
    sol_part = state.ast_dispatcher.find("object_assignments",
                                         state.solution_ast).get(
                                             index, fallback())

    # test object exists
    state.do_test(
        DefinedProcessTest(
            index,
            state.student_process,
            FeedbackComponent(missing_msg, append_message.kwargs),
        ))

    child = part_to_child(stu_part,
                          sol_part,
                          append_message,
                          state,
                          node_name="object_assignments")

    return child
コード例 #27
0
ファイル: check_function.py プロジェクト: datacamp/pythonwhat
def check_function(
    state,
    name,
    index=0,
    missing_msg=None,
    params_not_matched_msg=None,
    expand_msg=None,
    signature=True,
):
    """Check whether a particular function is called.

    ``check_function()`` is typically followed by:

    - ``check_args()`` to check whether the arguments were specified.
      In turn, ``check_args()`` can be followed by ``has_equal_value()`` or ``has_equal_ast()``
      to assert that the arguments were correctly specified.
    - ``has_equal_value()`` to check whether rerunning the function call coded by the student
      gives the same result as calling the function call as in the solution.

    Checking function calls is a tricky topic. Please visit the
    `dedicated article <articles/checking_function_calls.html>`_ for more explanation,
    edge cases and best practices.

    Args:
        name (str): the name of the function to be tested. When checking functions in packages, always
            use the 'full path' of the function.
        index (int): index of the function call to be checked. Defaults to 0.
        missing_msg (str): If specified, this overrides an automatically generated feedback message in case
            the student did not call the function correctly.
        params_not_matched_msg (str): If specified, this overrides an automatically generated feedback message
            in case the function parameters were not successfully matched.
        expand_msg (str): If specified, this overrides any messages that are prepended by previous SCT chains.
        signature (Signature): Normally, check_function() can figure out what the function signature is,
            but it might be necessary to use ``sig_from_params()`` to manually build a signature and pass this along.
        state (State): State object that is passed from the SCT Chain (don't specify this).

    :Examples:

        Student code and solution code::

            import numpy as np
            arr = np.array([1, 2, 3, 4, 5])
            np.mean(arr)

        SCT::

            # Verify whether arr was correctly set in np.mean
            Ex().check_function('numpy.mean').check_args('a').has_equal_value()

            # Verify whether np.mean(arr) produced the same result
            Ex().check_function('numpy.mean').has_equal_value()
    """

    append_missing = missing_msg is None
    append_params_not_matched = params_not_matched_msg is None
    if missing_msg is None:
        missing_msg = MISSING_MSG
    if expand_msg is None:
        expand_msg = PREPEND_MSG
    if params_not_matched_msg is None:
        params_not_matched_msg = SIG_ISSUE_MSG

    stu_out = state.ast_dispatcher.find("function_calls", state.student_ast)
    sol_out = state.ast_dispatcher.find("function_calls", state.solution_ast)

    student_mappings = state.ast_dispatcher.find("mappings", state.student_ast)

    fmt_kwargs = {
        "times": get_times(index + 1),
        "ord": get_ord(index + 1),
        "index": index,
        "mapped_name": get_mapped_name(name, student_mappings),
    }

    # Get Parts ----
    # Copy, otherwise signature binding overwrites sol_out[name][index]['args']
    with debugger(state):
        try:
            sol_parts = {**sol_out[name][index]}
        except KeyError:
            state.report(
                "`check_function()` couldn't find a call of `%s()` in the solution code. Make sure you get the mapping right!"
                % name)
        except IndexError:
            state.report(
                "`check_function()` couldn't find %s calls of `%s()` in your solution code."
                % (index + 1, name))

    try:
        # Copy, otherwise signature binding overwrites stu_out[name][index]['args']
        stu_parts = {**stu_out[name][index]}
    except (KeyError, IndexError):
        state.report(missing_msg, fmt_kwargs, append=append_missing)

    # Signatures -----
    if signature:
        signature = None if isinstance(signature, bool) else signature
        get_sig = partial(
            getSignatureInProcess,
            name=name,
            signature=signature,
            manual_sigs=state.get_manual_sigs(),
        )

        try:
            sol_sig = get_sig(mapped_name=sol_parts["name"],
                              process=state.solution_process)
            sol_parts["args"] = bind_args(sol_sig, sol_parts["args"])
        except Exception as e:
            with debugger(state):
                state.report(
                    "`check_function()` couldn't match the %s call of `%s` to its signature:\n%s "
                    % (get_ord(index + 1), name, e))

        try:
            stu_sig = get_sig(mapped_name=stu_parts["name"],
                              process=state.student_process)
            stu_parts["args"] = bind_args(stu_sig, stu_parts["args"])
        except Exception:
            state.to_child(highlight=stu_parts["node"]).report(
                params_not_matched_msg,
                fmt_kwargs,
                append=append_params_not_matched)

    # three types of parts: pos_args, keywords, args (e.g. these are bound to sig)
    append_message = FeedbackComponent(expand_msg, fmt_kwargs)
    child = part_to_child(stu_parts,
                          sol_parts,
                          append_message,
                          state,
                          node_name="function_calls")
    return child