Esempio n. 1
0
 def set_information(self, key, value):
     self.information[key] = utils.shorten_str(str(value))
Esempio n. 2
0
def has_expr(
        state,
        incorrect_msg=None,
        error_msg=None,
        undefined_msg=None,
        append=None,
        extra_env=None,
        context_vals=None,
        pre_code=None,
        expr_code=None,
        name=None,
        copy=True,
        func=None,
        override=None,
        test=None,  # todo: default or arg before state
):

    if (
            append is None
    ):  # if not specified, set to False if incorrect_msg was manually specified
        append = incorrect_msg is None
    if incorrect_msg is None:
        if name:
            incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
        elif expr_code:
            incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
        else:
            incorrect_msg = DEFAULT_INCORRECT_MSG
    if undefined_msg is None:
        undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
    if error_msg is None:
        if test == "error":
            error_msg = DEFAULT_ERROR_MSG_INV
        else:
            error_msg = DEFAULT_ERROR_MSG

    if state.solution_code is not None and isinstance(expr_code, str):
        expr_code = expr_code.replace("__focus__", state.solution_code)

    get_func = partial(
        evalCalls[test],
        extra_env=extra_env,
        context_vals=context_vals,
        pre_code=pre_code,
        expr_code=expr_code,
        name=name,
        copy=copy,
    )

    if override is not None:
        # don't bother with running expression and fetching output/value
        # eval_sol, str_sol = eval
        eval_sol, str_sol = override, str(override)
    else:
        eval_sol, str_sol = get_func(
            tree=state.solution_ast,
            process=state.solution_process,
            context=state.solution_context,
            env=state.solution_env,
        )

        if (test == "error") ^ isinstance(eval_sol, Exception):
            raise InstructorError.from_message(
                "Evaluating expression raised error in solution process (or didn't raise if testing for one). "
                "Error: {} - {}".format(type(eval_sol), str_sol))
        if isinstance(eval_sol, ReprFail):
            raise InstructorError.from_message(
                "Couldn't extract the value for the highlighted expression from the solution process: "
                + eval_sol.info)

    eval_stu, str_stu = get_func(
        tree=state.student_ast,
        process=state.student_process,
        context=state.student_context,
        env=state.student_env,
    )

    # kwargs ---
    fmt_kwargs = {
        "stu_part": state.student_parts,
        "sol_part": state.solution_parts,
        "name": name,
        "test": test,
        "test_desc": "" if test == "value" else "the %s " % test,
        "expr_code": expr_code,
    }

    fmt_kwargs["stu_eval"] = utils.shorten_str(str(eval_stu))
    fmt_kwargs["sol_eval"] = utils.shorten_str(str(eval_sol))
    if incorrect_msg == DEFAULT_INCORRECT_MSG and (
            fmt_kwargs["stu_eval"] is None or fmt_kwargs["sol_eval"] is None
            or fmt_kwargs["stu_eval"] == fmt_kwargs["sol_eval"]):
        incorrect_msg = "Expected something different."

    # tests ---
    # error in process
    if (test == "error") ^ isinstance(eval_stu, Exception):
        fmt_kwargs["stu_str"] = str_stu
        state.report(error_msg, fmt_kwargs, append=append)

    # name is undefined after running expression
    if isinstance(eval_stu, UndefinedValue):
        state.report(undefined_msg, fmt_kwargs, append=append)

    # test equality of results
    state.do_test(
        EqualTest(
            eval_stu,
            eval_sol,
            FeedbackComponent(incorrect_msg, fmt_kwargs, append=append),
            func,
        ))

    return state
Esempio n. 3
0
def has_expr(incorrect_msg=DEFAULT_INCORRECT_MSG,
             error_msg=DEFAULT_ERROR_MSG,
             undefined_msg=DEFAULT_UNDEFINED_MSG,
             extra_env=None,
             context_vals=None,
             pre_code=None,
             expr_code=None,
             name=None,
             copy=True,
             func=None,
             state=None,
             test=None):

    rep = Reporter.active_reporter

    get_func = partial(evalCalls[test],
                       extra_env=extra_env,
                       context_vals=context_vals,
                       pre_code=pre_code,
                       expr_code=expr_code,
                       name=name,
                       copy=copy,
                       do_exec=True if test == 'output' else False)

    eval_sol, str_sol = get_func(tree=state.solution_tree,
                                 process=state.solution_process,
                                 context=state.solution_context,
                                 env=state.solution_env)

    if (test == 'error') ^ isinstance(str_sol, Exception):
        raise ValueError(
            "Evaluating expression raised error in solution process (or not an error if testing for one). "
            "Error: {} - {}".format(type(str_sol), str_sol))
    if isinstance(eval_sol, ReprFail):
        raise ValueError(
            "Couldn't figure out the value of a default argument: " +
            eval_sol.info)

    eval_stu, str_stu = get_func(tree=state.student_tree,
                                 process=state.student_process,
                                 context=state.student_context,
                                 env=state.student_env)

    # kwargs ---
    fmt_kwargs = {
        'stu_part': state.student_parts,
        'sol_part': state.solution_parts,
        'name': name,
        'test': test,
        'extra_env': str(extra_env) if extra_env else "",
        'context_vals': context_vals
    }
    fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
    fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))

    # tests ---
    # error in process
    if (test == 'error') ^ isinstance(str_stu, Exception):
        _msg = state.build_message(error_msg, fmt_kwargs)
        feedback = Feedback(_msg, state)
        rep.do_test(Test(feedback))

    # name is undefined after running expression
    if isinstance(str_stu, UndefinedValue):
        _msg = state.build_message(undefined_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, state)))

    # test equality of results
    _msg = state.build_message(incorrect_msg, fmt_kwargs)
    rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))

    return state
Esempio n. 4
0
def has_expr(incorrect_msg=None,
             error_msg=None,
             undefined_msg=None,
             append=None,
             extra_env=None,
             context_vals=None,
             pre_code=None,
             expr_code=None,
             name=None,
             copy=True,
             func=None,
             override=None,
             state=None,
             test=None):

    if append is None:  # if not specified, set to False if incorrect_msg was manually specified
        append = incorrect_msg is None
    if incorrect_msg is None:
        if name:
            incorrect_msg = DEFAULT_INCORRECT_NAME_MSG
        elif expr_code:
            incorrect_msg = DEFAULT_INCORRECT_EXPR_CODE_MSG
        else:
            incorrect_msg = DEFAULT_INCORRECT_MSG
    if undefined_msg is None:
        undefined_msg = DEFAULT_UNDEFINED_NAME_MSG
    if error_msg is None:
        if test == 'error':
            error_msg = DEFAULT_ERROR_MSG_INV
        else:
            error_msg = DEFAULT_ERROR_MSG

    rep = Reporter.active_reporter

    get_func = partial(evalCalls[test],
                       extra_env=extra_env,
                       context_vals=context_vals,
                       pre_code=pre_code,
                       expr_code=expr_code,
                       name=name,
                       copy=copy)

    if override is not None:
        # don't bother with running expression and fetching output/value
        # eval_sol, str_sol = eval
        eval_sol, str_sol = override, str(override)
    else:
        eval_sol, str_sol = get_func(tree=state.solution_tree,
                                     process=state.solution_process,
                                     context=state.solution_context,
                                     env=state.solution_env)

        if (test == 'error') ^ isinstance(eval_sol, Exception):
            raise InstructorError(
                "Evaluating expression raised error in solution process (or not an error if testing for one). "
                "Error: {} - {}".format(type(eval_sol), str_sol))
        if isinstance(eval_sol, ReprFail):
            raise InstructorError(
                "Couldn't extract the value for the highlighted expression from the solution process: "
                + eval_sol.info)

    eval_stu, str_stu = get_func(tree=state.student_tree,
                                 process=state.student_process,
                                 context=state.student_context,
                                 env=state.student_env)

    # kwargs ---
    fmt_kwargs = {
        'stu_part': state.student_parts,
        'sol_part': state.solution_parts,
        'name': name,
        'test': test,
        'test_desc': '' if test == 'value' else 'the %s ' % test,
        'expr_code': expr_code
    }

    fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
    fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))
    if incorrect_msg == DEFAULT_INCORRECT_MSG and \
        ( fmt_kwargs['stu_eval'] is None or
          fmt_kwargs['sol_eval'] is None or
          fmt_kwargs['stu_eval'] == fmt_kwargs['sol_eval'] ):
        incorrect_msg = "Expected something different."

    # tests ---
    # error in process
    if (test == 'error') ^ isinstance(eval_stu, Exception):
        fmt_kwargs['stu_str'] = str_stu
        _msg = state.build_message(error_msg, fmt_kwargs, append=append)
        feedback = Feedback(_msg, state)
        rep.do_test(Test(feedback))

    # name is undefined after running expression
    if isinstance(eval_stu, UndefinedValue):
        _msg = state.build_message(undefined_msg, fmt_kwargs, append=append)
        rep.do_test(Test(Feedback(_msg, state)))

    # test equality of results
    _msg = state.build_message(incorrect_msg, fmt_kwargs, append=append)
    rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, state), func))

    return state
Esempio n. 5
0
def test_operator(index=1,
                  eq_condition="equal",
                  used=None,
                  do_eval=True,
                  not_found_msg=None,
                  incorrect_op_msg=None,
                  incorrect_result_msg=None,
                  state=None):
    """THIS FUNCTION IS DEPRECATED

    Test if operator groups match.

    This function compares an operator group in the student's code with the corresponding one in the solution
    code. It will cause the reporter to fail if the corresponding operators do not match. The fail message
    that is returned will depend on the sort of fail. We say that one operator group correpsonds to a group of
    operators that is evaluated to one value (e.g. 3 + 5 * (1/3)).

    Args:
        index (int): Index of the operator group to be checked. Defaults to 1.
        eq_condition (str): how results of operators are compared. Currently, only "equal" is supported,
            meaning that the result in student and solution process should have exactly the same value.
        used(List[str]): A list of operators that have to be in the group. Valid operators are: "+", "-",
          "*", "/", "%", "**", "<<", ">>", "|", "^", "&" and "//". If the list is None, operators that are
          in the group in the solution have to be in the student code. Defaults to None.
        do_eval (bool): Boolean representing whether the group should be evaluated and compared or not.
          Defaults to True.
        not_found_msg (str): Feedback message if not enough operators groups are found in the student's code.
        incorrect_op_msg (str): Feedback message if the wrong operators are used in the student's code.
        incorrect_result_msg (str): Feedback message if the operator group evaluates to the wrong result in
          the student's code.

    :Example:

        Student code::

            1 + 5 * (3+5)
            1 + 1 * 238

        Solution code::

            3.1415 + 5
            1 + 238

        SCT::

            test_operator(index = 2, used = ["+"]) # pass
            test_operator(index = 2) # fail
            test_operator(index = 1, incorrect_op_msg = "Use the correct operators") # fail
            test_operator(index = 1, used = [], incorrect_result_msg = "Incorrect result") # fail

    """
    rep = Reporter.active_reporter
    rep.set_tag("fun", "test_operator")

    # Indexing starts at 1 for the pythonwhat user.
    index = index - 1
    eq_map = {"equal": EqualTest}

    if eq_condition not in eq_map:
        raise NameError("%r not a valid equality condition " % eq_condition)

    # Parses through code and extracts a data structure holding all operator groups.
    student_ops = state.student_operators
    solution_ops = state.solution_operators

    # Check if number of student operations is greater than index (which is decreased with 1 by now)
    _msg = state.build_message(not_found_msg or "You didn't define enough operations in your code.")
    rep.do_test(
        BiggerTest(
            len(student_ops),
            index,
            _msg))

    expr_student, used_student = student_ops[index]

    # Throw error if solution code is invalid with SCT
    if index > len(solution_ops) + 1:
        raise IndexError("index not found in solution: %d" % index)

    expr_solution, used_solution = solution_ops[index]

    build_incorrect_msg = "The highlighted operation"

    used_student = set(used_student)
    used_solution = set(used_solution)
    if used is not None:
        used = set(used)
    else:
        used = used_solution

    for op in used:
        if incorrect_op_msg is None:
            incorrect_op_msg = build_incorrect_msg + (" is missing a `%s` operation." % op)

        _msg = state.build_message(incorrect_op_msg)
        rep.do_test(DefinedCollTest(op, used_student,
            Feedback(_msg, expr_student)))

    if (do_eval):


        eval_solution, str_solution = getResultInProcess(process = state.solution_process, tree = expr_solution)

        if str_solution is None:
            raise ValueError("Running the operation in the solution environment raised an error")
        if isinstance(eval_solution, ReprFail):
            raise ValueError("Couldn't find the result of the operation in the solution process: " + eval_solution.info)

        eval_student, str_student = getResultInProcess(process = state.student_process, tree = expr_student)

        # Compare the evaluated operation groups
        if incorrect_result_msg is None:
            if isinstance(str_student, Exception):
                stud_patt = "an error"
            else:
                stud_patt = "`%s`" % utils.shorten_str(str_student)

            incorrect_result_msg = build_incorrect_msg + (" evaluates to %s, should be `%s`." % (stud_patt, utils.shorten_str(str_solution)))

        _msg = state.build_message(incorrect_result_msg)
        rep.do_test(eq_map[eq_condition](
            eval_student, eval_solution, Feedback(_msg, expr_student)))
Esempio n. 6
0
 def set_information(self, key, value):
     self.information[key] = utils.shorten_str(str(value))
Esempio n. 7
0
def has_expr(
        incorrect_msg="__JINJA__:Unexpected expression {{test}}: expected `{{sol_eval}}`, got `{{stu_eval}}`{{' with values ' + extra_env if extra_env}}.",
        error_msg="Running an expression in the student process caused an issue.",
        undefined_msg="FMT:Have you defined `{name}` without errors?",
        extra_env=None,
        context_vals=None,
        expr_code=None,
        pre_code=None,
        keep_objs_in_env=None,
        name=None,
        highlight=None,
        copy=True,
        func=None,
        state=None,
        test=None):
    """Run student and solution code, compare returned value, printed output, or errors.

    Args:
        incorrect_msg (str): feedback message if the output of the expression in the solution doesn't match
          the one of the student. This feedback message will be expanded if it is used in the context of
          another test function, like test_if_else.
        error_msg (str): feedback message if there was an error when running the student code.
          Note that when testing for an error, this message is displayed when none is raised.
        undefined_msg (str): feedback message if the name argument is defined, but a variable 
          with that name doesn't exist after running the student code.
        extra_env (dict): set variables to the extra environment. They will update the student
          and solution environment in the active state before the student/solution code in the active
          state is ran. This argument should contain a dictionary with the keys the names of
          the variables you want to set, and the values are the values of these variables.
        context_vals (list): set variables which are bound in a for loop to certain values. This argument is
          only useful if you use the function in a test_for_loop. It contains a list with the values
          of the bound variables.
        expr_code (str): if this variable is not None, the expression in the student/solution code will not
          be ran. Instead, the given piece of code will be ran in the student as well as the solution environment
          and the result will be compared.
        pre_code (str): the code in string form that should be executed before the expression is executed.
          This is the ideal place to set a random seed, for example.
        keep_obj_in_env (list()): a list of variable names that should be hold in the copied environment where
          the expression is evaluated. All primitive types are copied automatically, other objects have to
          be passed explicitely.
        name (str): the name of a variable, or expression, whose value will be tested after running the
          student and solution code. This could be thought of as post code.
        copy (bool): whether to try to deep copy objects in the environment, such as lists, that could
          accidentally be mutated. Disable to speed up SCTs. Disabling may lead to cryptic mutation issues.
        func: custom binary function of form f(stu_result, sol_result), for equality testing.
    """
    rep = Reporter.active_reporter

    # run function to highlight a block of code
    if callable(highlight):
        try:
            highlight = highlight(state=state).student_tree
        except:
            pass
    highlight = highlight or state.highlight

    get_func = partial(evalCalls[test],
                       extra_env=extra_env,
                       context_vals=context_vals,
                       pre_code=pre_code,
                       expr_code=expr_code,
                       keep_objs_in_env=keep_objs_in_env,
                       name=name,
                       copy=copy,
                       do_exec=True if test == 'output' else False)

    eval_sol, str_sol = get_func(tree=state.solution_tree,
                                 process=state.solution_process,
                                 context=state.solution_context)

    if (test == 'error') ^ isinstance(str_sol, Exception):
        raise ValueError(
            "evaluating expression raised error in solution process (or not an error if testing for one). "
            "Error: %s - %s" % (type(str_sol), str_sol))
    if isinstance(eval_sol, ReprFail):
        raise ValueError(
            "Couldn't figure out the value of a default argument: " +
            eval_sol.info)

    eval_stu, str_stu = get_func(tree=state.student_tree,
                                 process=state.student_process,
                                 context=state.student_context)

    # kwargs ---
    fmt_kwargs = {
        'stu_part': state.student_parts,
        'sol_part': state.solution_parts,
        'name': name,
        'test': test,
        'extra_env': str(extra_env) if extra_env else "",
        'context_vals': context_vals
    }
    fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
    fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))

    # tests ---
    # error in process
    if (test == 'error') ^ isinstance(str_stu, Exception):
        _msg = state.build_message(error_msg, fmt_kwargs)
        feedback = Feedback(_msg, highlight)
        rep.do_test(Test(feedback))

    # name is undefined after running expression
    if isinstance(str_stu, UndefinedValue):
        _msg = state.build_message(undefined_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, highlight)))

    # test equality of results
    _msg = state.build_message(incorrect_msg, fmt_kwargs)
    rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight), func))

    return state
Esempio n. 8
0
def has_expr(
        incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.",
        error_msg="Running an expression in the student process caused an issue.",
        undefined_msg="FMT:Have you defined `{name}` without errors?",
        extra_env=None,
        context_vals=None,
        expr_code=None,
        pre_code=None,
        keep_objs_in_env=None,
        name=None,
        highlight=None,
        state=None,
        test=None):
    rep = Reporter.active_reporter

    # run function to highlight a block of code
    if callable(highlight):
        try:
            highlight = highlight(state=state).student_tree
        except:
            pass
    highlight = highlight or state.highlight

    get_func = partial(evalCalls[test],
                       extra_env=extra_env,
                       context_vals=context_vals,
                       pre_code=pre_code,
                       expr_code=expr_code,
                       keep_objs_in_env=keep_objs_in_env,
                       name=name,
                       do_exec=True if test == 'output' else False)

    eval_sol, str_sol = get_func(tree=state.solution_tree,
                                 process=state.solution_process,
                                 context=state.solution_context)

    if (test == 'error') ^ isinstance(str_sol, Exception):
        raise ValueError(
            "evaluating expression raised error in solution process (or not an error if testing for one). "
            "Error: %s - %s" % (type(str_sol), str_sol))
    if isinstance(eval_sol, ReprFail):
        raise ValueError(
            "Couldn't figure out the value of a default argument: " +
            eval_sol.info)

    eval_stu, str_stu = get_func(tree=state.student_tree,
                                 process=state.student_process,
                                 context=state.student_context)

    # kwargs ---
    fmt_kwargs = {
        'stu_part': state.student_parts,
        'sol_part': state.solution_parts,
        'name': name,
        'test': test,
        'extra_env': " " + str(extra_env or ""),
        'context_vals': context_vals
    }
    fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
    fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))

    # tests ---
    # error in process
    if (test == 'error') ^ isinstance(str_stu, Exception):
        _msg = state.build_message(error_msg, fmt_kwargs)
        feedback = Feedback(_msg, highlight)
        rep.do_test(Test(feedback))

    # name is undefined after running expression
    if isinstance(str_stu, UndefinedValue):
        _msg = state.build_message(undefined_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, highlight)))

    # test equality of results
    _msg = state.build_message(incorrect_msg, fmt_kwargs)
    rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight)))

    return state
Esempio n. 9
0
def has_expr(incorrect_msg="FMT:Unexpected expression {test}: expected `{sol_eval}`, got `{stu_eval}` with values{extra_env}.",
             error_msg="Running an expression in the student process caused an issue.",
             undefined_msg="FMT:Have you defined `{name}` without errors?",
             extra_env=None,
             context_vals=None,
             expr_code=None,
             pre_code=None,
             keep_objs_in_env=None,
             name=None,
             highlight=None,
             state=None,
             test=None):
    rep = Reporter.active_reporter

    # run function to highlight a block of code
    if callable(highlight):
        try:    highlight = highlight(state=state).student_tree
        except: pass
    highlight = highlight or state.highlight

    get_func = partial(evalCalls[test], 
                       extra_env = extra_env,
                       context_vals = context_vals,
                       pre_code = pre_code,
                       expr_code = expr_code,
                       keep_objs_in_env = keep_objs_in_env,
                       name=name,
                       do_exec = True if test == 'output' else False)
    
    eval_sol, str_sol = get_func(tree = state.solution_tree,
                                 process = state.solution_process,
                                 context = state.solution_context)

    if (test == 'error') ^ isinstance(str_sol, Exception):
        raise ValueError("evaluating expression raised error in solution process (or not an error if testing for one). "
                         "Error: %s - %s"%(type(str_sol), str_sol))
    if isinstance(eval_sol, ReprFail):
        raise ValueError("Couldn't figure out the value of a default argument: " + eval_sol.info)

    eval_stu, str_stu = get_func(tree = state.student_tree,
                                 process = state.student_process,
                                 context = state.student_context)

    # kwargs ---
    fmt_kwargs = {'stu_part': state.student_parts, 'sol_part': state.solution_parts, 
                  'name': name, 'test': test,
                  'extra_env': " "+str(extra_env or ""), 'context_vals': context_vals}
    fmt_kwargs['stu_eval'] = utils.shorten_str(str(eval_stu))
    fmt_kwargs['sol_eval'] = utils.shorten_str(str(eval_sol))

    # tests ---
    # error in process
    if (test == 'error') ^ isinstance(str_stu, Exception):
        _msg = state.build_message(error_msg, fmt_kwargs)
        feedback = Feedback(_msg, highlight)
        rep.do_test(Test(feedback))

    # name is undefined after running expression
    if isinstance(str_stu, UndefinedValue):
        _msg = state.build_message(undefined_msg, fmt_kwargs)
        rep.do_test(Test(Feedback(_msg, highlight)))

    # test equality of results
    _msg = state.build_message(incorrect_msg, fmt_kwargs)
    rep.do_test(EqualTest(eval_stu, eval_sol, Feedback(_msg, highlight)))

    return state