Ejemplo n.º 1
0
def import_captured(name, our_code=False):
    """
    Import the module `name`, capturing stdout, and any exceptions that happen.
    Returns the module, and a dict of results.

    If `our_code` is true, then the code is edX-authored, and any exception output
    can include full context.  If `our_code` is false, then this is student-submitted
    code, and should have only student-provided information visible in exception 
    traces.  This isn't a security precaution, it just keeps us from showing confusing
    and unhelpful information to students.
    """
    result = {
        'status': 'notrun',
    }
    try:
        with graderutil.captured_stdout() as stdout:
            mod = __import__(name)
    except:
        result['status'] = 'error'
        if our_code:
            exc = graderutil.format_exception()
        else:
            exc = graderutil.format_exception(main_file=name, hide_file=True)
        result['exception'] = exc
        mod = None
    else:
        result['status'] = 'ok'
    result['stdout'] = stdout.getvalue()
    return mod, result
Ejemplo n.º 2
0
def import_captured(name, our_code=False):
    """
    Import the module `name`, capturing stdout, and any exceptions that happen.
    Returns the module, and a dict of results.

    If `our_code` is true, then the code is edX-authored, and any exception output
    can include full context.  If `our_code` is false, then this is student-submitted
    code, and should have only student-provided information visible in exception 
    traces.  This isn't a security precaution, it just keeps us from showing confusing
    and unhelpful information to students.
    """
    result = {
        'status': 'notrun',
    }
    try:
        with graderutil.captured_stdout() as stdout:
            mod = __import__(name)
    except:
        result['status'] = 'error'
        if our_code:
            exc = graderutil.format_exception()
        else:
            exc = graderutil.format_exception(main_file=name, hide_file=True)
        result['exception'] = exc
        mod = None
    else:
        result['status'] = 'ok'
    result['stdout'] = stdout.getvalue()
    return mod, result
Ejemplo n.º 3
0
def run(grader_name, submission_name, seed=1):
    """
    `grader_name`: importable module name of the grader
    `submission_name`: importable module name of the submission
    `seed`: A value to seed randomness with.

    Returns a data structure:

    {
    'grader': {
        'status': 'ok',     # or 'error', 'nograder'
        'stdout': 'whatever grader printed',
        'exception': 'a stack trace if error'
        },
    'submission': {
        'status': 'ok',     # or 'error', 'caught'
        'stdout': 'whatever the submission printed',
        'exception': 'a stack trace if error'
        },
    'results': [
        ["Test short desc", "test detailed description", "test output..."],
        ...
        ],
    'exceptions': 0,    # or however many were caught.
    }

    """

    output = {
        'grader': {
            'status': 'notrun',
        },
        'submission': {
            'status': 'notrun',
        },
        'results': [],
        'exceptions': 0,
    }

    # Use a private random number generator, so student code won't accidentally
    # mess it up.  (if they mess it up deliberately, we don't care--it only
    # hurts them).
    gradelib.rand = random.Random(seed)
    # Also seed the random singleton in case the exercise uses random numbers.
    random.seed(seed + 1)

    grader_mod, results = import_captured(grader_name, our_code=True)
    if grader_mod:
        try:
            grader = grader_mod.grader
        except:
            results['status'] = 'error'
            results['exception'] = graderutil.format_exception()
            output['exceptions'] += 1
    else:
        output['exceptions'] += 1
    output['grader'].update(results)

    if output['grader']['status'] == 'ok':
        submission, results = import_captured(submission_name)
        output['submission'].update(results)

        if submission and output['submission']['status'] == 'ok':
            # results is a list of ("short description", "detailed desc", "output") tuples.
            try:
                for test in grader.tests():
                    with graderutil.captured_stdout() as test_stdout:
                        try:
                            exception_output = ""
                            test(submission)
                        except gradelib.EndTest:
                            grader.caught_end_test()
                        except:
                            # The error could be either the grader code or the submission code,
                            # so hide information.
                            exception_output = graderutil.format_exception(
                                main_file=submission_name, hide_file=True)
                            output['exceptions'] += 1
                        else:
                            exception_output = ""
                        # Get the output, including anything printed, and any exception.
                        test_output = test_stdout.getvalue()
                        if test_output and test_output[-1] != '\n':
                            test_output += '\n'
                        test_output += exception_output
                    output['results'].append(
                        (test.short_description, test.detailed_description,
                         test_output))
            except:
                output['grader']['status'] = 'error'
                output['grader']['exception'] = graderutil.format_exception()
                output['exceptions'] += 1
        else:
            output['exceptions'] += 1

        if grader.uncaught_end_tests():
            # We raised EndTest more than we caught them, the student must be
            # catching them, inadvertently or not.
            output['submission']['exception'] = _(
                "Your code interfered with our grader.  Don't use bare 'except' clauses."
            )
            output['submission']['status'] = 'caught'
    return output
Ejemplo n.º 4
0
def run(grader_name, submission_name, seed=1):
    """
    `grader_name`: importable module name of the grader
    `submission_name`: importable module name of the submission
    `seed`: A value to seed randomness with.

    Returns a data structure:

    {
    'grader': {
        'status': 'ok',     # or 'error', 'nograder'
        'stdout': 'whatever grader printed',
        'exception': 'a stack trace if error'
        },
    'submission': {
        'status': 'ok',     # or 'error', 'caught'
        'stdout': 'whatever the submission printed',
        'exception': 'a stack trace if error'
        },
    'results': [
        ["Test short desc", "test detailed description", "test output..."],
        ...
        ],
    'exceptions': 0,    # or however many were caught.
    }

    """

    output = {
        'grader': {
            'status': 'notrun',
        },
        'submission': {
            'status': 'notrun',
        },
        'results': [],
        'exceptions': 0,
    }

    # Use a private random number generator, so student code won't accidentally
    # mess it up.  (if they mess it up deliberately, we don't care--it only
    # hurts them).
    gradelib.rand = random.Random(seed)
    # Also seed the random singleton in case the exercise uses random numbers.
    random.seed(seed+1)

    grader_mod, results = import_captured(grader_name, our_code=True)
    if grader_mod:
        try:
            grader = grader_mod.grader
        except:
            results['status'] = 'error'
            results['exception'] = graderutil.format_exception()
            output['exceptions'] += 1
    else:
        output['exceptions'] += 1
    output['grader'].update(results)

    if output['grader']['status'] == 'ok':
        submission, results = import_captured(submission_name)
        output['submission'].update(results)

        if submission and output['submission']['status'] == 'ok':
            # results is a list of ("short description", "detailed desc", "output") tuples.
            try:
                for test in grader.tests():
                    with graderutil.captured_stdout() as test_stdout:
                        try:
                            exception_output = ""
                            test(submission)
                        except gradelib.EndTest:
                            grader.caught_end_test()
                        except:
                            # The error could be either the grader code or the submission code,
                            # so hide information.
                            exception_output = graderutil.format_exception(main_file=submission_name, hide_file=True)
                            output['exceptions'] += 1
                        else:
                            exception_output = ""
                        # Get the output, including anything printed, and any exception.
                        test_output = test_stdout.getvalue()
                        if test_output and test_output[-1] != '\n':
                            test_output += '\n'
                        test_output += exception_output
                    output['results'].append((test.short_description, test.detailed_description, test_output))
            except:
                output['grader']['status'] = 'error'
                output['grader']['exception'] = graderutil.format_exception()
                output['exceptions'] += 1
        else:
            output['exceptions'] += 1

        if grader.uncaught_end_tests():
            # We raised EndTest more than we caught them, the student must be 
            # catching them, inadvertently or not.
            output['submission']['exception'] = _("Your code interfered with our grader.  Don't use bare 'except' clauses.")
            output['submission']['status'] = 'caught'
    return output