def test_commands_api(self): clear_report() student_code = 'word = input("Give me a word")\nprint(word+"!")' set_source(student_code) self.assertFalse(commands.get_output()) commands.queue_input("Hello") self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(["Give me a word", "Hello!"], commands.get_output()) commands.queue_input("World", "Again") self.assertIsInstance(commands.run(), Sandbox) self.assertEqual( commands.get_output(), ["Give me a word", "Hello!", "Give me a word", "World!"]) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), [ "Give me a word", "Hello!", "Give me a word", "World!", "Give me a word", "Again!" ]) commands.reset_output() commands.queue_input("Dogs", "Are", "Great") self.assertIsInstance(commands.run(), Sandbox) self.assertIsInstance(commands.run(), Sandbox) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), [ "Give me a word", "Dogs!", "Give me a word", "Are!", "Give me a word", "Great!" ]) commands.reset_output() commands.queue_input(json.dumps("Virginia,Trend")) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), ["Give me a word", '"Virginia,Trend"!'])
def test_compatibility_api(self): student_code = 'word = input("Give me a word")\nprint(word+"!")' set_source(student_code) self.assertFalse(compatibility.get_output()) compatibility.queue_input("Hello") self.assertIsNone(compatibility.run_student()) self.assertEqual(compatibility.get_output(), ["Give me a word", "Hello!"]) compatibility.queue_input("World", "Again") self.assertIsNone(compatibility.run_student()) self.assertEqual( compatibility.get_output(), ["Give me a word", "Hello!", "Give me a word", "World!"]) self.assertIsNone(compatibility.run_student()) self.assertEqual(compatibility.get_output(), [ "Give me a word", "Hello!", "Give me a word", "World!", "Give me a word", "Again!" ]) compatibility.reset_output() compatibility.queue_input("Dogs", "Are", "Great") self.assertIsNone(compatibility.run_student()) self.assertIsNone(compatibility.run_student()) self.assertIsNone(compatibility.run_student()) self.assertEqual(compatibility.get_output(), [ "Give me a word", "Dogs!", "Give me a word", "Are!", "Give me a word", "Great!" ])
def test_runtime_error_inputs(self): student_code = dedent(''' def x(): value = input("Gimme a number") return 7 % value x ''') student = Sandbox() set_source(student_code) student.run() result = student.call("x", inputs=["0"]) print([[c.kind, c.inputs] for c in student._context]) self.assertEqual( """A TypeError occurred: Unsupported operand type(s) for %: 'int' and 'str' I ran the code: x() And I entered as input: 0 The traceback was: Line 4 of file answer.py in x return 7 % value Type errors occur when you use an operator or function on the wrong type of value. For example, using `+` to add to a list (instead of `.append`), or dividing a string by a number. Suggestion: To fix a type error, you should trace through your code. Make sure each expression has the type you expect it to have. """.strip(), student.feedback.message)
def test_find_matches(self): set_source("fun = 1 + 1\nfun2 = 2 + 2") parse_program() matches = find_matches("_var_ = __expr__") self.assertTrue(type(matches) == list, "find_matches did not return a list") self.assertTrue(type(matches[0]) == AstMap, "find_matches does not contain an AstMap") self.assertTrue(len(matches) == 2, "find_matches does not return the correct number of matches")
def test_error_context(self): student_code = dedent(''' def get_input(): return int(input("Gimme the number")) ''') set_source(student_code, "student.py") student = Sandbox() student.run() result = student.call("get_input", inputs="Banana") print("--", [c.inputs for c in student._context]) self.assertEqual(3, student.feedback.location.line) self.assertEqual( dedent("""A ValueError occurred: Invalid literal for int() with base 10: 'banana' I ran the code: get_input() And I entered as input: Banana The traceback was: Line 3 of file student.py in get_input return int(input("Gimme the number")) A ValueError occurs when you pass the wrong type of value to a function. For example, you try to convert a string without numbers to an integer (like `int('Five')`). Suggestion: Read the error message to see which function had the issue. Check what type the function expects. Then trace your code to make sure you pass in that type.""" ).strip(), student.feedback.message)
def test_blocked_module_import(self): student_code = dedent(''' import os os ''') set_source(student_code) exception = commands.run() self.assertIsNotNone(exception)
def test_sandboxing_open(self): student_code = dedent(''' print(open("test_sandbox.py").read()) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "The filename you passed to 'open' is restricted.")
def test_sandboxing_pedal(self): student_code = dedent(''' from pedal.report import MAIN_REPORT print(MAIN_REPORT) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_weird_module_behavior(self): student_code = dedent(''' import pprint pprint.pprint(5) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertIsNone(student.exception)
def test_parse_program(self): # noinspection PyBroadException try: parse_program() self.assertTrue(False, "Program did not throw exception") except Exception: self.assertTrue(True, "Should NOT FAIL") set_source("fun = 1 + 0") parse_program() self.assertTrue('cait' in MAIN_REPORT, "No parsing happened")
def test_sandboxing_devious_open1(self): student_code = dedent(''' __builtins__['globals']()['open']("test_sandbox.py") # Fine to import anything else ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "You are not allowed to call 'globals'.")
def __init__(self, std_code=None, report=None): if report is None and std_code is None: self.report = MAIN_REPORT elif report is not None and std_code is not None: raise Exception("New code should generate new reports") elif std_code is not None: self.report = Report() set_source(std_code, self.report) if 'cait' not in self.report: self._initialize_report()
def test_mock_turtle(self): student_code = dedent(''' import turtle turtle.forward(100) turtle.mainloop() ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertIsNone(student.exception) self.assertEqual(student.modules.turtles.calls[0][0], 'forward') self.assertEqual(student.modules.turtles.calls[0][1][0], 100)
def test_find_matches(self): set_source("fun = 1 + 1\nfun2 = 2 + 2") parse_program() matches = find_matches("_var_ = __expr__") self.assertTrue( type(matches) == list, "find_matches did not return a list") self.assertTrue( type(matches[0]) == AstMap, "find_matches does not contain an AstMap") self.assertTrue( len(matches) == 2, "find_matches does not return the correct number of matches")
def test_matplotlib_compatibility(self): student_code = dedent(''' import matplotlib.pyplot as plt plt.plot([1,2,3]) plt.title("My line plot") plt.show() plt.hist([1,2,3]) plt.show() ''') set_source(student_code) exception = compatibility.run_student() plt2 = compatibility.get_plots() self.assertEqual(len(plt2), 2)
def test_sandboxing_sys_modules(self): clear_report() student_code = dedent(''' import sys # Might try to bypass us del sys.modules['pedal'] from pedal.report import MAIN_REPORT print(MAIN_REPORT) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_good_unique_calls(self): student_code = dedent(""" def x(n): if n > 0: return x(n-1) return 0 x(5) x(4) """) set_source(student_code) commands.start_trace() commands.run() self.assertEqual(commands.count_unique_calls('x'), 6)
def make_exam(code): """ Args: code: """ clear_report() set_seed(0) set_source(code) student = run() pool_1.choose().ask() pool_2.choose().ask() if pool_1.answered and pool_2.answered: pool_3.choose().ask()
def test_sandboxing_open(self): student_code = dedent(''' print(open("test_sandbox.py").read()) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "The filename you passed to 'open' is restricted.") # Then can we turn it back on? student.allow_function('open') student.run(student_code, filename='student.py') self.assertIsNone(student.exception) # And off again student.block_function('open') student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "You are not allowed to call 'open'.")
def find_file(filename, sections=False, independent=False, report=None): if report is None: report = MAIN_REPORT try: with open(filename, 'r') as student_file: source.set_source(student_file.read(), filename=filename, sections=sections, independent=independent, report=report) except IOError: message = ("The given filename ('{filename}') was either not found" " or could not be opened. Please make sure the file is" " available.").format(filename=filename) report.attach('Source File Not Found', category='Syntax', tool='VPL', group=0 if sections else None, mistake={'message': message}) report['source']['success'] = False
def test_old_style_api(self): set_source("a = open('file.txt')") ast = parse_program() calls = ast.find_all("Call") self.assertEqual(len(calls), 1) self.assertEqual(calls[0].func.ast_name, 'Name') self.assertEqual(calls[0].func.id, 'open') self.assertEqual(len(calls[0].args), 1) clear_report() set_source("def a():\n pass\na()") ast = parse_program() defs = ast.find_all("FunctionDef") self.assertEqual(len(defs), 1) print(defs[0].body) print(defs[0].name) print(defs[0].args) print(defs[0].__getattr__('name')) self.assertEqual(defs[0]._name, "a") print("SUCCESS")
def test_find_match(self): set_source("fun = 1 + 1") parse_program() match = find_match("_var_ = __expr__") self.assertTrue(type(match) == AstMap, "Match not found")
import sys from pedal.report import * from pedal.source import set_source set_source("1+''", "answer.py") from pedal.sandbox.sandbox import Sandbox from pedal.sandbox import compatibility student = MAIN_REPORT['sandbox']['run'] = Sandbox() student.report_exceptions_mode = True print(len(sys.modules.keys()), sorted(sys.modules.keys())) old = set(sys.modules.keys()) compatibility.run_student(raise_exceptions=True) #import pedal.mistakes.instructor_append as ins_app #print(ins_app) from pedal.mistakes import instructor_append new = set(sys.modules.keys()) print(len(sys.modules.keys()), sorted(sys.modules.keys())) print(new - old) print(student.output) print(student.exception_position) from pedal.resolvers import simple
import time stopwatch = time.time() def click(phase): global stopwatch diff = time.time() - stopwatch print("Phase {}: {} secs".format(phase, round(diff, 2))) stopwatch = time.time() import pedal click("Imported pedal") from pedal.source import set_source click("Imported source") set_source("a = 0") click("Set source") from pedal.tifa import tifa_analysis click("Imported Tifa") tifa_analysis() click("Ran Tifa") from pedal.cait import parse_program click("Imported cait") ast = parse_program() click("Parsed program") if ast.find_all("Assign"): print(ast.find_all("Assign"))
def to_source(source): MAIN_REPORT.clear() set_source(source) parse_program() tifa_analysis()
def test_invalid_code(self): set_source("float('0') + 1") parse_program() matches = find_match("_var_ = __expr__") self.assertIsNone(matches)
def __enter__(self): clear_report() set_source(self.code) tifa_analysis() compatibility.run_student(raise_exceptions=True) return self
import sys from pedal.report import * from pedal.source import set_source set_source( """ import matplotlib.pyplot as plt print("Hello world") plt.hist([1,2,3]) plt.xlabel("Hello") plt.show() """, "answer.py") from pedal.sandbox.sandbox import Sandbox from pedal.sandbox import compatibility student = MAIN_REPORT['sandbox']['run'] = Sandbox() student.report_exceptions_mode = True compatibility.run_student(raise_exceptions=False) print(compatibility.get_output()) print(compatibility.get_plots()) from pedal.resolvers import simple SUCCESS, SCORE, CATEGORY, LABEL, MESSAGE, DATA, HIDE = simple.resolve() print(CATEGORY, LABEL, MESSAGE)
from pedal.report import * from pedal.source import set_source print("TEST") CODE = """ def x(): print("ALPHA") return {'x': 5} """ set_source(CODE, "answer.py") from pedal.assertions import assert_equal from pedal.sandbox.sandbox import Sandbox from pedal.sandbox import compatibility student = MAIN_REPORT['sandbox']['run'] = Sandbox() compatibility.run_student(raise_exceptions=False) print("FIRE") #assert_equal(student.call('x'), 1.5) y=student.call('x') print("AAAA") print(y.keys())
print("Phase {}: {} secs".format(phase, round(diff, 2))) stopwatch = time.time() # import time; stopwatch = time.time(); # print("Phase {}: {} secs".format(phase, round(time.time() - stopwatch, 2))) ; stopwatch = time.time() import pedal click("Imported pedal") from pedal.source import set_source click("Imported source") set_source("a = 0") click("Set source") from pedal.tifa import tifa_analysis click("Imported Tifa") tifa_analysis() click("Ran Tifa") from pedal.cait import parse_program click("Imported cait") ast = parse_program() click("Parsed program")
def test_old_style_api(self): set_source("a = open('file.txt')") std_ast = parse_program() calls = std_ast.find_all("Call") self.assertEqual(len(calls), 1) self.assertEqual(calls[0].func.ast_name, 'Name') self.assertEqual(calls[0].func.id, 'open') self.assertEqual(len(calls[0].args), 1) clear_report() set_source("def a():\n pass\na()") std_ast = parse_program() defs = std_ast.find_all("FunctionDef") self.assertEqual(len(defs), 1) clear_report() set_source("1 < 1") std_ast = parse_program() compares = std_ast.find_all("Compare") self.assertEqual(len(compares), 1) clear_report() set_source("for x in y:\n x") std_ast = parse_program() loops = std_ast.find_all("For") self.assertEqual(len(loops), 1) self.assertEqual(loops[0].target.ast_name, "Name") # Multiple assignment clear_report() set_source("a, b = 0, 1") std_ast = parse_program() assigns = std_ast.find_all("Assign") self.assertEqual(len(assigns), 1) self.assertEqual(len(assigns[0].targets), 1) self.assertEqual(assigns[0].targets[0].ast_name, 'Tuple') self.assertEqual(assigns[0].targets[0].elts[0].id, 'a') clear_report() set_source('from pprint import *') parse_program()
def test_commands_exceptions(self): student_code = '1 + "Banana"' set_source(student_code) commands.run() self.assertIsNotNone(commands.get_exception())