def test_commands_api(self):
clear_report()
student_code = 'word = input("Give me a word")\nprint(word+"!")'
set_source(student_code)
self.assertFalse(commands.get_output())
commands.queue_input("Hello")
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(["Give me a word", "Hello!"], commands.get_output())
commands.queue_input("World", "Again")
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(
commands.get_output(),
["Give me a word", "Hello!", "Give me a word", "World!"])
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(), [
"Give me a word", "Hello!", "Give me a word", "World!",
"Give me a word", "Again!"
])
commands.reset_output()
commands.queue_input("Dogs", "Are", "Great")
self.assertIsInstance(commands.run(), Sandbox)
self.assertIsInstance(commands.run(), Sandbox)
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(), [
"Give me a word", "Dogs!", "Give me a word", "Are!",
"Give me a word", "Great!"
])
commands.reset_output()
commands.queue_input(json.dumps("Virginia,Trend"))
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(),
["Give me a word", '"Virginia,Trend"!'])
def test_compatibility_api(self):
student_code = 'word = input("Give me a word")\nprint(word+"!")'
set_source(student_code)
self.assertFalse(compatibility.get_output())
compatibility.queue_input("Hello")
self.assertIsNone(compatibility.run_student())
self.assertEqual(compatibility.get_output(),
["Give me a word", "Hello!"])
compatibility.queue_input("World", "Again")
self.assertIsNone(compatibility.run_student())
self.assertEqual(
compatibility.get_output(),
["Give me a word", "Hello!", "Give me a word", "World!"])
self.assertIsNone(compatibility.run_student())
self.assertEqual(compatibility.get_output(), [
"Give me a word", "Hello!", "Give me a word", "World!",
"Give me a word", "Again!"
])
compatibility.reset_output()
compatibility.queue_input("Dogs", "Are", "Great")
self.assertIsNone(compatibility.run_student())
self.assertIsNone(compatibility.run_student())
self.assertIsNone(compatibility.run_student())
self.assertEqual(compatibility.get_output(), [
"Give me a word", "Dogs!", "Give me a word", "Are!",
"Give me a word", "Great!"
])
def test_runtime_error_inputs(self):
student_code = dedent('''
def x():
value = input("Gimme a number")
return 7 % value
x
''')
student = Sandbox()
set_source(student_code)
student.run()
result = student.call("x", inputs=["0"])
print([[c.kind, c.inputs] for c in student._context])
self.assertEqual(
"""A TypeError occurred:
Unsupported operand type(s) for %: 'int' and 'str'
I ran the code:
x()
And I entered as input:
0
The traceback was:
Line 4 of file answer.py in x
return 7 % value
Type errors occur when you use an operator or function on the wrong type of value. For example, using `+` to add to a list (instead of `.append`), or dividing a string by a number.
Suggestion: To fix a type error, you should trace through your code. Make sure each expression has the type you expect it to have.
""".strip(), student.feedback.message)
def test_find_matches(self):
set_source("fun = 1 + 1\nfun2 = 2 + 2")
parse_program()
matches = find_matches("_var_ = __expr__")
self.assertTrue(type(matches) == list, "find_matches did not return a list")
self.assertTrue(type(matches[0]) == AstMap, "find_matches does not contain an AstMap")
self.assertTrue(len(matches) == 2, "find_matches does not return the correct number of matches")
def test_error_context(self):
student_code = dedent('''
def get_input():
return int(input("Gimme the number"))
''')
set_source(student_code, "student.py")
student = Sandbox()
student.run()
result = student.call("get_input", inputs="Banana")
print("--", [c.inputs for c in student._context])
self.assertEqual(3, student.feedback.location.line)
self.assertEqual(
dedent("""A ValueError occurred:
Invalid literal for int() with base 10: 'banana'
I ran the code:
get_input()
And I entered as input:
Banana
The traceback was:
Line 3 of file student.py in get_input
return int(input("Gimme the number"))
A ValueError occurs when you pass the wrong type of value to a function. For example, you try to convert a string without numbers to an integer (like `int('Five')`).
Suggestion: Read the error message to see which function had the issue. Check what type the function expects. Then trace your code to make sure you pass in that type."""
).strip(), student.feedback.message)
def test_blocked_module_import(self):
student_code = dedent('''
import os
os
''')
set_source(student_code)
exception = commands.run()
self.assertIsNotNone(exception)
def test_sandboxing_open(self):
student_code = dedent('''
print(open("test_sandbox.py").read())
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception),
"The filename you passed to 'open' is restricted.")
def test_sandboxing_pedal(self):
student_code = dedent('''
from pedal.report import MAIN_REPORT
print(MAIN_REPORT)
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_weird_module_behavior(self):
student_code = dedent('''
import pprint
pprint.pprint(5)
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertIsNone(student.exception)
def test_parse_program(self):
# noinspection PyBroadException
try:
parse_program()
self.assertTrue(False, "Program did not throw exception")
except Exception:
self.assertTrue(True, "Should NOT FAIL")
set_source("fun = 1 + 0")
parse_program()
self.assertTrue('cait' in MAIN_REPORT, "No parsing happened")
def test_parse_program(self):
# noinspection PyBroadException
try:
parse_program()
self.assertTrue(False, "Program did not throw exception")
except Exception:
self.assertTrue(True, "Should NOT FAIL")
set_source("fun = 1 + 0")
parse_program()
self.assertTrue('cait' in MAIN_REPORT, "No parsing happened")
def test_sandboxing_devious_open1(self):
student_code = dedent('''
__builtins__['globals']()['open']("test_sandbox.py")
# Fine to import anything else
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception),
"You are not allowed to call 'globals'.")
def __init__(self, std_code=None, report=None):
if report is None and std_code is None:
self.report = MAIN_REPORT
elif report is not None and std_code is not None:
raise Exception("New code should generate new reports")
elif std_code is not None:
self.report = Report()
set_source(std_code, self.report)
if 'cait' not in self.report:
self._initialize_report()
def test_mock_turtle(self):
student_code = dedent('''
import turtle
turtle.forward(100)
turtle.mainloop()
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertIsNone(student.exception)
self.assertEqual(student.modules.turtles.calls[0][0], 'forward')
self.assertEqual(student.modules.turtles.calls[0][1][0], 100)
def test_find_matches(self):
set_source("fun = 1 + 1\nfun2 = 2 + 2")
parse_program()
matches = find_matches("_var_ = __expr__")
self.assertTrue(
type(matches) == list, "find_matches did not return a list")
self.assertTrue(
type(matches[0]) == AstMap,
"find_matches does not contain an AstMap")
self.assertTrue(
len(matches) == 2,
"find_matches does not return the correct number of matches")
def test_matplotlib_compatibility(self):
student_code = dedent('''
import matplotlib.pyplot as plt
plt.plot([1,2,3])
plt.title("My line plot")
plt.show()
plt.hist([1,2,3])
plt.show()
''')
set_source(student_code)
exception = compatibility.run_student()
plt2 = compatibility.get_plots()
self.assertEqual(len(plt2), 2)
def test_sandboxing_sys_modules(self):
clear_report()
student_code = dedent('''
import sys
# Might try to bypass us
del sys.modules['pedal']
from pedal.report import MAIN_REPORT
print(MAIN_REPORT)
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_good_unique_calls(self):
student_code = dedent("""
def x(n):
if n > 0:
return x(n-1)
return 0
x(5)
x(4)
""")
set_source(student_code)
commands.start_trace()
commands.run()
self.assertEqual(commands.count_unique_calls('x'), 6)
def make_exam(code):
"""
Args:
code:
"""
clear_report()
set_seed(0)
set_source(code)
student = run()
pool_1.choose().ask()
pool_2.choose().ask()
if pool_1.answered and pool_2.answered:
pool_3.choose().ask()
def test_sandboxing_open(self):
student_code = dedent('''
print(open("test_sandbox.py").read())
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception),
"The filename you passed to 'open' is restricted.")
# Then can we turn it back on?
student.allow_function('open')
student.run(student_code, filename='student.py')
self.assertIsNone(student.exception)
# And off again
student.block_function('open')
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception),
"You are not allowed to call 'open'.")
def find_file(filename, sections=False, independent=False, report=None):
if report is None:
report = MAIN_REPORT
try:
with open(filename, 'r') as student_file:
source.set_source(student_file.read(),
filename=filename,
sections=sections,
independent=independent,
report=report)
except IOError:
message = ("The given filename ('{filename}') was either not found"
" or could not be opened. Please make sure the file is"
" available.").format(filename=filename)
report.attach('Source File Not Found',
category='Syntax',
tool='VPL',
group=0 if sections else None,
mistake={'message': message})
report['source']['success'] = False
def test_old_style_api(self):
set_source("a = open('file.txt')")
ast = parse_program()
calls = ast.find_all("Call")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].func.ast_name, 'Name')
self.assertEqual(calls[0].func.id, 'open')
self.assertEqual(len(calls[0].args), 1)
clear_report()
set_source("def a():\n pass\na()")
ast = parse_program()
defs = ast.find_all("FunctionDef")
self.assertEqual(len(defs), 1)
print(defs[0].body)
print(defs[0].name)
print(defs[0].args)
print(defs[0].__getattr__('name'))
self.assertEqual(defs[0]._name, "a")
print("SUCCESS")
def test_old_style_api(self):
set_source("a = open('file.txt')")
ast = parse_program()
calls = ast.find_all("Call")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].func.ast_name, 'Name')
self.assertEqual(calls[0].func.id, 'open')
self.assertEqual(len(calls[0].args), 1)
clear_report()
set_source("def a():\n pass\na()")
ast = parse_program()
defs = ast.find_all("FunctionDef")
self.assertEqual(len(defs), 1)
print(defs[0].body)
print(defs[0].name)
print(defs[0].args)
print(defs[0].__getattr__('name'))
self.assertEqual(defs[0]._name, "a")
print("SUCCESS")
def test_find_match(self):
set_source("fun = 1 + 1")
parse_program()
match = find_match("_var_ = __expr__")
self.assertTrue(type(match) == AstMap, "Match not found")
import sys
from pedal.report import *
from pedal.source import set_source
set_source("1+''", "answer.py")
from pedal.sandbox.sandbox import Sandbox
from pedal.sandbox import compatibility
student = MAIN_REPORT['sandbox']['run'] = Sandbox()
student.report_exceptions_mode = True
print(len(sys.modules.keys()), sorted(sys.modules.keys()))
old = set(sys.modules.keys())
compatibility.run_student(raise_exceptions=True)
#import pedal.mistakes.instructor_append as ins_app
#print(ins_app)
from pedal.mistakes import instructor_append
new = set(sys.modules.keys())
print(len(sys.modules.keys()), sorted(sys.modules.keys()))
print(new - old)
print(student.output)
print(student.exception_position)
from pedal.resolvers import simple
import time
stopwatch = time.time()
def click(phase):
global stopwatch
diff = time.time() - stopwatch
print("Phase {}: {} secs".format(phase, round(diff, 2)))
stopwatch = time.time()
import pedal
click("Imported pedal")
from pedal.source import set_source
click("Imported source")
set_source("a = 0")
click("Set source")
from pedal.tifa import tifa_analysis
click("Imported Tifa")
tifa_analysis()
click("Ran Tifa")
from pedal.cait import parse_program
click("Imported cait")
ast = parse_program()
click("Parsed program")
if ast.find_all("Assign"):
print(ast.find_all("Assign"))
def to_source(source):
MAIN_REPORT.clear()
set_source(source)
parse_program()
tifa_analysis()
def test_find_match(self):
set_source("fun = 1 + 1")
parse_program()
match = find_match("_var_ = __expr__")
self.assertTrue(type(match) == AstMap, "Match not found")
def test_invalid_code(self):
set_source("float('0') + 1")
parse_program()
matches = find_match("_var_ = __expr__")
self.assertIsNone(matches)
def __enter__(self):
clear_report()
set_source(self.code)
tifa_analysis()
compatibility.run_student(raise_exceptions=True)
return self
import sys
from pedal.report import *
from pedal.source import set_source
set_source(
"""
import matplotlib.pyplot as plt
print("Hello world")
plt.hist([1,2,3])
plt.xlabel("Hello")
plt.show()
""", "answer.py")
from pedal.sandbox.sandbox import Sandbox
from pedal.sandbox import compatibility
student = MAIN_REPORT['sandbox']['run'] = Sandbox()
student.report_exceptions_mode = True
compatibility.run_student(raise_exceptions=False)
print(compatibility.get_output())
print(compatibility.get_plots())
from pedal.resolvers import simple
SUCCESS, SCORE, CATEGORY, LABEL, MESSAGE, DATA, HIDE = simple.resolve()
print(CATEGORY, LABEL, MESSAGE)
def test_invalid_code(self):
set_source("float('0') + 1")
parse_program()
matches = find_match("_var_ = __expr__")
self.assertIsNone(matches)
from pedal.report import *
from pedal.source import set_source
print("TEST")
CODE = """
def x():
print("ALPHA")
return {'x': 5}
"""
set_source(CODE, "answer.py")
from pedal.assertions import assert_equal
from pedal.sandbox.sandbox import Sandbox
from pedal.sandbox import compatibility
student = MAIN_REPORT['sandbox']['run'] = Sandbox()
compatibility.run_student(raise_exceptions=False)
print("FIRE")
#assert_equal(student.call('x'), 1.5)
y=student.call('x')
print("AAAA")
print(y.keys())
print("Phase {}: {} secs".format(phase, round(diff, 2)))
stopwatch = time.time()
# import time; stopwatch = time.time();
# print("Phase {}: {} secs".format(phase, round(time.time() - stopwatch, 2))) ; stopwatch = time.time()
import pedal
click("Imported pedal")
from pedal.source import set_source
click("Imported source")
set_source("a = 0")
click("Set source")
from pedal.tifa import tifa_analysis
click("Imported Tifa")
tifa_analysis()
click("Ran Tifa")
from pedal.cait import parse_program
click("Imported cait")
ast = parse_program()
click("Parsed program")
def test_old_style_api(self):
set_source("a = open('file.txt')")
std_ast = parse_program()
calls = std_ast.find_all("Call")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].func.ast_name, 'Name')
self.assertEqual(calls[0].func.id, 'open')
self.assertEqual(len(calls[0].args), 1)
clear_report()
set_source("def a():\n pass\na()")
std_ast = parse_program()
defs = std_ast.find_all("FunctionDef")
self.assertEqual(len(defs), 1)
clear_report()
set_source("1 < 1")
std_ast = parse_program()
compares = std_ast.find_all("Compare")
self.assertEqual(len(compares), 1)
clear_report()
set_source("for x in y:\n x")
std_ast = parse_program()
loops = std_ast.find_all("For")
self.assertEqual(len(loops), 1)
self.assertEqual(loops[0].target.ast_name, "Name")
# Multiple assignment
clear_report()
set_source("a, b = 0, 1")
std_ast = parse_program()
assigns = std_ast.find_all("Assign")
self.assertEqual(len(assigns), 1)
self.assertEqual(len(assigns[0].targets), 1)
self.assertEqual(assigns[0].targets[0].ast_name, 'Tuple')
self.assertEqual(assigns[0].targets[0].elts[0].id, 'a')
clear_report()
set_source('from pprint import *')
parse_program()
def test_commands_exceptions(self):
student_code = '1 + "Banana"'
set_source(student_code)
commands.run()
self.assertIsNotNone(commands.get_exception())
