def test_gently_and_set_success(self):
clear_report()
gently("What have you done?")
set_success()
final = simple.resolve()
self.assertFalse(final.success)
self.assertEqual(final.message, 'What have you done?')
def test_unmessaged_tifa(self):
clear_report()
contextualize_report('import random\nrandom')
verify()
tifa_analysis()
final = simple.resolve()
self.assertEqual(SUCCESS_MESSAGE, final.title+"\n"+final.message)
def test_sectional_success(self):
clear_report()
contextualize_report('a=0\n##### Part 1\nprint("A")\n##### Part 2\nprint("B")')
separate_into_sections(independent=True)
# Part 0
verify()
commands.clear_sandbox()
commands.run()
# Part 1
next_section()
verify()
commands.clear_sandbox()
commands.run()
give_partial(.2)
# Part 2
next_section()
verify()
commands.clear_sandbox()
commands.run()
give_partial(.2)
# Resolve everything
finals = sectional.resolve()
self.assertEqual("""# Global
FeedbackSourceSection
Feedback separated into groups
# 1
Complete
Great work!
# 2
Complete
Great work!""",
"\n".join(f"# {g.section_number if g is not None else 'Global'}\n{f.title}\n{f.message}"
for g, f in finals.items()))
def test_hidden_error(self):
clear_report()
contextualize_report('import pedal')
verify()
tifa_analysis()
final = simple.resolve()
self.assertNotEqual("No errors reported.", final.message)
def test_commands_api(self):
clear_report()
student_code = 'word = input("Give me a word")\nprint(word+"!")'
set_source(student_code)
self.assertFalse(commands.get_output())
commands.queue_input("Hello")
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(["Give me a word", "Hello!"], commands.get_output())
commands.queue_input("World", "Again")
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(
commands.get_output(),
["Give me a word", "Hello!", "Give me a word", "World!"])
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(), [
"Give me a word", "Hello!", "Give me a word", "World!",
"Give me a word", "Again!"
])
commands.reset_output()
commands.queue_input("Dogs", "Are", "Great")
self.assertIsInstance(commands.run(), Sandbox)
self.assertIsInstance(commands.run(), Sandbox)
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(), [
"Give me a word", "Dogs!", "Give me a word", "Are!",
"Give me a word", "Great!"
])
commands.reset_output()
commands.queue_input(json.dumps("Virginia,Trend"))
self.assertIsInstance(commands.run(), Sandbox)
self.assertEqual(commands.get_output(),
["Give me a word", '"Virginia,Trend"!'])
def test_gently_order(self):
clear_report()
gently('A great and exciting message!')
gently('A boring message that we should not show.')
final = simple.resolve()
self.assertFalse(final.success)
self.assertEqual(final.message, 'A great and exciting message!')
def test_empty(self):
clear_report()
contextualize_report(' ')
verify()
tifa_analysis()
commands.run()
final = simple.resolve()
self.assertEqual(Feedback.CATEGORIES.SYNTAX, final.category)
self.assertEqual("No Source Code", final.title)
self.assertEqual("Source code file is blank.", final.message)
def test_success_suppression(self):
clear_report()
contextualize_report('a=0\na')
verify()
tifa_analysis()
set_success()
suppress(label='set_success')
final = simple.resolve()
self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category)
self.assertEqual(SUCCESS_MESSAGE, final.title+"\n"+final.message)
def test_success(self):
clear_report()
contextualize_report('a=0\na')
verify()
tifa_analysis()
set_success()
final = simple.resolve()
self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category)
self.assertEqual("Complete", final.title)
self.assertEqual("Great work!", final.message)
def test_runtime_suppression(self):
clear_report()
contextualize_report('import json\njson.loads("0")+"1"')
verify()
tifa_analysis()
commands.run()
suppress("Runtime")
final = simple.resolve()
self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category)
self.assertEqual(SUCCESS_TEXT, final.message)
def test_analyzer_suppression(self):
clear_report()
contextualize_report('1+"Hello"')
verify()
tifa_analysis()
commands.run()
suppress("analyzer")
final = simple.resolve()
self.assertEqual("runtime", final.category)
self.assertEqual("Type Error", final.title)
def __enter__(self):
clear_report(report=self.report)
contextualize_report(self.code, report=self.report)
verify(report=self.report)
if self.run_tifa:
tifa_analysis(report=self.report)
# TODO: Clean this up
self.student = get_sandbox(self.report)
self.report['sandbox']['sandbox'].tracer_style = self.tracer_style
commands.run()
return self
def test_sandboxing_sys_modules(self):
clear_report()
student_code = dedent('''
import sys
# Might try to bypass us
del sys.modules['pedal']
from pedal.report import MAIN_REPORT
print(MAIN_REPORT)
''')
set_source(student_code)
student = Sandbox()
student.run(student_code, filename='student.py')
self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_combining_scores_complex(self):
clear_report()
contextualize_report('a=0\nprint(a)')
# These are added
feedback(activate=True, valence=1, score="+4%", category='instructor')
# These are skipped
feedback(activate=False, valence=1, score="+5%", category='instructor')
# These are skipped
feedback(activate=True, valence=-1, score="+8%", category='instructor')
# These are added
feedback(activate=False, valence=-1, score="+7%", category='instructor')
# Calculate final result
final = simple.resolve()
self.assertEqual(.11, final.score)
def test_simple_system_correct(self):
clear_report()
vpl = VPLEnvironment(main_code='1+2')
with io.StringIO() as f, redirect_stdout(f):
log("Message Printed")
vpl.resolve()
self.assertEqual("""<|--
-System Notes
Message Printed
-Complete
Great work!
--|>
Grade :=>> 1
""", f.getvalue())
def test_simple_system(self):
clear_report()
vpl = VPLEnvironment(main_code='1+2')
with io.StringIO() as f, redirect_stdout(f):
log("Message Printed")
assert_equal(1, 2)
vpl.resolve()
self.assertEqual("""<|--
-System Notes
Message Printed
-Failed Instructor Test
Student code failed instructor test.
>1 != 2
--|>
Grade :=>> 0
""", f.getvalue())
def test_old_style_api(self):
contextualize_report("a = open('file.txt')")
std_ast = parse_program()
calls = std_ast.find_all("Call")
self.assertEqual(len(calls), 1)
self.assertEqual(calls[0].func.ast_name, 'Name')
self.assertEqual(calls[0].func.id, 'open')
self.assertEqual(len(calls[0].args), 1)
clear_report()
contextualize_report("def a():\n pass\na()")
std_ast = parse_program()
defs = std_ast.find_all("FunctionDef")
self.assertEqual(len(defs), 1)
clear_report()
contextualize_report("1 < 1")
std_ast = parse_program()
compares = std_ast.find_all("Compare")
self.assertEqual(len(compares), 1)
clear_report()
contextualize_report("for x in y:\n x")
std_ast = parse_program()
loops = std_ast.find_all("For")
self.assertEqual(len(loops), 1)
self.assertEqual(loops[0].target.ast_name, "Name")
# Multiple assignment
clear_report()
contextualize_report("a, b = 0, 1")
std_ast = parse_program()
assigns = std_ast.find_all("Assign")
self.assertEqual(len(assigns), 1)
self.assertEqual(len(assigns[0].targets), 1)
self.assertEqual(assigns[0].targets[0].ast_name, 'Tuple')
self.assertEqual(assigns[0].targets[0].elts[0].id, 'a')
clear_report()
contextualize_report('from pprint import *')
parse_program()
def test_gently_vs_runtime(self):
# Runtime > Gently
clear_report()
contextualize_report('import json\njson.loads("0")+"1"')
verify()
tifa_analysis()
commands.run()
gently("I have a gentle opinion, but you don't want to hear it.")
final = simple.resolve()
print(final.label)
self.assertEqual(Feedback.CATEGORIES.RUNTIME, final.category)
# Runtime < Explain
clear_report()
contextualize_report('import json\njson.loads("0")+"1"')
verify()
tifa_analysis()
commands.run()
explain("LISTEN TO ME")
final = simple.resolve()
self.assertEqual(Feedback.CATEGORIES.INSTRUCTOR, final.category)
def test_resolve(self):
clear_report()
vpl = VPLEnvironment(main_file=here+'datafiles/student_example.py')
separate_into_sections()
# Part 0 - usually skipped
# Part 1
vpl.next_section()
# Part 2
vpl.next_section()
compliment('Hey, not a bad job!')
# Part 3
vpl.next_section()
# Part 4
vpl.next_section()
# Resolve entire thing
with io.StringIO() as f, redirect_stdout(f):
vpl.sectional_resolve()
self.assertEqual("""<|--
-Part 0
No feedback for this section
-Part 1
No feedback for this section
-Part 2
Complete
Great work!
-Part 3
Syntax Error
Bad syntax on line 15
The traceback was:
Line 15 of file datafiles/student_example.py
>a syntax error in this section!
Suggestion: Check line 15, the line before it, and the line after it.
-Part 4
No feedback for this section
--|>
Grade :=>> 1
""", f.getvalue())
def test_sectional_error(self):
clear_report()
contextualize_report('a=0\n##### Part 1\nprint("A")\n##### Part 2\nsyntax error')
separate_into_sections(independent=True)
# Part 0
verify()
commands.clear_sandbox()
commands.run()
# Part 1
next_section()
verify()
commands.clear_sandbox()
commands.run()
give_partial(.2)
# Part 2
next_section()
verify()
commands.clear_sandbox()
commands.run()
give_partial(.2)
# Resolve everything
finals = sectional.resolve()
self.assertEqual("""# Global
FeedbackSourceSection
Feedback separated into groups
# 1
Complete
Great work!
# 2
Syntax Error
Bad syntax on line 5
The traceback was:
Line 5 of file answer.py
syntax error
Suggestion: Check line 5, the line before it, and the line after it.""",
"\n".join(f"# {g.section_number if g is not None else 'Global'}\n{f.title}\n{f.message}"
for g, f in finals.items()))
def test_do_nothing(self):
clear_report()
final = simple.resolve()
self.assertTrue(final.success)
self.assertEqual(final.message, SUCCESS_TEXT)
def test_gently(self):
clear_report()
gently('You should always create unit tests.')
final = simple.resolve()
self.assertFalse(final.success)
self.assertEqual(final.message, 'You should always create unit tests.')
def test_set_success(self):
clear_report()
set_success()
final = simple.resolve()
self.assertTrue(final.success)
self.assertEqual(final.message, 'Great work!')
