def test_gently_and_set_success(self): clear_report() gently("What have you done?") set_success() final = simple.resolve() self.assertFalse(final.success) self.assertEqual(final.message, 'What have you done?')
def test_unmessaged_tifa(self): clear_report() contextualize_report('import random\nrandom') verify() tifa_analysis() final = simple.resolve() self.assertEqual(SUCCESS_MESSAGE, final.title+"\n"+final.message)
def test_sectional_success(self): clear_report() contextualize_report('a=0\n##### Part 1\nprint("A")\n##### Part 2\nprint("B")') separate_into_sections(independent=True) # Part 0 verify() commands.clear_sandbox() commands.run() # Part 1 next_section() verify() commands.clear_sandbox() commands.run() give_partial(.2) # Part 2 next_section() verify() commands.clear_sandbox() commands.run() give_partial(.2) # Resolve everything finals = sectional.resolve() self.assertEqual("""# Global FeedbackSourceSection Feedback separated into groups # 1 Complete Great work! # 2 Complete Great work!""", "\n".join(f"# {g.section_number if g is not None else 'Global'}\n{f.title}\n{f.message}" for g, f in finals.items()))
def test_hidden_error(self): clear_report() contextualize_report('import pedal') verify() tifa_analysis() final = simple.resolve() self.assertNotEqual("No errors reported.", final.message)
def test_commands_api(self): clear_report() student_code = 'word = input("Give me a word")\nprint(word+"!")' set_source(student_code) self.assertFalse(commands.get_output()) commands.queue_input("Hello") self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(["Give me a word", "Hello!"], commands.get_output()) commands.queue_input("World", "Again") self.assertIsInstance(commands.run(), Sandbox) self.assertEqual( commands.get_output(), ["Give me a word", "Hello!", "Give me a word", "World!"]) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), [ "Give me a word", "Hello!", "Give me a word", "World!", "Give me a word", "Again!" ]) commands.reset_output() commands.queue_input("Dogs", "Are", "Great") self.assertIsInstance(commands.run(), Sandbox) self.assertIsInstance(commands.run(), Sandbox) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), [ "Give me a word", "Dogs!", "Give me a word", "Are!", "Give me a word", "Great!" ]) commands.reset_output() commands.queue_input(json.dumps("Virginia,Trend")) self.assertIsInstance(commands.run(), Sandbox) self.assertEqual(commands.get_output(), ["Give me a word", '"Virginia,Trend"!'])
def test_gently_order(self): clear_report() gently('A great and exciting message!') gently('A boring message that we should not show.') final = simple.resolve() self.assertFalse(final.success) self.assertEqual(final.message, 'A great and exciting message!')
def test_empty(self): clear_report() contextualize_report(' ') verify() tifa_analysis() commands.run() final = simple.resolve() self.assertEqual(Feedback.CATEGORIES.SYNTAX, final.category) self.assertEqual("No Source Code", final.title) self.assertEqual("Source code file is blank.", final.message)
def test_success_suppression(self): clear_report() contextualize_report('a=0\na') verify() tifa_analysis() set_success() suppress(label='set_success') final = simple.resolve() self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category) self.assertEqual(SUCCESS_MESSAGE, final.title+"\n"+final.message)
def test_success(self): clear_report() contextualize_report('a=0\na') verify() tifa_analysis() set_success() final = simple.resolve() self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category) self.assertEqual("Complete", final.title) self.assertEqual("Great work!", final.message)
def test_runtime_suppression(self): clear_report() contextualize_report('import json\njson.loads("0")+"1"') verify() tifa_analysis() commands.run() suppress("Runtime") final = simple.resolve() self.assertEqual(Feedback.CATEGORIES.COMPLETE, final.category) self.assertEqual(SUCCESS_TEXT, final.message)
def test_analyzer_suppression(self): clear_report() contextualize_report('1+"Hello"') verify() tifa_analysis() commands.run() suppress("analyzer") final = simple.resolve() self.assertEqual("runtime", final.category) self.assertEqual("Type Error", final.title)
def __enter__(self): clear_report(report=self.report) contextualize_report(self.code, report=self.report) verify(report=self.report) if self.run_tifa: tifa_analysis(report=self.report) # TODO: Clean this up self.student = get_sandbox(self.report) self.report['sandbox']['sandbox'].tracer_style = self.tracer_style commands.run() return self
def test_sandboxing_sys_modules(self): clear_report() student_code = dedent(''' import sys # Might try to bypass us del sys.modules['pedal'] from pedal.report import MAIN_REPORT print(MAIN_REPORT) ''') set_source(student_code) student = Sandbox() student.run(student_code, filename='student.py') self.assertEqual(str(student.exception), "You cannot import pedal!")
def test_combining_scores_complex(self): clear_report() contextualize_report('a=0\nprint(a)') # These are added feedback(activate=True, valence=1, score="+4%", category='instructor') # These are skipped feedback(activate=False, valence=1, score="+5%", category='instructor') # These are skipped feedback(activate=True, valence=-1, score="+8%", category='instructor') # These are added feedback(activate=False, valence=-1, score="+7%", category='instructor') # Calculate final result final = simple.resolve() self.assertEqual(.11, final.score)
def test_simple_system_correct(self): clear_report() vpl = VPLEnvironment(main_code='1+2') with io.StringIO() as f, redirect_stdout(f): log("Message Printed") vpl.resolve() self.assertEqual("""<|-- -System Notes Message Printed -Complete Great work! --|> Grade :=>> 1 """, f.getvalue())
def test_simple_system(self): clear_report() vpl = VPLEnvironment(main_code='1+2') with io.StringIO() as f, redirect_stdout(f): log("Message Printed") assert_equal(1, 2) vpl.resolve() self.assertEqual("""<|-- -System Notes Message Printed -Failed Instructor Test Student code failed instructor test. >1 != 2 --|> Grade :=>> 0 """, f.getvalue())
def test_old_style_api(self): contextualize_report("a = open('file.txt')") std_ast = parse_program() calls = std_ast.find_all("Call") self.assertEqual(len(calls), 1) self.assertEqual(calls[0].func.ast_name, 'Name') self.assertEqual(calls[0].func.id, 'open') self.assertEqual(len(calls[0].args), 1) clear_report() contextualize_report("def a():\n pass\na()") std_ast = parse_program() defs = std_ast.find_all("FunctionDef") self.assertEqual(len(defs), 1) clear_report() contextualize_report("1 < 1") std_ast = parse_program() compares = std_ast.find_all("Compare") self.assertEqual(len(compares), 1) clear_report() contextualize_report("for x in y:\n x") std_ast = parse_program() loops = std_ast.find_all("For") self.assertEqual(len(loops), 1) self.assertEqual(loops[0].target.ast_name, "Name") # Multiple assignment clear_report() contextualize_report("a, b = 0, 1") std_ast = parse_program() assigns = std_ast.find_all("Assign") self.assertEqual(len(assigns), 1) self.assertEqual(len(assigns[0].targets), 1) self.assertEqual(assigns[0].targets[0].ast_name, 'Tuple') self.assertEqual(assigns[0].targets[0].elts[0].id, 'a') clear_report() contextualize_report('from pprint import *') parse_program()
def test_gently_vs_runtime(self): # Runtime > Gently clear_report() contextualize_report('import json\njson.loads("0")+"1"') verify() tifa_analysis() commands.run() gently("I have a gentle opinion, but you don't want to hear it.") final = simple.resolve() print(final.label) self.assertEqual(Feedback.CATEGORIES.RUNTIME, final.category) # Runtime < Explain clear_report() contextualize_report('import json\njson.loads("0")+"1"') verify() tifa_analysis() commands.run() explain("LISTEN TO ME") final = simple.resolve() self.assertEqual(Feedback.CATEGORIES.INSTRUCTOR, final.category)
def test_resolve(self): clear_report() vpl = VPLEnvironment(main_file=here+'datafiles/student_example.py') separate_into_sections() # Part 0 - usually skipped # Part 1 vpl.next_section() # Part 2 vpl.next_section() compliment('Hey, not a bad job!') # Part 3 vpl.next_section() # Part 4 vpl.next_section() # Resolve entire thing with io.StringIO() as f, redirect_stdout(f): vpl.sectional_resolve() self.assertEqual("""<|-- -Part 0 No feedback for this section -Part 1 No feedback for this section -Part 2 Complete Great work! -Part 3 Syntax Error Bad syntax on line 15 The traceback was: Line 15 of file datafiles/student_example.py >a syntax error in this section! Suggestion: Check line 15, the line before it, and the line after it. -Part 4 No feedback for this section --|> Grade :=>> 1 """, f.getvalue())
def test_sectional_error(self): clear_report() contextualize_report('a=0\n##### Part 1\nprint("A")\n##### Part 2\nsyntax error') separate_into_sections(independent=True) # Part 0 verify() commands.clear_sandbox() commands.run() # Part 1 next_section() verify() commands.clear_sandbox() commands.run() give_partial(.2) # Part 2 next_section() verify() commands.clear_sandbox() commands.run() give_partial(.2) # Resolve everything finals = sectional.resolve() self.assertEqual("""# Global FeedbackSourceSection Feedback separated into groups # 1 Complete Great work! # 2 Syntax Error Bad syntax on line 5 The traceback was: Line 5 of file answer.py syntax error Suggestion: Check line 5, the line before it, and the line after it.""", "\n".join(f"# {g.section_number if g is not None else 'Global'}\n{f.title}\n{f.message}" for g, f in finals.items()))
def test_do_nothing(self): clear_report() final = simple.resolve() self.assertTrue(final.success) self.assertEqual(final.message, SUCCESS_TEXT)
def test_gently(self): clear_report() gently('You should always create unit tests.') final = simple.resolve() self.assertFalse(final.success) self.assertEqual(final.message, 'You should always create unit tests.')
def test_set_success(self): clear_report() set_success() final = simple.resolve() self.assertTrue(final.success) self.assertEqual(final.message, 'Great work!')