def test_incorrect_answer(self): user_answer = dedent(""" def add(a, b): return a - b """) get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5")
def test_correct_answer(self): user_answer = dedent(""" def add(a, b): return a + b """) get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer")
def test_correct_answer(self): user_answer = "def add(a, b):\n\treturn a + b" "" get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer")
def test_name_error(self): user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error"))
def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass""" test_case_data = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg)
def test_incorrect_answer(self): user_answer = "def add(a, b):\n\treturn a - b" test_case_data = [{"func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5")
def test_infinite_loop(self): user_answer = dedent(""" def add(a, b): while True: pass """) timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT) get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), timeout_msg)
def test_type_error(self): user_answer = dedent(""" def add(a): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error"))
def test_recursion_error(self): user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = ["Traceback", "call", "RuntimeError", "maximum recursion depth exceeded"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error"))
def test_indent_error(self): user_answer = dedent(""" def add(a, b): return a + b """) indent_error_msg = ["Traceback", "call", "File", "line", "<string>", "IndentationError", "indented block"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assertIn(msg, result.get("error"))
def test_value_error(self): user_answer = dedent(""" def add(a, b): c = 'a' return int(a) + int(b) + int(c) """) value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"] get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer) result = get_evaluator.evaluate() err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error"))
def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" "" test_case_data = [{ "func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertFalse(result.get("success")) self.assertEquals(result.get("error"), self.timeout_msg)
def test_incorrect_answer(self): user_answer = "def add(a, b):\n\treturn a - b" test_case_data = [{ "func_name": "add", "expected_answer": "5", "test_id": u'null', "pos_args": ["3", "2"], "kw_args": {} }] get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertFalse(result.get("success")) self.assertEqual(result.get("error"), "AssertionError in: assert add(3, 2) == 5")
def test_correct_answer(self): user_answer = "def add(a, b):\n\treturn a + b""" get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None) result = get_class.evaluate() self.assertTrue(result.get("success")) self.assertEqual(result.get("error"), "Correct answer")