def test_incorrect_answer(self):
     user_answer = dedent("""
     def add(a, b):
         return a - b
     """)
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEqual(result.get("error"), "AssertionError  in: assert add(3, 2) == 5")
 def test_correct_answer(self):
     user_answer = dedent("""
     def add(a, b):
         return a + b
     """)
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     self.assertTrue(result.get("success"))
     self.assertEqual(result.get("error"), "Correct answer")
 def test_correct_answer(self):
     user_answer = "def add(a, b):\n\treturn a + b" ""
     get_class = PythonCodeEvaluator(self.test_case_data,
                                     self.test,
                                     self.language,
                                     user_answer,
                                     ref_code_path=None,
                                     in_dir=None)
     result = get_class.evaluate()
     self.assertTrue(result.get("success"))
     self.assertEqual(result.get("error"), "Correct answer")
 def test_name_error(self):
     user_answer = ""
     name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     err = result.get("error").splitlines()
     self.assertFalse(result.get("success"))
     self.assertEqual(2, len(err))
     for msg in name_error_msg:
         self.assertIn(msg, result.get("error"))
 def test_infinite_loop(self):
     user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"""
     test_case_data = [{"func_name": "add", 
                              "expected_answer": "5", 
                              "test_id": u'null', 
                              "pos_args": ["3", "2"], 
                              "kw_args": {}
                             }]
     get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
     result = get_class.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEquals(result.get("error"), self.timeout_msg)
 def test_incorrect_answer(self):
     user_answer = "def add(a, b):\n\treturn a - b"
     test_case_data = [{"func_name": "add", 
                              "expected_answer": "5", 
                              "test_id": u'null', 
                              "pos_args": ["3", "2"], 
                              "kw_args": {}
                             }]
     get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
     result = get_class.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEqual(result.get("error"), "AssertionError  in: assert add(3, 2) == 5")
 def test_infinite_loop(self):
     user_answer = dedent("""
     def add(a, b):
         while True:
             pass
     """)
     timeout_msg = ("Code took more than {0} seconds to run. "
                    "You probably have an infinite loop in your code.").format(SERVER_TIMEOUT)
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEquals(result.get("error"), timeout_msg)
 def test_type_error(self):
     user_answer = dedent("""
     def add(a):
         return a + b
     """)
     type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument"]
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     err = result.get("error").splitlines()
     self.assertFalse(result.get("success"))
     self.assertEqual(2, len(err))
     for msg in type_error_msg:
         self.assertIn(msg, result.get("error"))
 def test_recursion_error(self):
     user_answer = dedent("""
     def add(a, b):
         return add(3, 3)
     """)
     recursion_error_msg = ["Traceback", "call", "RuntimeError",
                             "maximum recursion depth exceeded"]
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     err = result.get("error").splitlines()
     self.assertFalse(result.get("success"))
     self.assertEqual(2, len(err))
     for msg in recursion_error_msg:
         self.assertIn(msg, result.get("error"))
 def test_indent_error(self):
     user_answer = dedent("""
     def add(a, b):
     return a + b
     """)
     indent_error_msg = ["Traceback", "call", "File", "line", "<string>",
                         "IndentationError", "indented block"]
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     err = result.get("error").splitlines()
     self.assertFalse(result.get("success"))
     self.assertEqual(5, len(err))
     for msg in indent_error_msg:
         self.assertIn(msg, result.get("error"))
 def test_value_error(self):
     user_answer = dedent("""
     def add(a, b):
         c = 'a'
         return int(a) + int(b) + int(c)
     """)
     value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base"]
     get_evaluator = PythonCodeEvaluator(self.test_case_data, self.test,
                                         self.language, user_answer)
     result = get_evaluator.evaluate()
     err = result.get("error").splitlines()
     self.assertFalse(result.get("success"))
     self.assertEqual(2, len(err))
     for msg in value_error_msg:
         self.assertIn(msg, result.get("error"))
 def test_infinite_loop(self):
     user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" ""
     test_case_data = [{
         "func_name": "add",
         "expected_answer": "5",
         "test_id": u'null',
         "pos_args": ["3", "2"],
         "kw_args": {}
     }]
     get_class = PythonCodeEvaluator(self.test_case_data,
                                     self.test,
                                     self.language,
                                     user_answer,
                                     ref_code_path=None,
                                     in_dir=None)
     result = get_class.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEquals(result.get("error"), self.timeout_msg)
 def test_incorrect_answer(self):
     user_answer = "def add(a, b):\n\treturn a - b"
     test_case_data = [{
         "func_name": "add",
         "expected_answer": "5",
         "test_id": u'null',
         "pos_args": ["3", "2"],
         "kw_args": {}
     }]
     get_class = PythonCodeEvaluator(self.test_case_data,
                                     self.test,
                                     self.language,
                                     user_answer,
                                     ref_code_path=None,
                                     in_dir=None)
     result = get_class.evaluate()
     self.assertFalse(result.get("success"))
     self.assertEqual(result.get("error"),
                      "AssertionError  in: assert add(3, 2) == 5")
 def test_correct_answer(self):
     user_answer = "def add(a, b):\n\treturn a + b"""
     get_class = PythonCodeEvaluator(self.test_case_data, self.test, self.language, user_answer, ref_code_path=None, in_dir=None)
     result = get_class.evaluate()
     self.assertTrue(result.get("success"))
     self.assertEqual(result.get("error"), "Correct answer")