def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg) user_answer = dedent(""" def add(a, b); return a + b """) syntax_error_msg = [ "Traceback", "call", "File", "line", "<string>", "SyntaxError", "invalid syntax" ] get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error"))
def test_indent_error(self): # Given user_answer = dedent(""" def add(a, b): return a + b """) indent_error_msg = [ "Traceback", "call", "File", "line", "<string>", "IndentationError", "indented block" ] kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assertIn(msg, result.get("error"))
def test_recursion_error(self): # Given user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = [ "Traceback", "call", "maximum recursion depth exceeded" ] kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) for msg in recursion_error_msg: self.assertIn(msg, result.get("error"))
def test_value_error(self): # Given user_answer = dedent(""" def add(a, b): c = 'a' return int(a) + int(b) + int(c) """) value_error_msg = [ "Traceback", "call", "ValueError", "invalid literal", "base" ] kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(9, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error"))
def test_file_based_assert(self): # Given self.test_case_data = [{ "test_case": "assert(ans()=='2')", "weight": 0.0 }] self.file_paths = [('/tmp/test.txt', False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) # Then self.assertIn("Correct answer", result.get('error')) self.assertTrue(result.get('success'))
def test_single_testcase_error(self): # Given """ Tests the user answer with just an incorrect test case """ user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{ "test_case": 's="abbb"\nasert palindrome(s)==False', "weight": 0.0 }] syntax_error_msg = [ "Traceback", "call", "File", "line", "<string>", "SyntaxError", "invalid syntax" ] kwargs = { 'user_answer': user_answer, 'test_case_data': test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(6, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error"))
def test_multiple_testcase_error(self): """ Tests the user answer with an correct test case first and then with an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{ "test_case": 'assert(palindrome("abba")==True)', "weight": 0.0 }, { "test_case": 's="abbb"\nassert palindrome(S)==False', "weight": 0.0 }] name_error_msg = [ "Traceback", "call", "NameError", "name 'S' is not defined" ] kwargs = { 'user_answer': user_answer, 'test_case_data': test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(7, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error"))
def test_partial_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)" test_case_data = [{ "test_case": 'assert(add(-1,2)==1)', 'weight': 1.0 }, { "test_case": 'assert(add(-1,-2)==-3)', 'weight': 1.0 }, { "test_case": 'assert(add(1,2)==3)', 'weight': 2.0 }] kwargs = { 'user_answer': user_answer, 'test_case_data': test_case_data, 'file_paths': self.file_paths, 'partial_grading': True } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('weight'), 2.0) self.assertIn('AssertionError in: assert(add(-1,2)==1)', result.get('error')) self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', result.get('error'))
def test_correct_answer(self): user_answer = "def add(a,b):\n\treturn a + b" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer")
def test_infinite_loop(self): user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg)
def test_incorrect_answer(self): user_answer = "def add(a,b):\n\treturn a - b" get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)" )
def test_correct_answer(self): user_answer = "def add(a,b):\n\treturn a + b" get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertTrue(result.get('success')) self.assertEqual(result.get('error'), "Correct answer")
def test_incorrect_answer(self): user_answer = "def add(a,b):\n\treturn a - b" get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), "AssertionError in: assert(add(1,2)==3)")
def test_name_error(self): user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error"))
def test_file_based_assert(self): self.test_case_data = [{"test_case": "assert(ans()=='2')"}] self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success'))
def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) # Then self.assertTrue(result.get('success')) self.assertIn("Correct answer", result.get('error'))
def test_file_based_assert(self): self.test_case_data = [{"test_case": "assert(ans()=='2')"}] self.file_paths = [(os.getcwd() + "/yaksh/test.txt", False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) self.assertEqual(result.get('error'), "Correct answer") self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = "def add(a, b):\n\twhile True:\n\t\tpass" kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) # Then self.assertFalse(result.get('success')) self.assertEqual(result.get('error'), self.timeout_msg)
def test_name_error(self): user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error"))
def test_type_error(self): user_answer = dedent(""" def add(a): return a + b """) type_error_msg = [ "Traceback", "call", "TypeError", "exactly", "argument" ] get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(3, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error"))
def test_recursion_error(self): user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = ["Traceback", "call", "RuntimeError", "maximum recursion depth exceeded" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error"))
def test_value_error(self): user_answer = dedent(""" def add(a, b): c = 'a' return int(a) + int(b) + int(c) """) value_error_msg = [ "Traceback", "call", "ValueError", "invalid literal", "base" ] get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(4, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error"))
def test_type_error(self): user_answer = dedent(""" def add(a): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "exactly", "argument" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error"))
def test_name_error(self): # Given user_answer = "" name_error_msg = ["Traceback", "call", "NameError", "name", "defined"] kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(9, len(err)) for msg in name_error_msg: self.assertIn(msg, result.get("error"))
def test_recursion_error(self): user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = [ "Traceback", "call", "RuntimeError", "maximum recursion depth exceeded" ] get_class = PythonAssertionEvaluator() kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(969, len(err)) for msg in recursion_error_msg: self.assertIn(msg, result.get("error"))
def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) # Then self.assertFalse(result.get('success')) self.assertIn('AssertionError in: assert(add(1,2)==3)', result.get('error')) self.assertIn('AssertionError in: assert(add(-1,2)==1)', result.get('error')) self.assertIn('AssertionError in: assert(add(-1,-2)==-3)', result.get('error'))
def test_value_error(self): user_answer = dedent(""" def add(a, b): c = 'a' return int(a) + int(b) + int(c) """) value_error_msg = ["Traceback", "call", "ValueError", "invalid literal", "base" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(2, len(err)) for msg in value_error_msg: self.assertIn(msg, result.get("error"))
def test_indent_error(self): user_answer = dedent(""" def add(a, b): return a + b """) indent_error_msg = ["Traceback", "call", "File", "line", "<string>", "IndentationError", "indented block" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assertIn(msg, result.get("error"))
def test_type_error(self): # Given user_answer = dedent(""" def add(a): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "argument"] kwargs = { 'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths, 'partial_grading': False } # When evaluator = PythonAssertionEvaluator() result = evaluator.evaluate(**kwargs) err = result.get("error").splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(9, len(err)) for msg in type_error_msg: self.assertIn(msg, result.get("error"))
def test_syntax_error(self): user_answer = dedent(""" def add(a, b); return a + b """) syntax_error_msg = ["Traceback", "call", "File", "line", "<string>", "SyntaxError", "invalid syntax" ] get_class = PythonAssertionEvaluator() kwargs = {'user_answer': user_answer, 'test_case_data': self.test_case_data, 'file_paths': self.file_paths } result = get_class.evaluate(**kwargs) err = result.get("error").splitlines() self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in syntax_error_msg: self.assertIn(msg, result.get("error"))