def test_indent_error(self):
# Given
user_answer = dedent("""
def add(a, b):
return a + b
""")
indent_error_msg = [
"Traceback", "call", "File", "line", "<string>",
"IndentationError", "indented block"
]
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
self.assertIn(msg, result.get("error"))
def test_value_error(self):
# Given
user_answer = dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = [
"Traceback", "call", "ValueError", "invalid literal", "base"
]
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(9, len(err))
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
def test_recursion_error(self):
# Given
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = [
"Traceback", "call", "maximum recursion depth exceeded"
]
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
for msg in recursion_error_msg:
self.assertIn(msg, result.get("error"))
def test_file_based_assert(self):
# Given
self.test_case_data = [{
"test_case": "assert(ans()=='2')",
"weight": 0.0
}]
self.file_paths = [('/tmp/test.txt', False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
return f.read()[0]
""")
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
# Then
self.assertIn("Correct answer", result.get('error'))
self.assertTrue(result.get('success'))
def test_single_testcase_error(self):
# Given
""" Tests the user answer with just an incorrect test case """
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{
"test_case": 's="abbb"\nasert palindrome(s)==False',
"weight": 0.0
}]
syntax_error_msg = [
"Traceback", "call", "File", "line", "<string>", "SyntaxError",
"invalid syntax"
]
kwargs = {
'user_answer': user_answer,
'test_case_data': test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(6, len(err))
for msg in syntax_error_msg:
self.assertIn(msg, result.get("error"))
def test_multiple_testcase_error(self):
""" Tests the user answer with an correct test case
first and then with an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{
"test_case": 'assert(palindrome("abba")==True)',
"weight": 0.0
}, {
"test_case": 's="abbb"\nassert palindrome(S)==False',
"weight": 0.0
}]
name_error_msg = [
"Traceback", "call", "NameError", "name 'S' is not defined"
]
kwargs = {
'user_answer': user_answer,
'test_case_data': test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(7, len(err))
for msg in name_error_msg:
self.assertIn(msg, result.get("error"))
def test_partial_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn abs(a) + abs(b)"
test_case_data = [{
"test_case": 'assert(add(-1,2)==1)',
'weight': 1.0
}, {
"test_case": 'assert(add(-1,-2)==-3)',
'weight': 1.0
}, {
"test_case": 'assert(add(1,2)==3)',
'weight': 2.0
}]
kwargs = {
'user_answer': user_answer,
'test_case_data': test_case_data,
'file_paths': self.file_paths,
'partial_grading': True
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
# Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('weight'), 2.0)
self.assertIn('AssertionError in: assert(add(-1,2)==1)',
result.get('error'))
self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error'))
def test_correct_answer(self):
user_answer = "def add(a,b):\n\treturn a + b"
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer")
def test_infinite_loop(self):
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
def test_correct_answer(self):
user_answer = "def add(a,b):\n\treturn a + b"
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertTrue(result.get('success'))
self.assertEqual(result.get('error'), "Correct answer")
def test_incorrect_answer(self):
user_answer = "def add(a,b):\n\treturn a - b"
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'),
"AssertionError in: assert(add(1,2)==3)"
)
def test_incorrect_answer(self):
user_answer = "def add(a,b):\n\treturn a - b"
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'),
"AssertionError in: assert(add(1,2)==3)")
def test_name_error(self):
user_answer = ""
name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(3, len(err))
for msg in name_error_msg:
self.assertIn(msg, result.get("error"))
def test_file_based_assert(self):
self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
self.file_paths = [(os.getcwd()+"/yaksh/test.txt", False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
return f.read()[0]
""")
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_file_based_assert(self):
self.test_case_data = [{"test_case": "assert(ans()=='2')"}]
self.file_paths = [(os.getcwd() + "/yaksh/test.txt", False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
return f.read()[0]
""")
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertEqual(result.get('error'), "Correct answer")
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
# Given
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
# Then
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
def test_correct_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertIn("Correct answer", result.get('error'))
def test_name_error(self):
user_answer = ""
name_error_msg = ["Traceback",
"call",
"NameError",
"name",
"defined"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(2, len(err))
for msg in name_error_msg:
self.assertIn(msg, result.get("error"))
def test_recursion_error(self):
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = ["Traceback",
"call",
"RuntimeError",
"maximum recursion depth exceeded"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(2, len(err))
for msg in recursion_error_msg:
self.assertIn(msg, result.get("error"))
def test_type_error(self):
user_answer = dedent("""
def add(a):
return a + b
""")
type_error_msg = [
"Traceback", "call", "TypeError", "exactly", "argument"
]
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(3, len(err))
for msg in type_error_msg:
self.assertIn(msg, result.get("error"))
def test_recursion_error(self):
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = [
"Traceback", "call", "RuntimeError",
"maximum recursion depth exceeded"
]
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(969, len(err))
for msg in recursion_error_msg:
self.assertIn(msg, result.get("error"))
def test_name_error(self):
# Given
user_answer = ""
name_error_msg = ["Traceback", "call", "NameError", "name", "defined"]
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(9, len(err))
for msg in name_error_msg:
self.assertIn(msg, result.get("error"))
def test_value_error(self):
user_answer = dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = [
"Traceback", "call", "ValueError", "invalid literal", "base"
]
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(4, len(err))
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
def test_type_error(self):
user_answer = dedent("""
def add(a):
return a + b
""")
type_error_msg = ["Traceback",
"call",
"TypeError",
"exactly",
"argument"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(2, len(err))
for msg in type_error_msg:
self.assertIn(msg, result.get("error"))
def test_infinite_loop(self):
user_answer = "def add(a, b):\n\twhile True:\n\t\tpass"
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
self.assertFalse(result.get('success'))
self.assertEqual(result.get('error'), self.timeout_msg)
user_answer = dedent("""
def add(a, b);
return a + b
""")
syntax_error_msg = [
"Traceback", "call", "File", "line", "<string>", "SyntaxError",
"invalid syntax"
]
get_class = PythonAssertionEvaluator()
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in syntax_error_msg:
self.assertIn(msg, result.get("error"))
def test_value_error(self):
user_answer = dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = ["Traceback",
"call",
"ValueError",
"invalid literal",
"base"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(2, len(err))
for msg in value_error_msg:
self.assertIn(msg, result.get("error"))
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
# Then
self.assertFalse(result.get('success'))
self.assertIn('AssertionError in: assert(add(1,2)==3)',
result.get('error'))
self.assertIn('AssertionError in: assert(add(-1,2)==1)',
result.get('error'))
self.assertIn('AssertionError in: assert(add(-1,-2)==-3)',
result.get('error'))
def test_indent_error(self):
user_answer = dedent("""
def add(a, b):
return a + b
""")
indent_error_msg = ["Traceback", "call",
"File",
"line",
"<string>",
"IndentationError",
"indented block"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
self.assertIn(msg, result.get("error"))
def test_syntax_error(self):
user_answer = dedent("""
def add(a, b);
return a + b
""")
syntax_error_msg = ["Traceback",
"call",
"File",
"line",
"<string>",
"SyntaxError",
"invalid syntax"
]
get_class = PythonAssertionEvaluator()
kwargs = {'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths
}
result = get_class.evaluate(**kwargs)
err = result.get("error").splitlines()
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in syntax_error_msg:
self.assertIn(msg, result.get("error"))
def test_type_error(self):
# Given
user_answer = dedent("""
def add(a):
return a + b
""")
type_error_msg = ["Traceback", "call", "TypeError", "argument"]
kwargs = {
'user_answer': user_answer,
'test_case_data': self.test_case_data,
'file_paths': self.file_paths,
'partial_grading': False
}
# When
evaluator = PythonAssertionEvaluator()
result = evaluator.evaluate(**kwargs)
err = result.get("error").splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(9, len(err))
for msg in type_error_msg:
self.assertIn(msg, result.get("error"))
