def test_cpp_infinite_loop(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
while(0==0){
cout<<"abc";}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
def test_incorrect_answer(self):
# Given
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a = s.nextInt();
int b = s.nextInt();
System.out.print(a);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
result_error = result.get('error')[0].get('error_msg')
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result_error)
self.assertTrue(lines_of_error > 0)
def test_only_stdout(self):
# Given
self.test_case_data = [{
'expected_output': '11',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
class Test
{public static void main(String[] args){
int a = 5;
int b = 6;
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
exec(user_answer, globals())
if add(1,2) == 3:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a = s.nextInt();
int b = s.nextInt();
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_file_based_answer(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "2",
"weight": 0.0
}]
self.file_paths = [(self.tmp_file, False)]
user_answer = dedent("""
with open("test.txt") as f:
a = f.read()
print(a[0])
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self):
# Given
user_answer = dedent("""\
a = "this should be a string."
print(type(a).__name__)
""")
test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "str",
"weight": 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_infinite_loop(self):
# Given
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
" do echo "
" > /dev/null ; done")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_indent_error(self):
# Given
user_answer = dedent("""
def add(a, b):
return a + b
""")
indent_error_msg = [
"Traceback", "call", "File", "line", "<string>",
"IndentationError", "indented block"
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]["traceback"].splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(5, len(err))
for msg in indent_error_msg:
self.assert_correct_output(msg,
result.get("error")[0]['traceback'])
def test_incorrect_answer(self):
# Given
user_answer = dedent(""" #!/bin/bash
read A
read B
echo -n `expr $A - $B`
""")
test_case_data = [{
'expected_output': '11',
'expected_input': '5\n6',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
result_error = result.get('error')[0].get('error_msg')
self.assert_correct_output("Incorrect", result_error)
self.assertFalse(result.get('success'))
def test_stdout_only(self):
# Given
user_answer = dedent(""" #!/bin/bash
A=6
B=4
echo -n `expr $A + $B`
""")
test_case_data = [{
'expected_output': '10',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_array_input(self):
# Given
user_answer = dedent(""" readarray arr;
COUNTER=0
while [ $COUNTER -lt 3 ]; do
echo -n "${arr[$COUNTER]}"
let COUNTER=COUNTER+1
done
""")
test_case_data = [{
'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
'expected_input': '1,2,3\n4,5,6\n7,8,9',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_file_based_assert(self):
# Given
self.file_paths = [(self.f_path, False)]
self.tc_data = dedent("""
#!/bin/bash
cat $1
""")
self.tc_data_args = "test.txt"
self.test_case_data = [{
"test_case": self.tc_data,
"test_case_args": self.tc_data_args,
"test_case_type": "standardtestcase",
"weight": 0.0
}]
user_answer = ("#!/bin/bash\ncat $1")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_cpp_only_stdout(self):
# Given
self.test_case_data = [{
'expected_output': '11',
'expected_input': '',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=5,b=6;
cout<<a+b;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_correct_answer_string(self):
# Given
self.test_case_data = [{
"test_case_type":
"stdiobasedtestcase",
"expected_input":
("the quick brown fox jumps over the lazy dog\nthe"),
"expected_output":
"2",
"weight":
0.0
}]
user_answer = dedent("""
from six.moves import input
a = str(input())
b = str(input())
print(a.count(b))
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_recursion_error(self):
# Given
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = "maximum recursion depth exceeded"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['message']
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(recursion_error_msg, err)
def test_incorrect_answer_integer(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "1\n2",
"expected_output": "3",
"weight": 0.0
}]
user_answer = dedent("""
a = int(input())
b = int(input())
print(a-b)
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output(
"Incorrect Answer: Line number(s) 1 did not match.",
result.get('error')[0].get('error_msg'))
def test_type_error(self):
# Given
user_answer = dedent("""
def add(a):
return a + b
""")
type_error_msg = ["Traceback", "call", "TypeError", "argument"]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in type_error_msg:
self.assert_correct_output(msg, err)
def test_infinite_loop(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "1\n2",
"expected_output": "3",
"weight": 0.0
}]
timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
user_answer = "while True:\n\tpass"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assert_correct_output(timeout_msg,
result.get("error")[0]["message"])
self.assertFalse(result.get('success'))
def test_value_error(self):
# Given
user_answer = dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = [
"Traceback", "call", "ValueError", "invalid literal", "base"
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in value_error_msg:
self.assert_correct_output(msg, err)
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
for error in result.get("error"):
self.assertEqual(error['exception'], 'AssertionError')
self.assertEqual(
error['message'],
"Expected answer from the test case did not match the output")
error_testcase_list = [tc['test_case'] for tc in result.get('error')]
self.assertEqual(error_testcase_list, given_test_case_list)
def test_file_based_assert(self):
# Given
self.test_case_data = [{
"test_case_type": "standardtestcase",
"test_case": "assert(ans()=='2')",
"weight": 0.0
}]
self.file_paths = [(self.tmp_file, False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
return f.read()[0]
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
# Given
user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_single_testcase_error(self):
""" Tests the user answer with just an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{
"test_case_type": "standardtestcase",
"test_case": 's="abbb"\nasert palindrome(s)==False',
"weight": 0.0
}]
syntax_error_msg = [
"Traceback", "call", "SyntaxError", "invalid syntax"
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in syntax_error_msg:
self.assert_correct_output(msg, err)
def test_array_input(self):
# Given
self.test_case_data = [{
'expected_output': '561',
'expected_input': '5\n6\n1',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a[] = new int[3];
for (int i=0;i<3;i++){
a[i] = s.nextInt();
System.out.print(a[i]);}
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self):
# Given
user_answer = dedent("""\
def strchar(s):
a = "should be a string"
return type(a)
""")
test_case_data = [
{
"test_case_type": "standardtestcase",
"test_case": 'assert(strchar("hello")==str)',
"weight": 0.0
},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_infinite_loop(self):
# Given
user_answer = dedent("""
class Test
{public static void main(String[] args){
while(0==0)
{
System.out.print("a");}
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_correct_answer_list(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "1,2,3\n5,6,7",
"expected_output": "[1, 2, 3, 5, 6, 7]",
"weight": 0.0
}]
user_answer = dedent("""
from six.moves import input
input_a = input()
input_b = input()
a = [int(i) for i in input_a.split(',')]
b = [int(i) for i in input_b.split(',')]
print(a+b)
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_string_input(self):
# Given
self.test_case_data = [{
'expected_output': 'HelloWorld',
'expected_input': 'Hello\nWorld',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
String a = s.nextLine();
String b = s.nextLine();
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_cpp_error(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=10;
cout<<a
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output("Compilation Error", result.get("error"))