def test_infinite_loop(self):
# Given
self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "1\n2",
"expected_output": "3",
"weight": 0.0
}]
timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
user_answer = "while True:\n\tpass"
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assert_correct_output(timeout_msg,
result.get("error")[0]["message"]
)
self.assertFalse(result.get('success'))
def test_infinite_loop(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
while(0==0){
printf("abc");}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg, result.get("error"))
def test_array_input(self):
self.test_case_data = [{'expected_output': '561',
'expected_input': '5\n6\n1',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a[] = new int[3];
for (int i=0;i<3;i++){
a[i] = s.nextInt();
System.out.print(a[i]);}
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
def test_array_input(self):
# Given
user_answer = dedent(""" readarray arr;
COUNTER=0
while [ $COUNTER -lt 3 ]; do
echo -n "${arr[$COUNTER]}"
let COUNTER=COUNTER+1
done
""")
test_case_data = [{
'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
'expected_input': '1,2,3\n4,5,6\n7,8,9',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a,b;
scanf("%d%d",&a,&b);
printf("%d",a+b);
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": "1\n2",
"expected_output": "3",
"weight": 0.0
}]
timeout_msg = ("Code took more than {0} seconds to run. "
"You probably have an infinite loop in"
" your code.").format(SERVER_TIMEOUT)
user_answer = "while True:\n\tpass"
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assert_correct_output(timeout_msg,
result.get("error")[0]["message"]
)
self.assertFalse(result.get('success'))
def test_incorrect_answer_without_test_case_args(self):
# Given
user_answer = "echo 'hello'"
tc_data = "echo 'hello world'"
self.test_case_data = [{
"test_case": tc_data,
"test_case_args": "",
"test_case_type": "standardtestcase",
"weight": 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
def test_correct_answer(self):
user_answer = dedent(""" #!/bin/bash
read A
read B
echo -n `expr $A + $B`
"""
)
test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a,b;
scanf("%d%d",&a,&b);
printf("%d",a+b);
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_assignment_upload(self):
# Given
user_answer = "Assignment Upload"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
with open("test.txt") as f:
data = f.read()
if data == '2':
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "hooktestcase",
"hook_code": hook_code,"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'assign_files': [(self.tmp_file, False)],
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_incorrect_answer_without_test_case_args(self):
# Given
user_answer = "echo 'hello'"
tc_data = "echo 'hello world'"
self.test_case_data = [
{"test_case": tc_data,
"test_case_args": "",
"test_case_type": "standardtestcase",
"weight": 0.0
}
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
exec(user_answer, globals())
if add(1,2) == 3:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "hooktestcase",
"hook_code": hook_code,"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self):
# Given
user_answer = dedent("""\
a = "this should be a string."
print(type(a).__name__)
""")
test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "str",
"weight": 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
for error in result.get("error"):
self.assertEqual(error['exception'], 'AssertionError')
self.assertEqual(error['message'],
"Expected answer from the test case did not match the output"
)
error_testcase_list = [tc['test_case'] for tc in result.get('error')]
self.assertEqual(error_testcase_list, given_test_case_list)
def test_correct_answer_string(self):
# Given
self.test_case_data = [{
"test_case_type": "stdiobasedtestcase",
"expected_input": ("the quick brown fox jumps over "
"the lazy dog\nthe"),
"expected_output": "2",
"weight": 0.0
}]
user_answer = dedent("""
from six.moves import input
a = str(input())
b = str(input())
print(a.count(b))
"""
)
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_error(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a=10;
printf("%d",a)
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output("Compilation Error", result.get("error"))
def test_file_based_answer(self):
# Given
self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "2",
"weight": 0.0
}]
self.file_paths = [(self.tmp_file, False)]
user_answer = dedent("""
with open("test.txt") as f:
a = f.read()
print(a[0])
"""
)
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_file_based_assert(self):
# Given
self.file_paths = [(self.f_path, False)]
self.tc_data = dedent("""
#!/bin/bash
cat $1
""")
self.tc_data_args = "test.txt"
self.test_case_data = [
{"test_case": self.tc_data,
"test_case_args": self.tc_data_args,
"test_case_type": "standardtestcase",
"weight": 0.0
}
]
user_answer = ("#!/bin/bash\ncat $1")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_assert_with_hook(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
assert_test_case = "assert add(1,2) == 3"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if "return a + b" in user_answer:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": assert_test_case, 'weight': 1.0},
{"test_case_type": "hooktestcase",
"hook_code": hook_code, 'weight': 1.0},
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 2.0)
def test_array_input(self):
# Given
user_answer = dedent(""" readarray arr;
COUNTER=0
while [ $COUNTER -lt 3 ]; do
echo -n "${arr[$COUNTER]}"
let COUNTER=COUNTER+1
done
"""
)
test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
'expected_input': '1,2,3\n4,5,6\n7,8,9',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
# Given
user_answer = dedent("""\
#include<stdio.h>
int main(void){
while(0==0){
printf("abc");}
}""")
hook_code = dedent("""\
def check_answer(user_answer):
with open("Test.c", "w+") as f:
f.write(user_answer)
import subprocess
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
def _run_command(cmd):
proc = subprocess.Popen("{}".format(cmd),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout,stderr = proc.communicate()
return stdout,stderr
cmds = ["gcc Test.c", "./a.out"]
for cmd in cmds:
stdout, stderr = _run_command(cmd)
if stdout.decode("utf-8") == "Hello, world!":
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output(self.timeout_msg, result.get('error'))
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_incorrect_answer(self):
# Given
user_answer = dedent(""" #!/bin/bash
read A
read B
echo -n `expr $A - $B`
"""
)
test_case_data = [{'expected_output': '11',
'expected_input': '5\n6',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
result_error = result.get('error')[0].get('error_msg')
self.assert_correct_output("Incorrect", result_error)
self.assertFalse(result.get('success'))
def test_stdout_only(self):
# Given
user_answer = dedent(""" #!/bin/bash
A=6
B=4
echo -n `expr $A + $B`
""")
test_case_data = [{
'expected_output': '10',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_stdout_only(self):
# Given
user_answer = dedent(""" #!/bin/bash
A=6
B=4
echo -n `expr $A + $B`
"""
)
test_case_data = [{'expected_output': '10',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a=10;
printf("%d",a);
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_infinite_loop(self):
# Given
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
" do echo "" > /dev/null ; done")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"]
)
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_cpp_correct_answer(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a,b;
cin>>a>>b;
cout<<a+b;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_infinite_loop(self):
# Given
user_answer = ("class Test {\n\tint square_num(int a)"
" {\n\t\twhile(0==0){\n\t\t}\n\t}\n}")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"]
)
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_only_stdout(self):
self.test_case_data = [{'expected_output': '11',
'expected_input': '',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
class Test
{public static void main(String[] args){
int a = 5;
int b = 6;
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
def test_correct_answer(self):
# Given
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a = s.nextInt();
int b = s.nextInt();
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_correct_answer_list(self):
# Given
self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "1,2,3\n5,6,7",
"expected_output": "[1, 2, 3, 5, 6, 7]",
"weight": 0.0
}]
user_answer = dedent("""
from six.moves import input
input_a = input()
input_b = input()
a = [int(i) for i in input_a.split(',')]
b = [int(i) for i in input_b.split(',')]
print(a+b)
"""
)
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_error(self):
# Given
user_answer = dedent("""
class Test
{
System.out.print("a");
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
for error in errors:
self.assertEqual(error.get('exception'), 'CompilationError')
def test_incorrect_answer_integer(self):
# Given
self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "1\n2",
"expected_output": "3",
"weight": 0.0
}]
user_answer = dedent("""
a = int(input())
b = int(input())
print(a-b)
"""
)
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output(
"Incorrect Answer: Line number(s) 1 did not match.",
result.get('error')[0].get('error_msg')
)
def test_array_input(self):
# Given
self.test_case_data = [{'expected_output': '561',
'expected_input': '5\n6\n1',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a[3],i;
for(i=0;i<3;i++){
scanf("%d",&a[i]);}
for(i=0;i<3;i++){
printf("%d",a[i]);}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
for error in result.get("error"):
self.assertEqual(error['exception'], 'AssertionError')
self.assertEqual(
error['message'],
"Expected answer from the test case did not match the output"
)
error_testcase_list = [tc['test_case'] for tc in result.get('error')]
self.assertEqual(error_testcase_list, given_test_case_list)
def test_string_input(self):
# Given
self.test_case_data = [{'expected_output': 'abc',
'expected_input': 'abc',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<stdio.h>
int main(void){
char a[4];
scanf("%s",a);
printf("%s",a);
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
exec(user_answer, globals())
if add(1,2) == 3:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "hooktestcase",
"hook_code": hook_code, "weight": 1.0
}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output('Incorrect Answer', result.get('error'))
def test_infinite_loop(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
while(0==0){
printf("abc");}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"]
)
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_assignment_upload(self):
# Given
user_answer = "Assignment Upload"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
with open("test.txt") as f:
data = f.read()
if data == '2':
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "hooktestcase",
"hook_code": hook_code, "weight": 1.0
}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'assign_files': [(self.tmp_file, False)],
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_cpp_correct_answer(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a,b;
cin>>a>>b;
cout<<a+b;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_cpp_incorrect_answer(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=10;
cout<<a;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
result_error = result.get('error')[0].get('error_msg')
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result_error)
self.assertTrue(lines_of_error > 0)
def test_cpp_incorrect_answer(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=10;
cout<<a;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
result_error = result.get('error')[0].get('error_msg')
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result_error)
self.assertTrue(lines_of_error > 0)
def test_file_based_assert(self):
# Given
self.file_paths = [(self.f_path, False)]
self.tc_data = dedent("""
#!/bin/bash
cat $1
""")
self.tc_data_args = "test.txt"
self.test_case_data = [{
"test_case": self.tc_data,
"test_case_args": self.tc_data_args,
"test_case_type": "standardtestcase",
"weight": 0.0
}]
user_answer = ("#!/bin/bash\ncat $1")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_cpp_error(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=10;
cout<<a
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
errors = result.get('error')
# Then
self.assertFalse(result.get("success"))
for error in errors:
self.assertEqual(error['exception'], 'CompilationError')
def test_incorrect_answer(self):
# Given
user_answer = dedent(""" #!/bin/bash
read A
read B
echo -n `expr $A - $B`
""")
test_case_data = [{
'expected_output': '11',
'expected_input': '5\n6',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
result_error = result.get('error')[0].get('error_msg')
self.assert_correct_output("Incorrect", result_error)
self.assertFalse(result.get('success'))
def test_cpp_infinite_loop(self):
# Given
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
while(0==0){
cout<<"abc";}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"]
)
def test_infinite_loop(self):
# Given
user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
" do echo "
" > /dev/null ; done")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'bash'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_cpp_only_stdout(self):
# Given
self.test_case_data = [{'expected_output': '11',
'expected_input': '',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
int a=5,b=6;
cout<<a+b;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
}, 'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_array_input(self):
# Given
self.test_case_data = [{'expected_output': '561',
'expected_input': '5\n6\n1',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a[3],i;
for(i=0;i<3;i++){
scanf("%d",&a[i]);}
for(i=0;i<3;i++){
printf("%d",a[i]);}
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_type_error(self):
# Given
user_answer = dedent("""
def add(a):
return a + b
""")
type_error_msg = ["Traceback",
"call",
"TypeError",
"argument"
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in type_error_msg:
self.assert_correct_output(msg, err)
def test_error(self):
# Given
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a=10;
printf("%d",a)
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output("Compilation Error", result.get("error"))
def test_value_error(self):
# Given
user_answer = dedent("""
def add(a, b):
c = 'a'
return int(a) + int(b) + int(c)
""")
value_error_msg = ["Traceback",
"call",
"ValueError",
"invalid literal",
"base"
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
errors = result.get("error")
# Then
self.assertFalse(result.get("success"))
for msg in value_error_msg:
self.assert_correct_output(msg, errors[0]['traceback'])
for index, error in enumerate(errors):
self.assertEqual(error['test_case'],
self.test_case_data[index]['test_case'])
def test_only_stdout(self):
# Given
self.test_case_data = [{'expected_output': '11',
'expected_input': '',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<stdio.h>
int main(void){
int a=5,b=6;
printf("%d",a+b);
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_file_based_assert(self):
# Given
self.test_case_data = [{"test_case_type": "standardtestcase",
"test_case": "assert(ans()=='2')",
"weight": 0.0}
]
self.file_paths = [(self.tmp_file, False)]
user_answer = dedent("""
def ans():
with open("test.txt") as f:
return f.read()[0]
""")
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_cpp_string_input(self):
# Given
self.test_case_data = [{'expected_output': 'abc',
'expected_input': 'abc',
'weight': 0.0,
'test_case_type': 'stdiobasedtestcase',
}]
user_answer = dedent("""
#include<iostream>
using namespace std;
int main(void){
char a[4];
cin>>a;
cout<<a;
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_single_testcase_error(self):
""" Tests the user answer with just an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": 's="abbb"\nasert palindrome(s)==False',
"weight": 0.0
}
]
syntax_error_msg = ["Traceback",
"call",
"SyntaxError",
"invalid syntax"
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in syntax_error_msg:
self.assert_correct_output(msg, err)
def test_incorrect_answer(self):
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
int a = s.nextInt();
int b = s.nextInt();
System.out.print(a);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_multiple_testcase_error(self):
""" Tests the user answer with an correct test case
first and then with an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": 'assert(palindrome("abba")==True)',
"weight": 0.0},
{"test_case_type": "standardtestcase",
"test_case": 's="abbb"\n'
'assert palindrome(S)==False',
"weight": 0.0}
]
name_error_msg = ["Traceback",
"call",
"NameError",
"name 'S' is not defined"
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in name_error_msg:
self.assertIn(msg, err)
def test_string_input(self):
self.test_case_data = [{'expected_output': 'HelloWorld',
'expected_input': 'Hello\nWorld',
'test_case_type': 'stdiobasedtestcase',
'weight': 0.0
}]
user_answer = dedent("""
import java.util.Scanner;
class Test
{public static void main(String[] args){
Scanner s = new Scanner(System.in);
String a = s.nextLine();
String b = s.nextLine();
System.out.print(a+b);
}}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
self.assertTrue(result.get('success'))
def test_file_based_answer(self):
# Given
self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "2",
"weight": 0.0
}]
self.file_paths = [(self.tmp_file, False)]
user_answer = dedent("""
with open("test.txt") as f:
a = f.read()
print(a[0])
"""
)
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': self.test_case_data
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
