def test_unicode_literal_bug(self):
# Given
user_answer = dedent("""\
def strchar(s):
a = "should be a string"
return type(a)
""")
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": 'assert(strchar("hello")==str)',
"weight": 0.0}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_incorrect_answer(self):
# Given
user_answer = ("class Test {\n\tint square_num(int a) "
"{\n\treturn a;\n\t}\n}")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
errors = result.get('error')
# Then
self.assertFalse(result.get('success'))
for error in errors:
self.assertEqual(error.get('exception'), 'AssertionError')
def test_recursion_error(self):
# Given
user_answer = dedent("""
def add(a, b):
return add(3, 3)
""")
recursion_error_msg = "maximum recursion depth exceeded"
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['message']
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(recursion_error_msg, err)
def test_incorrect_answer(self):
# Given
user_answer = "class Test {\n\tint square_num(int a) {\n\treturn a;\n\t}\n}"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
lines_of_error = len(result.get('error')[0].splitlines())
self.assertFalse(result.get('success'))
self.assert_correct_output("Incorrect", result.get('error'))
self.assertTrue(lines_of_error > 1)
def test_infinite_loop(self):
user_answer = ("funcprot(0)\nfunction[c]=add(a,b)"
"\n\tc=a;\nwhile(1==1)\nend\nendfunction")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'scilab'
},
'test_case_data': self.test_case_data,
}
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_unicode_literal_bug(self):
# Given
user_answer = dedent("""\
a = "this should be a string."
print(type(a).__name__)
""")
test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "str",
"weight": 0.0
}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_assignment_upload(self):
# Given
user_answer = "Assignment Upload"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
with open("test.txt") as f:
data = f.read()
if data == '2':
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'assign_files': [(self.tmp_file, False)],
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_incorrect_answer(self):
# Given
user_answer = "def add(a,b):\n\treturn a - b"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
exec(user_answer, globals())
if add(1,2) == 3:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
"weight": 1.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output('Incorrect Answer', result.get('error'))
def test_assert_with_hook(self):
# Given
user_answer = "def add(a,b):\n\treturn a + b"
assert_test_case = "assert add(1,2) == 3"
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if "return a + b" in user_answer:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": assert_test_case, 'weight': 1.0},
{"test_case_type": "hooktestcase",
"hook_code": hook_code, 'weight': 1.0},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 2.0)
def test_multiple_testcase_error(self):
""" Tests the user answer with an correct test case
first and then with an incorrect test case """
# Given
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{
"test_case_type": "standardtestcase",
"test_case": 'assert(palindrome("abba")==True)',
"weight": 0.0
}, {
"test_case_type": "standardtestcase",
"test_case": 's="abbb"\n'
'assert palindrome(S)==False',
"weight": 0.0
}]
name_error_msg = [
"Traceback", "call", "NameError", "name 'S' is not defined"
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]['traceback']
# Then
self.assertFalse(result.get("success"))
for msg in name_error_msg:
self.assertIn(msg, err)
def test_get_error_lineno(self):
user_answer = dedent("""\
print(1/0)
""")
test_case_data = [{"test_case_type": "stdiobasedtestcase",
"expected_input": "",
"expected_output": "1",
"weight": 0.0
}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
error = result.get("error")[0]
self.assertEqual(error['line_no'], 1)
self.assertEqual(error['exception'], "ZeroDivisionError")
def test_name_error(self):
# Given
user_answer = ""
name_error_msg = ["Traceback",
"call",
"NameError",
"name",
"defined"
]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
err = result.get("error")[0]["traceback"]
for msg in name_error_msg:
self.assertIn(msg, err)
def test_error(self):
# Given
user_answer = dedent("""
class Test
{
System.out.print("a");
}""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assertTrue("Compilation Error" in '\n'.join(result.get("error")))
def test_infinite_loop(self):
# Given
user_answer = "int add(int a, int b)\n{while(1>0){}}"
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
self.assert_correct_output(self.timeout_msg, result.get("error"))
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_incorrect_answer_with_nose_assert(self):
user_answer = dedent("""\
def add(a, b):
return a - b
""")
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": 'assert_equal(add(1, 2), 3)',
"weight": 0.0}]
kwargs = {'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get("success"))
error = result.get("error")[0]
self.assertEqual(error['exception'], 'AssertionError')
self.assertEqual(error['message'], '-1 != 3')
def test_single_testcase_error(self):
# Given
""" Tests the user answer with just an incorrect test case """
user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
test_case_data = [{
"test_case_type": "standardtestcase",
"test_case": 's="abbb"\nasert palindrome(s)==False',
"weight": 0.0
}]
syntax_error_msg = [
"Traceback", "call", "File", "line", "<string>", "SyntaxError",
"invalid syntax"
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'python'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
error_as_str = ''.join(result.get("error"))
err = error_as_str.splitlines()
# Then
self.assertFalse(result.get("success"))
self.assertEqual(13, len(err))
for msg in syntax_error_msg:
self.assert_correct_output(msg, result.get("error"))
def test_infinite_loop(self):
# Given
user_answer = dedent("""\
class Test
{public static void main(String[] args){
while(0==0)
{
System.out.print("a");}
}}""")
hook_code = dedent("""\
def check_answer(user_answer):
with open("Test.java", "w+") as f:
f.write(user_answer)
import subprocess
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
def _run_command(cmd):
proc = subprocess.Popen(
"{}".format(cmd), shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout,stderr = proc.communicate()
return stdout,stderr
cmds = ["javac Test.java", "java Test"]
for cmd in cmds:
stdout, stderr = _run_command(cmd)
if stdout.decode("utf-8") == "Hello, world!":
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
"weight": 1.0,
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertFalse(result.get('success'))
self.assert_correct_output(self.timeout_msg,
result.get("error")[0]["message"])
parent_proc = Process(os.getpid()).children()
if parent_proc:
children_procs = Process(parent_proc[0].pid)
self.assertFalse(any(children_procs.children(recursive=True)))
def test_multiple_hooks(self):
# Given
user_answer = dedent("""\
class Test
{public static void main(String[] args){
System.out.print("Hello, world!");
}}
""")
hook_code_1 = dedent("""\
def check_answer(user_answer):
with open("Test.java", "w+") as f:
f.write(user_answer)
import subprocess
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
def _run_command(cmd):
proc = subprocess.Popen(
"{}".format(cmd), shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout,stderr = proc.communicate()
return stdout,stderr
cmds = ["javac Test.java", "java Test"]
for cmd in cmds:
stdout, stderr = _run_command(cmd)
if stdout.decode("utf-8") == "Hello, world!":
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
hook_code_2 = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if ('System.out.print("Hello, world!");' in
user_answer):
success, err, mark_fraction = True, "", 0.5
return success, err, mark_fraction
""")
test_case_data = [
{
"test_case_type": "hooktestcase",
"hook_code": hook_code_1,
'weight': 1.0
},
{
"test_case_type": "hooktestcase",
"hook_code": hook_code_2,
'weight': 1.0
},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'java'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 1.5)
def test_assert_with_hook(self):
# Given
user_answer = ("class Test {\n\tint square_num(int a)"
" {\n\treturn a*a;\n\t}\n}")
assert_test_case = dedent("""
class main
{
public static <E> void check(E expect, E result)
{
if(result.equals(expect))
{
System.out.println("Correct:Output expected "+expect+
" and got "+result);
}
else
{
System.out.println("Incorrect:Output expected "+expect+
" but got "+result);
System.exit(1);
}
}
public static void main(String arg[])
{
Test t = new Test();
int result, input, output;
input = 0; output = 0;
result = t.square_num(input);
System.out.println("Input submitted to the function: "+
input);
check(output, result);
input = 5; output = 25;
result = t.square_num(input);
System.out.println("Input submitted to the function: "+
input);
check(output, result);
input = 6; output = 36;
result = t.square_num(input);
System.out.println("Input submitted to the function: "+
input);
check(output, result);
}
}
""")
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if "return a*a" in user_answer:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [
{
"test_case_type": "standardtestcase",
"test_case": assert_test_case,
'weight': 1.0
},
{
"test_case_type": "hooktestcase",
"hook_code": hook_code,
'weight': 1.0
},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'java'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 2.0)
def test_file_based_assert(self):
# Given
self.file_paths = [(self.f_path, False)]
self.tc_data = dedent("""
class main
{
public static <E> void check(E expect, E result)
{
if(result.equals(expect))
{
System.out.println("Correct:Output expected "+expect+
" and got "+result);
}
else
{
System.out.println("Incorrect:Output expected "+expect+
" but got "+result);
System.exit(1);
}
}
public static void main(String arg[])
{
String result = "";
Test t = new Test();
try{
result = t.readFile();}
catch(Exception e){
System.out.print(e);
}
check("2", result);
}
}
""")
self.test_case_data = [{
"test_case": self.tc_data,
"test_case_type": "standardtestcase",
"weight": 0.0
}]
user_answer = dedent("""
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
class Test{
String readFile() throws IOException {
BufferedReader br = new BufferedReader(new FileReader("test.txt"));
try {
StringBuilder sb = new StringBuilder();
String line = br.readLine();
while (line != null) {
sb.append(line);
line = br.readLine();}
return sb.toString();
} finally {
br.close();
}}}
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'java'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get("success"))
def test_incorrect_testcase(self):
# Given
self.tc_data = dedent("""
#include <stdio.h>
#include <stdlib.h>
extern int add(int, int);
template <class T>
void check(T expect, T result)
{
if (expect == result)
{
printf("Correct: Expected %d got %d ",expect,result);
}
else
{
printf("Incorrect: Expected %d got %d ",expect,result);
exit (1);
}
}
int main(void)
{
int result;
result = add(0,0);
printf("Input submitted to the function: 0, 0");
check(0, result);
result = add(2,3);
printf("Input submitted to the function: 2 3");
check(5,result)
printf("All Correct");
return 0;
}
""")
user_answer = dedent("""\
int add(int a, int b)
{
return a+b;
}""")
self.test_case_data = [{
"test_case": self.tc_data,
"test_case_type": "standardtestcase",
"weight": 0.0
}]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
err = result.get('error')[0]
lines_of_error = len(err.splitlines())
self.assertFalse(result.get('success'))
self.assertTrue(lines_of_error > 1)
self.assertIn("Test case Error", err)
def test_assert_with_hook(self):
# Given
user_answer = "int add(int a, int b)\n{return a+b;}"
assert_test_case = dedent("""\
#include <stdio.h>
#include <stdlib.h>
extern int add(int, int);
template <class T>
void check(T expect, T result)
{
if (expect == result)
{
printf("Correct: Expected %d got %d ",expect,result);
}
else
{
printf("Incorrect: Expected %d got %d ",expect,result);
exit (1);
}
}
int main(void)
{
int result;
result = add(0,0);
printf("Input submitted to the function: 0, 0");
check(0, result);
result = add(2,3);
printf("Input submitted to the function: 2 3");
check(5,result);
printf("All Correct");
return 0;
}
""")
hook_code = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if "return a+b;" in user_answer:
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
"""
)
test_case_data = [{"test_case_type": "standardtestcase",
"test_case": assert_test_case,
'weight': 1.0
},
{"test_case_type": "hooktestcase",
"hook_code": hook_code, 'weight': 1.0},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'cpp'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 2.0)
def test_file_based_assert(self):
# Given
self.file_paths = [(self.f_path, False)]
self.tc_data = dedent("""
#include <stdio.h>
#include <stdlib.h>
extern int ans();
template <class T>
void check(T expect,T result)
{
if (expect == result)
{
printf("Correct: Expected %d got %d ",expect,result);
}
else
{
printf("Incorrect: Expected %d got %d ",expect,result);
exit (0);
}
}
int main(void)
{
int result;
result = ans();
check(50, result);
}
""")
self.test_case_data = [{"test_case": self.tc_data,
"test_case_type": "standardtestcase",
"weight": 0.0
}]
user_answer = dedent("""
#include<stdio.h>
char ans()
{
FILE *fp;
char buff[255];
fp = fopen("test.txt", "r");
fscanf(fp, "%s", buff);
fclose(fp);
return buff[0];
}
""")
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': False,
'language': 'cpp'
},
'test_case_data': self.test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
def test_multiple_hooks(self):
# Given
user_answer = dedent(""" #!/bin/bash
echo -n Hello, world!
""")
hook_code_1 = dedent("""\
def check_answer(user_answer):
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
if "echo -n Hello, world!" in user_answer:
success, err, mark_fraction = True, "", 0.5
return success, err, mark_fraction
""")
hook_code_2 = dedent("""\
def check_answer(user_answer):
import subprocess
import sys
success = False
err = "Incorrect Answer"
mark_fraction = 0.0
proc = subprocess.Popen(user_answer, shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
)
stdout,stderr = proc.communicate()
if stdout.decode('utf-8') == "Hello, world!":
success, err, mark_fraction = True, "", 1.0
return success, err, mark_fraction
""")
test_case_data = [
{
"test_case_type": "hooktestcase",
"hook_code": hook_code_1,
'weight': 1.0
},
{
"test_case_type": "hooktestcase",
"hook_code": hook_code_2,
'weight': 1.0
},
]
kwargs = {
'metadata': {
'user_answer': user_answer,
'file_paths': self.file_paths,
'partial_grading': True,
'language': 'bash'
},
'test_case_data': test_case_data,
}
# When
grader = Grader(self.in_dir)
result = grader.evaluate(kwargs)
# Then
self.assertTrue(result.get('success'))
self.assertEqual(result.get("weight"), 1.5)
