def test_cpp_infinite_loop(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ while(0==0){ cout<<"abc";} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"])
def test_incorrect_answer(self): # Given user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a = s.nextInt(); int b = s.nextInt(); System.out.print(a); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then lines_of_error = len(result.get('error')[0].get('error_line_numbers')) result_error = result.get('error')[0].get('error_msg') self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect", result_error) self.assertTrue(lines_of_error > 0)
def test_only_stdout(self): # Given self.test_case_data = [{ 'expected_output': '11', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" class Test {public static void main(String[] args){ int a = 5; int b = 6; System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" hook_code = dedent("""\ def check_answer(user_answer): success = False err = "Incorrect Answer" mark_fraction = 0.0 exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """) test_case_data = [{ "test_case_type": "hooktestcase", "hook_code": hook_code, "weight": 1.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': True, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_correct_answer(self): # Given user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a = s.nextInt(); int b = s.nextInt(); System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_file_based_answer(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "2", "weight": 0.0 }] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" with open("test.txt") as f: a = f.read() print(a[0]) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ a = "this should be a string." print(type(a).__name__) """) test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "str", "weight": 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_infinite_loop(self): # Given user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo " " > /dev/null ; done") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_indent_error(self): # Given user_answer = dedent(""" def add(a, b): return a + b """) indent_error_msg = [ "Traceback", "call", "File", "line", "<string>", "IndentationError", "indented block" ] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) err = result.get("error")[0]["traceback"].splitlines() # Then self.assertFalse(result.get("success")) self.assertEqual(5, len(err)) for msg in indent_error_msg: self.assert_correct_output(msg, result.get("error")[0]['traceback'])
def test_incorrect_answer(self): # Given user_answer = dedent(""" #!/bin/bash read A read B echo -n `expr $A - $B` """) test_case_data = [{ 'expected_output': '11', 'expected_input': '5\n6', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then result_error = result.get('error')[0].get('error_msg') self.assert_correct_output("Incorrect", result_error) self.assertFalse(result.get('success'))
def test_stdout_only(self): # Given user_answer = dedent(""" #!/bin/bash A=6 B=4 echo -n `expr $A + $B` """) test_case_data = [{ 'expected_output': '10', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_array_input(self): # Given user_answer = dedent(""" readarray arr; COUNTER=0 while [ $COUNTER -lt 3 ]; do echo -n "${arr[$COUNTER]}" let COUNTER=COUNTER+1 done """) test_case_data = [{ 'expected_output': '1 2 3\n4 5 6\n7 8 9\n', 'expected_input': '1,2,3\n4,5,6\n7,8,9', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_file_based_assert(self): # Given self.file_paths = [(self.f_path, False)] self.tc_data = dedent(""" #!/bin/bash cat $1 """) self.tc_data_args = "test.txt" self.test_case_data = [{ "test_case": self.tc_data, "test_case_args": self.tc_data_args, "test_case_type": "standardtestcase", "weight": 0.0 }] user_answer = ("#!/bin/bash\ncat $1") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_cpp_only_stdout(self): # Given self.test_case_data = [{ 'expected_output': '11', 'expected_input': '', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=5,b=6; cout<<a+b; }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_correct_answer_string(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2", "weight": 0.0 }] user_answer = dedent(""" from six.moves import input a = str(input()) b = str(input()) print(a.count(b)) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_recursion_error(self): # Given user_answer = dedent(""" def add(a, b): return add(3, 3) """) recursion_error_msg = "maximum recursion depth exceeded" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) err = result.get("error")[0]['message'] # Then self.assertFalse(result.get("success")) self.assert_correct_output(recursion_error_msg, err)
def test_incorrect_answer_integer(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] user_answer = dedent(""" a = int(input()) b = int(input()) print(a-b) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) self.assert_correct_output( "Incorrect Answer: Line number(s) 1 did not match.", result.get('error')[0].get('error_msg'))
def test_type_error(self): # Given user_answer = dedent(""" def add(a): return a + b """) type_error_msg = ["Traceback", "call", "TypeError", "argument"] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) for msg in type_error_msg: self.assert_correct_output(msg, err)
def test_infinite_loop(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assert_correct_output(timeout_msg, result.get("error")[0]["message"]) self.assertFalse(result.get('success'))
def test_value_error(self): # Given user_answer = dedent(""" def add(a, b): c = 'a' return int(a) + int(b) + int(c) """) value_error_msg = [ "Traceback", "call", "ValueError", "invalid literal", "base" ] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) for msg in value_error_msg: self.assert_correct_output(msg, err)
def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) given_test_case_list = [tc["test_case"] for tc in self.test_case_data] for error in result.get("error"): self.assertEqual(error['exception'], 'AssertionError') self.assertEqual( error['message'], "Expected answer from the test case did not match the output") error_testcase_list = [tc['test_case'] for tc in result.get('error')] self.assertEqual(error_testcase_list, given_test_case_list)
def test_file_based_assert(self): # Given self.test_case_data = [{ "test_case_type": "standardtestcase", "test_case": "assert(ans()=='2')", "weight": 0.0 }] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" def ans(): with open("test.txt") as f: return f.read()[0] """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = "class Test {\n\tint square_num(int a) {\n\t\twhile(0==0){\n\t\t}\n\t}\n}" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_single_testcase_error(self): """ Tests the user answer with just an incorrect test case """ # Given user_answer = "def palindrome(a):\n\treturn a == a[::-1]" test_case_data = [{ "test_case_type": "standardtestcase", "test_case": 's="abbb"\nasert palindrome(s)==False', "weight": 0.0 }] syntax_error_msg = [ "Traceback", "call", "SyntaxError", "invalid syntax" ] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) err = result.get("error")[0]['traceback'] # Then self.assertFalse(result.get("success")) for msg in syntax_error_msg: self.assert_correct_output(msg, err)
def test_array_input(self): # Given self.test_case_data = [{ 'expected_output': '561', 'expected_input': '5\n6\n1', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a[] = new int[3]; for (int i=0;i<3;i++){ a[i] = s.nextInt(); System.out.print(a[i]);} }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ def strchar(s): a = "should be a string" return type(a) """) test_case_data = [ { "test_case_type": "standardtestcase", "test_case": 'assert(strchar("hello")==str)', "weight": 0.0 }, ] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_infinite_loop(self): # Given user_answer = dedent(""" class Test {public static void main(String[] args){ while(0==0) { System.out.print("a");} }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_correct_answer_list(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]", "weight": 0.0 }] user_answer = dedent(""" from six.moves import input input_a = input() input_b = input() a = [int(i) for i in input_a.split(',')] b = [int(i) for i in input_b.split(',')] print(a+b) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_string_input(self): # Given self.test_case_data = [{ 'expected_output': 'HelloWorld', 'expected_input': 'Hello\nWorld', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); String a = s.nextLine(); String b = s.nextLine(); System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_cpp_error(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=10; cout<<a }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output("Compilation Error", result.get("error"))