def test_infinite_loop(self): # Given self.test_case_data = [{"test_case_type": "stdiobasedtestcase", "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = {'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assert_correct_output(timeout_msg, result.get("error")[0]["message"] ) self.assertFalse(result.get('success'))
def test_cpp_error(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=10; cout<<a }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output("Compilation Error", result.get("error"))
def test_array_input(self): self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a[] = new int[3]; for (int i=0;i<3;i++){ a[i] = s.nextInt(); System.out.print(a[i]);} }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } grader = Grader(self.in_dir) result = grader.evaluate(kwargs) self.assertTrue(result.get('success'))
def test_cpp_infinite_loop(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ while(0==0){ cout<<"abc";} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"] )
def test_string_input(self): # Given self.test_case_data = [{ 'expected_output': 'abc', 'expected_input': 'abc', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> int main(void){ char a[4]; scanf("%s",a); printf("%s",a); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_correct_answer(self): # Given user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a = s.nextInt(); int b = s.nextInt(); System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = dedent(""" #include<stdio.h> int main(void){ while(0==0){ printf("abc");} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"] ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_correct_answer(self): user_answer = dedent(""" #!/bin/bash read A read B echo -n `expr $A + $B` """ ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } grader = Grader(self.in_dir) result = grader.evaluate(kwargs) self.assertTrue(result.get('success'))
def test_correct_answer(self): # Given user_answer = dedent(""" #include<stdio.h> int main(void){ int a,b; scanf("%d%d",&a,&b); printf("%d",a+b); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_assignment_upload(self): # Given user_answer = "Assignment Upload" hook_code = dedent("""\ def check_answer(user_answer): success = False err = "Incorrect Answer" mark_fraction = 0.0 with open("test.txt") as f: data = f.read() if data == '2': success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ ) test_case_data = [{"test_case_type": "hooktestcase", "hook_code": hook_code,"weight": 1.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'assign_files': [(self.tmp_file, False)], 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_incorrect_answer_without_test_case_args(self): # Given user_answer = "echo 'hello'" tc_data = "echo 'hello world'" self.test_case_data = [ {"test_case": tc_data, "test_case_args": "", "test_case_type": "standardtestcase", "weight": 0.0 } ] kwargs = {'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success'))
def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" hook_code = dedent("""\ def check_answer(user_answer): success = False err = "Incorrect Answer" mark_fraction = 0.0 exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """ ) test_case_data = [{"test_case_type": "hooktestcase", "hook_code": hook_code,"weight": 1.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': True, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ a = "this should be a string." print(type(a).__name__) """) test_case_data = [{"test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "str", "weight": 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) given_test_case_list = [tc["test_case"] for tc in self.test_case_data] for error in result.get("error"): self.assertEqual(error['exception'], 'AssertionError') self.assertEqual(error['message'], "Expected answer from the test case did not match the output" ) error_testcase_list = [tc['test_case'] for tc in result.get('error')] self.assertEqual(error_testcase_list, given_test_case_list)
def test_incorrect_answer(self): # Given user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a = s.nextInt(); int b = s.nextInt(); System.out.print(a); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then lines_of_error = len(result.get('error')[0].get('error_line_numbers')) result_error = result.get('error')[0].get('error_msg') self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect", result_error) self.assertTrue(lines_of_error > 0)
def test_error(self): # Given user_answer = dedent(""" #include<stdio.h> int main(void){ int a=10; printf("%d",a) }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output("Compilation Error", result.get("error"))
def test_string_input(self): # Given self.test_case_data = [{'expected_output': 'HelloWorld', 'expected_input': 'Hello\nWorld', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); String a = s.nextLine(); String b = s.nextLine(); System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_file_based_assert(self): # Given self.file_paths = [(self.f_path, False)] self.tc_data = dedent(""" #!/bin/bash cat $1 """) self.tc_data_args = "test.txt" self.test_case_data = [ {"test_case": self.tc_data, "test_case_args": self.tc_data_args, "test_case_type": "standardtestcase", "weight": 0.0 } ] user_answer = ("#!/bin/bash\ncat $1") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_array_input(self): # Given self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> int main(void){ int a[3],i; for(i=0;i<3;i++){ scanf("%d",&a[i]);} for(i=0;i<3;i++){ printf("%d",a[i]);} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_array_input(self): # Given user_answer = dedent(""" readarray arr; COUNTER=0 while [ $COUNTER -lt 3 ]; do echo -n "${arr[$COUNTER]}" let COUNTER=COUNTER+1 done """ ) test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n', 'expected_input': '1,2,3\n4,5,6\n7,8,9', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_cpp_incorrect_answer(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=10; cout<<a; }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then lines_of_error = len(result.get('error')[0].get('error_line_numbers')) result_error = result.get('error')[0].get('error_msg') self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect", result_error) self.assertTrue(lines_of_error > 0)
def test_incorrect_answer(self): # Given user_answer = dedent(""" #!/bin/bash read A read B echo -n `expr $A - $B` """ ) test_case_data = [{'expected_output': '11', 'expected_input': '5\n6', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then result_error = result.get('error')[0].get('error_msg') self.assert_correct_output("Incorrect", result_error) self.assertFalse(result.get('success'))
def test_stdout_only(self): # Given user_answer = dedent(""" #!/bin/bash A=6 B=4 echo -n `expr $A + $B` """ ) test_case_data = [{'expected_output': '10', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_cpp_correct_answer(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a,b; cin>>a>>b; cout<<a+b; }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo "" > /dev/null ; done") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"] ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_cpp_only_stdout(self): # Given self.test_case_data = [{ 'expected_output': '11', 'expected_input': '', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=5,b=6; cout<<a+b; }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ def strchar(s): a = "should be a string" return type(a) """) test_case_data = [ { "test_case_type": "standardtestcase", "test_case": 'assert(strchar("hello")==str)', "weight": 0.0 }, ] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_only_stdout(self): self.test_case_data = [{'expected_output': '11', 'expected_input': '', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" class Test {public static void main(String[] args){ int a = 5; int b = 6; System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } grader = Grader(self.in_dir) result = grader.evaluate(kwargs) self.assertTrue(result.get('success'))
def test_correct_answer_list(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1,2,3\n5,6,7", "expected_output": "[1, 2, 3, 5, 6, 7]", "weight": 0.0 }] user_answer = dedent(""" from six.moves import input input_a = input() input_b = input() a = [int(i) for i in input_a.split(',')] b = [int(i) for i in input_b.split(',')] print(a+b) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_array_input(self): # Given self.test_case_data = [{'expected_output': '561', 'expected_input': '5\n6\n1', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a[] = new int[3]; for (int i=0;i<3;i++){ a[i] = s.nextInt(); System.out.print(a[i]);} }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_correct_answer_string(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": ("the quick brown fox jumps over the lazy dog\nthe"), "expected_output": "2", "weight": 0.0 }] user_answer = dedent(""" from six.moves import input a = str(input()) b = str(input()) print(a.count(b)) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_only_stdout(self): # Given self.test_case_data = [{'expected_output': '11', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" class Test {public static void main(String[] args){ int a = 5; int b = 6; System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_incorrect_answer_integer(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] user_answer = dedent(""" a = int(input()) b = int(input()) print(a-b) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) self.assert_correct_output( "Incorrect Answer: Line number(s) 1 did not match.", result.get('error')[0].get('error_msg'))
def test_infinite_loop(self): # Given user_answer = ("class Test {\n\tint square_num(int a)" " {\n\t\twhile(0==0){\n\t\t}\n\t}\n}") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"] ) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_file_based_answer(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "2", "weight": 0.0 }] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" with open("test.txt") as f: a = f.read() print(a[0]) """) kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_error(self): # Given user_answer = dedent(""" class Test { System.out.print("a"); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) errors = result.get('error') # Then self.assertFalse(result.get("success")) for error in errors: self.assertEqual(error.get('exception'), 'CompilationError')
def test_infinite_loop(self): # Given self.test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "1\n2", "expected_output": "3", "weight": 0.0 }] timeout_msg = ("Code took more than {0} seconds to run. " "You probably have an infinite loop in" " your code.").format(SERVER_TIMEOUT) user_answer = "while True:\n\tpass" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assert_correct_output(timeout_msg, result.get('error')) self.assertFalse(result.get('success'))
def test_string_input(self): # Given self.test_case_data = [{'expected_output': 'abc', 'expected_input': 'abc', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> int main(void){ char a[4]; scanf("%s",a); printf("%s",a); }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_unicode_literal_bug(self): # Given user_answer = dedent("""\ a = "this should be a string." print(type(a).__name__) """) test_case_data = [{ "test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "str", "weight": 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_incorrect_answer(self): # Given user_answer = "def add(a,b):\n\treturn a - b" kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success')) self.assert_correct_output('AssertionError in:\n assert(add(1,2)==3)', result.get('error')) self.assert_correct_output( 'AssertionError in:\n assert(add(-1,2)==1)', result.get('error')) self.assert_correct_output( 'AssertionError in:\n assert(add(-1,-2)==-3)', result.get('error'))
def test_cpp_error(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=10; cout<<a }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) errors = result.get('error') # Then self.assertFalse(result.get("success")) for error in errors: self.assertEqual(error['exception'], 'CompilationError')
def test_correct_answer(self): # Given user_answer = "def add(a,b):\n\treturn a + b" hook_code = dedent("""\ def check_answer(user_answer): success = False err = "Incorrect Answer" mark_fraction = 0.0 exec(user_answer, globals()) if add(1,2) == 3: success, err, mark_fraction = True, "", 1.0 return success, err, mark_fraction """) test_case_data = [{ "test_case_type": "hooktestcase", "hook_code": hook_code, "weight": 1.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': True, 'language': 'python' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_cpp_only_stdout(self): # Given self.test_case_data = [{'expected_output': '11', 'expected_input': '', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ int a=5,b=6; cout<<a+b; }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = ("#!/bin/bash\nwhile [ 1 ] ;" " do echo " " > /dev/null ; done") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_array_input(self): # Given self.test_case_data = [{ 'expected_output': '561', 'expected_input': '5\n6\n1', 'weight': 0.0, 'test_case_type': 'stdiobasedtestcase', }] user_answer = dedent(""" #include<stdio.h> int main(void){ int a[3],i; for(i=0;i<3;i++){ scanf("%d",&a[i]);} for(i=0;i<3;i++){ printf("%d",a[i]);} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_file_based_assert(self): # Given self.file_paths = [(self.f_path, False)] self.tc_data = dedent(""" #!/bin/bash cat $1 """) self.tc_data_args = "test.txt" self.test_case_data = [{ "test_case": self.tc_data, "test_case_args": self.tc_data_args, "test_case_type": "standardtestcase", "weight": 0.0, "hidden": True }] user_answer = ("#!/bin/bash\ncat $1") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get("success"))
def test_infinite_loop(self): # Given user_answer = dedent(""" #include<stdio.h> int main(void){ while(0==0){ printf("abc");} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_array_input(self): # Given user_answer = dedent(""" readarray arr; COUNTER=0 while [ $COUNTER -lt 3 ]; do echo -n "${arr[$COUNTER]}" let COUNTER=COUNTER+1 done """) test_case_data = [{ 'expected_output': '1 2 3\n4 5 6\n7 8 9\n', 'expected_input': '1,2,3\n4,5,6\n7,8,9', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0, }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_incorrect_answer(self): # Given user_answer = dedent(""" #!/bin/bash read A read B echo -n `expr $A - $B` """) test_case_data = [{ 'expected_output': '11', 'expected_input': '5\n6', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0, 'hidden': True }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then result_error = result.get('error')[0].get('error_msg') self.assert_correct_output("Incorrect", result_error) self.assertTrue(result.get('error')[0]['hidden']) self.assertFalse(result.get('success'))
def test_cpp_infinite_loop(self): # Given user_answer = dedent(""" #include<iostream> using namespace std; int main(void){ while(0==0){ cout<<"abc";} }""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'cpp' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"])
def test_stdout_only(self): # Given user_answer = dedent(""" #!/bin/bash A=6 B=4 echo -n `expr $A + $B` """) test_case_data = [{ 'expected_output': '10', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))
def test_infinite_loop(self): # Given user_answer = ("class Test {\n\tint square_num(int a)" " {\n\t\twhile(0==0){\n\t\t}\n\t}\n}") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get("success")) self.assert_correct_output(self.timeout_msg, result.get("error")[0]["message"]) parent_proc = Process(os.getpid()).children() if parent_proc: children_procs = Process(parent_proc[0].pid) self.assertFalse(any(children_procs.children(recursive=True)))
def test_incorrect_answer_without_test_case_args(self): # Given user_answer = "echo 'hello'" tc_data = "echo 'hello world'" self.test_case_data = [{ "test_case": tc_data, "test_case_args": "", "test_case_type": "standardtestcase", "weight": 0.0 }] kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'bash' }, 'test_case_data': self.test_case_data, } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertFalse(result.get('success'))
def test_incorrect_answer(self): user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); int a = s.nextInt(); int b = s.nextInt(); System.out.print(a); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } grader = Grader(self.in_dir) result = grader.evaluate(kwargs) lines_of_error = len(result.get('error')[0].splitlines()) self.assertFalse(result.get('success')) self.assert_correct_output("Incorrect", result.get('error')) self.assertTrue(lines_of_error > 1)
def test_string_input(self): self.test_case_data = [{'expected_output': 'HelloWorld', 'expected_input': 'Hello\nWorld', 'test_case_type': 'stdiobasedtestcase', 'weight': 0.0 }] user_answer = dedent(""" import java.util.Scanner; class Test {public static void main(String[] args){ Scanner s = new Scanner(System.in); String a = s.nextLine(); String b = s.nextLine(); System.out.print(a+b); }}""") kwargs = { 'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'java' }, 'test_case_data': self.test_case_data, } grader = Grader(self.in_dir) result = grader.evaluate(kwargs) self.assertTrue(result.get('success'))
def test_file_based_answer(self): # Given self.test_case_data = [{"test_case_type": "stdiobasedtestcase", "expected_input": "", "expected_output": "2", "weight": 0.0 }] self.file_paths = [(self.tmp_file, False)] user_answer = dedent(""" with open("test.txt") as f: a = f.read() print(a[0]) """ ) kwargs = {'metadata': { 'user_answer': user_answer, 'file_paths': self.file_paths, 'partial_grading': False, 'language': 'python' }, 'test_case_data': self.test_case_data } # When grader = Grader(self.in_dir) result = grader.evaluate(kwargs) # Then self.assertTrue(result.get('success'))