def test_infinite_loop(self):
        # Given
        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
                            "expected_input": "1\n2",
                            "expected_output": "3",
                            "weight": 0.0
                            }]
        timeout_msg = ("Code took more than {0} seconds to run. "
                            "You probably have an infinite loop in"
                            " your code.").format(SERVER_TIMEOUT)
        user_answer = "while True:\n\tpass"

        kwargs = {'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'python'
                    },
                    'test_case_data': self.test_case_data
                }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assert_correct_output(timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
        self.assertFalse(result.get('success'))
示例#2
0
    def test_infinite_loop(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        while(0==0){
        printf("abc");}
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg, result.get("error"))
示例#3
0
    def test_array_input(self):
        self.test_case_data = [{'expected_output': '561',
                                'expected_input': '5\n6\n1',
                                'test_case_type': 'stdiobasedtestcase',
                                'weight': 0.0
                                }]
        user_answer = dedent("""
        import java.util.Scanner;
        class Test
        {public static void main(String[] args){
         Scanner s = new Scanner(System.in);
         int a[] = new int[3];
         for (int i=0;i<3;i++){
         a[i] = s.nextInt();
         System.out.print(a[i]);}
        }}""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    },
                    'test_case_data': self.test_case_data,
                  }

        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        self.assertTrue(result.get('success'))
示例#4
0
    def test_array_input(self):
        # Given
        user_answer = dedent(""" readarray arr;
                                 COUNTER=0
                                 while [  $COUNTER -lt 3 ]; do
                                     echo -n "${arr[$COUNTER]}"
                                     let COUNTER=COUNTER+1 
                                 done
                            """)
        test_case_data = [{
            'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
            'expected_input': '1,2,3\n4,5,6\n7,8,9',
            'test_case_type': 'stdiobasedtestcase',
            'weight': 0.0
        }]
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#5
0
    def test_correct_answer(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a,b;
        scanf("%d%d",&a,&b);
        printf("%d",a+b);
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#6
0
    def test_infinite_loop(self):
        # Given
        self.test_case_data = [{
            "test_case_type": "stdiobasedtestcase",
            "expected_input": "1\n2",
            "expected_output": "3",
            "weight": 0.0
            }]
        timeout_msg = ("Code took more than {0} seconds to run. "
                       "You probably have an infinite loop in"
                       " your code.").format(SERVER_TIMEOUT)
        user_answer = "while True:\n\tpass"

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assert_correct_output(timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
        self.assertFalse(result.get('success'))
示例#7
0
    def test_incorrect_answer_without_test_case_args(self):
        # Given
        user_answer = "echo 'hello'"
        tc_data = "echo 'hello world'"
        self.test_case_data = [{
            "test_case": tc_data,
            "test_case_args": "",
            "test_case_type": "standardtestcase",
            "weight": 0.0
        }]
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': self.test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
示例#8
0
    def test_correct_answer(self):
        user_answer = dedent(""" #!/bin/bash
                             read A
                             read B
                             echo -n `expr $A + $B`
                             """
                             )
        test_case_data = [{'expected_output': '11',
            'expected_input': '5\n6',
            'test_case_type': 'stdiobasedtestcase',
            'weight': 0.0
            }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': test_case_data,
                  }

        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        self.assertTrue(result.get('success'))
    def test_correct_answer(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a,b;
        scanf("%d%d",&a,&b);
        printf("%d",a+b);
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_assignment_upload(self):
        # Given
        user_answer = "Assignment Upload"
        hook_code = dedent("""\
                            def check_answer(user_answer):
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                with open("test.txt") as f:
                                    data = f.read()
                                if data == '2':
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """
                            )
        test_case_data = [{"test_case_type": "hooktestcase",
                           "hook_code": hook_code,"weight": 1.0
                            }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'assign_files': [(self.tmp_file, False)],
                    'partial_grading': False,
                    'language': 'python'
                    },
                    'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#11
0
    def test_incorrect_answer_without_test_case_args(self):
        # Given
        user_answer = "echo 'hello'"
        tc_data = "echo 'hello world'"
        self.test_case_data = [
            {"test_case": tc_data,
             "test_case_args": "",
             "test_case_type": "standardtestcase",
             "weight": 0.0
             }
        ]
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'bash'
                  }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
    def test_correct_answer(self):
        # Given
        user_answer = "def add(a,b):\n\treturn a + b"
        hook_code = dedent("""\
                            def check_answer(user_answer):
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                exec(user_answer, globals())
                                if add(1,2) == 3:
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """
                            )

        test_case_data = [{"test_case_type": "hooktestcase",
                           "hook_code": hook_code,"weight": 1.0
                            }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': True,
                    'language': 'python'
                    },
                    'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
 def test_unicode_literal_bug(self):
     # Given
     user_answer = dedent("""\
                          a = "this should be a string."
                          print(type(a).__name__)
                         """)
     test_case_data = [{"test_case_type": "stdiobasedtestcase",
                        "expected_input": "",
                        "expected_output": "str",
                        "weight": 0.0
                        }]
     kwargs = {
               'metadata': {
                 'user_answer': user_answer,
                 'file_paths': self.file_paths,
                 'partial_grading': False,
                 'language': 'python'
                 },
                 'test_case_data': test_case_data,
             }
     # When
     grader = Grader(self.in_dir)
     result = grader.evaluate(kwargs)
     # Then
     self.assertTrue(result.get("success"))
    def test_incorrect_answer(self):
        # Given
        user_answer = "def add(a,b):\n\treturn a - b"
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'python'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
        given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
        for error in result.get("error"):
            self.assertEqual(error['exception'], 'AssertionError')
            self.assertEqual(error['message'],
                 "Expected answer from the test case did not match the output"
                 )
        error_testcase_list = [tc['test_case'] for tc in result.get('error')]
        self.assertEqual(error_testcase_list, given_test_case_list)
示例#15
0
    def test_correct_answer_string(self):
        # Given
        self.test_case_data = [{
            "test_case_type": "stdiobasedtestcase",
            "expected_input": ("the quick brown fox jumps over "
                               "the lazy dog\nthe"),
            "expected_output": "2",
            "weight": 0.0
            }]
        user_answer = dedent("""
                                from six.moves import input
                                a = str(input())
                                b = str(input())
                                print(a.count(b))
                             """
                             )

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_error(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a=10;
        printf("%d",a)
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output("Compilation Error", result.get("error"))
示例#17
0
    def test_file_based_answer(self):
        # Given
        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                "expected_input": "",
                                "expected_output": "2",
                                "weight": 0.0
                                }]
        self.file_paths = [(self.tmp_file, False)]

        user_answer = dedent("""
                            with open("test.txt") as f:
                                a = f.read()
                                print(a[0])
                             """
                             )
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_file_based_assert(self):
        # Given
        self.file_paths = [(self.f_path, False)]
        self.tc_data = dedent("""
            #!/bin/bash
            cat $1
            """)
        self.tc_data_args = "test.txt"
        self.test_case_data = [
            {"test_case": self.tc_data,
                "test_case_args": self.tc_data_args,
                "test_case_type": "standardtestcase",
                "weight": 0.0
            }
        ]
        user_answer = ("#!/bin/bash\ncat $1")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get("success"))
示例#19
0
    def test_assert_with_hook(self):
        # Given
        user_answer = "def add(a,b):\n\treturn a + b"
        assert_test_case = "assert add(1,2) == 3"
        hook_code = dedent("""\
                            def check_answer(user_answer):
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                if "return a + b" in user_answer:
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """
                           )

        test_case_data = [{"test_case_type": "standardtestcase",
                           "test_case": assert_test_case, 'weight': 1.0},
                          {"test_case_type": "hooktestcase",
                           "hook_code": hook_code, 'weight': 1.0},
                          ]
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': True,
                  'language': 'python'},
                  'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
        self.assertEqual(result.get("weight"), 2.0)
    def test_array_input(self):
        # Given
        user_answer = dedent(""" readarray arr;
                                 COUNTER=0
                                 while [  $COUNTER -lt 3 ]; do
                                     echo -n "${arr[$COUNTER]}"
                                     let COUNTER=COUNTER+1 
                                 done
                            """
                             )
        test_case_data = [{'expected_output': '1 2 3\n4 5 6\n7 8 9\n',
                            'expected_input': '1,2,3\n4,5,6\n7,8,9',
                            'test_case_type': 'stdiobasedtestcase',
                            'weight': 0.0
                           }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_infinite_loop(self):
        # Given
        user_answer = dedent("""\
                             #include<stdio.h>
                             int main(void){
                             while(0==0){
                             printf("abc");}
                             }""")

        hook_code = dedent("""\
                            def check_answer(user_answer):
                                with open("Test.c", "w+") as f:
                                    f.write(user_answer)
                                import subprocess
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                def _run_command(cmd):
                                    proc = subprocess.Popen("{}".format(cmd),
                                                            shell=True,
                                                            stdout=subprocess.PIPE,
                                                            stderr=subprocess.PIPE
                                                            )
                                    stdout,stderr = proc.communicate()
                                    return stdout,stderr
                                cmds = ["gcc Test.c", "./a.out"]
                                for cmd in cmds:
                                    stdout, stderr = _run_command(cmd)
                                if stdout.decode("utf-8") == "Hello, world!":
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """)

        test_case_data = [{
            "test_case_type": "hooktestcase",
            "hook_code": hook_code,
            "weight": 1.0
        }]

        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'cpp'
            },
            'test_case_data': test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
        self.assert_correct_output(self.timeout_msg, result.get('error'))
        parent_proc = Process(os.getpid()).children()
        if parent_proc:
            children_procs = Process(parent_proc[0].pid)
            self.assertFalse(any(children_procs.children(recursive=True)))
    def test_incorrect_answer(self):
        # Given
        user_answer = dedent(""" #!/bin/bash
                             read A
                             read B
                             echo -n `expr $A - $B`
                             """
                             )
        test_case_data = [{'expected_output': '11',
            'expected_input': '5\n6',
            'test_case_type': 'stdiobasedtestcase',
            'weight': 0.0
            }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        result_error = result.get('error')[0].get('error_msg')
        self.assert_correct_output("Incorrect", result_error)
        self.assertFalse(result.get('success'))
示例#23
0
    def test_stdout_only(self):
        # Given
        user_answer = dedent(""" #!/bin/bash
                             A=6
                             B=4
                             echo -n `expr $A + $B`
                             """)
        test_case_data = [{
            'expected_output': '10',
            'test_case_type': 'stdiobasedtestcase',
            'weight': 0.0
        }]
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_stdout_only(self):
        # Given
        user_answer = dedent(""" #!/bin/bash
                             A=6
                             B=4
                             echo -n `expr $A + $B`
                             """
                             )
        test_case_data = [{'expected_output': '10',
                           'test_case_type': 'stdiobasedtestcase',
                           'weight': 0.0
                           }]
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#25
0
    def test_incorrect_answer(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a=10;
        printf("%d",a);
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        lines_of_error = len(result.get('error')[0].splitlines())
        self.assertFalse(result.get('success'))
        self.assert_correct_output("Incorrect", result.get('error'))
        self.assertTrue(lines_of_error > 1)
    def test_infinite_loop(self):
        # Given
        user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
            " do echo "" > /dev/null ; done")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'bash'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
        parent_proc = Process(os.getpid()).children()
        if parent_proc:
            children_procs = Process(parent_proc[0].pid)
            self.assertFalse(any(children_procs.children(recursive=True)))
示例#27
0
    def test_cpp_correct_answer(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a,b;
        cin>>a>>b;
        cout<<a+b;
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#28
0
    def test_infinite_loop(self):
        # Given
        user_answer = ("class Test {\n\tint square_num(int a)"
                       " {\n\t\twhile(0==0){\n\t\t}\n\t}\n}")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
        parent_proc = Process(os.getpid()).children()
        if parent_proc:
            children_procs = Process(parent_proc[0].pid)
            self.assertFalse(any(children_procs.children(recursive=True)))
示例#29
0
    def test_only_stdout(self):
        self.test_case_data = [{'expected_output': '11',
                                'expected_input': '',
                                'test_case_type': 'stdiobasedtestcase',
                                'weight': 0.0
                               }]
        user_answer = dedent("""
        class Test
        {public static void main(String[] args){
         int a = 5;
         int b = 6;
         System.out.print(a+b);
        }}""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    },
                    'test_case_data': self.test_case_data,
                  }

        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        self.assertTrue(result.get('success'))
示例#30
0
    def test_correct_answer(self):
        # Given
        user_answer = dedent("""
        import java.util.Scanner;
        class Test
        {public static void main(String[] args){
         Scanner s = new Scanner(System.in);
         int a = s.nextInt();
         int b = s.nextInt();
         System.out.print(a+b);
        }}""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#31
0
    def test_correct_answer_list(self):
        # Given
        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                "expected_input": "1,2,3\n5,6,7",
                                "expected_output": "[1, 2, 3, 5, 6, 7]",
                                "weight": 0.0
                                }]
        user_answer = dedent("""
                                from six.moves import input
                                input_a = input()
                                input_b = input()
                                a = [int(i) for i in input_a.split(',')]
                                b = [int(i) for i in input_b.split(',')]
                                print(a+b)
                             """
                             )

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#32
0
    def test_error(self):
        # Given
        user_answer = dedent("""
        class Test
        {
         System.out.print("a");
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        errors = result.get('error')

        # Then
        self.assertFalse(result.get("success"))
        for error in errors:
            self.assertEqual(error.get('exception'), 'CompilationError')
示例#33
0
    def test_incorrect_answer_integer(self):
        # Given
        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                "expected_input": "1\n2",
                                "expected_output": "3",
                                "weight": 0.0
                                }]
        user_answer = dedent("""
                                a = int(input())
                                b = int(input())
                                print(a-b)
                             """
                             )
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
        self.assert_correct_output(
            "Incorrect Answer: Line number(s) 1 did not match.",
            result.get('error')[0].get('error_msg')
        )
示例#34
0
    def test_array_input(self):
        # Given
        self.test_case_data = [{'expected_output': '561',
                                'expected_input': '5\n6\n1',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a[3],i;
        for(i=0;i<3;i++){
        scanf("%d",&a[i]);}
        for(i=0;i<3;i++){
        printf("%d",a[i]);}
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#35
0
    def test_incorrect_answer(self):
        # Given
        user_answer = "def add(a,b):\n\treturn a - b"
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
        given_test_case_list = [tc["test_case"] for tc in self.test_case_data]
        for error in result.get("error"):
            self.assertEqual(error['exception'], 'AssertionError')
            self.assertEqual(
                error['message'],
                "Expected answer from the test case did not match the output"
                )
        error_testcase_list = [tc['test_case'] for tc in result.get('error')]
        self.assertEqual(error_testcase_list, given_test_case_list)
示例#36
0
    def test_string_input(self):
        # Given
        self.test_case_data = [{'expected_output': 'abc',
                                'expected_input': 'abc',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        char a[4];
        scanf("%s",a);
        printf("%s",a);
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#37
0
    def test_incorrect_answer(self):
        # Given
        user_answer = "def add(a,b):\n\treturn a - b"
        hook_code = dedent("""\
                            def check_answer(user_answer):
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                exec(user_answer, globals())
                                if add(1,2) == 3:
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """
                           )

        test_case_data = [{"test_case_type": "hooktestcase",
                           "hook_code": hook_code, "weight": 1.0
                           }]

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get('success'))
        self.assert_correct_output('Incorrect Answer', result.get('error'))
示例#38
0
    def test_infinite_loop(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        while(0==0){
        printf("abc");}
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
        parent_proc = Process(os.getpid()).children()
        if parent_proc:
            children_procs = Process(parent_proc[0].pid)
            self.assertFalse(any(children_procs.children(recursive=True)))
示例#39
0
    def test_assignment_upload(self):
        # Given
        user_answer = "Assignment Upload"
        hook_code = dedent("""\
                            def check_answer(user_answer):
                                success = False
                                err = "Incorrect Answer"
                                mark_fraction = 0.0
                                with open("test.txt") as f:
                                    data = f.read()
                                if data == '2':
                                    success, err, mark_fraction = True, "", 1.0
                                return success, err, mark_fraction
                            """
                           )

        test_case_data = [{"test_case_type": "hooktestcase",
                           "hook_code": hook_code, "weight": 1.0
                           }]
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'assign_files': [(self.tmp_file, False)],
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#40
0
    def test_cpp_correct_answer(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a,b;
        cin>>a>>b;
        cout<<a+b;
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
    def test_cpp_incorrect_answer(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a=10;
        cout<<a;
        }""")
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'cpp'
            },
            'test_case_data': self.test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
        result_error = result.get('error')[0].get('error_msg')
        self.assertFalse(result.get('success'))
        self.assert_correct_output("Incorrect", result_error)
        self.assertTrue(lines_of_error > 0)
示例#42
0
    def test_cpp_incorrect_answer(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a=10;
        cout<<a;
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        lines_of_error = len(result.get('error')[0].get('error_line_numbers'))
        result_error = result.get('error')[0].get('error_msg')
        self.assertFalse(result.get('success'))
        self.assert_correct_output("Incorrect", result_error)
        self.assertTrue(lines_of_error > 0)
示例#43
0
    def test_file_based_assert(self):
        # Given
        self.file_paths = [(self.f_path, False)]
        self.tc_data = dedent("""
            #!/bin/bash
            cat $1
            """)
        self.tc_data_args = "test.txt"
        self.test_case_data = [{
            "test_case": self.tc_data,
            "test_case_args": self.tc_data_args,
            "test_case_type": "standardtestcase",
            "weight": 0.0
        }]
        user_answer = ("#!/bin/bash\ncat $1")
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': self.test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get("success"))
示例#44
0
    def test_cpp_error(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a=10;
        cout<<a
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        errors = result.get('error')

        # Then
        self.assertFalse(result.get("success"))
        for error in errors:
            self.assertEqual(error['exception'], 'CompilationError')
示例#45
0
    def test_incorrect_answer(self):
        # Given
        user_answer = dedent(""" #!/bin/bash
                             read A
                             read B
                             echo -n `expr $A - $B`
                             """)
        test_case_data = [{
            'expected_output': '11',
            'expected_input': '5\n6',
            'test_case_type': 'stdiobasedtestcase',
            'weight': 0.0
        }]
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        result_error = result.get('error')[0].get('error_msg')
        self.assert_correct_output("Incorrect", result_error)
        self.assertFalse(result.get('success'))
示例#46
0
    def test_cpp_infinite_loop(self):
        # Given
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        while(0==0){
        cout<<"abc";}
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg,
                                   result.get("error")[0]["message"]
                                   )
示例#47
0
    def test_infinite_loop(self):
        # Given
        user_answer = ("#!/bin/bash\nwhile [ 1 ] ;"
                       " do echo "
                       " > /dev/null ; done")
        kwargs = {
            'metadata': {
                'user_answer': user_answer,
                'file_paths': self.file_paths,
                'partial_grading': False,
                'language': 'bash'
            },
            'test_case_data': self.test_case_data,
        }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output(self.timeout_msg,
                                   result.get("error")[0]["message"])
        parent_proc = Process(os.getpid()).children()
        if parent_proc:
            children_procs = Process(parent_proc[0].pid)
            self.assertFalse(any(children_procs.children(recursive=True)))
示例#48
0
    def test_cpp_only_stdout(self):
        # Given
        self.test_case_data = [{'expected_output': '11',
                               'expected_input': '',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        int a=5,b=6;
        cout<<a+b;
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    }, 'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#49
0
    def test_array_input(self):
        # Given
        self.test_case_data = [{'expected_output': '561',
                                'expected_input': '5\n6\n1',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a[3],i;
        for(i=0;i<3;i++){
        scanf("%d",&a[i]);}
        for(i=0;i<3;i++){
        printf("%d",a[i]);}
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#50
0
    def test_type_error(self):
        # Given
        user_answer = dedent("""
        def add(a):
            return a + b
        """)
        type_error_msg = ["Traceback",
                          "call",
                          "TypeError",
                          "argument"
                          ]

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        err = result.get("error")[0]['traceback']

        # Then
        self.assertFalse(result.get("success"))
        for msg in type_error_msg:
            self.assert_correct_output(msg, err)
示例#51
0
    def test_error(self):
        # Given
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a=10;
        printf("%d",a)
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertFalse(result.get("success"))
        self.assert_correct_output("Compilation Error", result.get("error"))
示例#52
0
    def test_value_error(self):
        # Given
        user_answer = dedent("""
        def add(a, b):
            c = 'a'
            return int(a) + int(b) + int(c)
        """)
        value_error_msg = ["Traceback",
                           "call",
                           "ValueError",
                           "invalid literal",
                           "base"
                           ]

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        errors = result.get("error")

        # Then
        self.assertFalse(result.get("success"))
        for msg in value_error_msg:
            self.assert_correct_output(msg, errors[0]['traceback'])
        for index, error in enumerate(errors):
            self.assertEqual(error['test_case'],
                             self.test_case_data[index]['test_case'])
示例#53
0
    def test_only_stdout(self):
        # Given
        self.test_case_data = [{'expected_output': '11',
                               'expected_input': '',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<stdio.h>
        int main(void){
        int a=5,b=6;
        printf("%d",a+b);
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#54
0
    def test_file_based_assert(self):
        # Given
        self.test_case_data = [{"test_case_type": "standardtestcase",
                                "test_case": "assert(ans()=='2')",
                                "weight": 0.0}
                               ]
        self.file_paths = [(self.tmp_file, False)]
        user_answer = dedent("""
            def ans():
                with open("test.txt") as f:
                    return f.read()[0]
            """)

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#55
0
    def test_cpp_string_input(self):
        # Given
        self.test_case_data = [{'expected_output': 'abc',
                                'expected_input': 'abc',
                                'weight': 0.0,
                                'test_case_type': 'stdiobasedtestcase',
                                }]
        user_answer = dedent("""
        #include<iostream>
        using namespace std;
        int main(void){
        char a[4];
        cin>>a;
        cout<<a;
        }""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'cpp'
                    },
                    'test_case_data': self.test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))
示例#56
0
    def test_single_testcase_error(self):
        """ Tests the user answer with just an incorrect test case """

        # Given
        user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
        test_case_data = [{"test_case_type": "standardtestcase",
                           "test_case": 's="abbb"\nasert palindrome(s)==False',
                           "weight": 0.0
                           }
                          ]
        syntax_error_msg = ["Traceback",
                            "call",
                            "SyntaxError",
                            "invalid syntax"
                            ]

        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        err = result.get("error")[0]['traceback']

        # Then
        self.assertFalse(result.get("success"))
        for msg in syntax_error_msg:
            self.assert_correct_output(msg, err)
示例#57
0
    def test_incorrect_answer(self):
        user_answer = dedent("""
        import java.util.Scanner;
        class Test
        {public static void main(String[] args){
         Scanner s = new Scanner(System.in);
         int a = s.nextInt();
         int b = s.nextInt();
         System.out.print(a);
        }}""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    },
                    'test_case_data': self.test_case_data,
                  }

        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        lines_of_error = len(result.get('error')[0].splitlines())
        self.assertFalse(result.get('success'))
        self.assert_correct_output("Incorrect", result.get('error'))
        self.assertTrue(lines_of_error > 1)
示例#58
0
    def test_multiple_testcase_error(self):
        """ Tests the user answer with an correct test case
         first and then with an incorrect test case """
        # Given
        user_answer = "def palindrome(a):\n\treturn a == a[::-1]"
        test_case_data = [{"test_case_type": "standardtestcase",
                           "test_case": 'assert(palindrome("abba")==True)',
                           "weight": 0.0},
                          {"test_case_type": "standardtestcase",
                           "test_case": 's="abbb"\n'
                                        'assert palindrome(S)==False',
                           "weight": 0.0}
                          ]
        name_error_msg = ["Traceback",
                          "call",
                          "NameError",
                          "name 'S' is not defined"
                          ]
        kwargs = {'metadata': {
                  'user_answer': user_answer,
                  'file_paths': self.file_paths,
                  'partial_grading': False,
                  'language': 'python'},
                  'test_case_data': test_case_data,
                  }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)
        err = result.get("error")[0]['traceback']

        # Then
        self.assertFalse(result.get("success"))
        for msg in name_error_msg:
            self.assertIn(msg, err)
示例#59
0
    def test_string_input(self):
        self.test_case_data = [{'expected_output': 'HelloWorld',
                                'expected_input': 'Hello\nWorld',
                                'test_case_type': 'stdiobasedtestcase',
                                'weight': 0.0
                               }]
        user_answer = dedent("""
        import java.util.Scanner;
        class Test
        {public static void main(String[] args){
         Scanner s = new Scanner(System.in);
         String a = s.nextLine();
         String b = s.nextLine();
         System.out.print(a+b);
        }}""")
        kwargs = {
                  'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'java'
                    },
                    'test_case_data': self.test_case_data,
                  }

        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        self.assertTrue(result.get('success'))
    def test_file_based_answer(self):
        # Given
        self.test_case_data = [{"test_case_type": "stdiobasedtestcase",
                                "expected_input": "",
                                "expected_output": "2",
                                "weight": 0.0
                                }]
        self.file_paths = [(self.tmp_file, False)]

        user_answer = dedent("""
                            with open("test.txt") as f:
                                a = f.read()
                                print(a[0])
                             """
                             )
        kwargs = {'metadata': {
                    'user_answer': user_answer,
                    'file_paths': self.file_paths,
                    'partial_grading': False,
                    'language': 'python'
                    },
                    'test_case_data': self.test_case_data
                }

        # When
        grader = Grader(self.in_dir)
        result = grader.evaluate(kwargs)

        # Then
        self.assertTrue(result.get('success'))