def build_details_str(exec_res: ExecResult, input_file: str, output_file: str) -> str: res = "" def append(text: str, end='\n'): nonlocal res res += text + end append(with_color("[Input]", Fore.LIGHTMAGENTA_EX)) with open(input_file, "r") as f: append(f.read(), end='') append(with_color("[Expected]", Fore.LIGHTMAGENTA_EX)) with open(output_file, "r") as f: append(f.read(), end='') append(with_color("[Received]", Fore.LIGHTMAGENTA_EX)) append(exec_res.output, end='') if exec_res.status != ExecStatus.NORMAL: append( with_color("Aborted ({})\n".format(exec_res.status.name), Fore.LIGHTYELLOW_EX)) if exec_res.has_stderr(): append(with_color("[Error]", Fore.LIGHTYELLOW_EX)) append(exec_res.stderr, end='') return res
def build_details_str(exec_res: ExecResult, input_file: str, output_file: str) -> str: res = "" def append(text: str, end='\n'): nonlocal res res += text + end with open(output_file, "r") as f: expected_output = f.read() append(with_color("[Input]", Fore.LIGHTMAGENTA_EX)) with open(input_file, "r") as f: append(f.read(), end='') append(with_color("[Expected]", Fore.LIGHTMAGENTA_EX)) append(expected_output, end='') append(with_color("[Received]", Fore.LIGHTMAGENTA_EX)) append(exec_res.output, end='') if exec_res.status != ExecStatus.NORMAL: append(with_color("Aborted ({})\n".format( exec_res.status.name), Fore.LIGHTYELLOW_EX)) if exec_res.has_stderr(): append(with_color("[Error]", Fore.LIGHTYELLOW_EX)) append(exec_res.stderr, end='') return res
def test_build_details_str__on_runtime_failure(self): in_out = 'correct\n' stderr = '' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('Aborted ({})\n'.format(ExecStatus.RE.name), Fore.LIGHTYELLOW_EX) + '\n') io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str(ExecResult( ExecStatus.RE, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result)
def test_build_details_str__show_testcase_if_there_is_stderr(self): in_out = 'correct\n' stderr = 'stderr\n' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Error]', Fore.LIGHTYELLOW_EX) + '\n' + stderr) io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str(ExecResult( ExecStatus.NORMAL, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result)
class TestTester(unittest.TestCase): def test_multiple_exec_files(self): all_ok = tester.main( '', ['-d', os.path.join(RESOURCE_DIR, "test_multiple_exec_files")]) self.assertTrue(all_ok) def test_run_single_test(self): test_dir = os.path.join(RESOURCE_DIR, "test_run_single_test") self.assertTrue(tester.main('', ['-d', test_dir, "-n", "1"])) self.assertFalse(tester.main('', ['-d', test_dir, "-n", "2"])) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file(self, os_mock, is_file_mock): self.assertTrue(is_executable_file('a.out')) @patch('os.access', return_value=False) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__not_executable(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('a.out')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__source_code(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('A.cpp')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__text(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('in.txt')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=False) def test_is_executable_file__directory(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('directory')) @patch("platform.system", return_value="Windows") @patch("os.environ.get", return_value=".EXE;.out") def test_is_executable_file__windows(self, platform_system_mock, os_environ_get_mock): self.assertTrue(is_executable_file("A.eXe")) self.assertTrue(is_executable_file("A.out")) self.assertFalse(is_executable_file("A.exe.bak")) @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', '', 0)) def test_run_for_samples(self, run_program_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, False), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', 'stderr', 0)) def test_run_for_samples__with_stderr(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, True), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'wrong', '', 0)) def test_run_for_samples__wrong_answer(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(0, False), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'wrong', '', 0)) def test_run_for_samples__stop_execution_on_first_failure( self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): sample_pair_list = [('in_1.txt', 'out_1.txt'), ('in_2.txt', 'out_2.txt')] self.assertEqual( TestSummary(0, False), tester.run_for_samples('a.out', sample_pair_list, 1, True)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', 'stderr', 0)) def test_run_for_samples__skip_stderr(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, True), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1, skip_io_on_success=True)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(0, build_details_str_mock.call_count) def test_build_details_str(self): in_out = 'correct\n' output = 'wrong\n' stderr = 'stderr\n' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + output + with_color('[Error]', Fore.LIGHTYELLOW_EX) + '\n' + stderr) io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.NORMAL, output, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result) def test_build_details_str__show_testcase_if_there_is_stderr(self): in_out = 'correct\n' stderr = 'stderr\n' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Error]', Fore.LIGHTYELLOW_EX) + '\n' + stderr) io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.NORMAL, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result) def test_build_details_str__on_runtime_failure(self): in_out = 'correct\n' stderr = '' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('Aborted ({})\n'.format(ExecStatus.RE.name), Fore.LIGHTYELLOW_EX) + '\n') io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.RE, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result)
class TestTester(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() def tearDown(self): shutil.rmtree(self.temp_dir) def test_multiple_exec_files(self): all_ok = tester.main( '', ['-d', os.path.join(RESOURCE_DIR, "test_multiple_exec_files")]) self.assertTrue(all_ok) def test_run_single_test(self): test_dir = os.path.join(RESOURCE_DIR, "test_run_single_test") self.assertTrue(tester.main('', ['-d', test_dir, "-n", "1"])) self.assertFalse(tester.main('', ['-d', test_dir, "-n", "2"])) def test_run_single_test_decimal_addition(self): test_dir = os.path.join(RESOURCE_DIR, "test_run_single_test_decimal_addition") self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "1", "-v", "0.01", "-j", "absolute_or_relative" ])) self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "2", "-v", "0.01", "--judge-type", "absolute_or_relative" ])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "1", "-v", "0.01", "-j", "absolute"])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "2", "-v", "0.01", "-j", "absolute"])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "1", "-v", "0.01", "-j", "relative"])) self.assertFalse( tester.main( '', ['-d', test_dir, "-n", "2", "-v", "0.01", "-j", "relative"])) def test_run_single_test_decimal_multiplication(self): test_dir = os.path.join(RESOURCE_DIR, "test_run_single_test_decimal_multiplication") self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "1", "-v", "0.01", "-j", "absolute_or_relative" ])) self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "2", "--error-value", "0.01", "-j", "absolute_or_relative" ])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "1", "-v", "0.01", "-j", "absolute"])) self.assertFalse( tester.main( '', ['-d', test_dir, "-n", "2", "-v", "0.01", "-j", "absolute"])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "1", "-v", "0.01", "-j", "relative"])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "2", "-v", "0.01", "-j", "relative"])) def test_run_single_test_decimal_mixed(self): test_dir = os.path.join(RESOURCE_DIR, "test_run_single_test_decimal_mixed") self.assertFalse( tester.main('', [ '-d', test_dir, "-n", "1", "-v", "0.01", "-j", "absolute_or_relative" ])) self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "2", "-v", "0.01", "-j", "absolute_or_relative" ])) self.assertFalse( tester.main('', [ '-d', test_dir, "-n", "1", "-v", "0.0001", "-j", "absolute_or_relative" ])) self.assertFalse( tester.main('', [ '-d', test_dir, "-n", "2", "-v", "0.0001", "-j", "absolute_or_relative" ])) def test_run_single_test_multisolution(self): test_dir = os.path.join(self.temp_dir, "test") shutil.copytree( os.path.join(RESOURCE_DIR, "test_run_single_test_multisolution"), test_dir) setter_main('', ['-d', test_dir, "-j", "multisolution"]) metadata = Metadata.load_from(os.path.join(test_dir, "metadata.json")) self.assertTrue(isinstance(metadata.judge_method, MultiSolutionJudge)) # Already set setter_main('', ['-d', test_dir, "--lang", "cpp"]) metadata = Metadata.load_from(os.path.join(test_dir, "metadata.json")) self.assertTrue(metadata.lang.name == 'cpp') self.assertTrue( tester.main('', ['-d', test_dir, '-n', '1', '-c', "True"])) self.assertTrue( tester.main('', ['-d', test_dir, "-n", "2", "-c", "True"])) self.assertTrue( tester.main('', ['-d', test_dir, "-n", "3", "-c", "True"])) self.assertTrue( tester.main('', ['-d', test_dir, "-n", "4", "-c", "True"])) def test_run_single_test_interactive(self): test_dir = os.path.join(self.temp_dir, "test") shutil.copytree( os.path.join(RESOURCE_DIR, "test_run_single_test_interactive"), test_dir) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "1", "-j", "interactive", '-c', "True" ])) self.assertTrue( tester.main( '', ['-d', test_dir, "-n", "2", "-j", "interactive", '-c', "True" ])) def test_compiler_and_tester(self): test_dir = os.path.join(self.temp_dir, "test") shutil.copytree(os.path.join(RESOURCE_DIR, "test_compiler_and_tester"), test_dir) for lang in ALL_LANGUAGES: setter_main('', ["--lang", lang.name, '-d', test_dir]) metadata = Metadata.load_from( os.path.join(test_dir, "metadata.json")) compile_main_and_judge_programs(metadata, force_compile=True, cwd=test_dir) for i in [1, 2, 3, 4]: self.assertTrue( tester.main('', [ '-d', test_dir, "-n", "{:d}".format(i), "-j", "normal" ])) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file(self, os_mock, is_file_mock): self.assertTrue(is_executable_file('a.out')) @patch('os.access', return_value=False) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__not_executable(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('a.out')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__source_code(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('A.cpp')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=True) def test_is_executable_file__text(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('in.txt')) @patch('os.access', return_value=True) @patch('pathlib.Path.is_file', return_value=False) def test_is_executable_file__directory(self, os_mock, is_file_mock): self.assertFalse(is_executable_file('directory')) @patch("platform.system", return_value="Windows") @patch("os.environ.get", return_value=".EXE;.out") def test_is_executable_file__windows(self, platform_system_mock, os_environ_get_mock): self.assertTrue(is_executable_file("A.eXe")) self.assertTrue(is_executable_file("A.out")) self.assertFalse(is_executable_file("A.exe.bak")) @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', '', 0)) def test_run_for_samples(self, run_program_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, False), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', 'stderr', 0)) def test_run_for_samples__with_stderr(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, True), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'wrong', '', 0)) def test_run_for_samples__wrong_answer(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(0, False), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'wrong', '', 0)) def test_run_for_samples__stop_execution_on_first_failure( self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): sample_pair_list = [('in_1.txt', 'out_1.txt'), ('in_2.txt', 'out_2.txt')] self.assertEqual( TestSummary(0, False), tester.run_for_samples('a.out', sample_pair_list, 1, knock_out=True)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(1, build_details_str_mock.call_count) @patch('atcodertools.tools.tester.build_details_str', return_value='') @patch('atcodertools.tools.tester.run_program', return_value=ExecResult(ExecStatus.NORMAL, 'correct', 'stderr', 0)) def test_run_for_samples__skip_stderr(self, run_program_mock: MagicMock, build_details_str_mock: MagicMock): io_mock = mock_open(read_data='correct') with patch('atcodertools.tools.tester.open', io_mock): self.assertEqual( TestSummary(1, True), tester.run_for_samples('a.out', [('in_1.txt', 'out_1.txt')], 1, skip_io_on_success=True)) self.assertEqual(1, run_program_mock.call_count) self.assertEqual(0, build_details_str_mock.call_count) def test_build_details_str(self): in_out = 'correct\n' output = 'wrong\n' stderr = 'stderr\n' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + output + with_color('[Error]', Fore.LIGHTYELLOW_EX) + '\n' + stderr) io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.NORMAL, output, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result) def test_build_details_str__show_testcase_if_there_is_stderr(self): in_out = 'correct\n' stderr = 'stderr\n' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Error]', Fore.LIGHTYELLOW_EX) + '\n' + stderr) io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.NORMAL, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result) def test_build_details_str__on_runtime_failure(self): in_out = 'correct\n' stderr = '' expected = (with_color('[Input]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Expected]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('[Received]', Fore.LIGHTMAGENTA_EX) + '\n' + in_out + with_color('Aborted ({})\n'.format(ExecStatus.RE.name), Fore.LIGHTYELLOW_EX) + '\n') io_mock = mock_open(read_data=in_out) with patch('atcodertools.tools.tester.open', io_mock): result = build_details_str( ExecResult(ExecStatus.RE, in_out, stderr), 'in.txt', 'out.txt') self.assertEqual(expected, result)