def _TestSolutionWithChallengeCases(self, solution, ui): """Test a wrong solution which has specified challenge cases.""" all_testcases = self.ListTestCases() challenge_infiles = solution.challenge_cases testcases = [] for infile in challenge_infiles: matched_testcases = [ testcase for testcase in all_testcases if fnmatch.fnmatch(os.path.basename(testcase.infile), infile) ] if not matched_testcases: ui.errors.Error(solution, 'Challenge case not found: %s' % infile) result = test.TestsetResult(self, solution, []) result.Finalize(False, 'Challenge case not found: %s' % infile) yield result testcases.extend( [t for t in matched_testcases if t.infile not in testcases]) # Try challenge cases. result = test.TestsetResult(self, solution, testcases) yield taskgraph.TaskBranch([ self._TestSolutionWithChallengeCasesOne( solution, testcase, result, ui) for testcase in testcases ], unsafe_interrupt=True) if not result.IsFinalized(): result.Finalize(False, 'Unexpectedly accepted all challenge cases') ui.errors.Error(solution, result.detail) yield result
def TestSolution(self, solution, ui): """Test a single solution.""" if not (yield self.Build(ui)): result = test.TestsetResult(self, solution, []) result.Finalize(False, 'Failed to build tests') yield [result] if not (yield solution.Build(ui)): result = test.TestsetResult(self, solution, []) result.Finalize(False, 'Compile Error') yield [result] ui.console.PrintAction('TEST', solution, progress=True) if not solution.IsCorrect() and solution.challenge_cases: result = yield self._TestSolutionWithChallengeCases(solution, ui) else: result = yield self._TestSolutionWithAllCases(solution, ui) status_row = [result.detail] if result.IsCached(): status_row += [' ', '(cached)'] ui.console.PrintAction('TEST', solution, *status_row) if solution.IsCorrect() and not result.expected: assert result.notable_testcase judgefile = (os.path.splitext(result.notable_testcase.infile)[0] + consts.JUDGE_EXT) log = files.ReadFile(judgefile) ui.console.PrintLog(log) yield [result]
def _TestSolutionWithMergedTests(self, solution, ui): testcases = self.GetMergedTestCases() result = test.TestsetResult(self, solution, testcases) # Try all cases. yield taskgraph.TaskBranch([ self._TestSolutionWithAllCasesOne(solution, testcase, result, ui) for testcase in testcases], unsafe_interrupt=True) if not result.IsFinalized(): result.Finalize(True, 'okay') yield result
def _TestSolutionWithChallengeCases(self, solution, ui): """Test a wrong solution which has specified challenge cases.""" all_testcases = self.ListTestCases() challenge_infiles = [ os.path.join(self.out_dir, infile) for infile in set(solution.challenge_cases) ] testcases = [] for infile in challenge_infiles: matched_testcases = [ testcase for testcase in all_testcases if testcase.infile == infile ] if not matched_testcases: ui.errors.Error(solution, 'Challenge case not found: %s' % infile) result = test.TestsetResult(self, solution, []) result.Finalize(False, 'Challenge case not found: %s' % infile) yield result elif len(matched_testcases) >= 2: ui.errors.Error(solution, 'Multiple challenge cases found: %s' % infile) result = test.TestsetResult(self, solution, []) result.Finalize(False, 'Multiple challenge cases found: %s' % infile) yield result testcases.append(matched_testcases[0]) # Try challenge cases. result = test.TestsetResult(self, solution, testcases) yield taskgraph.TaskBranch([ self._TestSolutionWithChallengeCasesOne( solution, testcase, result, ui) for testcase in testcases ], unsafe_interrupt=True) if not result.IsFinalized(): result.Finalize(True, 'Expectedly failed all challenge cases') yield result
def _TestSolutionWithAllCases(self, solution, ui): """Test a solution without challenge cases. The solution can be marked as wrong but without challenge cases. """ testcases = self.ListTestCases() result = test.TestsetResult(self, solution, testcases) # Try all cases. yield taskgraph.TaskBranch([ self._TestSolutionWithAllCasesOne(solution, testcase, result, ui) for testcase in testcases ], unsafe_interrupt=True) if not result.IsFinalized(): if solution.IsCorrect(): result.Finalize(True, result.GetTimeStats(ui)) else: result.Finalize(False, 'Unexpectedly accepted all test cases') yield result
def _TestSolutionWithAllCasesOne(self, solution, testcase, result, ui): """Test a solution without challenge cases. The solution can be marked as wrong but without challenge cases. """ case_result = yield self._TestOneCase(solution, testcase, ui) result.results[testcase] = case_result if case_result.verdict not in (test.TestCaseResult.AC, test.TestCaseResult.WA, test.TestCaseResult.TLE, test.TestCaseResult.RE): result.Finalize(False, '%s: Judge Error' % os.path.basename(testcase.infile), notable_testcase=testcase) ui.errors.Error(solution, result.detail) if ui.options.keep_going: yield False else: raise taskgraph.Bailout([False]) elif case_result.verdict != test.TestCaseResult.AC: expected = not solution.IsCorrect() r = test.TestsetResult(result.testset, result.solution, result.testcases) r.Finalize( expected, '%s: %s' % (os.path.basename(testcase.infile), case_result.verdict), notable_testcase=testcase) result.Finalize( expected, '%s: %s' % (os.path.basename(testcase.infile), case_result.verdict), notable_testcase=testcase) if solution.IsCorrect(): if case_result.verdict == test.TestCaseResult.WA: judgefile = os.path.join( solution.out_dir, os.path.splitext(os.path.basename(testcase.infile))[0] + consts.JUDGE_EXT) ui.errors.Error( solution, '%s\n judge log: %s' % (r.detail, judgefile)) else: ui.errors.Error(solution, r.detail) elif (solution.expected_verdicts is not None and case_result.verdict not in solution.expected_verdicts): r = test.TestsetResult(result.testset, result.solution, result.testcases) r.Finalize( False, '%s: Unexpected Verdict (%s)' % (os.path.basename(testcase.infile), case_result.verdict), notable_testcase=testcase) ui.errors.Error(solution, r.detail) if case_result.verdict == test.TestCaseResult.WA: judgefile = os.path.join( solution.out_dir, os.path.splitext(os.path.basename(testcase.infile))[0] + consts.JUDGE_EXT) ui.errors.Error( solution, '%s\n judge log: %s' % (r.detail, judgefile)) else: ui.errors.Error(solution, r.detail) if ui.options.keep_going: yield False else: raise taskgraph.Bailout([False]) ui.console.PrintAction('TEST', solution, '%s: PASSED' % os.path.basename(testcase.infile), progress=True) yield True