def _TestSolutionWithChallengeCases(self, solution, ui):
"""Test a wrong solution which has specified challenge cases."""
all_testcases = self.ListTestCases()
challenge_infiles = solution.challenge_cases
testcases = []
for infile in challenge_infiles:
matched_testcases = [
testcase for testcase in all_testcases
if fnmatch.fnmatch(os.path.basename(testcase.infile), infile)
]
if not matched_testcases:
ui.errors.Error(solution,
'Challenge case not found: %s' % infile)
result = test.TestsetResult(self, solution, [])
result.Finalize(False, 'Challenge case not found: %s' % infile)
yield result
testcases.extend(
[t for t in matched_testcases if t.infile not in testcases])
# Try challenge cases.
result = test.TestsetResult(self, solution, testcases)
yield taskgraph.TaskBranch([
self._TestSolutionWithChallengeCasesOne(
solution, testcase, result, ui) for testcase in testcases
],
unsafe_interrupt=True)
if not result.IsFinalized():
result.Finalize(False, 'Unexpectedly accepted all challenge cases')
ui.errors.Error(solution, result.detail)
yield result
def TestSolution(self, solution, ui):
"""Test a single solution."""
if not (yield self.Build(ui)):
result = test.TestsetResult(self, solution, [])
result.Finalize(False, 'Failed to build tests')
yield [result]
if not (yield solution.Build(ui)):
result = test.TestsetResult(self, solution, [])
result.Finalize(False, 'Compile Error')
yield [result]
ui.console.PrintAction('TEST', solution, progress=True)
if not solution.IsCorrect() and solution.challenge_cases:
result = yield self._TestSolutionWithChallengeCases(solution, ui)
else:
result = yield self._TestSolutionWithAllCases(solution, ui)
status_row = [result.detail]
if result.IsCached():
status_row += [' ', '(cached)']
ui.console.PrintAction('TEST', solution, *status_row)
if solution.IsCorrect() and not result.expected:
assert result.notable_testcase
judgefile = (os.path.splitext(result.notable_testcase.infile)[0] +
consts.JUDGE_EXT)
log = files.ReadFile(judgefile)
ui.console.PrintLog(log)
yield [result]
def _TestSolutionWithMergedTests(self, solution, ui):
testcases = self.GetMergedTestCases()
result = test.TestsetResult(self, solution, testcases)
# Try all cases.
yield taskgraph.TaskBranch([
self._TestSolutionWithAllCasesOne(solution, testcase, result, ui)
for testcase in testcases],
unsafe_interrupt=True)
if not result.IsFinalized():
result.Finalize(True, 'okay')
yield result
def _TestSolutionWithChallengeCases(self, solution, ui):
"""Test a wrong solution which has specified challenge cases."""
all_testcases = self.ListTestCases()
challenge_infiles = [
os.path.join(self.out_dir, infile)
for infile in set(solution.challenge_cases)
]
testcases = []
for infile in challenge_infiles:
matched_testcases = [
testcase for testcase in all_testcases
if testcase.infile == infile
]
if not matched_testcases:
ui.errors.Error(solution,
'Challenge case not found: %s' % infile)
result = test.TestsetResult(self, solution, [])
result.Finalize(False, 'Challenge case not found: %s' % infile)
yield result
elif len(matched_testcases) >= 2:
ui.errors.Error(solution,
'Multiple challenge cases found: %s' % infile)
result = test.TestsetResult(self, solution, [])
result.Finalize(False,
'Multiple challenge cases found: %s' % infile)
yield result
testcases.append(matched_testcases[0])
# Try challenge cases.
result = test.TestsetResult(self, solution, testcases)
yield taskgraph.TaskBranch([
self._TestSolutionWithChallengeCasesOne(
solution, testcase, result, ui) for testcase in testcases
],
unsafe_interrupt=True)
if not result.IsFinalized():
result.Finalize(True, 'Expectedly failed all challenge cases')
yield result
def _TestSolutionWithAllCases(self, solution, ui):
"""Test a solution without challenge cases.
The solution can be marked as wrong but without challenge cases.
"""
testcases = self.ListTestCases()
result = test.TestsetResult(self, solution, testcases)
# Try all cases.
yield taskgraph.TaskBranch([
self._TestSolutionWithAllCasesOne(solution, testcase, result, ui)
for testcase in testcases
],
unsafe_interrupt=True)
if not result.IsFinalized():
if solution.IsCorrect():
result.Finalize(True, result.GetTimeStats(ui))
else:
result.Finalize(False, 'Unexpectedly accepted all test cases')
yield result
def _TestSolutionWithAllCasesOne(self, solution, testcase, result, ui):
"""Test a solution without challenge cases.
The solution can be marked as wrong but without challenge cases.
"""
case_result = yield self._TestOneCase(solution, testcase, ui)
result.results[testcase] = case_result
if case_result.verdict not in (test.TestCaseResult.AC,
test.TestCaseResult.WA,
test.TestCaseResult.TLE,
test.TestCaseResult.RE):
result.Finalize(False,
'%s: Judge Error' %
os.path.basename(testcase.infile),
notable_testcase=testcase)
ui.errors.Error(solution, result.detail)
if ui.options.keep_going:
yield False
else:
raise taskgraph.Bailout([False])
elif case_result.verdict != test.TestCaseResult.AC:
expected = not solution.IsCorrect()
r = test.TestsetResult(result.testset, result.solution,
result.testcases)
r.Finalize(
expected,
'%s: %s' %
(os.path.basename(testcase.infile), case_result.verdict),
notable_testcase=testcase)
result.Finalize(
expected,
'%s: %s' %
(os.path.basename(testcase.infile), case_result.verdict),
notable_testcase=testcase)
if solution.IsCorrect():
if case_result.verdict == test.TestCaseResult.WA:
judgefile = os.path.join(
solution.out_dir,
os.path.splitext(os.path.basename(testcase.infile))[0]
+ consts.JUDGE_EXT)
ui.errors.Error(
solution,
'%s\n judge log: %s' % (r.detail, judgefile))
else:
ui.errors.Error(solution, r.detail)
elif (solution.expected_verdicts is not None
and case_result.verdict not in solution.expected_verdicts):
r = test.TestsetResult(result.testset, result.solution,
result.testcases)
r.Finalize(
False,
'%s: Unexpected Verdict (%s)' %
(os.path.basename(testcase.infile), case_result.verdict),
notable_testcase=testcase)
ui.errors.Error(solution, r.detail)
if case_result.verdict == test.TestCaseResult.WA:
judgefile = os.path.join(
solution.out_dir,
os.path.splitext(os.path.basename(testcase.infile))[0]
+ consts.JUDGE_EXT)
ui.errors.Error(
solution,
'%s\n judge log: %s' % (r.detail, judgefile))
else:
ui.errors.Error(solution, r.detail)
if ui.options.keep_going:
yield False
else:
raise taskgraph.Bailout([False])
ui.console.PrintAction('TEST',
solution,
'%s: PASSED' %
os.path.basename(testcase.infile),
progress=True)
yield True