def test_get_operations_compilation_failed(self):
"""Test for a submission which failed to compile."""
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, False)
self.session.flush()
expected_operations = set(
self._scoring_operation(submission, dataset)
for dataset in submission.task.datasets if self.to_judge(dataset))
self.assertEqual(set(get_operations(self.session)),
expected_operations)
def test_get_operations_compilation_failed(self):
"""Test for a submission which failed to compile."""
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, False)
self.session.flush()
expected_operations = set(
self._scoring_operation(submission, dataset)
for dataset in submission.task.datasets if self.to_judge(dataset))
self.assertEqual(
set(get_operations(self.session)),
expected_operations)
def test_get_operations_evaluated(self):
"""Test for a submission which failed to compile."""
# A submission completely evaluated.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, True)
for result in results:
for codename, testcase in result.dataset.testcases.items():
self.add_evaluation(result, testcase)
result.set_evaluation_outcome()
self.session.flush()
expected_operations = set(
self._scoring_operation(submission, dataset)
for dataset in submission.task.datasets if self.to_judge(dataset))
self.assertEqual(set(get_operations(self.session)),
expected_operations)
def test_get_operations_evaluated(self):
"""Test for a submission which failed to compile."""
# A submission completely evaluated.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, True)
for result in results:
for testcase in result.dataset.testcases.values():
self.add_evaluation(result, testcase)
result.set_evaluation_outcome()
self.session.flush()
expected_operations = set(
self._scoring_operation(submission, dataset)
for dataset in submission.task.datasets if self.to_judge(dataset))
self.assertEqual(
set(get_operations(self.session)),
expected_operations)
def test_get_operations_no_operations(self):
"""Test for submissions without operations to do."""
# Submission to compile, without a result.
submission = self.add_submission(self.tasks[0], self.participation)
# Again to compile, but with a result.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation)
# With many failure during compilation.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation)
for result in results:
result.compilation_tries = 25
# Submission to evaluate.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, True)
# With many failures during evaluation.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, True)
for result in results:
result.evaluation_tries = 25
# Submission partially evaluated.
submission, results = self.add_submission_with_results(
self.tasks[0], self.participation, True)
evaluated_codenames = set()
for result in results:
# Pick one arbitrary testcase.
evaluated_codename = next(iter(result.dataset.testcases.keys()))
self.add_evaluation(
result, result.dataset.testcases[evaluated_codename])
evaluated_codenames.add(evaluated_codename)
self.session.flush()
self.assertEqual(set(get_operations(self.session)), set())