def to_submission(self, sr):
"""Fill detail of the submission result with the job result.
sr (SubmissionResult): the DB object to fill.
"""
# No need to check self.success because this method gets called
# only if it is True.
sr.evaluations += [
Evaluation(text=json.dumps(self.text, encoding='utf-8'),
outcome=self.outcome,
execution_time=self.plus.get('execution_time'),
execution_wall_clock_time=self.plus.get(
'execution_wall_clock_time'),
execution_memory=self.plus.get('execution_memory'),
evaluation_shard=self.shard,
evaluation_sandbox=":".join(self.sandboxes),
testcase=sr.dataset.testcases[
self.operation["testcase_codename"]])
]
submission = sr.submission
logger.metric("submission_one_testcase_evaluation_time",
submission_id=submission.id,
testcase_codename=self.operation["testcase_codename"],
dataset_id=sr.dataset_id,
language=submission.language,
task=submission.task_id,
participant=submission.participation_id,
value=(make_datetime() -
submission.timestamp).total_seconds())
def to_submission_evaluation(self, sr):
# Should not invalidate because evaluations will be added one
# by one now.
# No need to check self.success or job.success because this
# method gets called only if the first (and therefore the
# second!) is True.
# Should not mark evaluation completed, because not all
# testcases may be evaluated by this moment
for test_name, job in self.jobs.iteritems():
assert isinstance(job, EvaluationJob)
sr.evaluations += [
Evaluation(text=json.dumps(job.text, encoding='utf-8'),
outcome=job.outcome,
execution_time=job.plus.get('execution_time'),
execution_wall_clock_time=job.plus.get(
'execution_wall_clock_time'),
execution_memory=job.plus.get('execution_memory'),
evaluation_shard=job.shard,
evaluation_sandbox=":".join(job.sandboxes),
testcase=sr.dataset.testcases[test_name])
]
def add_evaluation(self, submission_result=None, testcase=None, **kwargs):
"""Add an evaluation."""
dataset = None
dataset = submission_result.dataset \
if submission_result is not None else dataset
dataset = testcase.dataset if testcase is not None else dataset
submission_result = submission_result \
if submission_result is not None else self.add_submission_result()
testcase = testcase if testcase is not None else self.add_testcase()
assert submission_result.dataset == testcase.dataset
args = {
"submission_result": submission_result,
"testcase": testcase,
}
args.update(kwargs)
evaluation = Evaluation(**args)
self.session.add(evaluation)
return evaluation
def to_submission(self, sr):
# Should not invalidate because evaluations will be added one
# by one now.
# No need to check self.success because this method gets called
# only if it is True.
sr.evaluations += [
Evaluation(text=json.dumps(self.text, encoding='utf-8'),
outcome=self.outcome,
execution_time=self.plus.get('execution_time'),
execution_wall_clock_time=self.plus.get(
'execution_wall_clock_time'),
execution_memory=self.plus.get('execution_memory'),
evaluation_shard=self.shard,
evaluation_sandbox=":".join(self.sandboxes),
testcase=sr.dataset.testcases[self.testcase_codename])
]
def to_submission(self, sr):
"""Fill detail of the submission result with the job result.
sr (SubmissionResult): the DB object to fill.
"""
# No need to check self.success because this method gets called
# only if it is True.
sr.evaluations += [Evaluation(
text=self.text,
outcome=self.outcome,
execution_time=self.plus.get('execution_time'),
execution_wall_clock_time=self.plus.get(
'execution_wall_clock_time'),
execution_memory=self.plus.get('execution_memory'),
evaluation_shard=self.shard,
evaluation_sandbox=":".join(self.sandboxes),
testcase=sr.dataset.testcases[self.operation.testcase_codename])]
def to_submission_evaluation(self, sr):
# This should actually be useless.
sr.invalidate_evaluation()
# No need to check self.success or job.success because this
# method gets called only if the first (and therefore the
# second!) is True.
sr.set_evaluation_outcome()
for test_name, job in self.jobs.iteritems():
assert isinstance(job, EvaluationJob)
sr.evaluations += [
Evaluation(text=json.dumps(job.text, encoding='utf-8'),
outcome=job.outcome,
execution_time=job.plus.get('execution_time'),
execution_wall_clock_time=job.plus.get(
'execution_wall_clock_time'),
execution_memory=job.plus.get('execution_memory'),
evaluation_shard=job.shard,
evaluation_sandbox=":".join(job.sandboxes),
testcase=sr.dataset.testcases[test_name])
]