def test_unsuccessful_jobs_fail_evaluation(self):
self.jobs[0].status = Job.FAILURE
self.jobs[0].save()
set_evaluation_inputs(evaluation_pk=self.evaluation.pk)
self.evaluation.refresh_from_db()
assert self.evaluation.status == self.evaluation.FAILURE
assert (self.evaluation.error_message ==
"The algorithm failed on one or more cases.")
def test_set_evaluation_inputs(self):
set_evaluation_inputs(evaluation_pk=self.evaluation.pk)
self.evaluation.refresh_from_db()
assert self.evaluation.status == self.evaluation.PENDING
assert self.evaluation.error_message == ""
assert self.evaluation.inputs.count() == 3
assert self.evaluation.input_prefixes == {
str(civ.pk): f"{alg.pk}/output/"
for alg, civ in zip(self.jobs, self.output_civs)
}
def test_unsuccessful_jobs_fail_evaluation(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
AlgorithmEvaluationFactory(status=AlgorithmEvaluation.SUCCESS,
submission=submission)
AlgorithmEvaluationFactory(status=AlgorithmEvaluation.FAILURE,
submission=submission)
set_evaluation_inputs(evaluation_pk=evaluation.pk)
evaluation.refresh_from_db()
assert evaluation.status == evaluation.FAILURE
assert (evaluation.output ==
"The algorithm failed to execute on 1 images.")
def test_unsuccessful_jobs_fail_evaluation(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = (
AlgorithmJobFactory(status=Job.SUCCESS),
AlgorithmJobFactory(status=Job.FAILURE),
)
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.FAILURE
assert (evaluation.error_message ==
"The algorithm failed to execute on 1 images.")
def test_set_evaluation_inputs(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = AlgorithmJobFactory.create_batch(2, status=Job.SUCCESS)
civs = ComponentInterfaceValueFactory.create_batch(2)
for alg, civ in zip(jobs, civs):
alg.outputs.set([civ])
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.PENDING
assert evaluation.error_message == ""
assert evaluation.inputs.count() == 1
def test_set_evaluation_inputs(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
algorithms = AlgorithmEvaluationFactory.create_batch(
2, status=AlgorithmEvaluation.SUCCESS, submission=submission)
civs = ComponentInterfaceValueFactory.create_batch(2)
for alg, civ in zip(algorithms, civs):
alg.outputs.set([civ])
set_evaluation_inputs(evaluation_pk=evaluation.pk)
evaluation.refresh_from_db()
assert evaluation.status == evaluation.PENDING
assert evaluation.output == ""
assert list(evaluation.inputs.all()) == civs