def test_algorithm_job_update_status():
alg = AlgorithmFactory()
user = UserFactory()
editor = UserFactory()
alg.add_user(user)
alg.add_editor(editor)
ai = AlgorithmImageFactory(algorithm=alg)
job = AlgorithmJobFactory(algorithm_image=ai, creator=user)
for status, _ in Job.STATUS_CHOICES:
job.update_status(status=status)
job.refresh_from_db()
assert job.status == status
remaining_recipients = {user.email, editor.email}
for email in mail.outbox:
remaining_recipients -= set(email.to)
assert (
email.subject ==
f"[{Site.objects.get_current().domain.lower()}] [{alg.title.lower()}] Job Failed"
)
assert (
f"Unfortunately your job for algorithm '{alg.title}' failed with an error"
in email.body)
assert remaining_recipients == set()
def test_update_started_adds_time():
j = AlgorithmJobFactory()
assert j.started_at is None
assert j.completed_at is None
j.update_status(status=j.EXECUTING)
j.refresh_from_db()
assert j.started_at is not None
assert j.completed_at is None
j.update_status(status=j.SUCCESS)
j.refresh_from_db()
assert j.started_at is not None
assert j.completed_at is not None
def test_mark_long_running_jobs_failed():
# Started jobs should be unaffected
j1 = EvaluationFactory()
j1.update_status(status=EvaluationJob.STARTED)
# Long running jobs should be marked as failed
j2 = EvaluationFactory()
j2.update_status(status=EvaluationJob.STARTED)
j2.started_at = timezone.now() - timedelta(days=1)
j2.save()
# A job that has not been started should not be marked as failed, even if
# if it is outside the celery task limit
j3 = EvaluationFactory()
j3.created -= timedelta(days=1)
j3.save()
# Algorithm jobs should not be affected
a = AlgorithmJobFactory()
a.update_status(status=AlgorithmJob.STARTED)
assert EvaluationJob.objects.all().count() == 3
assert (
AlgorithmJob.objects.filter(status=AlgorithmJob.STARTED).count() == 1
)
assert (
EvaluationJob.objects.filter(status=EvaluationJob.FAILURE).count() == 0
)
assert j1.status == EvaluationJob.STARTED
assert j2.status == EvaluationJob.STARTED
assert j3.status == EvaluationJob.PENDING
assert a.status == AlgorithmJob.STARTED
mark_long_running_jobs_failed(
app_label="evaluation", model_name="evaluation"
)
j1.refresh_from_db()
j2.refresh_from_db()
j3.refresh_from_db()
a.refresh_from_db()
assert j1.status == EvaluationJob.STARTED
assert j2.status == EvaluationJob.FAILURE
assert j3.status == EvaluationJob.PENDING
assert a.status == AlgorithmJob.STARTED
