def test_no_algorithm_image_does_nothing(self):
image = ImageFactory()
create_algorithm_jobs(
algorithm_image=None,
images=[image],
)
assert Job.objects.count() == 0
def create_algorithm_jobs_for_evaluation(*, evaluation_pk, max_jobs=1):
"""
Creates the algorithm jobs for the evaluation
By default the number of jobs are limited to allow for failures.
Once this task is called without limits the remaining jobs are
scheduled (if any), and the evaluation run.
Parameters
----------
evaluation_pk
The primary key of the evaluation
max_jobs
The maximum number of jobs to create
"""
Evaluation = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Evaluation")
evaluation = Evaluation.objects.get(pk=evaluation_pk)
# Only the challenge admins should be able to view these jobs, never
# the algorithm editors as these are participants - they must never
# be able to see the test data.
challenge_admins = [evaluation.submission.phase.challenge.admins_group]
if max_jobs is None:
# Once the algorithm has been run, score the submission. No emails as
# algorithm editors should not have access to the underlying images.
task_on_success = set_evaluation_inputs.signature(
kwargs={"evaluation_pk": str(evaluation.pk)}, immutable=True)
else:
# Run with 1 job and then if that goes well, come back and
# run all jobs. Note that setting None here is caught by
# the if statement to schedule `set_evaluation_inputs`
task_on_success = create_algorithm_jobs_for_evaluation.signature(
kwargs={
"evaluation_pk": str(evaluation.pk),
"max_jobs": None
},
immutable=True,
)
# If any of the jobs fail then mark the evaluation as failed.
task_on_failure = handle_failed_jobs.signature(
kwargs={"evaluation_pk": str(evaluation.pk)}, immutable=True)
create_algorithm_jobs(
algorithm_image=evaluation.submission.algorithm_image,
civ_sets=[{
*ai.values.all()
} for ai in evaluation.submission.phase.archive.items.prefetch_related(
"values__interface")],
creator=None,
extra_viewer_groups=challenge_admins,
extra_logs_viewer_groups=challenge_admins,
task_on_success=task_on_success,
task_on_failure=task_on_failure,
max_jobs=max_jobs,
)
evaluation.update_status(status=Evaluation.EXECUTING_PREREQUISITES)
def test_is_idempotent(self):
ai = AlgorithmImageFactory()
image = ImageFactory()
assert Job.objects.count() == 0
create_algorithm_jobs(algorithm_image=ai, images=[image])
assert Job.objects.count() == 1
jobs = create_algorithm_jobs(algorithm_image=ai, images=[image])
assert Job.objects.count() == 1
assert len(jobs) == 0
def test_jobs_workflow(self):
ai = AlgorithmImageFactory()
images = [ImageFactory(), ImageFactory()]
civ_sets = [{
ComponentInterfaceValueFactory(image=im,
interface=ai.algorithm.inputs.get())
} for im in images]
with capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(algorithm_image=ai, civ_sets=civ_sets)
assert len(callbacks) == 2
def test_gets_creator_from_session(self):
riu = RawImageUploadSessionFactory()
riu.image_set.add(ImageFactory(), ImageFactory())
create_algorithm_jobs(
algorithm_image=AlgorithmImageFactory(),
images=riu.image_set.all(),
creator=riu.creator,
)
j = Job.objects.first()
assert j.creator == riu.creator
def test_is_idempotent(self):
ai = AlgorithmImageFactory()
image = ImageFactory()
civ = ComponentInterfaceValueFactory(
image=image, interface=ai.algorithm.inputs.get())
assert Job.objects.count() == 0
create_algorithm_jobs(algorithm_image=ai, civ_sets=[{civ}])
assert Job.objects.count() == 1
jobs = create_algorithm_jobs(algorithm_image=ai, civ_sets=[{civ}])
assert Job.objects.count() == 1
assert len(jobs) == 0
def test_gets_creator_from_session(self):
ai = AlgorithmImageFactory()
riu = RawImageUploadSessionFactory()
riu.image_set.add(ImageFactory(), ImageFactory())
civ_sets = [{
ComponentInterfaceValueFactory(image=image,
interface=ai.algorithm.inputs.get())
} for image in riu.image_set.all()]
create_algorithm_jobs(algorithm_image=ai,
civ_sets=civ_sets,
creator=riu.creator)
j = Job.objects.first()
assert j.creator == riu.creator
def test_extra_viewer_groups(self):
ai = AlgorithmImageFactory()
image = ImageFactory()
groups = (GroupFactory(), GroupFactory(), GroupFactory())
jobs = create_algorithm_jobs(algorithm_image=ai,
images=[image],
extra_viewer_groups=groups)
for g in groups:
assert jobs[0].viewer_groups.filter(pk=g.pk).exists()
def test_extra_viewer_groups(self):
ai = AlgorithmImageFactory()
civ = ComponentInterfaceValueFactory(
interface=ai.algorithm.inputs.get())
groups = (GroupFactory(), GroupFactory(), GroupFactory())
jobs = create_algorithm_jobs(algorithm_image=ai,
civ_sets=[{civ}],
extra_viewer_groups=groups)
for g in groups:
assert jobs[0].viewer_groups.filter(pk=g.pk).exists()
def test_creates_job_correctly(self):
ai = AlgorithmImageFactory()
image = ImageFactory()
assert Job.objects.count() == 0
jobs = create_algorithm_jobs(algorithm_image=ai, images=[image])
assert Job.objects.count() == 1
j = Job.objects.first()
assert j.algorithm_image == ai
assert j.creator is None
assert (j.inputs.get(
interface__slug=DEFAULT_INPUT_INTERFACE_SLUG).image == image)
assert j.pk == jobs[0].pk
def test_create_jobs_is_idempotent():
image = ImageFactory()
ai = AlgorithmImageFactory()
user = UserFactory()
image.origin.algorithm_image = ai
image.origin.creator = user
image.origin.save()
assert Job.objects.count() == 0
create_algorithm_jobs(upload_session_pk=image.origin.pk)
assert Job.objects.count() == 1
j = Job.objects.all()[0]
assert j.algorithm_image == ai
assert j.creator == user
assert j.inputs.get(interface__slug="generic-medical-image").image == image
# Running the job twice should not result in new jobs
create_algorithm_jobs(upload_session_pk=image.origin.pk)
assert Job.objects.count() == 1
# Changing the algorithm image should create a new job
image.origin.algorithm_image = AlgorithmImageFactory()
image.origin.save()
create_algorithm_jobs(upload_session_pk=image.origin.pk)
assert Job.objects.count() == 2
def test_civ_existing_does_nothing(self):
default_input_interface = ComponentInterface.objects.get(
slug=DEFAULT_INPUT_INTERFACE_SLUG)
image = ImageFactory()
ai = AlgorithmImageFactory()
j = AlgorithmJobFactory(creator=None, algorithm_image=ai)
civ = ComponentInterfaceValueFactory(interface=default_input_interface,
image=image)
j.inputs.set([civ])
assert Job.objects.count() == 1
jobs = create_algorithm_jobs(algorithm_image=ai, images=[image])
assert Job.objects.count() == 1
assert len(jobs) == 0
def test_algorithm_with_invalid_output(client, algorithm_image, settings):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
assert Job.objects.count() == 0
# Create the algorithm image
algorithm_container, sha256 = algorithm_image
alg = AlgorithmImageFactory(image__from_path=algorithm_container,
image_sha256=sha256,
ready=True)
# Make sure the job fails when trying to upload an invalid file
detection_interface = ComponentInterfaceFactory(
store_in_database=False,
relative_path="some_text.txt",
slug="detection-json-file",
kind=ComponentInterface.Kind.ANY,
)
alg.algorithm.outputs.add(detection_interface)
alg.save()
image_file = ImageFileFactory(file__from_path=Path(__file__).parent /
"resources" / "input_file.tif")
civ = ComponentInterfaceValueFactory(image=image_file.image,
interface=alg.algorithm.inputs.get(),
file=None)
with capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(algorithm_image=alg, civ_sets=[{civ}])
recurse_callbacks(callbacks=callbacks)
jobs = Job.objects.filter(algorithm_image=alg,
inputs__image=image_file.image,
status=Job.FAILURE).all()
assert len(jobs) == 1
assert (jobs.first().error_message ==
"The file produced at /output/some_text.txt is not valid json")
assert len(jobs[0].outputs.all()) == 0
def test_create_jobs_is_limited(self):
user, editor = UserFactory(), UserFactory()
algorithm_image = AlgorithmImageFactory()
algorithm_image.algorithm.credits_per_job = 400
algorithm_image.algorithm.save()
algorithm_image.algorithm.add_editor(editor)
def create_upload(upload_creator):
riu = RawImageUploadSessionFactory(creator=upload_creator)
for _ in range(3):
ImageFactory(origin=riu),
riu.save()
interface = algorithm_image.algorithm.inputs.get()
return [{
ComponentInterfaceValueFactory(image=im, interface=interface)
} for im in riu.image_set.all()]
assert Job.objects.count() == 0
# Create an upload session as editor; should not be limited
upload = create_upload(editor)
create_algorithm_jobs(
algorithm_image=algorithm_image,
civ_sets=upload,
creator=editor,
)
assert Job.objects.count() == 3
# Create an upload session as user; should be limited
upload_2 = create_upload(user)
create_algorithm_jobs(
algorithm_image=algorithm_image,
civ_sets=upload_2,
creator=user,
)
# An additional 2 jobs should be created (standard nr of credits is 1000
# per user per month).
assert Job.objects.count() == 5
# As an editor you should not be limited
algorithm_image.algorithm.add_editor(user)
# The job that was skipped on the previous run should now be accepted
create_algorithm_jobs(
algorithm_image=algorithm_image,
civ_sets=upload_2,
creator=user,
)
assert Job.objects.count() == 6
def test_creates_job_correctly(self):
ai = AlgorithmImageFactory()
image = ImageFactory()
civ = ComponentInterfaceValueFactory(
image=image, interface=ai.algorithm.inputs.get())
assert Job.objects.count() == 0
jobs = create_algorithm_jobs(algorithm_image=ai, civ_sets=[{civ}])
assert Job.objects.count() == 1
j = Job.objects.first()
assert j.algorithm_image == ai
assert j.creator is None
assert (j.inputs.get(
interface__slug=DEFAULT_INPUT_INTERFACE_SLUG).image == image)
assert j.pk == jobs[0].pk
def test_no_jobs_workflow(self):
ai = AlgorithmImageFactory()
with capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(algorithm_image=ai, civ_sets=[])
assert len(callbacks) == 0
def test_no_images_does_nothing(self):
ai = AlgorithmImageFactory()
create_algorithm_jobs(algorithm_image=ai, images=[])
assert Job.objects.count() == 0
def test_algorithm(client, algorithm_image, settings):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
assert Job.objects.count() == 0
# Create the algorithm image
algorithm_container, sha256 = algorithm_image
alg = AlgorithmImageFactory(image__from_path=algorithm_container,
image_sha256=sha256,
ready=True)
# We should not be able to download image
with pytest.raises(NotImplementedError):
_ = alg.image.url
# Run the algorithm, it will create a results.json and an output.tif
image_file = ImageFileFactory(file__from_path=Path(__file__).parent /
"resources" / "input_file.tif")
civ = ComponentInterfaceValueFactory(image=image_file.image,
interface=alg.algorithm.inputs.get(),
file=None)
assert civ.interface.slug == "generic-medical-image"
with capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(algorithm_image=alg, civ_sets=[{civ}])
recurse_callbacks(callbacks=callbacks)
jobs = Job.objects.filter(algorithm_image=alg).all()
# There should be a single, successful job
assert len(jobs) == 1
assert jobs[0].stdout.endswith("Greetings from stdout\n")
assert jobs[0].stderr.endswith('("Hello from stderr")\n')
assert jobs[0].error_message == ""
assert jobs[0].status == jobs[0].SUCCESS
# The job should have two ComponentInterfaceValues,
# one for the results.json and one for output.tif
assert len(jobs[0].outputs.all()) == 2
json_result_interface = ComponentInterface.objects.get(
slug="results-json-file")
json_result_civ = jobs[0].outputs.get(interface=json_result_interface)
assert json_result_civ.value == {
"entity": "out.tif",
"metrics": {
"abnormal": 0.19,
"normal": 0.81
},
}
heatmap_interface = ComponentInterface.objects.get(slug="generic-overlay")
heatmap_civ = jobs[0].outputs.get(interface=heatmap_interface)
assert heatmap_civ.image.name == "output.tif"
# We add another ComponentInterface with file value and run the algorithm again
detection_interface = ComponentInterfaceFactory(
store_in_database=False,
relative_path="detection_results.json",
title="detection-json-file",
slug="detection-json-file",
kind=ComponentInterface.Kind.ANY,
)
alg.algorithm.outputs.add(detection_interface)
alg.save()
image_file = ImageFileFactory(file__from_path=Path(__file__).parent /
"resources" / "input_file.tif")
civ = ComponentInterfaceValueFactory(image=image_file.image,
interface=alg.algorithm.inputs.get(),
file=None)
with capture_on_commit_callbacks() as callbacks:
create_algorithm_jobs(algorithm_image=alg, civ_sets=[{civ}])
recurse_callbacks(callbacks=callbacks)
jobs = Job.objects.filter(algorithm_image=alg,
inputs__image=image_file.image).all()
# There should be a single, successful job
assert len(jobs) == 1
# The job should have three ComponentInterfaceValues,
# one with the detection_results store in the file
assert len(jobs[0].outputs.all()) == 3
detection_civ = jobs[0].outputs.get(interface=detection_interface)
assert not detection_civ.value
assert re.search("detection_results.*json$", detection_civ.file.name)
