def test_submission_evaluation(
client, evaluation_image, submission_file, settings
):
# Override the celery settings
settings.task_eager_propagates = (True,)
settings.task_always_eager = (True,)
# Upload a submission and create an evaluation
eval_container, sha256 = evaluation_image
method = MethodFactory(
image__from_path=eval_container, image_sha256=sha256, ready=True
)
# We should not be able to download methods
with pytest.raises(NotImplementedError):
_ = method.image.url
# This will create an evaluation, and we'll wait for it to be executed
with capture_on_commit_callbacks() as callbacks:
submission = SubmissionFactory(
predictions_file__from_path=submission_file, phase=method.phase
)
recurse_callbacks(callbacks=callbacks)
# The evaluation method should return the correct answer
assert len(submission.evaluation_set.all()) == 1
evaluation = submission.evaluation_set.first()
assert evaluation.stdout.endswith("Greetings from stdout\n")
assert evaluation.stderr.endswith('warn("Hello from stderr")\n')
assert evaluation.error_message == ""
assert evaluation.status == evaluation.SUCCESS
assert (
evaluation.outputs.get(interface__slug="metrics-json-file").value[
"acc"
]
== 0.5
)
# Try with a csv file
with capture_on_commit_callbacks() as callbacks:
submission = SubmissionFactory(
predictions_file__from_path=Path(__file__).parent
/ "resources"
/ "submission.csv",
phase=method.phase,
)
recurse_callbacks(callbacks=callbacks)
evaluation = submission.evaluation_set.first()
assert len(submission.evaluation_set.all()) == 1
assert evaluation.status == evaluation.SUCCESS
assert (
evaluation.outputs.get(interface__slug="metrics-json-file").value[
"acc"
]
== 0.5
)
def test_create_evaluation_is_idempotent(self):
s = SubmissionFactory(
phase=self.method.phase, algorithm_image=self.algorithm_image,
)
s.create_evaluation()
assert AlgorithmEvaluation.objects.count() == 2
def test_create_evaluation_is_idempotent(self):
with capture_on_commit_callbacks(execute=True):
s = SubmissionFactory(
phase=self.method.phase,
algorithm_image=self.algorithm_image,
)
s.create_evaluation()
assert Job.objects.count() == 2
def test_submission_download(client, two_challenge_sets):
"""Only the challenge admin should be able to download submissions."""
submission = SubmissionFactory(
phase=two_challenge_sets.challenge_set_1.challenge.phase_set.get(),
creator=two_challenge_sets.challenge_set_1.participant,
)
tests = [
# (
# image response + annotation response not test ground truth,
# user
# )
(403, None),
(403, two_challenge_sets.challenge_set_1.non_participant),
(302, two_challenge_sets.challenge_set_1.participant),
(403, two_challenge_sets.challenge_set_1.participant1),
(302, two_challenge_sets.challenge_set_1.creator),
(302, two_challenge_sets.challenge_set_1.admin),
(403, two_challenge_sets.challenge_set_2.non_participant),
(403, two_challenge_sets.challenge_set_2.participant),
(403, two_challenge_sets.challenge_set_2.participant1),
(403, two_challenge_sets.challenge_set_2.creator),
(403, two_challenge_sets.challenge_set_2.admin),
(302, two_challenge_sets.admin12),
(403, two_challenge_sets.participant12),
(302, two_challenge_sets.admin1participant2),
]
for test in tests:
response = get_view_for_user(url=submission.predictions_file.url,
client=client,
user=test[1])
assert response.status_code == test[0]
def test_non_zip_submission_failure(client, evaluation_image, submission_file,
settings):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
# Upload a submission and create an evaluation
eval_container, sha256 = evaluation_image
method = MethodFactory(image__from_path=eval_container,
image_sha256=sha256,
ready=True)
# Try with a 7z file
submission = SubmissionFactory(
predictions_file__from_path=Path(__file__).parent / "resources" /
"submission.7z",
phase=method.phase,
)
# The evaluation method should return the correct answer
assert len(submission.evaluation_set.all()) == 1
evaluation = submission.evaluation_set.first()
assert evaluation.error_message.endswith(
"7z-compressed files are not supported.")
assert evaluation.status == evaluation.FAILURE
def setUp(self):
interface = ComponentInterface.objects.get(
slug="generic-medical-image")
archive = ArchiveFactory()
ais = ArchiveItemFactory.create_batch(2)
archive.items.set(ais)
input_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
output_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
for ai, civ in zip(ais, input_civs):
ai.values.set([civ])
alg = AlgorithmImageFactory()
submission = SubmissionFactory(algorithm_image=alg)
submission.phase.archive = archive
submission.phase.save()
submission.phase.algorithm_inputs.set([interface])
jobs = []
for inpt, output in zip(input_civs, output_civs):
j = AlgorithmJobFactory(status=Job.SUCCESS, algorithm_image=alg)
j.inputs.set([inpt])
j.outputs.set([output])
jobs.append(j)
self.evaluation = EvaluationFactory(
submission=submission, status=Evaluation.EXECUTING_PREREQUISITES)
self.jobs = jobs
self.output_civs = output_civs
def test_algorithm_submission_creates_one_job_per_test_set_image(self):
SubmissionFactory(
phase=self.method.phase,
algorithm_image=self.algorithm_image,
)
assert Job.objects.count() == 2
assert [
inpt.image for ae in Job.objects.all() for inpt in ae.inputs.all()
] == self.images[:2]
def test_permission_filtered_views(self, client):
u = UserFactory()
p = PhaseFactory()
m = MethodFactory(phase=p)
s = SubmissionFactory(phase=p, creator=u)
e = EvaluationFactory(method=m,
submission=s,
rank=1,
status=Evaluation.SUCCESS)
for view_name, kwargs, permission, obj in [
("method-list", {}, "view_method", m),
("submission-list", {}, "view_submission", s),
("list", {}, "view_evaluation", e),
(
"leaderboard",
{
"slug": e.submission.phase.slug
},
"view_evaluation",
e,
),
]:
assign_perm(permission, u, obj)
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 200
assert obj in response.context[-1]["object_list"]
remove_perm(permission, u, obj)
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 200
assert obj not in response.context[-1]["object_list"]
def test_create_evaluation_is_idempotent(self):
s = SubmissionFactory(
phase=self.method.phase,
algorithm_image=self.algorithm_image,
)
with capture_on_commit_callbacks(execute=True):
create_evaluation(submission_pk=s.pk, max_initial_jobs=None)
create_evaluation(submission_pk=s.pk, max_initial_jobs=None)
assert Job.objects.count() == 2
def test_algorithm_submission_creates_one_job_per_test_set_image(self):
s = SubmissionFactory(
phase=self.method.phase,
algorithm_image=self.algorithm_image,
)
with capture_on_commit_callbacks(execute=True):
create_evaluation(submission_pk=s.pk, max_initial_jobs=None)
assert Job.objects.count() == 2
assert [
inpt.image for ae in Job.objects.all() for inpt in ae.inputs.all()
] == self.images[:2]
def test_submission_permissions(self):
"""
Challenge admins and submission creators should be able to view
submissions.
"""
s: Submission = SubmissionFactory()
assert get_groups_with_set_perms(s) == {
s.phase.challenge.admins_group: {"view_submission"}
}
assert get_users_with_perms(
s, attach_perms=True, with_group_users=False
) == {s.creator: ["view_submission"]}
def test_unsuccessful_jobs_fail_evaluation(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = (
AlgorithmJobFactory(status=Job.SUCCESS),
AlgorithmJobFactory(status=Job.FAILURE),
)
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.FAILURE
assert (evaluation.error_message ==
"The algorithm failed to execute on 1 images.")
def test_create_evaluation_is_idempotent(self):
s = SubmissionFactory(phase=self.method.phase,
algorithm_image=self.algorithm_image)
with capture_on_commit_callbacks(execute=False) as callbacks1:
create_evaluation(submission_pk=s.pk, max_initial_jobs=None)
with capture_on_commit_callbacks(execute=False) as callbacks2:
create_evaluation(submission_pk=s.pk, max_initial_jobs=None)
# Execute the callbacks non-recursively
for c in chain(callbacks1, callbacks2):
c()
assert Job.objects.count() == 2
def test_set_evaluation_inputs(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = AlgorithmJobFactory.create_batch(2, status=Job.SUCCESS)
civs = ComponentInterfaceValueFactory.create_batch(2)
for alg, civ in zip(jobs, civs):
alg.outputs.set([civ])
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.PENDING
assert evaluation.error_message == ""
assert evaluation.inputs.count() == 1
def test_submission_time_limit(client, two_challenge_sets):
phase = two_challenge_sets.challenge_set_1.challenge.phase_set.get()
phase.submission_limit = 10
phase.save()
SubmissionFactory(phase=phase,
creator=two_challenge_sets.challenge_set_1.participant)
def get_submission_view():
return get_view_for_user(
viewname="evaluation:submission-create",
client=client,
user=two_challenge_sets.challenge_set_1.participant,
reverse_kwargs={
"challenge_short_name":
two_challenge_sets.challenge_set_1.challenge.short_name,
"slug":
two_challenge_sets.challenge_set_1.challenge.phase_set.get().
slug,
},
)
assert "create 9 more" in get_submission_view().rendered_content
s = SubmissionFactory(
phase=phase, creator=two_challenge_sets.challenge_set_1.participant)
s.created = timezone.now() - timedelta(hours=23)
s.save()
assert "create 8 more" in get_submission_view().rendered_content
s = SubmissionFactory(
phase=phase, creator=two_challenge_sets.challenge_set_1.participant)
s.created = timezone.now() - timedelta(hours=25)
s.save()
assert "create 8 more" in get_submission_view().rendered_content
def test_evaluation_notifications(client, evaluation_image, submission_file,
settings):
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
# Try to upload a submission without a method in place
with capture_on_commit_callbacks(execute=True):
submission = SubmissionFactory(
predictions_file__from_path=submission_file)
# Missing should result in notification for admins of the challenge
# There are 2 notifications here. The second is about admin addition to the
# challenge, both notifications are for the admin.
for notification in Notification.objects.all():
assert notification.user == submission.phase.challenge.creator
assert "there is no valid evaluation method" in Notification.objects.filter(
message="missing method").get().print_notification(
user=submission.phase.challenge.creator)
# Add method and upload a submission
eval_container, sha256 = evaluation_image
method = MethodFactory(image__from_path=eval_container,
image_sha256=sha256,
ready=True)
# clear notifications for easier testing later
Notification.objects.all().delete()
# create submission and wait for it to be evaluated
with capture_on_commit_callbacks() as callbacks:
submission = SubmissionFactory(
predictions_file__from_path=submission_file, phase=method.phase)
recurse_callbacks(callbacks=callbacks)
# creator of submission and admins of challenge should get notification
# about successful submission
recipients = list(submission.phase.challenge.get_admins())
recipients.append(submission.creator)
assert Notification.objects.count() == len(recipients)
for recipient in recipients:
assert str(recipient) in str(Notification.objects.all())
result_string = format_html('<a href="{}">result</a>',
submission.get_absolute_url())
submission_string = format_html('<a href="{}">submission</a>',
submission.get_absolute_url())
challenge_string = format_html(
'<a href="{}">{}</a>',
submission.phase.challenge.get_absolute_url(),
submission.phase.challenge.short_name,
)
assert f"There is a new {result_string} for {challenge_string}" in Notification.objects.filter(
user=recipients[0]).get().print_notification(user=recipients[0])
assert f"Your {submission_string} to {challenge_string} succeeded" in Notification.objects.filter(
user=recipients[1]).get().print_notification(user=recipients[1])
Notification.objects.all().delete()
# update evaluation status to failed
evaluation = submission.evaluation_set.first()
evaluation.update_status(status=evaluation.FAILURE)
assert evaluation.status == evaluation.FAILURE
# notifications for admin and creator of submission
assert Notification.objects.count() == len(recipients)
for recipient in recipients:
assert str(recipient) in str(Notification.objects.all())
assert f"The {submission_string} from {user_profile_link(Notification.objects.filter(user=recipients[0]).get().actor)} to {challenge_string} failed" in Notification.objects.filter(
user=recipients[0]).get().print_notification(user=recipients[0])
assert f"Your {submission_string} to {challenge_string} failed" in Notification.objects.filter(
user=recipients[1]).get().print_notification(user=recipients[1])
# check that when admin unsubscribed from phase, they no longer
# receive notifications about activity related to that phase
Notification.objects.all().delete()
unfollow(user=submission.phase.challenge.creator, obj=submission.phase)
evaluation.update_status(status=evaluation.SUCCESS)
assert str(submission.phase.challenge.creator) not in str(
Notification.objects.all())