def setUp(self):
interface = ComponentInterface.objects.get(
slug="generic-medical-image")
archive = ArchiveFactory()
ais = ArchiveItemFactory.create_batch(2)
archive.items.set(ais)
input_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
output_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
for ai, civ in zip(ais, input_civs):
ai.values.set([civ])
alg = AlgorithmImageFactory()
submission = SubmissionFactory(algorithm_image=alg)
submission.phase.archive = archive
submission.phase.save()
submission.phase.algorithm_inputs.set([interface])
jobs = []
for inpt, output in zip(input_civs, output_civs):
j = AlgorithmJobFactory(status=Job.SUCCESS, algorithm_image=alg)
j.inputs.set([inpt])
j.outputs.set([output])
jobs.append(j)
self.evaluation = EvaluationFactory(
submission=submission, status=Evaluation.EXECUTING_PREREQUISITES)
self.jobs = jobs
self.output_civs = output_civs
def test_phase_filtered_views(self, client):
c = ChallengeFactory(hidden=False)
p1, p2 = PhaseFactory.create_batch(2, challenge=c)
e1 = EvaluationFactory(
method__phase=p1,
submission__phase=p1,
rank=1,
status=Evaluation.SUCCESS,
)
_ = EvaluationFactory(
method__phase=p2,
submission__phase=p2,
rank=1,
status=Evaluation.SUCCESS,
)
response = get_view_for_user(
client=client,
viewname="evaluation:leaderboard",
reverse_kwargs={
"challenge_short_name":
e1.submission.phase.challenge.short_name,
"slug": e1.submission.phase.slug,
},
)
assert response.status_code == 200
assert {e1.pk} == {o.pk for o in response.context[-1]["object_list"]}
class TestSetEvaluationInputs(TestCase):
def setUp(self):
interface = ComponentInterface.objects.get(
slug="generic-medical-image")
archive = ArchiveFactory()
ais = ArchiveItemFactory.create_batch(2)
archive.items.set(ais)
input_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
output_civs = ComponentInterfaceValueFactory.create_batch(
2, interface=interface)
for ai, civ in zip(ais, input_civs):
ai.values.set([civ])
alg = AlgorithmImageFactory()
submission = SubmissionFactory(algorithm_image=alg)
submission.phase.archive = archive
submission.phase.save()
submission.phase.algorithm_inputs.set([interface])
jobs = []
for inpt, output in zip(input_civs, output_civs):
j = AlgorithmJobFactory(status=Job.SUCCESS, algorithm_image=alg)
j.inputs.set([inpt])
j.outputs.set([output])
jobs.append(j)
self.evaluation = EvaluationFactory(
submission=submission, status=Evaluation.EXECUTING_PREREQUISITES)
self.jobs = jobs
self.output_civs = output_civs
def test_unsuccessful_jobs_fail_evaluation(self):
self.jobs[0].status = Job.FAILURE
self.jobs[0].save()
set_evaluation_inputs(evaluation_pk=self.evaluation.pk)
self.evaluation.refresh_from_db()
assert self.evaluation.status == self.evaluation.FAILURE
assert (self.evaluation.error_message ==
"The algorithm failed on one or more cases.")
def test_set_evaluation_inputs(self):
set_evaluation_inputs(evaluation_pk=self.evaluation.pk)
self.evaluation.refresh_from_db()
assert self.evaluation.status == self.evaluation.PENDING
assert self.evaluation.error_message == ""
assert self.evaluation.inputs.count() == 3
assert self.evaluation.input_prefixes == {
str(civ.pk): f"{alg.pk}/output/"
for alg, civ in zip(self.jobs, self.output_civs)
}
def test_unsuccessful_jobs_fail_evaluation(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = (
AlgorithmJobFactory(status=Job.SUCCESS),
AlgorithmJobFactory(status=Job.FAILURE),
)
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.FAILURE
assert (evaluation.error_message ==
"The algorithm failed to execute on 1 images.")
def test_published_evaluation_permissions(self, hidden_challenge):
"""
Challenge admins can change and view published evaluations.
If the challenge is hidden, only the participants can view published
evaluations, otherwise anyone can view published evaluations.
"""
e: Evaluation = EvaluationFactory(
submission__phase__auto_publish_new_results=True,
submission__phase__challenge__hidden=hidden_challenge,
)
if hidden_challenge:
viewer_group = e.submission.phase.challenge.participants_group
else:
viewer_group = Group.objects.get(
name=settings.REGISTERED_AND_ANON_USERS_GROUP_NAME)
assert e.published is True
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
viewer_group: {"view_evaluation"},
}
assert get_users_with_perms(e, with_group_users=False).count() == 0
def test_job_permissions_for_challenge(self):
ai = AlgorithmImageFactory(ready=True)
archive = ArchiveFactory()
evaluation = EvaluationFactory(submission__phase__archive=archive,
submission__algorithm_image=ai)
# Fake an image upload via a session
u = UserFactory()
s = UploadSessionFactory(creator=u)
im = ImageFactory()
s.image_set.set([im])
archive.images.set([im])
create_algorithm_jobs_for_evaluation(evaluation_pk=evaluation.pk)
job = Job.objects.get()
# Only the challenge admins and job viewers should be able to view the
# job. NOTE: NOT THE ALGORITHM EDITORS, they are the participants
# to the challenge and should not be able to see the test data
assert get_groups_with_set_perms(job) == {
evaluation.submission.phase.challenge.admins_group: {"view_job"},
job.viewers: {"view_job"},
}
# No-one should be able to change the job
assert (get_users_with_perms(job,
attach_perms=True,
with_group_users=False) == {})
# No-one should be in the viewers group
assert {*job.viewers.user_set.all()} == set()
def test_unpublishing_results_removes_permissions(self, hidden_challenge):
"""
If an evaluation is unpublished then the view permission should be
removed.
"""
e: Evaluation = EvaluationFactory(
submission__phase__auto_publish_new_results=True,
submission__phase__challenge__hidden=hidden_challenge,
)
if hidden_challenge:
viewer_group = e.submission.phase.challenge.participants_group
else:
viewer_group = Group.objects.get(
name=settings.REGISTERED_AND_ANON_USERS_GROUP_NAME)
assert e.published is True
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
viewer_group: {"view_evaluation"},
}
e.published = False
e.save()
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
}
def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.outputs.add(
ComponentInterfaceValue.objects.create(
interface=ComponentInterface.objects.get(
slug="metrics-json-file"),
value=r,
))
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_unhiding_challenge_updates_perms(self, settings):
"""If a challenge is unhidden then the viewer group should be updated"""
e: Evaluation = EvaluationFactory(
submission__phase__auto_publish_new_results=True,
submission__phase__challenge__hidden=True,
)
participants = e.submission.phase.challenge.participants_group
all_users = Group.objects.get(
name=settings.REGISTERED_AND_ANON_USERS_GROUP_NAME)
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
participants: {"view_evaluation"},
}
# Override the celery settings
settings.task_eager_propagates = (True, )
settings.task_always_eager = (True, )
e.submission.phase.challenge.hidden = False
e.submission.phase.challenge.save()
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
all_users: {"view_evaluation"},
}
def test_set_evaluation_inputs(self):
submission = SubmissionFactory()
evaluation = EvaluationFactory(submission=submission)
jobs = AlgorithmJobFactory.create_batch(2, status=Job.SUCCESS)
civs = ComponentInterfaceValueFactory.create_batch(2)
for alg, civ in zip(jobs, civs):
alg.outputs.set([civ])
set_evaluation_inputs(evaluation_pk=evaluation.pk,
job_pks=[j.pk for j in jobs])
evaluation.refresh_from_db()
assert evaluation.status == evaluation.PENDING
assert evaluation.error_message == ""
assert evaluation.inputs.count() == 1
def test_login_redirect(self, client):
e = EvaluationFactory()
for view_name, kwargs in [
("phase-create", {}),
("phase-update", {"slug": e.submission.phase.slug}),
("method-create", {}),
("method-list", {}),
("method-detail", {"pk": e.method.pk}),
("submission-create", {"slug": e.submission.phase.slug}),
("submission-create-legacy", {"slug": e.submission.phase.slug}),
("submission-list", {}),
("submission-detail", {"pk": e.submission.pk}),
("list", {}),
("update", {"pk": e.pk}),
]:
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name": e.submission.phase.challenge.short_name,
**kwargs,
},
user=None,
)
assert response.status_code == 302
assert response.url.startswith(
f"https://testserver/accounts/login/?next=http%3A//"
f"{e.submission.phase.challenge.short_name}.testserver/"
)
def test_civ_file_download(client):
"""Only viewers of the job should be allowed to download result files."""
detection_interface = ComponentInterface(
store_in_database=False,
relative_path="detection_results.json",
slug="detection-results",
title="Detection Results",
kind=ComponentInterface.Kind.ANY,
)
detection_interface.save()
output_civ = ComponentInterfaceValue.objects.create(
interface=detection_interface)
detection = {
"detected points": [{
"type": "Point",
"start": [0, 1, 2],
"end": [3, 4, 5]
}]
}
output_civ.file.save(
"detection_results.json",
ContentFile(
bytes(json.dumps(detection, ensure_ascii=True, indent=2),
"utf-8")),
)
user1, user2 = UserFactory(), UserFactory()
def has_correct_access(user_allowed, user_denied, url):
tests = [(403, None), (302, user_allowed), (403, user_denied)]
for test in tests:
response = get_view_for_user(url=url, client=client, user=test[1])
assert response.status_code == test[0]
# test algorithm
job = AlgorithmJobFactory(creator=user1)
job.algorithm_image.algorithm.outputs.add(detection_interface)
job.outputs.add(output_civ)
has_correct_access(user1, user2, job.outputs.first().file.url)
job.outputs.remove(output_civ)
# test evaluation
evaluation = EvaluationFactory()
evaluation.output_interfaces.add(detection_interface)
evaluation.outputs.add(output_civ)
assign_perm("view_evaluation", user1, evaluation)
has_correct_access(user1, user2, evaluation.outputs.first().file.url)
evaluation.outputs.remove(output_civ)
# test archive
archive = ArchiveFactory()
archive_item = ArchiveItemFactory(archive=archive)
archive_item.values.add(output_civ)
archive.add_editor(user1)
has_correct_access(user1, user2, archive_item.values.first().file.url)
archive.remove_editor(user1)
archive.add_user(user1)
has_correct_access(user1, user2, archive_item.values.first().file.url)
archive.remove_user(user1)
def test_public_private_default():
p = PhaseFactory()
r1 = EvaluationFactory(submission__phase=p)
assert r1.published is True
p.auto_publish_new_results = False
p.save()
r2 = EvaluationFactory(submission__phase=p)
assert r2.published is False
# The public/private status should only update on first save
r1.save()
assert r1.published is True
def test_notification_list_view_num_queries(client, django_assert_num_queries):
user1 = UserFactory()
phase = PhaseFactory()
eval = EvaluationFactory(submission__phase=phase,
status=Evaluation.FAILURE)
# delete all prior notifications for easier testing below
Notification.objects.all().delete()
# create notification
_ = NotificationFactory(
user=user1,
type=Notification.Type.EVALUATION_STATUS,
actor=eval.submission.creator,
message="failed",
action_object=eval,
target=phase,
)
notifications = Notification.objects.select_related(
"actor_content_type",
"target_content_type",
"action_object_content_type",
"user",
).all()
notifications_with_prefetched_fks = prefetch_generic_foreign_key_objects(
Notification.objects.select_related(
"actor_content_type",
"target_content_type",
"action_object_content_type",
"user",
).all())
try:
settings.DEBUG = True
notifications[0].target
# when the generic foreign keys have not been prefetched, accessing the
# action target, result in two db calls
assert len(connection.queries) == 2
reset_queries()
notifications_with_prefetched_fks[0].target
# when gfks have been prefetched, accessing the action target
# no longer requires any db calls
assert len(connection.queries) == 0
# related objects of the generic foreign keys have also been prefetched
notifications[0].action_object.submission.phase.challenge
assert len(connection.queries) == 5
reset_queries()
notifications_with_prefetched_fks[
0].action_object.submission.phase.challenge
assert len(connection.queries) == 0
finally:
settings.DEBUG = False
reset_queries()
def test_permission_filtered_views(self, client):
u = UserFactory()
p = PhaseFactory()
m = MethodFactory(phase=p)
s = SubmissionFactory(phase=p, creator=u)
e = EvaluationFactory(method=m,
submission=s,
rank=1,
status=Evaluation.SUCCESS)
for view_name, kwargs, permission, obj in [
("method-list", {}, "view_method", m),
("submission-list", {}, "view_submission", s),
("list", {}, "view_evaluation", e),
(
"leaderboard",
{
"slug": e.submission.phase.slug
},
"view_evaluation",
e,
),
]:
assign_perm(permission, u, obj)
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 200
assert obj in response.context[-1]["object_list"]
remove_perm(permission, u, obj)
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 200
assert obj not in response.context[-1]["object_list"]
def test_setting_display_all_metrics(client, challenge_set):
metrics = {"public": 3245.235, "secret": 4328.432, "extra": 2144.312}
phase = challenge_set.challenge.phase_set.get()
e = EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS,)
e.create_result(result=metrics)
phase.score_jsonpath = "public"
phase.extra_results_columns = [
{"title": "extra", "path": "extra", "order": "asc"}
]
phase.display_all_metrics = True
phase.save()
response = get_view_for_user(
client=client,
viewname="evaluation:detail",
challenge=challenge_set.challenge,
reverse_kwargs={"pk": e.pk},
user=challenge_set.challenge.creator,
)
assert response.status_code == 200
assert str(metrics["public"]) in response.rendered_content
assert str(metrics["extra"]) in response.rendered_content
assert str(metrics["secret"]) in response.rendered_content
phase.display_all_metrics = False
phase.save()
response = get_view_for_user(
client=client,
viewname="evaluation:detail",
challenge=challenge_set.challenge,
reverse_kwargs={"pk": e.pk},
user=challenge_set.challenge.creator,
)
assert response.status_code == 200
assert str(metrics["public"]) in response.rendered_content
assert str(metrics["extra"]) in response.rendered_content
assert str(metrics["secret"]) not in response.rendered_content
def test_challenge_update(client, two_challenge_sets,
django_assert_num_queries):
c1 = two_challenge_sets.challenge_set_1.challenge
c2 = two_challenge_sets.challenge_set_2.challenge
_ = EvaluationFactory(submission__phase__challenge=c1,
method__phase__challenge=c1)
_ = EvaluationFactory(submission__phase__challenge=c2,
method__phase__challenge=c2)
with django_assert_num_queries(4) as _:
update_challenge_results_cache()
# check the # queries stays the same even with more challenges & evaluations
c3 = ChallengeFactory()
_ = EvaluationFactory(submission__phase__challenge=c3,
method__phase__challenge=c3)
with django_assert_num_queries(4) as _:
update_challenge_results_cache()
def test_challenge_filtered_views(self, client):
c1, c2 = ChallengeFactory.create_batch(2, hidden=False)
u = UserFactory()
e1 = EvaluationFactory(
method__phase__challenge=c1,
submission__phase__challenge=c1,
submission__creator=u,
)
e2 = EvaluationFactory(
method__phase__challenge=c2,
submission__phase__challenge=c2,
submission__creator=u,
)
assign_perm("view_method", u, e1.method)
assign_perm("view_method", u, e2.method)
for view_name, obj in [
("method-list", e1.method),
("submission-list", e1.submission),
("list", e1),
]:
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e1.submission.phase.challenge.short_name,
},
user=u,
)
assert response.status_code == 200
assert {obj.pk
} == {o.pk
for o in response.context[-1]["object_list"]}
def test_unpublished_evaluation_permissions(self, hidden_challenge):
"""Only challenge admins can change and view unpublished evaluations."""
e: Evaluation = EvaluationFactory(
submission__phase__auto_publish_new_results=False,
submission__phase__challenge__hidden=hidden_challenge,
)
assert e.published is False
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
}
assert get_users_with_perms(e, with_group_users=False).count() == 0
def test_open_views(self, client):
e = EvaluationFactory(submission__phase__challenge__hidden=False)
for view_name, kwargs in [
("leaderboard", {"slug": e.submission.phase.slug}),
("detail", {"pk": e.pk}),
]:
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name": e.submission.phase.challenge.short_name,
**kwargs,
},
user=None,
)
assert response.status_code == 200
def test_duration():
j = AlgorithmJobFactory()
_ = EvaluationFactory()
jbs = Job.objects.with_duration()
assert jbs[0].duration is None
assert Job.objects.average_duration() is None
now = timezone.now()
j.started_at = now - timedelta(minutes=5)
j.completed_at = now
j.save()
jbs = Job.objects.with_duration()
assert jbs[0].duration == timedelta(minutes=5)
assert Job.objects.average_duration() == timedelta(minutes=5)
_ = AlgorithmJobFactory()
assert Job.objects.average_duration() == timedelta(minutes=5)
def test_hiding_phase_updates_perms(self):
e: Evaluation = EvaluationFactory(
submission__phase__auto_publish_new_results=True,
submission__phase__public=True,
submission__phase__challenge__hidden=False,
submission__creator=UserFactory(),
)
all_users = Group.objects.get(
name=settings.REGISTERED_AND_ANON_USERS_GROUP_NAME
)
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
all_users: {"view_evaluation"},
}
assert get_groups_with_set_perms(e.submission) == {
e.submission.phase.challenge.admins_group: {"view_submission"},
}
assert e.submission.creator.has_perm("view_submission", e.submission)
# Override the celery settings
settings.task_eager_propagates = (True,)
settings.task_always_eager = (True,)
with capture_on_commit_callbacks(execute=True):
e.submission.phase.public = False
e.submission.phase.save()
assert get_groups_with_set_perms(e) == {
e.submission.phase.challenge.admins_group: {
"change_evaluation",
"view_evaluation",
},
}
assert get_groups_with_set_perms(e.submission) == {
e.submission.phase.challenge.admins_group: {"view_submission"},
}
def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_job_permissions_for_challenge(self):
ai = AlgorithmImageFactory(ready=True)
archive = ArchiveFactory()
evaluation = EvaluationFactory(submission__phase__archive=archive,
submission__algorithm_image=ai)
# Fake an image upload via a session
u = UserFactory()
s = UploadSessionFactory(creator=u)
im = ImageFactory()
s.image_set.set([im])
civ = ComponentInterfaceValueFactory(
image=im, interface=ai.algorithm.inputs.get())
archive_item = ArchiveItemFactory(archive=archive)
with capture_on_commit_callbacks(execute=True):
archive_item.values.add(civ)
create_algorithm_jobs_for_evaluation(evaluation_pk=evaluation.pk)
job = Job.objects.get()
# Only the challenge admins and job viewers should be able to view the
# job and logs.
# NOTE: NOT THE *ALGORITHM* EDITORS, they are the participants
# to the challenge and should not be able to see the test data
assert get_groups_with_set_perms(job) == {
evaluation.submission.phase.challenge.admins_group: {
"view_job",
"view_logs",
},
job.viewers: {"view_job"},
}
# No-one should be able to change the job
assert (get_users_with_perms(job,
attach_perms=True,
with_group_users=False) == {})
# No-one should be in the viewers group
assert {*job.viewers.user_set.all()} == set()
def setup(self):
self.phase = PhaseFactory()
self.user = UserFactory()
evaluation_kwargs = {
"submission__creator": self.user,
"submission__phase": self.phase,
"status": Evaluation.SUCCESS,
}
now = timezone.now()
# Failed evaluations don't count
e = EvaluationFactory(
submission__creator=self.user,
submission__phase=self.phase,
status=Evaluation.FAILURE,
)
# Other users evaluations don't count
EvaluationFactory(
submission__creator=UserFactory(),
submission__phase=self.phase,
status=Evaluation.SUCCESS,
)
# Other phases don't count
EvaluationFactory(
submission__creator=self.user,
submission__phase=PhaseFactory(),
status=Evaluation.SUCCESS,
)
# Evaluations 1, 2 and 7 days ago
e = EvaluationFactory(**evaluation_kwargs)
e.submission.created = now - timedelta(days=1) + timedelta(hours=1)
e.submission.save()
e = EvaluationFactory(**evaluation_kwargs)
e.submission.created = now - timedelta(days=2) + timedelta(hours=1)
e.submission.save()
e = EvaluationFactory(**evaluation_kwargs)
e.submission.created = now - timedelta(days=7) + timedelta(hours=1)
e.submission.save()
def test_permission_required_views(self, client):
e = EvaluationFactory()
u = UserFactory()
VerificationFactory(user=u, is_verified=True)
for view_name, kwargs, permission, obj in [
(
"phase-create",
{},
"change_challenge",
e.submission.phase.challenge,
),
(
"phase-update",
{
"slug": e.submission.phase.slug
},
"change_phase",
e.submission.phase,
),
(
"method-create",
{},
"change_challenge",
e.submission.phase.challenge,
),
("method-detail", {
"pk": e.method.pk
}, "view_method", e.method),
(
"submission-create",
{
"slug": e.submission.phase.slug
},
"create_phase_submission",
e.submission.phase,
),
(
"submission-create-legacy",
{
"slug": e.submission.phase.slug
},
"change_challenge",
e.submission.phase.challenge,
),
(
"submission-detail",
{
"pk": e.submission.pk
},
"view_submission",
e.submission,
),
("update", {
"pk": e.pk
}, "change_evaluation", e),
("detail", {
"pk": e.pk
}, "view_evaluation", e),
]:
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 403
assign_perm(permission, u, obj)
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name":
e.submission.phase.challenge.short_name,
**kwargs,
},
user=u,
)
assert response.status_code == 200
remove_perm(permission, u, obj)
def test_hidden_phase_visible_for_admins_but_not_participants(client):
ch = ChallengeFactory()
u = UserFactory()
ch.add_participant(u)
visible_phase = ch.phase_set.first()
hidden_phase = PhaseFactory(challenge=ch, public=False)
e1 = EvaluationFactory(submission__phase=visible_phase,
submission__creator=u)
e2 = EvaluationFactory(submission__phase=hidden_phase,
submission__creator=u)
for view_name, kwargs, status in [
# phase non-specific pages
("list", {}, 200),
("submission-list", {}, 200),
# visible phase
("detail", {
"pk": e1.pk
}, 200),
("submission-create", {
"slug": visible_phase.slug
}, 200),
("submission-detail", {
"pk": e1.submission.pk
}, 200),
("leaderboard", {
"slug": visible_phase.slug
}, 200),
# hidden phase
("detail", {
"pk": e2.pk
}, 403),
("submission-create", {
"slug": hidden_phase.slug
}, 200),
("submission-detail", {
"pk": e2.submission.pk
}, 403),
("leaderboard", {
"slug": hidden_phase.slug
}, 200),
]:
# for participants only the visible phase tab is visible
# and they do not have access to the detail pages of their evals and
# submissions from the hidden phase, and do not see subs/evals from the hidden
# phase on the respective list pages
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name": ch.short_name,
**kwargs
},
user=u,
)
assert response.status_code == status
if status == 200:
assert f"{visible_phase.title}</a>" in response.rendered_content
assert f"{hidden_phase.title}</a>" not in response.rendered_content
if "list" in view_name:
assert (f"<td>{visible_phase.title}</td>"
in response.rendered_content)
assert (f"<td>{hidden_phase.title}</td>"
not in response.rendered_content)
# for the admin both phases are visible and they have access to submissions
# and evals from both phases
response = get_view_for_user(
client=client,
viewname=f"evaluation:{view_name}",
reverse_kwargs={
"challenge_short_name": ch.short_name,
**kwargs
},
user=ch.admins_group.user_set.first(),
)
assert response.status_code == 200
assert f"{visible_phase.title}</a>" in response.rendered_content
assert f"{hidden_phase.title}</a>" in response.rendered_content
if "list" in view_name:
assert (f"<td>{visible_phase.title}</td>"
in response.rendered_content)
assert (f"<td>{hidden_phase.title}</td>"
in response.rendered_content)
def test_evaluation_list(client, two_challenge_sets):
# participant 0, submission 1, challenge 1, etc
e_p_s1 = EvaluationFactory(
submission__phase=two_challenge_sets.challenge_set_1.challenge.
phase_set.get(),
submission__creator=two_challenge_sets.challenge_set_1.participant,
)
e_p_s2 = EvaluationFactory(
submission__phase=two_challenge_sets.challenge_set_1.challenge.
phase_set.get(),
submission__creator=two_challenge_sets.challenge_set_1.participant,
)
e_p1_s1 = EvaluationFactory(
submission__phase=two_challenge_sets.challenge_set_1.challenge.
phase_set.get(),
submission__creator=two_challenge_sets.challenge_set_1.participant1,
)
# participant12, submission 1 to each challenge
e_p12_s1_c1 = EvaluationFactory(
submission__phase=two_challenge_sets.challenge_set_1.challenge.
phase_set.get(),
submission__creator=two_challenge_sets.participant12,
)
e_p12_s1_c2 = EvaluationFactory(
submission__phase=two_challenge_sets.challenge_set_2.challenge.
phase_set.get(),
submission__creator=two_challenge_sets.participant12,
)
# Participants should only be able to see their own evaluations
response = get_view_for_user(
viewname="evaluation:list",
challenge=two_challenge_sets.challenge_set_1.challenge,
client=client,
user=two_challenge_sets.challenge_set_1.participant,
)
assert str(e_p_s1.pk) in response.rendered_content
assert str(e_p_s2.pk) in response.rendered_content
assert str(e_p1_s1.pk) not in response.rendered_content
assert str(e_p12_s1_c1.pk) not in response.rendered_content
assert str(e_p12_s1_c2.pk) not in response.rendered_content
# Admins should be able to see all evaluations
response = get_view_for_user(
viewname="evaluation:list",
challenge=two_challenge_sets.challenge_set_1.challenge,
client=client,
user=two_challenge_sets.challenge_set_1.admin,
)
assert str(e_p_s1.pk) in response.rendered_content
assert str(e_p_s2.pk) in response.rendered_content
assert str(e_p1_s1.pk) in response.rendered_content
assert str(e_p12_s1_c1.pk) in response.rendered_content
assert str(e_p12_s1_c2.pk) not in response.rendered_content
# Only evaluations relevant to this challenge should be listed
response = get_view_for_user(
viewname="evaluation:list",
challenge=two_challenge_sets.challenge_set_1.challenge,
client=client,
user=two_challenge_sets.participant12,
)
assert str(e_p12_s1_c1.pk) in response.rendered_content
assert str(e_p12_s1_c2.pk) not in response.rendered_content
assert str(e_p_s1.pk) not in response.rendered_content
assert str(e_p_s2.pk) not in response.rendered_content
assert str(e_p1_s1.pk) not in response.rendered_content
def test_results_display():
phase = PhaseFactory()
user1 = UserFactory()
user2 = UserFactory()
metrics = "metrics"
creator = "creator"
results = [
{
metrics: {
"b": 0.3
},
creator: user1
}, # Invalid result
{
metrics: {
"a": 0.6
},
creator: user1
},
{
metrics: {
"a": 0.4
},
creator: user1
},
{
metrics: {
"a": 0.2
},
creator: user1
},
{
metrics: {
"a": 0.1
},
creator: user2
},
{
metrics: {
"a": 0.5
},
creator: user2
},
{
metrics: {
"a": 0.3
},
creator: user2
},
]
queryset = [
EvaluationFactory(
submission__phase=phase,
submission__creator=r[creator],
status=Evaluation.SUCCESS,
) for r in results
]
for e, r in zip(queryset, results):
e.create_result(result=r[metrics])
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 3, 5, 6, 2, 4]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 0, 0, 1]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.BEST
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 0, 0, 0, 2, 0]
assert_ranks(queryset, expected_ranks)
# now test reverse order
phase.score_default_sort = phase.ASCENDING
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 1, 0, 0]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 1, 0, 0, 2]
assert_ranks(queryset, expected_ranks)
def test_calculate_ranks(django_assert_max_num_queries):
phase = PhaseFactory()
results = [
# Warning: Do not change this values without updating the
# expected_ranks below.
{
"a": 0.0,
"b": 0.0
},
{
"a": 0.5,
"b": 0.2
},
{
"a": 1.0,
"b": 0.3
},
{
"a": 0.7,
"b": 0.4
},
{
"a": 0.5,
"b": 0.5
},
# Following two are invalid as they are incomplete
{
"a": 1.0
},
{
"b": 0.3
},
# Add a valid, but unpublished result
{
"a": 0.1,
"b": 0.1
},
]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
# Unpublish the result
queryset[-1].published = False
queryset[-1].save()
expected = {
Phase.DESCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
},
Phase.ASCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
},
}
for score_method in (Phase.ABSOLUTE, Phase.MEDIAN, Phase.MEAN):
for a_order in (Phase.DESCENDING, Phase.ASCENDING):
for b_order in (Phase.DESCENDING, Phase.ASCENDING):
phase.score_jsonpath = "a"
phase.scoring_method_choice = score_method
phase.score_default_sort = a_order
phase.extra_results_columns = [{
"path": "b",
"title": "b",
"order": b_order
}]
phase.save()
with django_assert_max_num_queries(7):
calculate_ranks(phase_pk=phase.pk)
assert_ranks(
queryset,
expected[a_order][score_method][b_order]["ranks"],
expected[a_order][score_method][b_order]["rank_scores"],
)
