def test_challenge_card_status(
client,
phase1_submission_limit,
phase1_submissions_open,
phase1_submissions_close,
phase2_submission_limit,
phase2_submissions_open,
phase2_submissions_close,
expected_status,
phase_in_status,
):
ch = ChallengeFactory(hidden=False)
phase1 = ch.phase_set.first()
phase2 = PhaseFactory(challenge=ch)
u = UserFactory()
phase1.submission_limit = phase1_submission_limit
phase1.submissions_open_at = phase1_submissions_open
phase1.submissions_close_at = phase1_submissions_close
phase2.submission_limit = phase2_submission_limit
phase2.submissions_open_at = phase2_submissions_open
phase2.submissions_close_at = phase2_submissions_close
phase1.save()
phase2.save()
response = get_view_for_user(
client=client, viewname="challenges:list", user=u
)
if phase_in_status:
title = ch.phase_set.order_by("created").all()[phase_in_status].title
assert f"{expected_status} for {title}" in response.rendered_content
else:
assert expected_status in response.rendered_content
def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.outputs.add(
ComponentInterfaceValue.objects.create(
interface=ComponentInterface.objects.get(
slug="metrics-json-file"),
value=r,
))
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_open_for_submission(
submission_limit,
submissions_open,
submissions_close,
open_for_submissions,
expected_status,
):
phase = PhaseFactory()
phase.submission_limit = submission_limit
phase.submissions_open_at = submissions_open
phase.submissions_close_at = submissions_close
phase.save()
assert phase.open_for_submissions == open_for_submissions
assert expected_status in phase.submission_status_string
def test_public_private_default():
p = PhaseFactory()
r1 = EvaluationFactory(submission__phase=p)
assert r1.published is True
p.auto_publish_new_results = False
p.save()
r2 = EvaluationFactory(submission__phase=p)
assert r2.published is False
# The public/private status should only update on first save
r1.save()
assert r1.published is True
def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_calculate_ranks(django_assert_max_num_queries):
phase = PhaseFactory()
results = [
# Warning: Do not change this values without updating the
# expected_ranks below.
{
"a": 0.0,
"b": 0.0
},
{
"a": 0.5,
"b": 0.2
},
{
"a": 1.0,
"b": 0.3
},
{
"a": 0.7,
"b": 0.4
},
{
"a": 0.5,
"b": 0.5
},
# Following two are invalid as they are incomplete
{
"a": 1.0
},
{
"b": 0.3
},
# Add a valid, but unpublished result
{
"a": 0.1,
"b": 0.1
},
]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
# Unpublish the result
queryset[-1].published = False
queryset[-1].save()
expected = {
Phase.DESCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
},
Phase.ASCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
},
}
for score_method in (Phase.ABSOLUTE, Phase.MEDIAN, Phase.MEAN):
for a_order in (Phase.DESCENDING, Phase.ASCENDING):
for b_order in (Phase.DESCENDING, Phase.ASCENDING):
phase.score_jsonpath = "a"
phase.scoring_method_choice = score_method
phase.score_default_sort = a_order
phase.extra_results_columns = [{
"path": "b",
"title": "b",
"order": b_order
}]
phase.save()
with django_assert_max_num_queries(7):
calculate_ranks(phase_pk=phase.pk)
assert_ranks(
queryset,
expected[a_order][score_method][b_order]["ranks"],
expected[a_order][score_method][b_order]["rank_scores"],
)
def test_results_display():
phase = PhaseFactory()
user1 = UserFactory()
user2 = UserFactory()
metrics = "metrics"
creator = "creator"
results = [
{
metrics: {
"b": 0.3
},
creator: user1
}, # Invalid result
{
metrics: {
"a": 0.6
},
creator: user1
},
{
metrics: {
"a": 0.4
},
creator: user1
},
{
metrics: {
"a": 0.2
},
creator: user1
},
{
metrics: {
"a": 0.1
},
creator: user2
},
{
metrics: {
"a": 0.5
},
creator: user2
},
{
metrics: {
"a": 0.3
},
creator: user2
},
]
queryset = [
EvaluationFactory(
submission__phase=phase,
submission__creator=r[creator],
status=Evaluation.SUCCESS,
) for r in results
]
for e, r in zip(queryset, results):
e.create_result(result=r[metrics])
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 3, 5, 6, 2, 4]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 0, 0, 1]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.BEST
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 0, 0, 0, 2, 0]
assert_ranks(queryset, expected_ranks)
# now test reverse order
phase.score_default_sort = phase.ASCENDING
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 1, 0, 0]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 1, 0, 0, 2]
assert_ranks(queryset, expected_ranks)
