def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.outputs.add(
ComponentInterfaceValue.objects.create(
interface=ComponentInterface.objects.get(
slug="metrics-json-file"),
value=r,
))
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def assert_ranks(challenge, expected_ranks, queryset):
# Execute calculate_ranks manually
calculate_ranks(challenge_pk=challenge.pk)
for q, exp in zip(queryset, expected_ranks):
r = Result.objects.get(pk=q.pk)
assert r.rank == exp
return challenge
def test_null_results():
phase = PhaseFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_null_results():
challenge = ChallengeFactory()
results = [{"a": 0.6}, {"a": None}]
queryset = [
EvaluationFactory(submission__challenge=challenge,
status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
challenge.evaluation_config.score_jsonpath = "a"
challenge.evaluation_config.result_display_choice = Config.ALL
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [1, 0]
assert_ranks(queryset, expected_ranks)
def test_results_display():
challenge = ChallengeFactory()
user1 = UserFactory()
user2 = UserFactory()
metrics = "metrics"
creator = "creator"
results = [
{
metrics: {
"b": 0.3
},
creator: user1
}, # Invalid result
{
metrics: {
"a": 0.6
},
creator: user1
},
{
metrics: {
"a": 0.4
},
creator: user1
},
{
metrics: {
"a": 0.2
},
creator: user1
},
{
metrics: {
"a": 0.1
},
creator: user2
},
{
metrics: {
"a": 0.5
},
creator: user2
},
{
metrics: {
"a": 0.3
},
creator: user2
},
]
queryset = [
EvaluationFactory(
submission__challenge=challenge,
submission__creator=r[creator],
status=Evaluation.SUCCESS,
) for r in results
]
for e, r in zip(queryset, results):
e.create_result(result=r[metrics])
challenge.evaluation_config.score_jsonpath = "a"
challenge.evaluation_config.result_display_choice = Config.ALL
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [0, 1, 3, 5, 6, 2, 4]
assert_ranks(queryset, expected_ranks)
challenge.evaluation_config.result_display_choice = Config.MOST_RECENT
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [0, 0, 0, 2, 0, 0, 1]
assert_ranks(queryset, expected_ranks)
challenge.evaluation_config.result_display_choice = Config.BEST
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [0, 1, 0, 0, 0, 2, 0]
assert_ranks(queryset, expected_ranks)
# now test reverse order
challenge.evaluation_config.score_default_sort = (
challenge.evaluation_config.ASCENDING)
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [0, 0, 0, 2, 1, 0, 0]
assert_ranks(queryset, expected_ranks)
challenge.evaluation_config.result_display_choice = Config.MOST_RECENT
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
expected_ranks = [0, 0, 0, 1, 0, 0, 2]
assert_ranks(queryset, expected_ranks)
def test_calculate_ranks():
challenge = ChallengeFactory()
results = [
# Warning: Do not change this values without updating the
# expected_ranks below.
{
"a": 0.0,
"b": 0.0
},
{
"a": 0.5,
"b": 0.2
},
{
"a": 1.0,
"b": 0.3
},
{
"a": 0.7,
"b": 0.4
},
{
"a": 0.5,
"b": 0.5
},
# Following two are invalid if relative ranking is used
{
"a": 1.0
},
{
"b": 0.3
},
# Add a valid, but unpublished result
{
"a": 0.1,
"b": 0.1
},
]
queryset = [
EvaluationFactory(submission__challenge=challenge,
status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
# Unpublish the result
queryset[-1].published = False
queryset[-1].save()
expected = {
Config.DESCENDING: {
Config.ABSOLUTE: {
Config.DESCENDING: {
"ranks": [6, 4, 1, 3, 4, 1, 0, 0],
"rank_scores": [6, 4, 1, 3, 4, 1, 0, 0],
},
Config.ASCENDING: {
"ranks": [6, 4, 1, 3, 4, 1, 0, 0],
"rank_scores": [6, 4, 1, 3, 4, 1, 0, 0],
},
},
Config.MEDIAN: {
Config.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Config.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
Config.MEAN: {
Config.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Config.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
},
Config.ASCENDING: {
Config.ABSOLUTE: {
Config.DESCENDING: {
"ranks": [1, 2, 5, 4, 2, 5, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 5, 0, 0],
},
Config.ASCENDING: {
"ranks": [1, 2, 5, 4, 2, 5, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 5, 0, 0],
},
},
Config.MEDIAN: {
Config.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Config.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
Config.MEAN: {
Config.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Config.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
},
}
for score_method in (Config.ABSOLUTE, Config.MEDIAN, Config.MEAN):
for a_order in (Config.DESCENDING, Config.ASCENDING):
for b_order in (Config.DESCENDING, Config.ASCENDING):
challenge.evaluation_config.score_jsonpath = "a"
challenge.evaluation_config.scoring_method_choice = (
score_method)
challenge.evaluation_config.score_default_sort = a_order
challenge.evaluation_config.extra_results_columns = [{
"path":
"b",
"title":
"b",
"order":
b_order
}]
challenge.evaluation_config.save()
calculate_ranks(challenge_pk=challenge.pk)
assert_ranks(
queryset,
expected[a_order][score_method][b_order]["ranks"],
expected[a_order][score_method][b_order]["rank_scores"],
)
def test_calculate_ranks(django_assert_max_num_queries):
phase = PhaseFactory()
results = [
# Warning: Do not change this values without updating the
# expected_ranks below.
{
"a": 0.0,
"b": 0.0
},
{
"a": 0.5,
"b": 0.2
},
{
"a": 1.0,
"b": 0.3
},
{
"a": 0.7,
"b": 0.4
},
{
"a": 0.5,
"b": 0.5
},
# Following two are invalid as they are incomplete
{
"a": 1.0
},
{
"b": 0.3
},
# Add a valid, but unpublished result
{
"a": 0.1,
"b": 0.1
},
]
queryset = [
EvaluationFactory(submission__phase=phase, status=Evaluation.SUCCESS)
for _ in range(len(results))
]
for e, r in zip(queryset, results):
e.create_result(result=r)
# Unpublish the result
queryset[-1].published = False
queryset[-1].save()
expected = {
Phase.DESCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [5, 3, 1, 2, 3, 0, 0, 0],
"rank_scores": [5, 3, 1, 2, 3, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [5, 4, 1, 1, 1, 0, 0, 0],
"rank_scores": [5, 3.5, 2, 2, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [3, 2, 1, 3, 5, 0, 0, 0],
"rank_scores": [3, 2.5, 2, 3, 4, 0, 0, 0],
},
},
},
Phase.ASCENDING: {
Phase.ABSOLUTE: {
Phase.DESCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 5, 4, 2, 0, 0, 0],
"rank_scores": [1, 2, 5, 4, 2, 0, 0, 0],
},
},
Phase.MEDIAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
Phase.MEAN: {
Phase.DESCENDING: {
"ranks": [2, 2, 5, 2, 1, 0, 0, 0],
"rank_scores": [3, 3, 4, 3, 1.5, 0, 0, 0],
},
Phase.ASCENDING: {
"ranks": [1, 2, 4, 4, 3, 0, 0, 0],
"rank_scores": [1, 2, 4, 4, 3.5, 0, 0, 0],
},
},
},
}
for score_method in (Phase.ABSOLUTE, Phase.MEDIAN, Phase.MEAN):
for a_order in (Phase.DESCENDING, Phase.ASCENDING):
for b_order in (Phase.DESCENDING, Phase.ASCENDING):
phase.score_jsonpath = "a"
phase.scoring_method_choice = score_method
phase.score_default_sort = a_order
phase.extra_results_columns = [{
"path": "b",
"title": "b",
"order": b_order
}]
phase.save()
with django_assert_max_num_queries(7):
calculate_ranks(phase_pk=phase.pk)
assert_ranks(
queryset,
expected[a_order][score_method][b_order]["ranks"],
expected[a_order][score_method][b_order]["rank_scores"],
)
def test_results_display():
phase = PhaseFactory()
user1 = UserFactory()
user2 = UserFactory()
metrics = "metrics"
creator = "creator"
results = [
{
metrics: {
"b": 0.3
},
creator: user1
}, # Invalid result
{
metrics: {
"a": 0.6
},
creator: user1
},
{
metrics: {
"a": 0.4
},
creator: user1
},
{
metrics: {
"a": 0.2
},
creator: user1
},
{
metrics: {
"a": 0.1
},
creator: user2
},
{
metrics: {
"a": 0.5
},
creator: user2
},
{
metrics: {
"a": 0.3
},
creator: user2
},
]
queryset = [
EvaluationFactory(
submission__phase=phase,
submission__creator=r[creator],
status=Evaluation.SUCCESS,
) for r in results
]
for e, r in zip(queryset, results):
e.create_result(result=r[metrics])
phase.score_jsonpath = "a"
phase.result_display_choice = Phase.ALL
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 3, 5, 6, 2, 4]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 0, 0, 1]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.BEST
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 1, 0, 0, 0, 2, 0]
assert_ranks(queryset, expected_ranks)
# now test reverse order
phase.score_default_sort = phase.ASCENDING
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 2, 1, 0, 0]
assert_ranks(queryset, expected_ranks)
phase.result_display_choice = Phase.MOST_RECENT
phase.save()
calculate_ranks(phase_pk=phase.pk)
expected_ranks = [0, 0, 0, 1, 0, 0, 2]
assert_ranks(queryset, expected_ranks)