def calculate_ranks(*, phase_pk: uuid.UUID):
Phase = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Phase")
Evaluation = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Evaluation")
phase = Phase.objects.get(pk=phase_pk)
display_choice = phase.result_display_choice
score_method_choice = phase.scoring_method_choice
metrics = (
Metric(
path=phase.score_jsonpath,
reverse=(phase.score_default_sort == phase.DESCENDING),
),
*[
Metric(path=col["path"], reverse=col["order"] == phase.DESCENDING)
for col in phase.extra_results_columns
],
)
if score_method_choice == phase.ABSOLUTE:
def score_method(x):
return list(x)[0]
elif score_method_choice == phase.MEAN:
score_method = mean
elif score_method_choice == phase.MEDIAN:
score_method = median
else:
raise NotImplementedError
valid_evaluations = (Evaluation.objects.filter(
submission__phase=phase, published=True,
status=Evaluation.SUCCESS).order_by("-created").select_related(
"submission__creator").prefetch_related("outputs__interface"))
if display_choice == phase.MOST_RECENT:
valid_evaluations = filter_by_creators_most_recent(
evaluations=valid_evaluations)
elif display_choice == phase.BEST:
all_positions = rank_results(
evaluations=valid_evaluations,
metrics=metrics,
score_method=score_method,
)
valid_evaluations = filter_by_creators_best(
evaluations=valid_evaluations, ranks=all_positions.ranks)
final_positions = rank_results(
evaluations=valid_evaluations,
metrics=metrics,
score_method=score_method,
)
evaluations = Evaluation.objects.filter(submission__phase=phase)
_update_evaluations(evaluations=evaluations,
final_positions=final_positions)
def calculate_ranks(*, challenge_pk: uuid.UUID):
challenge = Challenge.objects.get(pk=challenge_pk)
display_choice = challenge.evaluation_config.result_display_choice
score_method_choice = challenge.evaluation_config.scoring_method_choice
metrics = (
Metric(
path=challenge.evaluation_config.score_jsonpath,
reverse=(
challenge.evaluation_config.score_default_sort
== Config.DESCENDING
),
),
)
if score_method_choice != Config.ABSOLUTE:
metrics += tuple(
Metric(path=col["path"], reverse=col["order"] == Config.DESCENDING)
for col in challenge.evaluation_config.extra_results_columns
)
if score_method_choice == Config.ABSOLUTE and len(metrics) == 1:
score_method = lambda x: list(x)[0]
elif score_method_choice == Config.MEAN:
score_method = mean
elif score_method_choice == Config.MEDIAN:
score_method = median
else:
raise NotImplementedError
valid_results = (
Result.objects.filter(
Q(job__submission__challenge=challenge), Q(published=True)
)
.order_by("-created")
.select_related("job__submission")
)
if display_choice == Config.MOST_RECENT:
valid_results = filter_by_creators_most_recent(results=valid_results)
elif display_choice == Config.BEST:
all_positions = rank_results(
results=valid_results, metrics=metrics, score_method=score_method
)
valid_results = filter_by_creators_best(
results=valid_results, ranks=all_positions.ranks
)
final_positions = rank_results(
results=valid_results, metrics=metrics, score_method=score_method
)
for res in Result.objects.filter(Q(job__submission__challenge=challenge)):
try:
rank = final_positions.ranks[res.pk]
rank_score = final_positions.rank_scores[res.pk]
rank_per_metric = final_positions.rank_per_metric[res.pk]
except KeyError:
# This result will be excluded from the display
rank = 0
rank_score = 0.0
rank_per_metric = {}
Result.objects.filter(pk=res.pk).update(
rank=rank, rank_score=rank_score, rank_per_metric=rank_per_metric
)
def calculate_ranks(*, challenge_pk: uuid.UUID): # noqa: C901
challenge = Challenge.objects.get(pk=challenge_pk)
display_choice = challenge.evaluation_config.result_display_choice
score_method_choice = challenge.evaluation_config.scoring_method_choice
Evaluation = apps.get_model( # noqa: N806
app_label="evaluation", model_name="Evaluation")
metrics = (Metric(
path=challenge.evaluation_config.score_jsonpath,
reverse=(challenge.evaluation_config.score_default_sort ==
challenge.evaluation_config.DESCENDING),
), )
if score_method_choice != challenge.evaluation_config.ABSOLUTE:
metrics += tuple(
Metric(
path=col["path"],
reverse=col["order"] == challenge.evaluation_config.DESCENDING,
) for col in challenge.evaluation_config.extra_results_columns)
if (score_method_choice == challenge.evaluation_config.ABSOLUTE
and len(metrics) == 1):
def score_method(x):
return list(x)[0]
elif score_method_choice == challenge.evaluation_config.MEAN:
score_method = mean
elif score_method_choice == challenge.evaluation_config.MEDIAN:
score_method = median
else:
raise NotImplementedError
valid_evaluations = (Evaluation.objects.filter(
submission__challenge=challenge,
published=True,
status=Evaluation.SUCCESS,
).order_by("-created").select_related(
"submission__creator").prefetch_related("outputs"))
if display_choice == challenge.evaluation_config.MOST_RECENT:
valid_evaluations = filter_by_creators_most_recent(
evaluations=valid_evaluations)
elif display_choice == challenge.evaluation_config.BEST:
all_positions = rank_results(
evaluations=valid_evaluations,
metrics=metrics,
score_method=score_method,
)
valid_evaluations = filter_by_creators_best(
evaluations=valid_evaluations, ranks=all_positions.ranks)
final_positions = rank_results(
evaluations=valid_evaluations,
metrics=metrics,
score_method=score_method,
)
for e in Evaluation.objects.filter(submission__challenge=challenge):
try:
rank = final_positions.ranks[e.pk]
rank_score = final_positions.rank_scores[e.pk]
rank_per_metric = final_positions.rank_per_metric[e.pk]
except KeyError:
# This result will be excluded from the display
rank = 0
rank_score = 0.0
rank_per_metric = {}
Evaluation.objects.filter(pk=e.pk).update(
rank=rank, rank_score=rank_score, rank_per_metric=rank_per_metric)