def calculate_ranks(*, phase_pk: uuid.UUID): Phase = apps.get_model( # noqa: N806 app_label="evaluation", model_name="Phase") Evaluation = apps.get_model( # noqa: N806 app_label="evaluation", model_name="Evaluation") phase = Phase.objects.get(pk=phase_pk) display_choice = phase.result_display_choice score_method_choice = phase.scoring_method_choice metrics = ( Metric( path=phase.score_jsonpath, reverse=(phase.score_default_sort == phase.DESCENDING), ), *[ Metric(path=col["path"], reverse=col["order"] == phase.DESCENDING) for col in phase.extra_results_columns ], ) if score_method_choice == phase.ABSOLUTE: def score_method(x): return list(x)[0] elif score_method_choice == phase.MEAN: score_method = mean elif score_method_choice == phase.MEDIAN: score_method = median else: raise NotImplementedError valid_evaluations = (Evaluation.objects.filter( submission__phase=phase, published=True, status=Evaluation.SUCCESS).order_by("-created").select_related( "submission__creator").prefetch_related("outputs__interface")) if display_choice == phase.MOST_RECENT: valid_evaluations = filter_by_creators_most_recent( evaluations=valid_evaluations) elif display_choice == phase.BEST: all_positions = rank_results( evaluations=valid_evaluations, metrics=metrics, score_method=score_method, ) valid_evaluations = filter_by_creators_best( evaluations=valid_evaluations, ranks=all_positions.ranks) final_positions = rank_results( evaluations=valid_evaluations, metrics=metrics, score_method=score_method, ) evaluations = Evaluation.objects.filter(submission__phase=phase) _update_evaluations(evaluations=evaluations, final_positions=final_positions)
def calculate_ranks(*, challenge_pk: uuid.UUID): challenge = Challenge.objects.get(pk=challenge_pk) display_choice = challenge.evaluation_config.result_display_choice score_method_choice = challenge.evaluation_config.scoring_method_choice metrics = ( Metric( path=challenge.evaluation_config.score_jsonpath, reverse=( challenge.evaluation_config.score_default_sort == Config.DESCENDING ), ), ) if score_method_choice != Config.ABSOLUTE: metrics += tuple( Metric(path=col["path"], reverse=col["order"] == Config.DESCENDING) for col in challenge.evaluation_config.extra_results_columns ) if score_method_choice == Config.ABSOLUTE and len(metrics) == 1: score_method = lambda x: list(x)[0] elif score_method_choice == Config.MEAN: score_method = mean elif score_method_choice == Config.MEDIAN: score_method = median else: raise NotImplementedError valid_results = ( Result.objects.filter( Q(job__submission__challenge=challenge), Q(published=True) ) .order_by("-created") .select_related("job__submission") ) if display_choice == Config.MOST_RECENT: valid_results = filter_by_creators_most_recent(results=valid_results) elif display_choice == Config.BEST: all_positions = rank_results( results=valid_results, metrics=metrics, score_method=score_method ) valid_results = filter_by_creators_best( results=valid_results, ranks=all_positions.ranks ) final_positions = rank_results( results=valid_results, metrics=metrics, score_method=score_method ) for res in Result.objects.filter(Q(job__submission__challenge=challenge)): try: rank = final_positions.ranks[res.pk] rank_score = final_positions.rank_scores[res.pk] rank_per_metric = final_positions.rank_per_metric[res.pk] except KeyError: # This result will be excluded from the display rank = 0 rank_score = 0.0 rank_per_metric = {} Result.objects.filter(pk=res.pk).update( rank=rank, rank_score=rank_score, rank_per_metric=rank_per_metric )
def calculate_ranks(*, challenge_pk: uuid.UUID): # noqa: C901 challenge = Challenge.objects.get(pk=challenge_pk) display_choice = challenge.evaluation_config.result_display_choice score_method_choice = challenge.evaluation_config.scoring_method_choice Evaluation = apps.get_model( # noqa: N806 app_label="evaluation", model_name="Evaluation") metrics = (Metric( path=challenge.evaluation_config.score_jsonpath, reverse=(challenge.evaluation_config.score_default_sort == challenge.evaluation_config.DESCENDING), ), ) if score_method_choice != challenge.evaluation_config.ABSOLUTE: metrics += tuple( Metric( path=col["path"], reverse=col["order"] == challenge.evaluation_config.DESCENDING, ) for col in challenge.evaluation_config.extra_results_columns) if (score_method_choice == challenge.evaluation_config.ABSOLUTE and len(metrics) == 1): def score_method(x): return list(x)[0] elif score_method_choice == challenge.evaluation_config.MEAN: score_method = mean elif score_method_choice == challenge.evaluation_config.MEDIAN: score_method = median else: raise NotImplementedError valid_evaluations = (Evaluation.objects.filter( submission__challenge=challenge, published=True, status=Evaluation.SUCCESS, ).order_by("-created").select_related( "submission__creator").prefetch_related("outputs")) if display_choice == challenge.evaluation_config.MOST_RECENT: valid_evaluations = filter_by_creators_most_recent( evaluations=valid_evaluations) elif display_choice == challenge.evaluation_config.BEST: all_positions = rank_results( evaluations=valid_evaluations, metrics=metrics, score_method=score_method, ) valid_evaluations = filter_by_creators_best( evaluations=valid_evaluations, ranks=all_positions.ranks) final_positions = rank_results( evaluations=valid_evaluations, metrics=metrics, score_method=score_method, ) for e in Evaluation.objects.filter(submission__challenge=challenge): try: rank = final_positions.ranks[e.pk] rank_score = final_positions.rank_scores[e.pk] rank_per_metric = final_positions.rank_per_metric[e.pk] except KeyError: # This result will be excluded from the display rank = 0 rank_score = 0.0 rank_per_metric = {} Evaluation.objects.filter(pk=e.pk).update( rank=rank, rank_score=rank_score, rank_per_metric=rank_per_metric)