def generate_rank_dict( queryset: Tuple[Result, ...], metric_paths: Tuple[str, ...], metric_reverse: Tuple[bool, ...], ): """ Generates a dictionary that contains the ranking of results based on a given metric path. """ rank = defaultdict(dict) pk_val = namedtuple("pk_val", ["pk", "val"]) if len(queryset) == 0: # No results to calculate return rank for (metric_path, reverse) in zip(metric_paths, metric_reverse): # Extract the value of the metric for this primary key and sort on the # value of the metric pk_vals = [ pk_val(str(res.pk), get_jsonpath(res.metrics, metric_path)) for res in queryset ] pk_vals.sort(key=lambda x: x.val, reverse=reverse) # Assign the ranks current_val = pk_vals[0].val current_rank = 1 for idx, result_pk_val in enumerate(pk_vals): # If the values of the metrics are the same, keep the rank # position the same if result_pk_val.val != current_val: current_val = result_pk_val.val current_rank = idx + 1 rank[result_pk_val.pk][metric_path] = current_rank return rank
def _get_rank_per_metric( *, results: Iterable[Result], metrics: Tuple[Metric, ...] ) -> Dict[str, Dict[str, float]]: """ Takes results and calculates the rank for each of the individual metrics Returns a dictionary where the key is the pk of the result, and the values is another dictionary where the key is the path of the metric and the value is the rank of this result for this metric """ metric_rank = {} for metric in metrics: # Extract the value of the metric for this primary key and sort on the # value of the metric metric_scores = { res.pk: get_jsonpath(res.metrics, metric.path) for res in results } metric_rank[metric.path] = _scores_to_ranks( scores=metric_scores, reverse=metric.reverse ) return { res.pk: { metric_path: ranks[res.pk] for metric_path, ranks in metric_rank.items() } for res in results }
def _get_rank_per_metric( *, evaluations: Iterable, metrics: Tuple[Metric, ...]) -> Dict[str, Dict[str, float]]: """ Takes results and calculates the rank for each of the individual metrics Returns a dictionary where the key is the pk of the result, and the values is another dictionary where the key is the path of the metric and the value is the rank of this result for this metric """ metric_rank = {} for metric in metrics: # Extract the value of the metric for this primary key and sort on the # value of the metric metric_scores = { e.pk: get_jsonpath( e.outputs.get(interface__slug="metrics-json-file").value, metric.path, ) for e in evaluations } metric_rank[metric.path] = _scores_to_ranks(scores=metric_scores, reverse=metric.reverse) return { e.pk: { metric_path: ranks[e.pk] for metric_path, ranks in metric_rank.items() } for e in evaluations }
def _get_rank_per_metric( *, results: Iterable[Result], metrics: Tuple[Metric, ...]) -> Dict[str, Dict[str, float]]: """ Takes results and calculates the rank for each of the individual metrics Returns a dictionary where the key is the pk of the result, and the values is another dictionary where the key is the path of the metric and the value is the rank of this result for this metric """ metric_rank = {} for metric in metrics: # Extract the value of the metric for this primary key and sort on the # value of the metric metric_scores = { res.pk: get_jsonpath(res.metrics, metric.path) for res in results } metric_rank[metric.path] = _scores_to_ranks(scores=metric_scores, reverse=metric.reverse) return { res.pk: { metric_path: ranks[res.pk] for metric_path, ranks in metric_rank.items() } for res in results }
def _filter_valid_results(*, results: Iterable[Result], metrics: Tuple[Metric, ...]) -> List[Result]: """ Ensure that all of the metrics are in every result """ return [ res for res in results if all( get_jsonpath(res.metrics, m.path) != "" for m in metrics) ]
def _filter_valid_results(*, evaluations: Iterable, metrics: Tuple[Metric, ...]) -> List: """Ensure that all of the metrics are in every result.""" return [ e for e in evaluations if all( get_jsonpath( e.outputs.get(interface__slug="metrics-json-file").value, m.path, ) not in ["", None] for m in metrics) ]
def _filter_valid_results( *, results: Iterable[Result], metrics: Tuple[Metric, ...] ) -> List[Result]: """ Ensure that all of the metrics are in every result """ return [ res for res in results if all( get_jsonpath(res.metrics, m.path) not in ["", None] for m in metrics ) ]
def test_get_jsonpath(): obj = {"spam": {"eggs": 42, "ham": {"beans": 84}}, "chips": 21} # Nested lookups assert get_jsonpath(obj=obj, jsonpath="chips") == 21 assert get_jsonpath(obj=obj, jsonpath="spam.ham.beans") == 84 # The path should have precedence assert get_jsonpath(obj=obj, jsonpath="spam.eggs") == 42 # Keys that don't exist assert get_jsonpath(obj=obj, jsonpath="foo") == "" assert get_jsonpath(obj=obj, jsonpath="spam.foo") == "" assert get_jsonpath(obj=obj, jsonpath="spam") == obj["spam"] assert get_jsonpath(obj=obj, jsonpath="") == ""
def test_get_jsonpath(): obj = { 'spam': {'eggs': 42, 'ham': {'beans': 84}}, 'chips': 21, } # Nested lookups assert get_jsonpath(obj=obj, jsonpath='chips') == 21 assert get_jsonpath(obj=obj, jsonpath='spam.ham.beans') == 84 # The path should have precedence assert get_jsonpath(obj=obj, jsonpath='spam.eggs') == 42 # Keys that don't exist assert get_jsonpath(obj=obj, jsonpath='foo') == '' assert get_jsonpath(obj=obj, jsonpath='spam.foo') == '' assert get_jsonpath(obj=obj, jsonpath='spam') == obj['spam'] assert get_jsonpath(obj=obj, jsonpath='') == ''