def generate_rank_dict(
queryset: Tuple[Result, ...],
metric_paths: Tuple[str, ...],
metric_reverse: Tuple[bool, ...],
):
"""
Generates a dictionary that contains the ranking of results based on a
given metric path.
"""
rank = defaultdict(dict)
pk_val = namedtuple("pk_val", ["pk", "val"])
if len(queryset) == 0:
# No results to calculate
return rank
for (metric_path, reverse) in zip(metric_paths, metric_reverse):
# Extract the value of the metric for this primary key and sort on the
# value of the metric
pk_vals = [
pk_val(str(res.pk), get_jsonpath(res.metrics, metric_path))
for res in queryset
]
pk_vals.sort(key=lambda x: x.val, reverse=reverse)
# Assign the ranks
current_val = pk_vals[0].val
current_rank = 1
for idx, result_pk_val in enumerate(pk_vals):
# If the values of the metrics are the same, keep the rank
# position the same
if result_pk_val.val != current_val:
current_val = result_pk_val.val
current_rank = idx + 1
rank[result_pk_val.pk][metric_path] = current_rank
return rank
def _get_rank_per_metric(
*, results: Iterable[Result], metrics: Tuple[Metric, ...]
) -> Dict[str, Dict[str, float]]:
"""
Takes results and calculates the rank for each of the individual metrics
Returns a dictionary where the key is the pk of the result, and the
values is another dictionary where the key is the path of the metric and
the value is the rank of this result for this metric
"""
metric_rank = {}
for metric in metrics:
# Extract the value of the metric for this primary key and sort on the
# value of the metric
metric_scores = {
res.pk: get_jsonpath(res.metrics, metric.path) for res in results
}
metric_rank[metric.path] = _scores_to_ranks(
scores=metric_scores, reverse=metric.reverse
)
return {
res.pk: {
metric_path: ranks[res.pk]
for metric_path, ranks in metric_rank.items()
}
for res in results
}
def _get_rank_per_metric(
*, evaluations: Iterable,
metrics: Tuple[Metric, ...]) -> Dict[str, Dict[str, float]]:
"""
Takes results and calculates the rank for each of the individual metrics
Returns a dictionary where the key is the pk of the result, and the
values is another dictionary where the key is the path of the metric and
the value is the rank of this result for this metric
"""
metric_rank = {}
for metric in metrics:
# Extract the value of the metric for this primary key and sort on the
# value of the metric
metric_scores = {
e.pk: get_jsonpath(
e.outputs.get(interface__slug="metrics-json-file").value,
metric.path,
)
for e in evaluations
}
metric_rank[metric.path] = _scores_to_ranks(scores=metric_scores,
reverse=metric.reverse)
return {
e.pk: {
metric_path: ranks[e.pk]
for metric_path, ranks in metric_rank.items()
}
for e in evaluations
}
def _get_rank_per_metric(
*, results: Iterable[Result],
metrics: Tuple[Metric, ...]) -> Dict[str, Dict[str, float]]:
"""
Takes results and calculates the rank for each of the individual metrics
Returns a dictionary where the key is the pk of the result, and the
values is another dictionary where the key is the path of the metric and
the value is the rank of this result for this metric
"""
metric_rank = {}
for metric in metrics:
# Extract the value of the metric for this primary key and sort on the
# value of the metric
metric_scores = {
res.pk: get_jsonpath(res.metrics, metric.path)
for res in results
}
metric_rank[metric.path] = _scores_to_ranks(scores=metric_scores,
reverse=metric.reverse)
return {
res.pk: {
metric_path: ranks[res.pk]
for metric_path, ranks in metric_rank.items()
}
for res in results
}
def _filter_valid_results(*, results: Iterable[Result],
metrics: Tuple[Metric, ...]) -> List[Result]:
""" Ensure that all of the metrics are in every result """
return [
res for res in results if all(
get_jsonpath(res.metrics, m.path) != "" for m in metrics)
]
def _filter_valid_results(*, evaluations: Iterable,
metrics: Tuple[Metric, ...]) -> List:
"""Ensure that all of the metrics are in every result."""
return [
e for e in evaluations if all(
get_jsonpath(
e.outputs.get(interface__slug="metrics-json-file").value,
m.path,
) not in ["", None] for m in metrics)
]
def _filter_valid_results(
*, results: Iterable[Result], metrics: Tuple[Metric, ...]
) -> List[Result]:
""" Ensure that all of the metrics are in every result """
return [
res
for res in results
if all(
get_jsonpath(res.metrics, m.path) not in ["", None]
for m in metrics
)
]
def test_get_jsonpath():
obj = {"spam": {"eggs": 42, "ham": {"beans": 84}}, "chips": 21}
# Nested lookups
assert get_jsonpath(obj=obj, jsonpath="chips") == 21
assert get_jsonpath(obj=obj, jsonpath="spam.ham.beans") == 84
# The path should have precedence
assert get_jsonpath(obj=obj, jsonpath="spam.eggs") == 42
# Keys that don't exist
assert get_jsonpath(obj=obj, jsonpath="foo") == ""
assert get_jsonpath(obj=obj, jsonpath="spam.foo") == ""
assert get_jsonpath(obj=obj, jsonpath="spam") == obj["spam"]
assert get_jsonpath(obj=obj, jsonpath="") == ""
def test_get_jsonpath():
obj = {
'spam': {'eggs': 42, 'ham': {'beans': 84}},
'chips': 21,
}
# Nested lookups
assert get_jsonpath(obj=obj, jsonpath='chips') == 21
assert get_jsonpath(obj=obj, jsonpath='spam.ham.beans') == 84
# The path should have precedence
assert get_jsonpath(obj=obj, jsonpath='spam.eggs') == 42
# Keys that don't exist
assert get_jsonpath(obj=obj, jsonpath='foo') == ''
assert get_jsonpath(obj=obj, jsonpath='spam.foo') == ''
assert get_jsonpath(obj=obj, jsonpath='spam') == obj['spam']
assert get_jsonpath(obj=obj, jsonpath='') == ''