def average_relevance_position(labels, predictions, weights=None, name=None):
"""Computes average relevance position (ARP).
This can also be named as average_relevance_rank, but this can be confusing
with mean_reciprocal_rank in acronyms. This name is more distinguishing and
has been used historically for binary relevance as average_click_position.
Args:
labels: A `Tensor` of the same shape as `predictions`.
predictions: A `Tensor` with shape [batch_size, list_size]. Each value is
the ranking score of the corresponding example.
weights: A `Tensor` of the same shape of predictions or [batch_size, 1]. The
former case is per-example and the latter case is per-list.
name: A string used as the name for this metric.
Returns:
A metric for the weighted average relevance position.
"""
metric = metrics_impl.ARPMetric(name)
with tf.compat.v1.name_scope(metric.name, 'average_relevance_position',
(labels, predictions, weights)):
# TODO: Add mask argument for metric.compute() call
per_list_arp, per_list_weights = metric.compute(labels, predictions,
weights)
return tf.compat.v1.metrics.mean(per_list_arp, per_list_weights)
def compute_mean(metric_key,
labels,
predictions,
weights=None,
topn=None,
name=None):
"""Returns the mean of the specified metric given the inputs.
Args:
metric_key: A key in `RankingMetricKey`.
labels: A `Tensor` of the same shape as `predictions` representing
relevance.
predictions: A `Tensor` with shape [batch_size, list_size]. Each value is
the ranking score of the corresponding example.
weights: A `Tensor` of the same shape of predictions or [batch_size, 1]. The
former case is per-example and the latter case is per-list.
topn: An `integer` specifying the cutoff of how many items are considered in
the metric.
name: A `string` used as the name for this metric.
Returns:
A scalar as the computed metric.
"""
metric_dict = {
RankingMetricKey.ARP: metrics_impl.ARPMetric(metric_key),
RankingMetricKey.MRR: metrics_impl.MRRMetric(metric_key, topn),
RankingMetricKey.NDCG: metrics_impl.NDCGMetric(name, topn),
RankingMetricKey.DCG: metrics_impl.DCGMetric(name, topn),
RankingMetricKey.PRECISION: metrics_impl.PrecisionMetric(name, topn),
RankingMetricKey.RECALL: metrics_impl.RecallMetric(name, topn),
RankingMetricKey.MAP:
metrics_impl.MeanAveragePrecisionMetric(name, topn),
RankingMetricKey.ORDERED_PAIR_ACCURACY: metrics_impl.OPAMetric(name),
RankingMetricKey.BPREF: metrics_impl.BPrefMetric(name, topn),
RankingMetricKey.HITS: metrics_impl.HitsMetric(metric_key, topn),
}
assert metric_key in metric_dict, ('metric_key %s not supported.' %
metric_key)
# TODO: Add mask argument for metric.compute() call
metric, weight = metric_dict[metric_key].compute(labels, predictions,
weights)
return tf.compat.v1.div_no_nan(tf.reduce_sum(input_tensor=metric * weight),
tf.reduce_sum(input_tensor=weight))
def __init__(self, name=None, dtype=None, **kwargs): super(ARPMetric, self).__init__(name=name, dtype=dtype, **kwargs) self._metric = metrics_impl.ARPMetric(name=name)