def compute(self, labels, predictions, weights):
"""See `_RankingMetric`."""
labels, predictions, weights, topn = _prepare_and_validate_params(
labels, predictions, weights, self._topn)
sorted_labels, sorted_weights = utils.sort_by_scores(predictions,
[labels, weights],
topn=topn)
# Relevance = 1.0 when labels >= 1.0.
sorted_relevance = tf.cast(tf.greater_equal(sorted_labels, 1.0),
dtype=tf.float32)
per_list_relevant_counts = tf.cumsum(sorted_relevance, axis=1)
per_list_cutoffs = tf.cumsum(tf.ones_like(sorted_relevance), axis=1)
per_list_precisions = tf.math.divide_no_nan(per_list_relevant_counts,
per_list_cutoffs)
total_precision = tf.reduce_sum(input_tensor=per_list_precisions *
sorted_weights * sorted_relevance,
axis=1,
keepdims=True)
total_relevance = tf.reduce_sum(input_tensor=sorted_weights *
sorted_relevance,
axis=1,
keepdims=True)
per_list_map = tf.math.divide_no_nan(total_precision, total_relevance)
# per_list_weights are computed from the whole list to avoid the problem of
# 0 when there is no relevant example in topn.
per_list_weights = _per_example_weights_to_per_list_weights(
weights, tf.cast(tf.greater_equal(labels, 1.0), dtype=tf.float32))
return per_list_map, per_list_weights
def _compute_per_list_metric(self, labels, predictions, weights, topn):
"""See `_DivRankingMetric`."""
sorted_labels, sorted_weights = utils.sort_by_scores(predictions,
[labels, weights],
topn=topn,
seed=self._seed)
alpha_dcg = _discounted_cumulative_gain(sorted_labels, sorted_weights,
self._gain_fn,
self._rank_discount_fn)
per_list_weights = self._compute_per_list_weights(weights, labels)
return tf.compat.v1.math.divide_no_nan(alpha_dcg, per_list_weights)
def compute(self, labels, predictions, weights):
"""See `_RankingMetric`."""
labels, predictions, weights, topn = _prepare_and_validate_params(
labels, predictions, weights, self._topn)
sorted_labels, sorted_weights = utils.sort_by_scores(predictions,
[labels, weights],
topn=topn)
dcg = _discounted_cumulative_gain(sorted_labels, sorted_weights,
self._gain_fn,
self._rank_discount_fn)
# Sorting over the weighted labels to get ideal ranking.
ideal_sorted_labels, ideal_sorted_weights = utils.sort_by_scores(
weights * labels, [labels, weights], topn=topn)
ideal_dcg = _discounted_cumulative_gain(ideal_sorted_labels,
ideal_sorted_weights,
self._gain_fn,
self._rank_discount_fn)
per_list_ndcg = tf.compat.v1.math.divide_no_nan(dcg, ideal_dcg)
per_list_weights = _per_example_weights_to_per_list_weights(
weights=weights,
relevance=self._gain_fn(tf.cast(labels, dtype=tf.float32)))
return per_list_ndcg, per_list_weights
def compute(self, labels, predictions, weights):
"""See `_RankingMetric`."""
list_size = tf.shape(input=predictions)[1]
labels, predictions, weights, topn = _prepare_and_validate_params(
labels, predictions, weights, list_size)
sorted_labels, sorted_weights = utils.sort_by_scores(predictions,
[labels, weights],
topn=topn)
relevance = sorted_labels * sorted_weights
position = tf.cast(tf.range(1, topn + 1), dtype=tf.float32)
# TODO: Consider to add a cap position topn + 1 when there is no
# relevant examples.
return position * tf.ones_like(relevance), relevance
def compute(self, labels, predictions, weights):
"""See `_RankingMetric`."""
labels, predictions, weights, topn = _prepare_and_validate_params(
labels, predictions, weights, self._topn)
sorted_labels, = utils.sort_by_scores(predictions, [labels], topn=topn)
sorted_list_size = tf.shape(input=sorted_labels)[1]
# Relevance = 1.0 when labels >= 1.0 to accommodate graded relevance.
relevance = tf.cast(tf.greater_equal(sorted_labels, 1.0),
dtype=tf.float32)
reciprocal_rank = 1.0 / tf.cast(tf.range(1, sorted_list_size + 1),
dtype=tf.float32)
# MRR has a shape of [batch_size, 1].
mrr = tf.reduce_max(input_tensor=relevance * reciprocal_rank,
axis=1,
keepdims=True)
per_list_weights = _per_example_weights_to_per_list_weights(
weights=weights,
relevance=tf.cast(tf.greater_equal(labels, 1.0), dtype=tf.float32))
return mrr, per_list_weights
def _compute_per_list_metric(self, labels, predictions, weights, topn):
"""See `_DivRankingMetric`."""
sorted_labels = utils.sort_by_scores(predictions, [labels],
topn=topn)[0]
# relevance shape = [batch_size, topn].
relevance = tf.reduce_sum(tf.cast(tf.greater_equal(sorted_labels, 1.0),
dtype=tf.float32),
axis=-1)
# num_subtopics shape = [batch_size, 1].
num_subtopics = tf.reduce_sum(tf.cast(tf.reduce_any(tf.greater_equal(
labels, 1.0),
axis=1,
keepdims=True),
dtype=tf.float32),
axis=-1)
return tf.compat.v1.math.divide_no_nan(
tf.reduce_sum(input_tensor=relevance, axis=1, keepdims=True),
tf.reduce_sum(input_tensor=tf.ones_like(relevance) * num_subtopics,
axis=1,
keepdims=True))
def _per_list_precision(labels, predictions, topn):
"""Computes the precision for each query in the batch.
Args:
labels: A `Tensor` of the same shape as `predictions`. A value >= 1 means a
relevant example.
predictions: A `Tensor` with shape [batch_size, list_size]. Each value is
the ranking score of the corresponding example.
topn: A cutoff for how many examples to consider for this metric.
Returns:
A `Tensor` of size [batch_size, 1] containing the precision of each query
respectively.
"""
sorted_labels = utils.sort_by_scores(predictions, [labels], topn=topn)[0]
# Relevance = 1.0 when labels >= 1.0.
relevance = tf.cast(tf.greater_equal(sorted_labels, 1.0), dtype=tf.float32)
per_list_precision = tf.compat.v1.math.divide_no_nan(
tf.reduce_sum(input_tensor=relevance, axis=1, keepdims=True),
tf.reduce_sum(input_tensor=tf.ones_like(relevance),
axis=1,
keepdims=True))
return per_list_precision
