def ndcg(labels, predictions,
metrics_collections=None,
updates_collections=None,
name=None,
top_k_int=1,
use_predicted_order=False):
# pylint: disable=unused-argument
"""
Compute full normalized discounted cumulative gain (ndcg) based on predictions
ndcg = dcg_k/idcg_k, k is a cut off ranking postion
There are a few variants of ndcg
The dcg (discounted cumulative gain) formula used in
twml.contrib.metrics.ndcg is
\sum_{i=1}^k \frac{2^{relevance\_score} -1}{\log_{2}(i + 1)}
k is the length of items to be ranked in a batch/query
Notice that whether k will be replaced with a fixed value requires discussions
The scores in predictions are transformed to order and relevance scores to calculate ndcg
A relevance score means how relevant a DataRecord is to a particular query
Args:
labels: the ground truth value.
predictions: the predicted values, whose shape must match labels. Ignored for CTR computation.
metrics_collections: optional list of collections to add this metric into.
updates_collections: optional list of collections to add the associated update_op into.
name: an optional variable_scope name.
Return:
ndcg: A `Tensor` representing the ndcg score.
update_op: A update operation used to accumulate data into this metric.
"""
with tf.variable_scope(name, 'ndcg', (labels, predictions)):
label_scores = tf.to_float(labels, name='label_to_float')
predicted_scores = tf.to_float(predictions, name='predictions_to_float')
total_ndcg = _metric_variable([], dtypes.float32, name='total_ndcg')
count_query = _metric_variable([], dtypes.float32, name='query_count')
# actual ndcg cutoff position top_k_int
max_prediction_size = array_ops.size(predicted_scores)
top_k_int = tf.minimum(max_prediction_size, top_k_int)
# the ndcg score of the batch
ndcg = math_fns.cal_ndcg(label_scores,
predicted_scores, top_k_int=top_k_int, use_predicted_order=use_predicted_order)
# add ndcg of the current batch to total_ndcg
update_total_op = state_ops.assign_add(total_ndcg, ndcg)
with ops.control_dependencies([ndcg]):
# count_query stores the number of queries
# count_query increases by 1 for each batch/query
update_count_op = state_ops.assign_add(count_query, 1)
mean_ndcg = math_fns.safe_div(total_ndcg, count_query, 'mean_ndcg')
update_op = math_fns.safe_div(update_total_op, update_count_op, 'update_mean_ndcg_op')
if metrics_collections:
ops.add_to_collections(metrics_collections, mean_ndcg)
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return mean_ndcg, update_op
def err(labels, predictions,
metrics_collections=None,
updates_collections=None,
name=None,
top_k_int=1,
use_predicted_order=False):
# pylint: disable=unused-argument
"""
Compute Expected Reciprocal Rank
The scores in predictions are transformed to order and relevance scores to calculate ndcg
A relevance score means how relevant a DataRecord is to a particular query
Args:
labels: the ground truth value.
predictions: the predicted values, whose shape must match labels. Ignored for CTR computation.
metrics_collections: optional list of collections to add this metric into.
updates_collections: optional list of collections to add the associated update_op into.
name: an optional variable_scope name.
Return:
err: A `Tensor` representing the err score.
update_op: A update operation used to accumulate data into this metric.
"""
with tf.variable_scope(name, 'err', (labels, predictions)):
label_scores = tf.to_float(labels, name='label_to_float_err')
predicted_scores = tf.to_float(predictions, name='predictions_to_float_err')
total_err = _metric_variable([], dtypes.float32, name='total_err')
count_query = _metric_variable([], dtypes.float32, name='query_count_err')
# actual err cutoff position top_k_int is equal to max_prediction_size
max_prediction_size = array_ops.size(predicted_scores)
top_k_int = tf.minimum(max_prediction_size, top_k_int)
# the err score of the batch
err_full = math_fns.cal_err(labels,
predicted_scores, top_k_int=top_k_int, use_predicted_order=use_predicted_order)
# add err of the current batch to total_err
update_total_op = state_ops.assign_add(total_err, err_full)
with ops.control_dependencies([err_full]):
# count_query stores the number of queries
# count_query increases by 1 for each batch/query
update_count_op = state_ops.assign_add(count_query, 1)
mean_err = math_fns.safe_div(total_err, count_query, 'mean_err')
update_op = math_fns.safe_div(update_total_op, update_count_op, 'update_mean_err_op')
if metrics_collections:
ops.add_to_collections(metrics_collections, mean_err)
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return mean_err, update_op
def _get_attentions(raw_scores):
"""
Used in attention weights in AttRank loss
for a query/batch/batchPreidictionRequest
(a rectified softmax function)
"""
not_consider = tf.less_equal(raw_scores, 0)
mask = tf.ones(tf.shape(raw_scores)) - tf.cast(not_consider,
dtype=tf.float32)
mask = tf.cast(mask, dtype=tf.float32)
expon_labels = mask * tf.exp(raw_scores)
expon_label_sum = tf.reduce_sum(expon_labels)
# expon_label_sum is safe as a denominator
attentions = math_fns.safe_div(expon_labels, expon_label_sum)
return attentions