def create_reciprocal_rank_lambda_weight(topn=None, smooth_fraction=0.):
"""Creates _LambdaWeight for MRR-like metric."""
return losses_impl.DCGLambdaWeight(topn,
gain_fn=lambda labels: labels,
rank_discount_fn=lambda rank: 1. / rank,
normalized=True,
smooth_fraction=smooth_fraction)
def test_softmax_loss(self):
with tf.Graph().as_default():
scores = [[1., 3., 2.], [1., 2., 3.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
with self.cached_session():
self.assertAlmostEqual(
ranking_losses._softmax_loss(labels, scores).eval(),
-(math.log(_softmax(scores[0])[2]) +
math.log(_softmax(scores[1])[2]) * 2.) / 2.,
places=5)
self.assertAlmostEqual(
ranking_losses._softmax_loss(labels, scores, weights).eval(),
-(math.log(_softmax(scores[0])[2]) * 2. +
math.log(_softmax(scores[1])[2]) * 2. * 1.) / 2.,
places=5)
# Test LambdaWeight.
lambda_weight = losses_impl.DCGLambdaWeight(
rank_discount_fn=lambda r: 1. / tf.math.log1p(r))
self.assertAlmostEqual(
ranking_losses._softmax_loss(
labels, scores, lambda_weight=lambda_weight).eval(),
-(math.log(_softmax(scores[0])[2]) / math.log(1. + 2.) +
math.log(_softmax(scores[1])[2]) * 2. / math.log(1. + 1.)) / 2.,
places=5)
def create_ndcg_lambda_weight(topn=None, smooth_fraction=0.):
"""Creates _LambdaWeight for NDCG metric."""
return losses_impl.DCGLambdaWeight(
topn,
gain_fn=lambda labels: tf.pow(2.0, labels) - 1.,
rank_discount_fn=lambda rank: 1. / tf.math.log1p(rank),
normalized=True,
smooth_fraction=smooth_fraction)
def test_smooth_fraction(self):
"""For the weights using absolute rank."""
with tf.Graph().as_default():
labels = [[2.0, 1.0, 0.0]]
ranks = [[1, 2, 3]]
lambda_weight = losses_impl.DCGLambdaWeight(smooth_fraction=1.0)
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 1. / 2., 2. * 2. / 3.], [1. / 2., 0., 1. / 6.],
[2. * 2. / 3., 1. / 6., 0.]]])
lambda_weight = losses_impl.DCGLambdaWeight(topn=1,
smooth_fraction=1.0)
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 1., 2.], [1., 0., 0.], [2., 0., 0.]]])
def test_invalid_labels(self):
with tf.Graph().as_default():
labels = [[2.0, 1.0, -1.0]]
ranks = [[1, 2, 3]]
lambda_weight = losses_impl.DCGLambdaWeight()
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 1. / 2., 0.], [1. / 2., 0., 0.], [0., 0., 0.]]])
def test_individual_weights(self):
with tf.Graph().as_default():
labels = [[1.0, 2.0]]
ranks = [[1, 2]]
lambda_weight = losses_impl.DCGLambdaWeight(normalized=True)
max_dcg = 2.5
with self.cached_session():
self.assertAllClose(
lambda_weight.individual_weights(labels, ranks).eval(),
[[1. / max_dcg / 1., 2. / max_dcg / 2.]])
def test_normalized(self):
with tf.Graph().as_default():
labels = [[1.0, 2.0]]
ranks = [[1, 2]]
lambda_weight = losses_impl.DCGLambdaWeight(normalized=True)
max_dcg = 2.5
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 1. / 2. / max_dcg], [1. / 2. / max_dcg, 0.]]])
def test_default(self):
"""For the weight using rank diff."""
with tf.Graph().as_default():
labels = [[2.0, 1.0, 0.0]]
ranks = [[1, 2, 3]]
lambda_weight = losses_impl.DCGLambdaWeight()
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 1. / 2., 2. * 1. / 6.], [1. / 2., 0., 1. / 2.],
[2. * 1. / 6., 1. / 2., 0.]]])
def test_gain_and_discount(self):
with tf.Graph().as_default():
labels = [[2.0, 1.0]]
ranks = [[1, 2]]
lambda_weight = losses_impl.DCGLambdaWeight(
gain_fn=lambda x: tf.pow(2., x) - 1.,
rank_discount_fn=lambda r: 1. / tf.math.log1p(r))
with self.cached_session():
self.assertAllClose(
lambda_weight.pair_weights(labels, ranks).eval(),
[[[0., 2. * (1. / math.log(2.) - 1. / math.log(3.))],
[2. * (1. / math.log(2.) - 1. / math.log(3.)), 0.]]])
def test_softmax_loss_should_support_lambda_weights(self):
with tf.Graph().as_default():
with self.cached_session():
scores = [[1., 3., 2.], [1., 2., 3.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.], [0., 0., 0.]]
reduction = tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS
lambda_weight = losses_impl.DCGLambdaWeight(
rank_discount_fn=lambda r: 1. / tf.math.log1p(r))
loss_fn = losses_impl.SoftmaxLoss(name=None,
lambda_weight=lambda_weight)
result = loss_fn.compute(labels, scores, None,
reduction).eval()
self.assertAlmostEqual(
result,
-(math.log(_softmax(scores[0])[2]) / math.log(1. + 2.) +
math.log(_softmax(scores[1])[2]) * 2. /
math.log(1. + 1.)) / 2.,
places=5)
def _check_make_pairwise_loss(self, loss_key):
"""Helper function to test `make_loss_fn`."""
with tf.Graph().as_default():
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.]]
listwise_weights = [[2.], [1.]]
listwise_weights_expanded = [[2.] * 3, [1.] * 3]
itemwise_weights = [[2., 3., 4.], [1., 1., 1.]]
default_weights = [1.] * 3
weights_feature_name = 'weights'
list_size = 3.
features = {}
loss_fn = ranking_losses.make_loss_fn(loss_key)
with self.cached_session():
# Individual lists.
self.assertAlmostEqual(
loss_fn([labels[0]], [scores[0]], features).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0], default_weights,
loss_key)
]),
places=5)
self.assertAlmostEqual(
loss_fn([labels[1]], [scores[1]], features).eval(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1], default_weights,
loss_key)
]),
places=5)
# Itemwise weights.
loss_fn = ranking_losses.make_loss_fn(
loss_key, weights_feature_name=weights_feature_name)
features[weights_feature_name] = [itemwise_weights[0]]
self.assertAlmostEqual(loss_fn([labels[0]], [scores[0]],
features).eval(),
_batch_aggregation([
_pairwise_loss(
labels[0], scores[0],
itemwise_weights[0], loss_key)
]),
places=5)
features[weights_feature_name] = [itemwise_weights[1]]
self.assertAlmostEqual(loss_fn([labels[1]], [scores[1]],
features).eval(),
_batch_aggregation([
_pairwise_loss(
labels[1], scores[1],
itemwise_weights[1], loss_key)
]),
places=5)
# Multiple lists.
features[weights_feature_name] = listwise_weights
self.assertAlmostEqual(
loss_fn(labels, scores, features).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
listwise_weights_expanded[0], loss_key),
_pairwise_loss(labels[1], scores[1],
listwise_weights_expanded[1], loss_key)
]),
places=5)
# Test LambdaWeight.
lambda_weight = losses_impl.DCGLambdaWeight(
rank_discount_fn=lambda r: 1. / tf.math.log1p(r),
smooth_fraction=1.)
loss_fn = ranking_losses.make_loss_fn(
loss_key,
weights_feature_name=weights_feature_name,
lambda_weight=lambda_weight)
self.assertAlmostEqual(
loss_fn(labels, scores, features).eval(),
_batch_aggregation([
_pairwise_loss(labels[0],
scores[0],
listwise_weights_expanded[0],
loss_key,
rank_discount_form='LOG'),
_pairwise_loss(labels[1],
scores[1],
listwise_weights_expanded[1],
loss_key,
rank_discount_form='LOG')
]) * list_size,
places=5)
# Test loss reduction method.
# Two reduction methods should return different loss values.
loss_fn_1 = ranking_losses.make_loss_fn(
loss_key, reduction=tf.compat.v1.losses.Reduction.SUM)
loss_fn_2 = ranking_losses.make_loss_fn(
loss_key, reduction=tf.compat.v1.losses.Reduction.MEAN)
self.assertNotAlmostEqual(
loss_fn_1(labels, scores, features).eval(),
loss_fn_2(labels, scores, features).eval())
def _check_pairwise_loss(self, loss_fn):
"""Helper function to test `loss_fn`."""
with tf.Graph().as_default():
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.]]
listwise_weights = [[2.], [1.]]
listwise_weights_expanded = [[2.] * 3, [1.] * 3]
itemwise_weights = [[2., 3., 4.], [1., 1., 1.]]
default_weights = [1.] * 3
list_size = 3.
loss_form_dict = {
ranking_losses._pairwise_hinge_loss:
ranking_losses.RankingLossKey.PAIRWISE_HINGE_LOSS,
ranking_losses._pairwise_logistic_loss:
ranking_losses.RankingLossKey.PAIRWISE_LOGISTIC_LOSS,
ranking_losses._pairwise_soft_zero_one_loss:
ranking_losses.RankingLossKey.PAIRWISE_SOFT_ZERO_ONE_LOSS,
}
loss_form = loss_form_dict[loss_fn]
with self.cached_session():
# Individual lists.
self.assertAlmostEqual(
loss_fn([labels[0]], [scores[0]]).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0], default_weights,
loss_form)
]),
places=5)
self.assertAlmostEqual(
loss_fn([labels[1]], [scores[1]]).eval(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1], default_weights,
loss_form)
]),
places=5)
# Itemwise weights.
self.assertAlmostEqual(
loss_fn([labels[0]], [scores[0]],
weights=[itemwise_weights[0]]).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
itemwise_weights[0], loss_form)
]),
places=5)
self.assertAlmostEqual(
loss_fn([labels[1]], [scores[1]],
weights=[itemwise_weights[1]]).eval(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1],
itemwise_weights[1], loss_form)
]),
places=5)
# Multiple lists.
self.assertAlmostEqual(
loss_fn(labels, scores, weights=listwise_weights).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
listwise_weights_expanded[0],
loss_form),
_pairwise_loss(labels[1], scores[1],
listwise_weights_expanded[1], loss_form)
]),
places=5)
# Test LambdaWeight.
lambda_weight = losses_impl.DCGLambdaWeight(
rank_discount_fn=lambda r: 1. / tf.math.log1p(r),
smooth_fraction=1.)
self.assertAlmostEqual(
loss_fn(labels,
scores,
weights=listwise_weights,
lambda_weight=lambda_weight).eval(),
_batch_aggregation([
_pairwise_loss(labels[0],
scores[0],
listwise_weights_expanded[0],
loss_form,
rank_discount_form='LOG'),
_pairwise_loss(labels[1],
scores[1],
listwise_weights_expanded[1],
loss_form,
rank_discount_form='LOG')
]) * list_size,
places=5)
# Test loss reduction method.
# Two reduction methods should return different loss values.
reduced_1 = loss_fn(
labels,
scores,
reduction=tf.compat.v1.losses.Reduction.SUM).eval()
reduced_2 = loss_fn(
labels,
scores,
reduction=tf.compat.v1.losses.Reduction.MEAN).eval()
self.assertNotAlmostEqual(reduced_1, reduced_2)
def _check_pairwise_loss(self, loss_form):
"""Helper function to test `loss_form`."""
with tf.Graph().as_default():
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.]]
listwise_weights = [[2.], [1.]]
listwise_weights_expanded = [[2.] * 3, [1.] * 3]
itemwise_weights = [[2., 3., 4.], [1., 1., 1.]]
default_weights = [1.] * 3
list_size = 3.
loss_fn = loss_form(name=None)
with self.cached_session():
# Individual lists.
reduction = tf.compat.v1.losses.Reduction.SUM_BY_NONZERO_WEIGHTS
self.assertAlmostEqual(
loss_fn.compute([labels[0]], [scores[0]], None,
reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0], default_weights,
loss_form)
]),
places=5)
self.assertAlmostEqual(
loss_fn.compute([labels[1]], [scores[1]], None,
reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1], default_weights,
loss_form)
]),
places=5)
# Itemwise weights.
self.assertAlmostEqual(
loss_fn.compute([labels[0]], [scores[0]],
weights=[itemwise_weights[0]],
reduction=reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
itemwise_weights[0], loss_form)
]),
places=5)
self.assertAlmostEqual(
loss_fn.compute([labels[1]], [scores[1]],
weights=[itemwise_weights[1]],
reduction=reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1],
itemwise_weights[1], loss_form)
]),
places=5)
# Multiple lists.
self.assertAlmostEqual(
loss_fn.compute(labels,
scores,
weights=listwise_weights,
reduction=reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
listwise_weights_expanded[0],
loss_form),
_pairwise_loss(labels[1], scores[1],
listwise_weights_expanded[1], loss_form)
]),
places=5)
# Test LambdaWeight.
lambda_weight = losses_impl.DCGLambdaWeight(
rank_discount_fn=lambda r: 1. / tf.math.log1p(r),
smooth_fraction=1.)
loss_fn = loss_form(name=None, lambda_weight=lambda_weight)
self.assertAlmostEqual(
loss_fn.compute(labels,
scores,
weights=listwise_weights,
reduction=reduction).eval(),
_batch_aggregation([
_pairwise_loss(labels[0],
scores[0],
listwise_weights_expanded[0],
loss_form,
rank_discount_form='LOG'),
_pairwise_loss(labels[1],
scores[1],
listwise_weights_expanded[1],
loss_form,
rank_discount_form='LOG')
]) * list_size,
places=5)
# Test loss reduction method.
# Two reduction methods should return different loss values.
reduced_1 = loss_fn.compute(
labels,
scores,
None,
reduction=tf.compat.v1.losses.Reduction.SUM).eval()
reduced_2 = loss_fn.compute(
labels,
scores,
None,
reduction=tf.compat.v1.losses.Reduction.MEAN).eval()
self.assertNotAlmostEqual(reduced_1, reduced_2)
