def _testComputeSoftmaxLoss(self, scores, labels, group_size):
"""Tests softmax loss given input """
loss = compute_softmax_loss(scores, labels)
self.assertAllClose(testing_utils.get_softmax_loss(
scores, labels, group_size),
loss,
atol=self.atol)
def testComputeSoftmaxLoss(self):
"""Tests softmax loss"""
scores = tf.placeholder(dtype=tf.float32, shape=[None, None])
labels = tf.placeholder(dtype=tf.float32, shape=[None, None])
group_size = tf.placeholder(dtype=tf.int32, shape=[None])
loss = compute_softmax_loss(scores, labels, group_size)
with self.test_session() as sess:
# a simple case
loss_v = sess.run(loss,
feed_dict={
scores: [[0, 1]],
labels: [[0, 1]],
group_size: [2],
})
self.assertAllClose(
test_utils.get_softmax_loss([[0, 1]], [[0, 1]], [2]), loss_v)
# a more complicated case
loss_v = sess.run(loss,
feed_dict={
scores: [[2, 1, 3, 0], [5, 2, 7, 9]],
labels: [[1, 0, 0, 2], [1, 3, 2, 0]],
group_size: [2, 3],
})
loss_v2 = test_utils.get_softmax_loss([[2, 1, 3, 0], [5, 2, 7, 9]],
[[1, 0, 0, 2], [1, 3, 2, 0]],
[2, 3])
self.assertAllClose(loss_v2, loss_v)
def testSoftmaxLossParityWithTfr(self):
"""Tests consistency of tfr softmax loss and inhouse softmax loss"""
labels = tf.constant([[1, 0, -1], [0, 0, 1]], dtype=tf.float32)
scores = tf.constant([[1, 2, 3], [2, 1, 1]], dtype=tf.float32)
loss_fn = tfr.losses.make_loss_fn('softmax_loss', lambda_weight=None)
tfr_loss = loss_fn(labels, scores, None)
loss = compute_softmax_loss(scores, labels)
loss = tf.reduce_sum(input_tensor=loss) / 2
self.assertEqual(tfr_loss, loss)
def compute_rank_clf_loss(hparams, scores, labels, group_size, weight):
"""
Compute ranking/classification loss
Note that the tfr loss is slightly different than our implementation: the tfr loss is sum over all loss and
devided by number of queries; our implementation is sum over all loss and devided by the number of larger than
0 labels.
"""
# Classification loss
if hparams.num_classes > 1:
labels = tf.cast(labels, tf.int32)
labels = tf.squeeze(
labels, -1) # Last dimension is max_group_size, which should be 1
return tf.losses.sparse_softmax_cross_entropy(logits=scores,
labels=labels,
weights=weight)
# Expand weight to [batch size, 1] so that in inhouse ranking loss it can be multiplied with loss which is
# [batch_size, max_group_size]
expanded_weight = tf.expand_dims(weight, axis=-1)
# Ranking losses
# tf-ranking loss
if hparams.use_tfr_loss:
weight_name = "weight"
loss_fn = tfr.losses.make_loss_fn(
hparams.tfr_loss_fn,
lambda_weight=hparams.tfr_lambda_weights,
weights_feature_name=weight_name)
loss = loss_fn(labels, scores, {weight_name: expanded_weight})
return loss
# our own implementation
if hparams.ltr_loss_fn == 'pairwise':
lambdarank = LambdaRank()
pairwise_loss, pairwise_mask = lambdarank(scores, labels, group_size)
loss = tf.reduce_sum(
tf.reduce_sum(pairwise_loss, axis=[1, 2]) *
expanded_weight) / tf.reduce_sum(pairwise_mask)
elif hparams.ltr_loss_fn == 'softmax':
loss = compute_softmax_loss(scores, labels,
group_size) * expanded_weight
is_positive_label = tf.cast(tf.greater(labels, 0), dtype=tf.float32)
loss = tf.div_no_nan(tf.reduce_sum(loss),
tf.reduce_sum(is_positive_label))
elif hparams.ltr_loss_fn == 'pointwise':
loss = compute_sigmoid_cross_entropy_loss(scores, labels,
group_size) * expanded_weight
loss = tf.reduce_mean(loss)
else:
raise ValueError(
'Currently only support pointwise/pairwise/softmax/softmax_cls.')
return loss
def testSoftmaxLossParityWithTfr(self):
"""Tests consistency of tfr softmax loss and inhouse softmax loss"""
group_size = tf.constant([2, 3], dtype=tf.int32)
labels = tf.constant([[1, 0, -1], [0, 0, 1]], dtype=tf.float32)
scores = tf.constant([[1, 2, 3], [2, 1, 1]], dtype=tf.float32)
loss_fn = tfr.losses.make_loss_fn('softmax_loss', lambda_weight=None)
tfr_loss = loss_fn(labels, scores, None)
loss = compute_softmax_loss(scores, labels, group_size)
loss = tf.reduce_sum(loss) / 2
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
loss1, loss2 = sess.run([tfr_loss, loss])
self.assertEqual(loss1, loss2)
def testSoftmaxInExtremeCase(self):
"""Tests softmax loss correctness when difference between padding score and valid score is large """
labels = [[
1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1
]]
scores = [[
-67.9612198, -70.6147385, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132
]]
group_size = tf.constant(
[sum(map(lambda q: q != -1, x)) for x in labels], dtype=tf.int32)
labels = tf.constant(labels, dtype=tf.float32)
scores = tf.constant(scores, dtype=tf.float32)
loss_fn = tfr.losses.make_loss_fn('softmax_loss', lambda_weight=None)
tfr_loss = loss_fn(labels, scores, None)
loss = compute_softmax_loss(scores, labels, group_size)
loss = tf.reduce_sum(loss) / 2
max_scores = tf.reduce_max(scores, axis=-1, keepdims=True)
mask = tf.sequence_mask(group_size,
maxlen=tf.shape(scores)[1],
dtype=tf.float32)
scores = scores - max_scores
exp_score_withmask = tf.exp(scores) * tf.cast(mask, dtype=tf.float32)
logsumexp = tf.log(tf.reduce_sum(
exp_score_withmask, axis=-1)) + tf.squeeze(max_scores, -1)
loss_ori = labels * (-scores + tf.expand_dims(logsumexp, axis=1))
with self.test_session() as sess:
sess.run(tf.global_variables_initializer())
loss1, loss2, logsumexp1, loss_ori1 = sess.run(
[tfr_loss, loss, logsumexp, loss_ori])
self.assertTrue(not math.isnan(loss2))
self.assertTrue(math.isinf(logsumexp1))
self.assertTrue(math.isnan(loss_ori1.sum()))
def _testSoftmaxInExtremeCase(self):
"""Tests softmax loss correctness when difference between padding score and valid score is large """
labels = [[
1, 0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, -1
]]
scores = [[
-67.9612198, -70.6147385, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132,
19.9753132, 19.9753132, 19.9753132, 19.9753132, 19.9753132
]]
group_size = tf.constant(
[sum(map(lambda q: q != -1, x)) for x in labels], dtype=tf.int32)
labels = tf.constant(labels, dtype=tf.float32)
scores = tf.constant(scores, dtype=tf.float32)
loss_fn = tfr.losses.make_loss_fn('softmax_loss', lambda_weight=None)
tfr_loss = loss_fn(labels, scores, None)
loss = compute_softmax_loss(scores, labels)
max_scores = tf.reduce_max(input_tensor=scores, axis=-1, keepdims=True)
mask = tf.sequence_mask(group_size,
maxlen=tf.shape(input=scores)[1],
dtype=tf.float32)
scores = scores - max_scores
exp_score_withmask = tf.exp(scores) * tf.cast(mask, dtype=tf.float32)
logsumexp = tf.math.log(
tf.reduce_sum(input_tensor=exp_score_withmask,
axis=-1)) + tf.squeeze(max_scores, -1)
loss_ori = labels * (-scores + tf.expand_dims(logsumexp, axis=1))
self.assertTrue(not math.isnan(loss.numpy()))
self.assertTrue(not math.isnan(tfr_loss.numpy()))
self.assertTrue(math.isinf(logsumexp.numpy()))
self.assertTrue(math.isnan(loss_ori.numpy().sum()))