def test_sparse_loss(self):
"""Test int labels."""
labels = tf.constant([0, 2, 1, 0])
activations = [[-0.5, 0.1, 2.0], [0.1, 1.5, -5.0], [4.0, -3.0, -6.0],
[-1.5, 0.7, 5.2]]
actual_loss = loss.bi_tempered_logistic_loss(activations,
tf.one_hot(labels, 3), 0.5,
1.5)
sparse_loss = loss.sparse_bi_tempered_logistic_loss(activations, labels,
0.5, 1.5)
with self.cached_session() as sess:
actual_loss_out = sess.run(actual_loss)
sparse_loss_out = sess.run(sparse_loss)
self.assertAllClose(actual_loss_out, sparse_loss_out)
labels = tf.constant([[0, 2], [1, 0]])
activations = [[[-0.5, 0.1, 2.0], [0.1, 1.5, -5.0]],
[[4.0, -3.0, -6.0], [-1.5, 0.7, 5.2]]]
actual_loss = loss.bi_tempered_logistic_loss(activations,
tf.one_hot(labels, 3), 0.5,
1.5)
sparse_loss = loss.sparse_bi_tempered_logistic_loss(activations, labels,
0.5, 1.5)
with self.cached_session() as sess:
actual_loss_out = sess.run(actual_loss)
sparse_loss_out = sess.run(sparse_loss)
self.assertAllClose(actual_loss_out, sparse_loss_out)
def test_loss_value(self):
"""Test the loss based on precomputed values."""
labels = tf.constant([[0.2, 0.3, 0.5], [0.6, 0.3, 0.1], [0.2, 0.8, 0.0]])
activations = [[-0.5, 0.1, 2.0], [0.1, 1.5, -5.0], [4.0, -3.0, -6.0]]
with self.cached_session():
actual_loss = loss.bi_tempered_logistic_loss(activations, labels, 0.5,
1.5)
self.assertAllClose(actual_loss.eval(),
[0.02301914, 0.18972909, 0.93874922])
actual_loss = loss.bi_tempered_logistic_loss(activations, labels, 0.5,
0.8, num_iters=20)
self.assertAllClose(actual_loss.eval(),
[0.21646356, 0.41836615, 1.33997854])
def test_constant_shift(self):
"""Test if adding a constant to all activations is vacuous."""
labels = tf.constant([[0.2, 0.3, 0.5], [0.4, 0.4, 0.2], [0.7, 0.2, 0.1]])
activations = tf.random.normal(shape=[3, 3])
bias = tf.random.normal(shape=[3, 1])
with self.cached_session() as sess:
actual_loss = loss.bi_tempered_logistic_loss(activations, labels, 0.5,
1.2)
shifted_loss = loss.bi_tempered_logistic_loss(activations + bias, labels,
0.5, 1.2)
self.assertEqual(actual_loss.shape, [3])
actual_loss_out, shifted_loss_out = sess.run([actual_loss, shifted_loss])
self.assertAllClose(actual_loss_out, shifted_loss_out)
def test_constant_shift(self):
"""Test if adding a constant to all activations is vacuous."""
labels = tf.constant([[0.2, 0.3, 0.5], [0.4, 0.4, 0.2],
[0.7, 0.2, 0.1]])
activations = tf.random.normal(shape=[3, 3])
bias = tf.random.normal(shape=[3, 1])
for t2 in [0.8, 1.2]:
actual_loss = loss.bi_tempered_logistic_loss(
activations, labels, 0.5, t2)
shifted_loss = loss.bi_tempered_logistic_loss(
activations + bias, labels, 0.5, t2)
actual_loss_out, shifted_loss_out = (actual_loss.numpy(),
shifted_loss.numpy())
self.assertAllClose(actual_loss_out, shifted_loss_out)
def test_label_smoothing(self):
"""Test label smoothing."""
labels = tf.constant([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
activations = [[-0.5, 0.1, 2.0], [0.1, 1.5, -5.0], [4.0, -3.0, -6.0]]
actual_loss = loss.bi_tempered_logistic_loss(
activations, labels, 0.5, 1.5, label_smoothing=0.1)
with self.cached_session() as sess:
actual_loss_out = sess.run(actual_loss)
self.assertAllClose(
actual_loss_out, [0.76652711, 0.08627685, 1.35443510], atol=1e-5)
def test_limit_case_logistic_loss(self):
"""Test for checking if t1 = t2 = 1.0 yields the logistic loss."""
labels = tf.constant([[1.0, 0.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
activations = tf.random.normal(shape=[3, 3])
with self.cached_session() as sess:
actual_loss = loss.bi_tempered_logistic_loss(activations, labels, 1.0,
1.0)
logistic_loss = tf.nn.softmax_cross_entropy_with_logits(
logits=activations, labels=labels)
actual_loss_out, logistic_loss_out = sess.run(
[actual_loss, logistic_loss])
self.assertAllClose(actual_loss_out, logistic_loss_out)
def test_dynamic_temperatures(self):
"""Test changing temperatures dynamically."""
t1 = tf.placeholder(tf.float32)
t2 = tf.placeholder(tf.float32)
labels = tf.constant([[0.2, 0.5, 0.3]])
activations = [[-0.5, 0.1, 2.0]]
actual_loss = loss.bi_tempered_logistic_loss(
activations, labels, t1, t2, num_iters=5)
t1_values = [1.0, 0.9, 0.8, 0.7]
t2_values = [1.0, 1.1, 1.2, 1.3]
loss_values = [[0.62870466], [0.45677936], [0.34298314], [0.26295574]]
loss_out = []
with self.cached_session() as sess:
for t1_value, t2_value in zip(t1_values, t2_values):
loss_out.append(sess.run(
actual_loss, feed_dict={t1: t1_value, t2: t2_value}))
self.assertAllClose(loss_values, loss_out, atol=1e-5)
def test_gradient_error(self):
"""Compare custom gradient with tf.gradient."""
labels = tf.constant([[0.4, 0.3, 0.3], [0.8, 0.1, 0.1], [0.0, 0.0, 1.0],
[0.0, 1.0, 0.0]])
activations = tf.random.normal(shape=[4, 3])
internal_loss = loss._internal_bi_tempered_logistic_loss(
activations, labels, 0.5, 1.5)
numerical_gradient = tf.gradients(internal_loss, activations)
actual_loss = loss.bi_tempered_logistic_loss(activations, labels, 0.5, 1.5)
actual_gradient = tf.gradients(actual_loss, activations)
with self.cached_session() as sess:
internal_loss_out, actual_loss_out = sess.run(
[internal_loss, actual_loss])
numerical_gradient_out, actual_gradient_out = sess.run(
[numerical_gradient[0], actual_gradient[0]])
self.assertEqual(actual_gradient_out.shape, (4, 3))
self.assertAllClose(actual_loss_out, internal_loss_out)
self.assertAllClose(
actual_gradient_out, numerical_gradient_out, atol=1e-5)
def test_gradient_error(self):
"""Compare custom gradient with tf.GradientTape."""
labels = tf.constant([[0.4, 0.3, 0.3], [0.8, 0.1, 0.1],
[0.0, 0.0, 1.0], [0.0, 1.0, 0.0]])
activations = tf.random.normal(shape=[4, 3])
for t1, t2 in [[0.5, 1.0], [1.0, 1.5], [0.5, 1.5]]:
with tf.GradientTape(persistent=True) as tape:
tape.watch(activations)
internal_loss = loss._internal_bi_tempered_logistic_loss(
activations, labels, t1, t2)
actual_loss = loss.bi_tempered_logistic_loss(
activations, labels, t1, t2)
numerical_gradient = tape.gradient(internal_loss, activations)
actual_gradient = tape.gradient(actual_loss, activations)
internal_loss_out, actual_loss_out = (internal_loss.numpy(),
actual_loss.numpy())
numerical_gradient_out, actual_gradient_out = (
numerical_gradient.numpy(), actual_gradient.numpy())
self.assertEqual(actual_gradient_out.shape, (4, 3))
self.assertAllClose(actual_loss_out, internal_loss_out, atol=1e-5)
self.assertAllClose(actual_gradient_out,
numerical_gradient_out,
atol=1e-4)
