def testUnknownDenominatorMode(self):
features = np.random.uniform(size=(10, 3, 20))
labels = np.eye(10, dtype=np.int32)
with self.assertRaisesRegex(ValueError, 'Invalid denominator_mode'):
losses.contrastive_loss(features,
labels,
denominator_mode='invalid')
def testUnknownSummationLocation(self):
features = np.random.uniform(size=(10, 3, 20))
labels = np.eye(10, dtype=np.int32)
with self.assertRaisesRegex(ValueError, 'Invalid summation_location'):
losses.contrastive_loss(features,
labels,
summation_location='invalid')
def testUnknownNumViewsDimension(self):
features = tf.keras.layers.Input(dtype=tf.float32,
batch_size=1,
shape=(None, 20))
with self.assertRaisesRegex(ValueError,
'features has unknown num_views'):
losses.contrastive_loss(features)
def testUnknownBatchSizeDimension(self):
features = tf.keras.layers.Input(dtype=tf.float32,
batch_size=None,
shape=(2, 20))
with self.assertRaisesRegex(ValueError,
'features has unknown batch_size'):
losses.contrastive_loss(features)
def testDefaultBehaviourSameAsAllLabelsDifferent(self):
features = np.random.uniform(size=(10, 3, 20))
labels = np.eye(10, dtype=np.int64)
loss = tf.reduce_mean(losses.contrastive_loss(features))
loss_without_labels = tf.reduce_mean(
losses.contrastive_loss(features, labels))
self.assertFalse(np.isnan(loss.numpy()))
self.assertFalse(np.isnan(loss_without_labels.numpy()))
self.assertEqual(loss.numpy(), loss_without_labels.numpy())
def testContrastModeOneVsAll(self):
# shape (2, 2, 3)
features = np.array([[[0, 0, 1], [0, 1, 0]], [[1., 0., 0.], [0., -1., 0.]]])
loss_one = tf.reduce_mean(
losses.contrastive_loss(
features, contrast_mode=enums.LossContrastMode.ONE_VIEW))
self.assertFalse(np.isnan(loss_one.numpy()))
expected_loss = 1.098612 # np.log(3.)
self.assertAlmostEqual(np.mean(loss_one.numpy()), expected_loss, places=6)
loss_all = tf.reduce_mean(
losses.contrastive_loss(
features, contrast_mode=enums.LossContrastMode.ALL_VIEWS))
self.assertFalse(np.isnan(loss_all.numpy()))
self.assertNotAlmostEqual(
np.mean(loss_all.numpy()), expected_loss, places=6)
def _compute_contrastive_loss(self):
"""Computes and returns the contrastive loss on the projection."""
with tf.name_scope('contrastive_loss'):
contrastive_params = self.hparams.loss_all_stages.contrastive
labels = (tf.one_hot(self.labels, self.num_classes)
if contrastive_params.use_labels else None)
projection = self.projection
projection_view_1, projection_view_2 = tf.split(projection,
2,
axis=-1)
contrastive_loss = losses.contrastive_loss(
tf.stack([projection_view_1, projection_view_2], axis=1),
labels=labels,
temperature=contrastive_params.temperature,
base_temperature=0.07,
contrast_mode=contrastive_params.contrast_mode,
summation_location=contrastive_params.summation_location,
denominator_mode=contrastive_params.denominator_mode,
positives_cap=contrastive_params.positives_cap)
if self.train:
self._add_scalar_summary('loss/contrastive_loss', contrastive_loss)
contrastive_loss = tf.reduce_mean(contrastive_loss)
return contrastive_loss
def testLossForOneView(self):
features = np.array([
[[0.01, 0.02, 0.14]],
[[0.86, 0.97, 0.33]],
[[0.32, 0.64, 0.28]],
])
labels = np.array([[0, 1, 0, 0], [0, 1, 0, 0], [0, 0, 1, 0]],
dtype=np.int32)
loss = losses.contrastive_loss(features, labels=labels, temperature=1.0)
pos0 = np.exp(np.dot(features[0, 0, :], features[1, 0, :]))
neg0 = np.exp(np.dot(features[0, 0, :], features[2, 0, :]))
loss0 = -np.log(pos0 / (pos0 + neg0))
pos1 = np.exp(np.dot(features[1, 0, :], features[0, 0, :]))
neg1 = np.exp(np.dot(features[1, 0, :], features[2, 0, :]))
loss1 = -np.log(pos1 / (pos1 + neg1))
expected_loss = np.array([loss0, loss1, 0.0])
self.assertTupleEqual(loss.numpy().shape, expected_loss.shape)
for index, (val1, val2) in enumerate(zip(loss.numpy(), expected_loss)):
self.assertAlmostEqual(
val1,
val2,
places=5,
msg=f'Lists not almost equal at index {index}: '
f'{loss.numpy()} != {expected_loss}')
def testLossForSummationLocationsAndDenominatorModes(self,
summation_location,
denominator_mode,
positives_cap,
expected_loss,
labels=(0, 0, 1)):
features = np.array([
[[0.01, 0.02, 0.14], [0.38, 0.61, 0.50]],
[[0.86, 0.97, 0.33], [0.26, 0.68, 0.45]],
[[0.32, 0.64, 0.28], [0.45, 0.74, 0.73]],
])
labels = tf.one_hot(labels, 2)
loss = losses.contrastive_loss(
features,
labels=labels,
summation_location=summation_location,
denominator_mode=denominator_mode,
positives_cap=positives_cap)
self.assertTupleEqual(loss.numpy().shape, (len(expected_loss),))
for index, (val1, val2) in enumerate(zip(loss.numpy(), expected_loss)):
self.assertAlmostEqual(
val1,
val2,
places=5,
msg=f'Lists not almost equal at index {index}: '
'{loss.numpy()} != {expected_loss}')
def testLossOnTPU(self):
# Calling tpu.replicate in Eager mode doesn't work. Wrapping in a graph
# implicitly disables Eager mode within its scope.
with tf.Graph().as_default():
features = tf.constant([
[[0.01, 0.02, 0.14], [0.38, 0.61, 0.50]],
[[0.86, 0.97, 0.33], [0.26, 0.68, 0.45]],
[[0.32, 0.64, 0.28], [0.45, 0.74, 0.73]],
[[0.45, 0.62, 0.07], [0.13, 0.28, 0.91]],
])
labels = tf.one_hot((0, 0, 1, 1), 2)
tpu_result = tf.compat.v1.tpu.replicate(
losses.contrastive_loss,
[[features[:2], labels[:2]], [features[2:], labels[2:]]])
# tpu_result should be a list of 2 lists, each containing a single float
# Tensor with shape [2].
self.assertLen(tpu_result, 2)
self.assertLen(tpu_result[0], 1)
self.assertLen(tpu_result[1], 1)
self.assertEqual([2], tpu_result[0][0].shape.as_list())
self.assertEqual([2], tpu_result[1][0].shape.as_list())
tpu_loss = tf.reshape(tpu_result, [4])
cpu_loss = losses.contrastive_loss(features, labels=labels)
cpu_partial_loss_1 = losses.contrastive_loss(
features[:2], labels=labels[:2])
cpu_partial_loss_2 = losses.contrastive_loss(
features[2:], labels=labels[2:])
cpu_partial_loss = tf.concat([cpu_partial_loss_1, cpu_partial_loss_2],
axis=0)
with self.cached_session() as sess:
sess.run(tf.compat.v1.tpu.initialize_system())
tpu_loss, cpu_loss, cpu_partial_loss = sess.run(
(tpu_loss, cpu_loss, cpu_partial_loss))
print(tpu_loss)
print(cpu_loss)
# Numerical precision isn't so high on TPU.
self.assertAllClose(tpu_loss, cpu_loss, atol=1e-2)
# Verify that the TPU computation is different than independently
# computing the two "local batches" on CPU, because of the internal
# cross_replica_concat.
self.assertNotAllClose(tpu_loss, cpu_partial_loss, atol=1e-2)
def testKerasLossVsNonKerasLoss(self):
features = np.random.uniform(0., 1., size=(12, 2, 20))
labels = np.eye(12, 15, dtype=np.int32)
loss_keras = losses.ContrastiveLoss()(labels, features)
loss_direct = tf.reduce_mean(
losses.contrastive_loss(features, labels=labels))
self.assertFalse(np.isnan(loss_direct.numpy()))
self.assertFalse(np.isnan(loss_keras.numpy()))
self.assertEqual(loss_direct.numpy(), loss_keras.numpy())
def testConvFeatures(self, features_shape): features_shape = tf.TensorShape(features_shape) features = tf.random.uniform(shape=features_shape) # Normalize embeddings to ensure the Loss does not return NaN values # for large feature sizes. normalization_axes = list(range(2, features_shape.rank)) normalized_features = tf.nn.l2_normalize(features, axis=normalization_axes) loss = tf.reduce_mean(losses.contrastive_loss(normalized_features)) self.assertFalse(np.isnan(loss.numpy()))
def testLossValueWithTemp(self):
sqrt2 = np.sqrt(2.)
sqrt6 = np.sqrt(6.)
features = np.array([[[0, 0, 1], [0, (2. * sqrt2) / 3., -1 / 3.]],
[[sqrt6 / 3., -sqrt2 / 3., -1. / 3],
[-sqrt6 / 3., -sqrt2 / 3., -1. / 3]]])
loss = losses.contrastive_loss(features, temperature=0.1)
self.assertFalse(np.isnan(loss.numpy()).any())
expected_loss = 0.1098612 # 0.1 * np.log(3.)
self.assertAlmostEqual(np.mean(loss.numpy()), expected_loss, places=5)
def testLossValueWithLabels(self):
sqrt2 = np.sqrt(2.)
sqrt6 = np.sqrt(6.)
features = np.array([[[0, 0, 1], [0, (2. * sqrt2) / 3., -1 / 3.]],
[[sqrt6 / 3., -sqrt2 / 3., -1. / 3],
[-sqrt6 / 3., -sqrt2 / 3., -1. / 3]]])
labels = np.eye(2, dtype=np.int32)
loss = losses.contrastive_loss(features, labels=labels)
self.assertFalse(np.isnan(loss.numpy()).any())
expected_loss = 1.098612 # np.log(3.)
self.assertAlmostEqual(np.mean(loss.numpy()), expected_loss, places=6)
def testLossValueWithLabelsAndPositives(self):
features = np.array([[[0, 0, 1], [0, 0, 1]], [[0, 1, 0], [0, 1, 0]],
[[1, 0, 0], [1, 0, 0]]])
labels = np.eye(3, dtype=np.int32)
# Make the label of sample 1 and 2 the same (= label 0)
labels[1] = labels[0]
loss = losses.contrastive_loss(features, labels).numpy()
self.assertFalse(np.isnan(loss).any())
expected_loss = [
1.57149910, # (3. * np.log(np.e + 4) - 1) / 3.
1.57149910, # (3. * np.log(np.e + 4) - 1) / 3.
0.90483244, # np.log(np.e + 4) - 1
]
self.assertAlmostEqual(loss[0], expected_loss[0], places=6)
self.assertAlmostEqual(loss[1], expected_loss[1], places=6)
self.assertAlmostEqual(loss[2], expected_loss[2], places=6)
def testIncorrectLabelsRank(self):
features = np.random.uniform(0., 1., size=(10, 3, 20))
labels = np.random.randint(5, size=(4, 4))
with self.assertRaisesRegex(ValueError, 'Invalid labels shape'):
losses.contrastive_loss(features, labels=labels)
def testIncorrectFeaturesRank(self):
features = np.zeros([1, 1])
with self.assertRaisesRegex(ValueError, 'Invalid features rank'):
losses.contrastive_loss(features)