def test_softmax_loss(self):
scores = [[1., 3., 2.], [1., 2., 3.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.SOFTMAX_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-(ln(_softmax(scores[0])[2]) + ln(_softmax(scores[1])[2]) * 2.) /
3.,
places=5)
self.assertAlmostEqual(loss(labels, scores, weights).numpy(),
-(ln(_softmax(scores[0])[2]) * 2. +
ln(_softmax(scores[1])[2]) * 2. * 1.) / 3.,
places=5)
# Test LambdaWeight.
rank_discount_fn = lambda r: 1. / tf.math.log1p(r)
lambda_weight = losses.DCGLambdaWeight(
rank_discount_fn=rank_discount_fn)
loss = losses.get(loss=losses.RankingLossKey.SOFTMAX_LOSS,
lambda_weight=lambda_weight)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-(ln(_softmax(scores[0])[2]) / ln(1. + 2.) +
ln(_softmax(scores[1])[2]) * 2. / ln(1. + 1.)) / 3.,
places=5)
def test_approx_ndcg_loss(self):
scores = [[1.4, -2.8, -0.4], [0., 1.8, 10.2], [1., 1.2, -3.2]]
# ranks= [[1, 3, 2], [3, 2, 1], [2, 1, 3]]
labels = [[0., 2., 1.], [1., 0., 3.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
example_weights = [[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]
norm_weights = [
normalize_weights(w, l) for w, l in zip(example_weights, labels)
]
loss = losses.get(loss=losses.RankingLossKey.APPROX_NDCG_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-((1 / (3 / ln(2) + 1 / ln(3))) * (3 / ln(4) + 1 / ln(3)) +
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 3.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
-(2 * (1 / (3 / ln(2) + 1 / ln(3))) * (3 / ln(4) + 1 / ln(3)) + 1 *
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 3.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, example_weights).numpy(),
-(norm_weights[0] * (1 / (3 / ln(2) + 1 / ln(3))) *
(3 / ln(4) + 1 / ln(3)) + norm_weights[1] *
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 3.,
places=5)
def test_gumbel_approx_ndcg_loss(self):
scores = [[1.4, -2.8, -0.4], [0., 1.8, 10.2], [1., 1.2, -3.2]]
labels = [[0., 2., 1.], [1., 0., 3.], [0., 0., 0.]]
# sampled_scores = [[-.291, -1.643, -2.826],
# [-.0866, -2.924, -3.530],
# [-12.42, -9.492, -7.939e-5],
# [-8.859, -6.830, -1.223e-3],
# [-.8930, -.5266, -45.80183],
# [-.6650, -.7220, -45.94149]]
# ranks = [[1, 2, 3],
# [1, 2, 3],
# [3, 2, 1],
# [3, 2, 1],
# [2, 1, 3],
# [1, 2, 3]]
# expanded_labels = [[0., 2., 1.],
# [0., 2., 1.],
# [1., 0., 3.],
# [1., 0., 3.],
# [0., 0., 0.],
# [0., 0., 0.]]
# expanded_weights = [[2.], [2.],
# [1.], [1.],
# [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.GUMBEL_APPROX_NDCG_LOSS,
sample_size=2,
seed=1)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-(2 * (1 / (3 / ln(2) + 1 / ln(3))) * (3 / ln(3) + 1 / ln(4)) + 2 *
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 6,
places=3)
def test_mean_squared_loss_with_invalid_labels(self):
scores = [[1., 3., 2.]]
labels = [[0., -1., 1.]]
loss = losses.get(loss=losses.RankingLossKey.MEAN_SQUARED_LOSS)
self.assertAlmostEqual(loss(labels, scores).numpy(), (1. + 1.) / 3.,
places=5)
def setUp(self):
super(KerasModelToEstimatorTest, self).setUp()
(context_feature_columns, example_feature_columns,
custom_objects) = _get_feature_columns()
self._context_feature_columns = context_feature_columns
self._example_feature_columns = example_feature_columns
# Remove label feature from example feature column.
del self._example_feature_columns[_LABEL_FEATURE]
self._custom_objects = custom_objects
self._network = _DummyUnivariateRankingNetwork(
context_feature_columns=self._context_feature_columns,
example_feature_columns=self._example_feature_columns)
self._loss = losses.get(
losses.RankingLossKey.SOFTMAX_LOSS,
reduction=tf.compat.v2.losses.Reduction.SUM_OVER_BATCH_SIZE)
self._eval_metrics = metrics.default_keras_metrics()
self._optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.1)
self._config = tf.estimator.RunConfig(keep_checkpoint_max=2,
save_checkpoints_secs=2)
self._data_file = os.path.join(tf.compat.v1.test.get_temp_dir(),
'test_elwc.tfrecord')
serialized_elwc_list = [
_ELWC_PROTO.SerializeToString(),
] * 20
if tf.io.gfile.exists(self._data_file):
tf.io.gfile.remove(self._data_file)
with tf.io.TFRecordWriter(self._data_file) as writer:
for serialized_elwc in serialized_elwc_list:
writer.write(serialized_elwc)
def test_softmax_loss_with_invalid_labels(self):
scores = [[1., 3., 2.]]
labels = [[0., -1., 1.]]
loss = losses.get(loss=losses.RankingLossKey.SOFTMAX_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(), -(ln(_softmax([1, 2])[1])), places=5)
def build_loss(self) -> tf.keras.losses.Loss:
"""See `AbstractPipeline`."""
if isinstance(self._hparams.loss, dict):
raise TypeError("In the simple pipeline, losses are expected to be "
"specified in a str instead of a dict.")
return losses.get(
loss=self._hparams.loss, reduction=self._hparams.loss_reduction)
def test_pairwise_logistic_loss_with_invalid_labels(self):
scores = [[1., 3., 2.]]
labels = [[0., -1., 1.]]
loss = losses.get(loss=losses.RankingLossKey.PAIRWISE_LOGISTIC_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(), ln(1 + math.exp(-1.)) / 3., places=5)
def test_sigmoid_cross_entropy_loss_with_invalid_labels(self):
scores = [[1., 3., 2.]]
labels = [[0., -1., 1.]]
loss = losses.get(loss=losses.RankingLossKey.SIGMOID_CROSS_ENTROPY_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
(ln(1. + math.exp(-2.)) + ln(1. + math.exp(1))) / 3.,
places=5)
def test_list_mle_loss_tie(self):
tf.random.set_seed(1)
scores = [[0., ln(2), ln(3)]]
labels = [[0., 0., 1.]]
loss = losses.get(loss=losses.RankingLossKey.LIST_MLE_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-((ln(3. / (3 + 2 + 1)) + ln(2. / (2 + 1)) + ln(1. / 1))),
places=5)
def build_loss(self) -> Dict[str, tf.keras.losses.Loss]:
"""See `AbstractPipeline`."""
reduction = self._hparams.loss_reduction
if isinstance(self._hparams.loss, str):
raise TypeError("In the multi-task pipeline, losses are expected to be "
"specified in a dict instead of a str.")
return {
task_name: losses.get(loss=loss_name, reduction=reduction)
for task_name, loss_name in self._hparams.loss.items()
}
def build_losses(self,
labels,
model_outputs,
aux_losses=None) -> tf.Tensor:
ranking_loss = tfr_losses.get(
loss=self.task_config.loss,
reduction=self.task_config.loss_reduction)
loss = ranking_loss(tf.cast(labels, tf.float32),
tf.cast(model_outputs, tf.float32))
if aux_losses:
loss += tf.add_n(aux_losses)
return tf_utils.safe_mean(loss)
def test_approx_mrr_loss(self):
scores = [[1.4, -2.8, -0.4], [0., 1.8, 10.2], [1., 1.2, -3.2]]
labels = [[0., 0., 1.], [1., 0., 1.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.APPROX_MRR_LOSS)
self.assertAlmostEqual(loss(labels, scores).numpy(),
-((1 / 2.) + 1 / 2. * (1 / 3. + 1 / 1.)) / 3.,
places=5)
self.assertAlmostEqual(loss(labels, scores, weights).numpy(),
-(2 * 1 / 2. + 1 * 1 / 2. * (1 / 3. + 1 / 1.)) /
3.,
places=5)
def test_list_mle_loss_lambda_weight(self):
scores = [[0., ln(3), ln(2)], [0., ln(2), ln(3)]]
labels = [[0., 2., 1.], [1., 0., 2.]]
lw = losses.ListMLELambdaWeight(
rank_discount_fn=lambda rank: tf.pow(2., 3 - rank) - 1.)
loss = losses.get(
loss=losses.RankingLossKey.LIST_MLE_LOSS, lambda_weight=lw)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-((3 * ln(3. / (3 + 2 + 1)) + 1 * ln(2. / (2 + 1)) + 0 * ln(1. / 1)) +
(3 * ln(3. / (3 + 2 + 1)) + 1 * ln(1. / (1 + 2)) + 0 * ln(2. / 2))) /
2,
places=5)
def test_gumbel_approx_ndcg_weighted_loss(self):
scores = [[1.4, -2.8, -0.4], [0., 1.8, 10.2], [1., 1.2, -3.2]]
labels = [[0., 2., 1.], [1., 0., 3.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.GUMBEL_APPROX_NDCG_LOSS,
sample_size=2,
seed=1)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
-(2 * 2 * (1 / (3 / ln(2) + 1 / ln(3))) *
(3 / ln(3) + 1 / ln(4)) + 1 * 2 * (1 / (7 / ln(2) + 1 / ln(3))) *
(7 / ln(2) + 1 / ln(4))) / 6,
places=3)
def setUp(self):
super(KerasModelToEstimatorTest, self).setUp()
self.context_feature_columns = _context_feature_columns()
self.example_feature_columns = _example_feature_columns()
self.features = _features()
self.network = _DummyUnivariateRankingNetwork(
context_feature_columns=self.context_feature_columns,
example_feature_columns=self.example_feature_columns)
self.loss = losses.get(
losses.RankingLossKey.SOFTMAX_LOSS,
reduction=tf.compat.v2.losses.Reduction.SUM_OVER_BATCH_SIZE)
self.eval_metrics = metrics.default_keras_metrics()
self.optimizer = tf.keras.optimizers.Adagrad(learning_rate=0.1)
self.config = tf.estimator.RunConfig(
keep_checkpoint_max=2, save_checkpoints_secs=2)
def test_unique_softmax_loss(self):
scores = [[1., 3., 2.], [1., 2., 3.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.UNIQUE_SOFTMAX_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-(ln(_softmax(scores[0])[2]) + ln(_softmax(scores[1][:2])[1]) +
ln(_softmax(scores[1])[2]) * 3.) / 9.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
-(ln(_softmax(scores[0])[2]) * 2. + ln(_softmax(scores[1][:2])[1])
* 1. + ln(_softmax(scores[1])[2]) * 3. * 1.) / 9.,
places=5)
def test_list_mle_loss(self):
scores = [[0., ln(3), ln(2)], [0., ln(2), ln(3)]]
labels = [[0., 2., 1.], [1., 0., 2.]]
weights = [[2.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.LIST_MLE_LOSS)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-((ln(3. / (3 + 2 + 1)) + ln(2. / (2 + 1)) + ln(1. / 1)) +
(ln(3. / (3 + 2 + 1)) + ln(1. / (1 + 2)) + ln(2. / 2))) / 2,
places=5)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
-(2 * (ln(3. / (3 + 2 + 1)) + ln(2. / (2 + 1)) + ln(1. / 1)) + 1 *
(ln(3. / (3 + 2 + 1)) + ln(1. / (1 + 2)) + ln(2. / 2))) / 2,
places=5)
def test_mean_squared_loss(self):
scores = [[0.2, 0.5, 0.3], [0.2, 0.3, 0.5], [0.2, 0.3, 0.5]]
labels = [[0., 0., 1.], [0., 0., 2.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(loss=losses.RankingLossKey.MEAN_SQUARED_LOSS)
self.assertAlmostEqual(loss(labels, scores).numpy(),
(_mean_squared_error(labels[0], scores[0]) +
_mean_squared_error(labels[1], scores[1]) +
_mean_squared_error(labels[2], scores[2])) /
9.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
(_mean_squared_error(labels[0], scores[0]) * 2.0 +
_mean_squared_error(labels[1], scores[1]) +
_mean_squared_error(labels[2], scores[2])) / 9.,
places=5)
def test_sigmoid_cross_entropy_loss(self):
scores = [[0.2, 0.5, 0.3], [0.2, 0.3, 0.5], [0.2, 0.3, 0.5]]
labels = [[0., 0., 1.], [0., 0., 2.], [0., 0., 0.]]
weights = [[2.], [1.], [1.]]
loss = losses.get(losses.RankingLossKey.SIGMOID_CROSS_ENTROPY_LOSS)
self.assertAlmostEqual(loss(labels, scores).numpy(),
(_sigmoid_cross_entropy(labels[0], scores[0]) +
_sigmoid_cross_entropy(labels[1], scores[1]) +
_sigmoid_cross_entropy(labels[2], scores[2])) /
9.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, weights).numpy(),
(_sigmoid_cross_entropy(labels[0], scores[0]) * 2.0 +
_sigmoid_cross_entropy(labels[1], scores[1]) +
_sigmoid_cross_entropy(labels[2], scores[2])) / 9.,
places=5)
def test_approx_ndcg_loss_sum_batch(self):
scores = [[1.4, -2.8, -0.4], [0., 1.8, 10.2], [1., 1.2, -3.2]]
# ranks= [[1, 3, 2], [3, 2, 1], [2, 1, 3]]
labels = [[0., 2., 1.], [1., 0., 3.], [0., 0., 0.]]
example_weights = [[1., 2., 3.], [4., 5., 6.], [7., 8., 9.]]
norm_wts = [
normalize_weights(wts, lbls)
for wts, lbls in zip(example_weights, labels)
]
loss = losses.get(
loss=losses.RankingLossKey.APPROX_NDCG_LOSS,
reduction=tf.losses.Reduction.SUM_OVER_BATCH_SIZE)
self.assertAlmostEqual(
loss(labels, scores).numpy(),
-((1 / (3 / ln(2) + 1 / ln(3))) * (3 / ln(4) + 1 / ln(3)) +
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 3.,
places=5)
self.assertAlmostEqual(
loss(labels, scores, example_weights).numpy(),
-(norm_wts[0] * (1 / (3 / ln(2) + 1 / ln(3))) *
(3 / ln(4) + 1 / ln(3)) + norm_wts[1] *
(1 / (7 / ln(2) + 1 / ln(3))) * (7 / ln(2) + 1 / ln(4))) / 3.,
places=5)
def _check_pairwise_loss(self, loss_form):
"""Helper function to test `loss_fn`."""
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 0., 2.]]
listwise_weights = [[2.], [1.]]
listwise_weights_expanded = [[2.] * 3, [1.] * 3]
itemwise_weights = [[2., 3., 4.], [1., 1., 1.]]
default_weights = [1.] * 3
list_size = 3.
loss_form_dict = {
'hinge':
losses.get(loss=losses.RankingLossKey.PAIRWISE_HINGE_LOSS,
name='hinge'),
'logistic':
losses.get(loss=losses.RankingLossKey.PAIRWISE_LOGISTIC_LOSS,
name='logistic'),
'soft_zero_one':
losses.get(loss=losses.RankingLossKey.PAIRWISE_SOFT_ZERO_ONE_LOSS,
name='soft_zero_one'),
}
loss_fn = loss_form_dict[loss_form]
# Individual lists.
self.assertAlmostEqual(loss_fn([labels[0]], [scores[0]]).numpy(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
default_weights, loss_form)
]),
places=5)
self.assertAlmostEqual(loss_fn([labels[1]], [scores[1]]).numpy(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1],
default_weights, loss_form)
]),
places=5)
# Itemwise weights.
self.assertAlmostEqual(loss_fn([labels[0]], [scores[0]],
sample_weight=[itemwise_weights[0]
]).numpy(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
itemwise_weights[0],
loss_form)
]),
places=5)
self.assertAlmostEqual(loss_fn([labels[1]], [scores[1]],
sample_weight=[itemwise_weights[1]
]).numpy(),
_batch_aggregation([
_pairwise_loss(labels[1], scores[1],
itemwise_weights[1],
loss_form)
]),
places=5)
# Multiple lists.
self.assertAlmostEqual(loss_fn(labels,
scores,
sample_weight=listwise_weights).numpy(),
_batch_aggregation([
_pairwise_loss(labels[0], scores[0],
listwise_weights_expanded[0],
loss_form),
_pairwise_loss(labels[1], scores[1],
listwise_weights_expanded[1],
loss_form)
]),
places=5)
# Test LambdaWeight.
rank_discount_fn = lambda r: 1. / tf.math.log1p(r)
lambda_weight = losses.DCGLambdaWeight(
rank_discount_fn=rank_discount_fn, smooth_fraction=1.)
loss_form_dict = {
'hinge':
losses.get(loss=losses.RankingLossKey.PAIRWISE_HINGE_LOSS,
name='hinge',
lambda_weight=lambda_weight),
'logistic':
losses.get(loss=losses.RankingLossKey.PAIRWISE_LOGISTIC_LOSS,
name='logistic',
lambda_weight=lambda_weight),
'soft_zero_one':
losses.get(loss=losses.RankingLossKey.PAIRWISE_SOFT_ZERO_ONE_LOSS,
name='soft_zero_one',
lambda_weight=lambda_weight),
}
loss_fn = loss_form_dict[loss_form]
self.assertAlmostEqual(loss_fn(labels,
scores,
sample_weight=listwise_weights).numpy(),
_batch_aggregation([
_pairwise_loss(labels[0],
scores[0],
listwise_weights_expanded[0],
loss_form,
rank_discount_form='LOG'),
_pairwise_loss(labels[1],
scores[1],
listwise_weights_expanded[1],
loss_form,
rank_discount_form='LOG')
]) * list_size,
places=5)
