def test_multi_dim_weighted_eval(self):
weights_feature_name = self._default_weights_feature_name
metric_fns = {
'metric/precision@1':
metrics_lib.make_ranking_metric_fn(
metrics_lib.RankingMetricKey.PRECISION, topn=1),
}
head = ranking_head.create_ranking_head(
loss_fn=_make_loss_fn(weights_feature_name),
eval_metric_fns=metric_fns)
weights = self._default_weights
# Create estimator spec.
spec = head.create_estimator_spec(
features={weights_feature_name: weights},
mode=model_fn.ModeKeys.EVAL,
logits=self._default_logits,
labels=self._default_labels)
expected_metrics = [
'labels_mean',
'logits_mean',
'metric/precision@1',
]
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
update_ops = {
k: spec.eval_metric_ops[k][1]
for k in spec.eval_metric_ops
}
loss, metrics = sess.run((spec.loss, update_ops))
self.assertAllClose(self._default_weighted_loss, loss)
self.assertItemsEqual(expected_metrics, metrics.keys())
def test_train_with_regularization_losses(self):
regularization_losses = [1.5, 0.5]
expected_regularization_loss = 2.
expected_train_result = b'my_train_op'
expected_loss = expected_regularization_loss + self._default_loss
def _train_op_fn(loss):
with ops.control_dependencies(
(check_ops.assert_equal(math_ops.to_float(expected_loss),
math_ops.to_float(loss),
name='assert_loss'), )):
return constant_op.constant(expected_train_result)
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
train_op_fn=_train_op_fn)
# Create estimator spec.
spec = head.create_estimator_spec(
features=self._default_features_dict,
mode=model_fn.ModeKeys.TRAIN,
logits=self._default_logits,
labels=self._default_labels,
regularization_losses=regularization_losses)
# Assert predictions, loss, and train_op.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
loss, train_result = sess.run((spec.loss, spec.train_op))
self.assertAllClose(expected_loss, loss)
self.assertEqual(expected_train_result, train_result)
def test_train_with_optimizer(self):
expected_train_result = b'my_train_op'
expected_loss = self._default_loss
class _Optimizer(object):
def minimize(self, loss, global_step):
del global_step
with ops.control_dependencies(
(check_ops.assert_equal(math_ops.to_float(expected_loss),
math_ops.to_float(loss),
name='assert_loss'), )):
return constant_op.constant(expected_train_result)
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
optimizer=_Optimizer())
# Create estimator spec.
spec = head.create_estimator_spec(features=self._default_features_dict,
mode=model_fn.ModeKeys.TRAIN,
logits=self._default_logits,
labels=self._default_labels)
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
loss, train_result = sess.run((spec.loss, spec.train_op))
self.assertAllClose(expected_loss, loss)
self.assertEqual(expected_train_result, train_result)
def test_train(self):
expected_train_result = b'my_train_op'
def _train_op_fn(loss):
with ops.control_dependencies(
(check_ops.assert_near(math_ops.to_float(self._default_loss),
math_ops.to_float(loss),
name='assert_loss'), )):
return constant_op.constant(expected_train_result)
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
train_op_fn=_train_op_fn)
# Create estimator spec.
spec = head.create_estimator_spec(features=self._default_features_dict,
mode=model_fn.ModeKeys.TRAIN,
logits=self._default_logits,
labels=self._default_labels)
# Assert spec contains expected tensors.
self.assertIsNotNone(spec.loss)
self.assertEqual({}, spec.eval_metric_ops)
self.assertIsNotNone(spec.train_op)
self.assertIsNone(spec.export_outputs)
# Assert predictions, loss, and train_op.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
loss, train_result = sess.run((spec.loss, spec.train_op))
self.assertAllClose(self._default_loss, loss)
self.assertEqual(expected_train_result, train_result)
def test_train_create_loss(self):
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn())
# Create loss.
training_loss = head.create_loss(features=self._default_features_dict,
mode=model_fn.ModeKeys.TRAIN,
logits=self._default_logits,
labels=self._default_labels)[0]
with self.cached_session():
_initialize_variables(self, monitored_session.Scaffold())
self.assertAllClose(self._default_loss, training_loss.eval())
def setUp(self):
super(GroupwiseRankingEstimatorTest, self).setUp()
ops.reset_default_graph()
self._model_dir = test.get_temp_dir()
gfile.MakeDirs(self._model_dir)
model_fn = model.make_groupwise_ranking_fn(
_group_score_fn,
group_size=2,
transform_fn=feature.make_identity_transform_fn(
['context', 'weight']),
ranking_head=head.create_ranking_head(
loss_fn=losses.make_loss_fn(
losses.RankingLossKey.PAIRWISE_HINGE_LOSS,
weights_feature_name='weight'),
optimizer=training.AdagradOptimizer(learning_rate=0.1)))
self._estimator = estimator.Estimator(model_fn, self._model_dir)
def test_multi_dim_weighted_train(self):
weights_feature_name = self._default_weights_feature_name
def _train_op_fn(loss):
return loss
head = ranking_head.create_ranking_head(
loss_fn=_make_loss_fn(weights_feature_name),
train_op_fn=_train_op_fn)
# Create estimator spec.
spec = head.create_estimator_spec(
features={weights_feature_name: self._default_weights},
mode=model_fn.ModeKeys.TRAIN,
logits=self._default_logits,
labels=self._default_labels)
# Assert predictions, loss, and train_op.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
loss, train_result = sess.run((spec.loss, spec.train_op))
self.assertAllClose(self._default_weighted_loss, loss)
self.assertAllClose(self._default_weighted_loss, train_result)
def test_eval(self):
metric_fns = {
'metric/precision@1':
metrics_lib.make_ranking_metric_fn(
metrics_lib.RankingMetricKey.PRECISION, topn=1),
}
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
eval_metric_fns=metric_fns)
# Create estimator spec.
spec = head.create_estimator_spec(features=self._default_features_dict,
mode=model_fn.ModeKeys.EVAL,
logits=self._default_logits,
labels=self._default_labels)
expected_metrics = [
'labels_mean',
'logits_mean',
'metric/precision@1',
]
# Assert spec contains expected tensors.
self.assertIsNotNone(spec.loss)
self.assertIsNone(spec.train_op)
self.assertIsNone(spec.export_outputs)
self.assertItemsEqual(expected_metrics, spec.eval_metric_ops.keys())
# Assert predictions, loss, and metrics.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
update_ops = {
k: spec.eval_metric_ops[k][1]
for k in spec.eval_metric_ops
}
loss, metrics = sess.run((spec.loss, update_ops))
self.assertAllClose(self._default_loss, loss)
self.assertItemsEqual(expected_metrics, metrics.keys())
def test_predict(self):
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn())
logits = [[1., 3.], [1., 2.]]
spec = head.create_estimator_spec(features=self._default_features_dict,
mode=model_fn.ModeKeys.PREDICT,
logits=logits)
# Assert spec contains expected tensors.
self.assertIsNone(spec.loss)
self.assertEqual({}, spec.eval_metric_ops)
self.assertIsNone(spec.train_op)
self.assertItemsEqual((self._default_signature, ),
spec.export_outputs.keys())
# Assert predictions.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
predictions = sess.run(spec.predictions)
self.assertAllClose(logits, predictions)
self.assertAllClose(
logits,
sess.run(spec.export_outputs[self._default_signature].value))
def test_name(self):
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
name='fake_head')
self.assertEqual('fake_head', head.name)