def test_make_precision_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
features = {}
m = metrics.make_ranking_metric_fn(metrics.RankingMetricKey.PRECISION)
m_top_1 = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.PRECISION, topn=1)
m_top_2 = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.PRECISION, topn=2)
self._check_metrics([
(m([labels[0]], [scores[0]], features), 1. / 3.),
(m_top_1([labels[0]], [scores[0]], features), 0. / 1.),
(m_top_2([labels[0]], [scores[0]], features), 1. / 2.),
(m(labels, scores, features), (1. / 3. + 2. / 3.) / 2.),
])
def test_multi_dim_weighted_eval(self):
weights_feature_name = self._default_weights_feature_name
metric_fns = {
'metric/precision@1':
metrics_lib.make_ranking_metric_fn(
metrics_lib.RankingMetricKey.PRECISION, topn=1),
}
head = ranking_head.create_ranking_head(
loss_fn=_make_loss_fn(weights_feature_name),
eval_metric_fns=metric_fns)
weights = self._default_weights
# Create estimator spec.
spec = head.create_estimator_spec(
features={weights_feature_name: weights},
mode=model_fn.ModeKeys.EVAL,
logits=self._default_logits,
labels=self._default_labels)
expected_metrics = [
'labels_mean',
'logits_mean',
'metric/precision@1',
]
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
update_ops = {
k: spec.eval_metric_ops[k][1]
for k in spec.eval_metric_ops
}
loss, metrics = sess.run((spec.loss, update_ops))
self.assertAllClose(self._default_weighted_loss, loss)
self.assertItemsEqual(expected_metrics, metrics.keys())
def test_make_average_relevance_position_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
weights = [[1., 2., 3.], [4., 5., 6.]]
weights_feature_name = 'weights'
features = {weights_feature_name: weights}
m = metrics.make_ranking_metric_fn(metrics.RankingMetricKey.ARP)
m_w = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.ARP,
weights_feature_name=weights_feature_name)
self._check_metrics([
(m([labels[0]], [scores[0]], features), 2.),
(m(labels, scores, features), (1. * 2. + 2. * 1. + 1. * 2.) / 4.),
(m_w(labels, scores, features),
(3. * 1. * 2. + 6. * 2. * 1. + 5 * 1. * 2.) / (3. + 12. + 5.)),
])
def test_make_mean_reciprocal_rank_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
weights = [[1., 2., 3.], [4., 5., 6.]]
weights_feature_name = 'weights'
features = {weights_feature_name: weights}
m = metrics.make_ranking_metric_fn(metrics.RankingMetricKey.MRR)
m_w = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.MRR,
weights_feature_name=weights_feature_name)
self._check_metrics([
(m([labels[0]], [scores[0]], features), 0.5),
(m(labels, scores, features), (0.5 + 1.0) / 2),
(m_w(labels, scores,
features), (6. * 0.5 + 15. * 1.) / (6. + 15.)),
])
def test_make_ordered_pair_accuracy_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
m = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.ORDERED_PAIR_ACCURACY)
self._check_metrics([
(m([labels[0]], [scores[0]], {}), 1. / 2.),
(m([labels[1]], [scores[1]], {}), 1.),
(m(labels, scores, {}), (1. + 3.) / (2. + 3.)),
])
def test_make_discounted_cumulative_gain_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
weights = [[1., 1., 1.], [2., 2., 1.]]
weights_feature_name = 'weights'
features = {weights_feature_name: weights}
m = metrics.make_ranking_metric_fn(metrics.RankingMetricKey.DCG)
m_w = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.DCG,
weights_feature_name=weights_feature_name)
expected_dcg_1 = _dcg(0., 1) + _dcg(1., 2) + _dcg(0., 3)
self._check_metrics([
(m([labels[0]], [scores[0]], features), expected_dcg_1),
])
expected_dcg_2 = _dcg(2., 1) + _dcg(1., 2)
expected_dcg_2_weighted = _dcg(2., 1) + _dcg(1., 2) * 2.
expected_weight_2 = ((4 - 1) * 1. + (2 - 1) * 2.) / (4 - 1 + 2 - 1)
self._check_metrics([
(m(labels, scores,
features), (expected_dcg_1 + expected_dcg_2) / 2.0),
(m_w(labels, scores,
features), (expected_dcg_1 + expected_dcg_2_weighted) /
(1. + expected_weight_2)),
])
def test_make_normalized_discounted_cumulative_gain_fn(self):
scores = [[1., 3., 2.], [1., 2., 3.]]
labels = [[0., 0., 1.], [0., 1., 2.]]
weights = [[1., 2., 3.], [4., 5., 6.]]
weights_feature_name = 'weights'
features = {weights_feature_name: weights[0]}
m = metrics.make_ranking_metric_fn(metrics.RankingMetricKey.NDCG)
expected_ndcg = (_dcg(0., 1) + _dcg(1., 2) + _dcg(0., 3)) / (
_dcg(1., 1) + _dcg(0., 2) + _dcg(0., 3))
self._check_metrics([
(m([labels[0]], [scores[0]], features), expected_ndcg),
])
expected_ndcg_1 = (_dcg(0., 1) + _dcg(1., 2) + _dcg(0., 3)) / (
_dcg(1., 1) + _dcg(0., 2) + _dcg(0., 3))
expected_ndcg_2 = 1.0
expected_ndcg = (expected_ndcg_1 + expected_ndcg_2) / 2.0
self._check_metrics([
(m(labels, scores, features), expected_ndcg),
])
# With weights.
m_top = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.NDCG,
weights_feature_name=weights_feature_name,
topn=1)
m_weight = metrics.make_ranking_metric_fn(
metrics.RankingMetricKey.NDCG,
weights_feature_name=weights_feature_name)
self._check_metrics([
(m_top([labels[0]], [scores[0]],
features), _dcg(0., 1, 2.) / _dcg(1., 1, 3.)),
(m_weight([labels[0]], [scores[0]], features),
(_dcg(0., 1, 2.) + _dcg(1., 2, 3.) + _dcg(0., 3, 1.)) /
(_dcg(1., 1, 3.) + _dcg(0., 2, 1.) + _dcg(0., 3, 2.))),
])
def test_eval(self):
metric_fns = {
'metric/precision@1':
metrics_lib.make_ranking_metric_fn(
metrics_lib.RankingMetricKey.PRECISION, topn=1),
}
head = ranking_head.create_ranking_head(loss_fn=_make_loss_fn(),
eval_metric_fns=metric_fns)
# Create estimator spec.
spec = head.create_estimator_spec(features=self._default_features_dict,
mode=model_fn.ModeKeys.EVAL,
logits=self._default_logits,
labels=self._default_labels)
expected_metrics = [
'labels_mean',
'logits_mean',
'metric/precision@1',
]
# Assert spec contains expected tensors.
self.assertIsNotNone(spec.loss)
self.assertIsNone(spec.train_op)
self.assertIsNone(spec.export_outputs)
self.assertItemsEqual(expected_metrics, spec.eval_metric_ops.keys())
# Assert predictions, loss, and metrics.
with self.cached_session() as sess:
_initialize_variables(self, spec.scaffold)
self.assertIsNone(spec.scaffold.summary_op)
update_ops = {
k: spec.eval_metric_ops[k][1]
for k in spec.eval_metric_ops
}
loss, metrics = sess.run((spec.loss, update_ops))
self.assertAllClose(self._default_loss, loss)
self.assertItemsEqual(expected_metrics, metrics.keys())