def metrics(self, regularization_losses=None):
"""Creates metrics. See `base_head.Head` for details."""
keys = metric_keys.MetricKeys
with ops.name_scope('metrics', values=(regularization_losses,)):
# Mean metric.
eval_metrics = {}
eval_metrics[self._loss_mean_key] = metrics.Mean(name=keys.LOSS_MEAN)
eval_metrics[self._accuracy_key] = metrics.Accuracy(name=keys.ACCURACY)
eval_metrics[self._precision_key] = metrics.Precision(name=keys.PRECISION)
eval_metrics[self._recall_key] = metrics.Recall(name=keys.RECALL)
eval_metrics[self._prediction_mean_key] = metrics.Mean(
name=keys.PREDICTION_MEAN)
eval_metrics[self._label_mean_key] = metrics.Mean(name=keys.LABEL_MEAN)
eval_metrics[self._accuracy_baseline_key] = (
metrics.Mean(name=keys.ACCURACY_BASELINE))
# The default summation_method is "interpolation" in the AUC metric.
eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC)
eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR', name=keys.AUC_PR)
if regularization_losses is not None:
eval_metrics[self._loss_regularization_key] = metrics.Mean(
name=keys.LOSS_REGULARIZATION)
for i, threshold in enumerate(self._thresholds):
eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy(
name=self._accuracy_keys[i], threshold=threshold)
eval_metrics[self._precision_keys[i]] = metrics.Precision(
name=self._precision_keys[i], thresholds=threshold)
eval_metrics[self._recall_keys[i]] = metrics.Recall(
name=self._recall_keys[i], thresholds=threshold)
return eval_metrics
def test_config(self):
r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9])
self.assertEqual(r_obj.name, 'my_recall')
self.assertEqual(len(r_obj.variables), 2)
self.assertEqual([v.name for v in r_obj.variables],
['true_positives:0', 'false_negatives:0'])
self.assertEqual(r_obj.thresholds, [0.4, 0.9])
def test_div_by_zero(self): r_obj = metrics.Recall() y_pred = constant_op.constant([0, 0, 0, 0]) y_true = constant_op.constant([0, 0, 0, 0]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertEqual(0, self.evaluate(result))
def test_unweighted_with_threshold(self): r_obj = metrics.Recall(thresholds=[0.5, 0.7]) y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
def test_unweighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result))
def metrics(self, regularization_losses=None):
"""Creates metrics. See `base_head.Head` for details."""
keys = metric_keys.MetricKeys
with ops.name_scope(None, 'metrics', (regularization_losses, )):
# Mean metric.
eval_metrics = {}
eval_metrics[self._loss_mean_key] = metrics.Mean(
name=keys.LOSS_MEAN)
# The default summation_method is "interpolation" in the AUC metric.
eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC)
eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR',
name=keys.AUC_PR)
if regularization_losses is not None:
eval_metrics[self._loss_regularization_key] = metrics.Mean(
name=keys.LOSS_REGULARIZATION)
for i, threshold in enumerate(self._thresholds):
eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy(
name=self._accuracy_keys[i], threshold=threshold)
eval_metrics[self._precision_keys[i]] = (metrics.Precision(
name=self._precision_keys[i], thresholds=threshold))
eval_metrics[self._recall_keys[i]] = metrics.Recall(
name=self._recall_keys[i], thresholds=threshold)
for i in range(len(self._classes_for_class_based_metrics)):
eval_metrics[self._prob_keys[i]] = metrics.Mean(
name=self._prob_keys[i])
eval_metrics[self._auc_keys[i]] = metrics.AUC(
name=self._auc_keys[i])
eval_metrics[self._auc_pr_keys[i]] = metrics.AUC(
curve='PR', name=self._auc_pr_keys[i])
return eval_metrics
def test_unweighted_all_incorrect(self): r_obj = metrics.Recall(thresholds=[0.5]) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs) y_true = constant_op.constant(1 - inputs) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0, self.evaluate(result))
def test_unweighted_top_k_and_threshold(self):
r_obj = metrics.Recall(thresholds=.7, top_k=2)
self.evaluate(variables.variables_initializer(r_obj.variables))
y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(0.25, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
def test_reset_states_recall(self):
r_obj = metrics.Recall()
model = _get_model([r_obj])
x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4))))
y = np.concatenate((np.ones((50, 1)), np.ones((50, 1))))
model.evaluate(x, y)
self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
model.evaluate(x, y)
self.assertEqual(self.evaluate(r_obj.true_positives), 50.)
self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
def test_extreme_thresholds(self):
r_obj = metrics.Recall(thresholds=[-1.0, 2.0]) # beyond values range
y_pred = math_ops.cast(constant_op.constant([1, 0, 1, 0],
shape=(1, 4)),
dtype=dtypes.float32)
y_true = math_ops.cast(constant_op.constant([0, 1, 1, 1],
shape=(1, 4)),
dtype=dtypes.float32)
self.evaluate(variables.variables_initializer(r_obj.variables))
result = r_obj(y_true, y_pred)
self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
def test_reset_states(self): r_obj = metrics.Recall() model = _get_simple_sequential_model([r_obj]) x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4)))) y = np.concatenate((np.ones((50, 1)), np.ones((50, 1)))) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.tp), 50.) self.assertEqual(self.evaluate(r_obj.fn), 50.) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.tp), 50.) self.assertEqual(self.evaluate(r_obj.fn), 50.)
def test_weighted(self):
r_obj = metrics.Recall()
y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
self.evaluate(variables.variables_initializer(r_obj.variables))
result = r_obj(
y_true,
y_pred,
sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
weighted_tp = 3.0 + 1.0
weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
expected_recall = weighted_tp / weighted_t
self.assertAlmostEqual(expected_recall, self.evaluate(result))
def test_value_is_idempotent(self):
r_obj = metrics.Recall(thresholds=[0.3, 0.72])
y_pred = random_ops.random_uniform(shape=(10, 3))
y_true = random_ops.random_uniform(shape=(10, 3))
update_op = r_obj.update_state(y_true, y_pred)
self.evaluate(variables.variables_initializer(r_obj.variables))
# Run several updates.
for _ in range(10):
self.evaluate(update_op)
# Then verify idempotency.
initial_recall = self.evaluate(r_obj.result())
for _ in range(10):
self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
def test_weighted_with_threshold(self):
r_obj = metrics.Recall(thresholds=[0.5, 1.])
y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
y_pred = constant_op.constant([[1, 0], [0.6, 0]],
shape=(2, 2),
dtype=dtypes.float32)
weights = constant_op.constant([[1, 4], [3, 2]],
shape=(2, 2),
dtype=dtypes.float32)
self.evaluate(variables.variables_initializer(r_obj.variables))
result = r_obj(y_true, y_pred, sample_weight=weights)
weighted_tp = 0 + 3.
weighted_positives = (0 + 3.) + (4. + 0.)
expected_recall = weighted_tp / weighted_positives
self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
def test_unweighted_top_k_and_class_id(self):
r_obj = metrics.Recall(class_id=2, top_k=2)
self.evaluate(variables.variables_initializer(r_obj.variables))
y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(1, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
self.assertAlmostEqual(0, self.evaluate(r_obj.fn))
y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5))
y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(0.5, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.tp))
self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
def test_config(self):
r_obj = metrics.Recall(
name='my_recall', thresholds=[0.4, 0.9], top_k=15, class_id=12)
self.assertEqual(r_obj.name, 'my_recall')
self.assertEqual(len(r_obj.variables), 2)
self.assertEqual([v.name for v in r_obj.variables],
['true_positives:0', 'false_negatives:0'])
self.assertEqual(r_obj.thresholds, [0.4, 0.9])
self.assertEqual(r_obj.top_k, 15)
self.assertEqual(r_obj.class_id, 12)
# Check save and restore config
r_obj2 = metrics.Recall.from_config(r_obj.get_config())
self.assertEqual(r_obj2.name, 'my_recall')
self.assertEqual(len(r_obj2.variables), 2)
self.assertEqual(r_obj2.thresholds, [0.4, 0.9])
self.assertEqual(r_obj2.top_k, 15)
self.assertEqual(r_obj2.class_id, 12)
def test_weighted_top_k(self):
r_obj = metrics.Recall(top_k=3)
y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5))
y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5))
self.evaluate(variables.variables_initializer(r_obj.variables))
self.evaluate(
r_obj(y_true1,
y_pred1,
sample_weight=constant_op.constant([[1, 4, 2, 3, 5]])))
y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5))
y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5))
result = r_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3))
tp = (2 + 5) + (3 + 3)
positives = (4 + 2 + 5) + (3 + 3 + 3 + 3)
expected_recall = tp / positives
self.assertAlmostEqual(expected_recall, self.evaluate(result))
def test_multiple_updates(self):
r_obj = metrics.Recall(thresholds=[0.5, 1.])
y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2))
y_pred = constant_op.constant([[1, 0], [0.6, 0]],
shape=(2, 2),
dtype=dtypes.float32)
weights = constant_op.constant([[1, 4], [3, 2]],
shape=(2, 2),
dtype=dtypes.float32)
self.evaluate(variables.variables_initializer(r_obj.variables))
update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
for _ in range(2):
self.evaluate(update_op)
weighted_tp = (0 + 3.) + (0 + 3.)
weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.))
expected_recall = weighted_tp / weighted_positives
self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()),
1e-3)
def test_unweighted_class_id(self):
r_obj = metrics.Recall(class_id=2)
self.evaluate(variables.variables_initializer(r_obj.variables))
y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(1, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives))
y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(0.5, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))
y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(0.5, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))