def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope('metrics', values=(regularization_losses,)): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean(name=keys.LOSS_MEAN) eval_metrics[self._accuracy_key] = metrics.Accuracy(name=keys.ACCURACY) eval_metrics[self._precision_key] = metrics.Precision(name=keys.PRECISION) eval_metrics[self._recall_key] = metrics.Recall(name=keys.RECALL) eval_metrics[self._prediction_mean_key] = metrics.Mean( name=keys.PREDICTION_MEAN) eval_metrics[self._label_mean_key] = metrics.Mean(name=keys.LABEL_MEAN) eval_metrics[self._accuracy_baseline_key] = ( metrics.Mean(name=keys.ACCURACY_BASELINE)) # The default summation_method is "interpolation" in the AUC metric. eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC) eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR', name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) eval_metrics[self._precision_keys[i]] = metrics.Precision( name=self._precision_keys[i], thresholds=threshold) eval_metrics[self._recall_keys[i]] = metrics.Recall( name=self._recall_keys[i], thresholds=threshold) return eval_metrics
def test_config(self): r_obj = metrics.Recall(name='my_recall', thresholds=[0.4, 0.9]) self.assertEqual(r_obj.name, 'my_recall') self.assertEqual(len(r_obj.variables), 2) self.assertEqual([v.name for v in r_obj.variables], ['true_positives:0', 'false_negatives:0']) self.assertEqual(r_obj.thresholds, [0.4, 0.9])
def test_div_by_zero(self): r_obj = metrics.Recall() y_pred = constant_op.constant([0, 0, 0, 0]) y_true = constant_op.constant([0, 0, 0, 0]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertEqual(0, self.evaluate(result))
def test_unweighted_with_threshold(self): r_obj = metrics.Recall(thresholds=[0.5, 0.7]) y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
def test_unweighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result))
def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope(None, 'metrics', (regularization_losses, )): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean( name=keys.LOSS_MEAN) # The default summation_method is "interpolation" in the AUC metric. eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC) eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR', name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) eval_metrics[self._precision_keys[i]] = (metrics.Precision( name=self._precision_keys[i], thresholds=threshold)) eval_metrics[self._recall_keys[i]] = metrics.Recall( name=self._recall_keys[i], thresholds=threshold) for i in range(len(self._classes_for_class_based_metrics)): eval_metrics[self._prob_keys[i]] = metrics.Mean( name=self._prob_keys[i]) eval_metrics[self._auc_keys[i]] = metrics.AUC( name=self._auc_keys[i]) eval_metrics[self._auc_pr_keys[i]] = metrics.AUC( curve='PR', name=self._auc_pr_keys[i]) return eval_metrics
def test_unweighted_all_incorrect(self): r_obj = metrics.Recall(thresholds=[0.5]) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs) y_true = constant_op.constant(1 - inputs) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0, self.evaluate(result))
def test_unweighted_top_k_and_threshold(self): r_obj = metrics.Recall(thresholds=.7, top_k=2) self.evaluate(variables.variables_initializer(r_obj.variables)) y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.25, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives)) self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
def test_reset_states_recall(self): r_obj = metrics.Recall() model = _get_model([r_obj]) x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4)))) y = np.concatenate((np.ones((50, 1)), np.ones((50, 1)))) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.true_positives), 50.) self.assertEqual(self.evaluate(r_obj.false_negatives), 50.) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.true_positives), 50.) self.assertEqual(self.evaluate(r_obj.false_negatives), 50.)
def test_extreme_thresholds(self): r_obj = metrics.Recall(thresholds=[-1.0, 2.0]) # beyond values range y_pred = math_ops.cast(constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32) y_true = math_ops.cast(constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
def test_reset_states(self): r_obj = metrics.Recall() model = _get_simple_sequential_model([r_obj]) x = np.concatenate((np.ones((50, 4)), np.zeros((50, 4)))) y = np.concatenate((np.ones((50, 1)), np.ones((50, 1)))) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.tp), 50.) self.assertEqual(self.evaluate(r_obj.fn), 50.) model.evaluate(x, y) self.assertEqual(self.evaluate(r_obj.tp), 50.) self.assertEqual(self.evaluate(r_obj.fn), 50.)
def test_weighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj( y_true, y_pred, sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) weighted_tp = 3.0 + 1.0 weighted_t = (2.0 + 3.0) + (4.0 + 1.0) expected_recall = weighted_tp / weighted_t self.assertAlmostEqual(expected_recall, self.evaluate(result))
def test_value_is_idempotent(self): r_obj = metrics.Recall(thresholds=[0.3, 0.72]) y_pred = random_ops.random_uniform(shape=(10, 3)) y_true = random_ops.random_uniform(shape=(10, 3)) update_op = r_obj.update_state(y_true, y_pred) self.evaluate(variables.variables_initializer(r_obj.variables)) # Run several updates. for _ in range(10): self.evaluate(update_op) # Then verify idempotency. initial_recall = self.evaluate(r_obj.result()) for _ in range(10): self.assertArrayNear(initial_recall, self.evaluate(r_obj.result()), 1e-3)
def test_weighted_with_threshold(self): r_obj = metrics.Recall(thresholds=[0.5, 1.]) y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) y_pred = constant_op.constant([[1, 0], [0.6, 0]], shape=(2, 2), dtype=dtypes.float32) weights = constant_op.constant([[1, 4], [3, 2]], shape=(2, 2), dtype=dtypes.float32) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred, sample_weight=weights) weighted_tp = 0 + 3. weighted_positives = (0 + 3.) + (4. + 0.) expected_recall = weighted_tp / weighted_positives self.assertArrayNear([expected_recall, 0], self.evaluate(result), 1e-3)
def test_unweighted_top_k_and_class_id(self): r_obj = metrics.Recall(class_id=2, top_k=2) self.evaluate(variables.variables_initializer(r_obj.variables)) y_pred = constant_op.constant([0.2, 0.6, 0.3, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(1, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.tp)) self.assertAlmostEqual(0, self.evaluate(r_obj.fn)) y_pred = constant_op.constant([1, 1, 0.9, 1, 1], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.tp)) self.assertAlmostEqual(1, self.evaluate(r_obj.fn))
def test_config(self): r_obj = metrics.Recall( name='my_recall', thresholds=[0.4, 0.9], top_k=15, class_id=12) self.assertEqual(r_obj.name, 'my_recall') self.assertEqual(len(r_obj.variables), 2) self.assertEqual([v.name for v in r_obj.variables], ['true_positives:0', 'false_negatives:0']) self.assertEqual(r_obj.thresholds, [0.4, 0.9]) self.assertEqual(r_obj.top_k, 15) self.assertEqual(r_obj.class_id, 12) # Check save and restore config r_obj2 = metrics.Recall.from_config(r_obj.get_config()) self.assertEqual(r_obj2.name, 'my_recall') self.assertEqual(len(r_obj2.variables), 2) self.assertEqual(r_obj2.thresholds, [0.4, 0.9]) self.assertEqual(r_obj2.top_k, 15) self.assertEqual(r_obj2.class_id, 12)
def test_weighted_top_k(self): r_obj = metrics.Recall(top_k=3) y_pred1 = constant_op.constant([0.2, 0.1, 0.4, 0, 0.2], shape=(1, 5)) y_true1 = constant_op.constant([0, 1, 1, 0, 1], shape=(1, 5)) self.evaluate(variables.variables_initializer(r_obj.variables)) self.evaluate( r_obj(y_true1, y_pred1, sample_weight=constant_op.constant([[1, 4, 2, 3, 5]]))) y_pred2 = constant_op.constant([0.2, 0.6, 0.4, 0.2, 0.2], shape=(1, 5)) y_true2 = constant_op.constant([1, 0, 1, 1, 1], shape=(1, 5)) result = r_obj(y_true2, y_pred2, sample_weight=constant_op.constant(3)) tp = (2 + 5) + (3 + 3) positives = (4 + 2 + 5) + (3 + 3 + 3 + 3) expected_recall = tp / positives self.assertAlmostEqual(expected_recall, self.evaluate(result))
def test_multiple_updates(self): r_obj = metrics.Recall(thresholds=[0.5, 1.]) y_true = constant_op.constant([[0, 1], [1, 0]], shape=(2, 2)) y_pred = constant_op.constant([[1, 0], [0.6, 0]], shape=(2, 2), dtype=dtypes.float32) weights = constant_op.constant([[1, 4], [3, 2]], shape=(2, 2), dtype=dtypes.float32) self.evaluate(variables.variables_initializer(r_obj.variables)) update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights) for _ in range(2): self.evaluate(update_op) weighted_tp = (0 + 3.) + (0 + 3.) weighted_positives = ((0 + 3.) + (4. + 0.)) + ((0 + 3.) + (4. + 0.)) expected_recall = weighted_tp / weighted_positives self.assertArrayNear([expected_recall, 0], self.evaluate(r_obj.result()), 1e-3)
def test_unweighted_class_id(self): r_obj = metrics.Recall(class_id=2) self.evaluate(variables.variables_initializer(r_obj.variables)) y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(1, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives)) self.assertAlmostEqual(0, self.evaluate(r_obj.false_negatives)) y_pred = constant_op.constant([0.2, 0.1, 0, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives)) self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives)) y_pred = constant_op.constant([0.2, 0.1, 0.6, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 0, 0, 0], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives)) self.assertAlmostEqual(1, self.evaluate(r_obj.false_negatives))