def test_metrics_correctness_with_iterator(self): model = keras.Sequential() model.add( keras.layers.Dense(8, activation='relu', input_dim=4, kernel_initializer='ones')) model.add( keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones')) model.compile(loss='binary_crossentropy', metrics=['accuracy', metrics_module.BinaryAccuracy()], optimizer=RMSPropOptimizer(learning_rate=0.001)) np.random.seed(123) x = np.random.randint(10, size=(100, 4)).astype(np.float32) y = np.random.randint(2, size=(100, 1)).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(np.around(outs[1], decimals=1), 0.5) self.assertEqual(np.around(outs[2], decimals=1), 0.5) y = np.zeros((100, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) iterator = dataset_ops.make_one_shot_iterator(dataset) outs = model.evaluate(iterator, steps=10) self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.)
def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope(None, 'metrics', (regularization_losses, )): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean( name=keys.LOSS_MEAN) # The default summation_method is "interpolation" in the AUC metric. eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC) eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR', name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) eval_metrics[self._precision_keys[i]] = (metrics.Precision( name=self._precision_keys[i], thresholds=threshold)) eval_metrics[self._recall_keys[i]] = metrics.Recall( name=self._recall_keys[i], thresholds=threshold) for i in range(len(self._classes_for_class_based_metrics)): eval_metrics[self._prob_keys[i]] = metrics.Mean( name=self._prob_keys[i]) eval_metrics[self._auc_keys[i]] = metrics.AUC( name=self._auc_keys[i]) eval_metrics[self._auc_pr_keys[i]] = metrics.AUC( curve='PR', name=self._auc_pr_keys[i]) return eval_metrics
def test_metrics_correctness_with_dataset(self): layers = [ keras.layers.Dense( 8, activation='relu', input_dim=4, kernel_initializer='ones'), keras.layers.Dense(1, activation='sigmoid', kernel_initializer='ones') ] model = testing_utils.get_model_from_layers(layers, (4,)) model.compile( loss='binary_crossentropy', metrics=['accuracy', metrics_module.BinaryAccuracy()], optimizer='rmsprop', run_eagerly=testing_utils.should_run_eagerly()) np.random.seed(123) x = np.random.randint(10, size=(100, 4)).astype(np.float32) y = np.random.randint(2, size=(100, 1)).astype(np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.batch(10) outs = model.evaluate(dataset, steps=10) self.assertEqual(np.around(outs[1], decimals=1), 0.5) self.assertEqual(np.around(outs[2], decimals=1), 0.5) y = np.zeros((100, 1), dtype=np.float32) dataset = dataset_ops.Dataset.from_tensor_slices((x, y)) dataset = dataset.repeat(100) dataset = dataset.batch(10) outs = model.evaluate(dataset, steps=10) self.assertEqual(outs[1], 0.) self.assertEqual(outs[2], 0.)
def test_binary_accuracy(self): acc_obj = metrics.BinaryAccuracy(name='my acc') # check config self.assertEqual(acc_obj.name, 'my acc') self.assertTrue(acc_obj.stateful) self.assertEqual(len(acc_obj.variables), 2) self.assertEqual(acc_obj.dtype, dtypes.float32) self.evaluate(variables.variables_initializer(acc_obj.variables)) # verify that correct value is returned update_op = acc_obj.update_state([[1], [0]], [[1], [0]]) self.evaluate(update_op) result = self.evaluate(acc_obj.result()) self.assertEqual(result, 1) # 2/2 # check y_pred squeeze update_op = acc_obj.update_state([[1], [1]], [[[1]], [[0]]]) self.evaluate(update_op) result = self.evaluate(acc_obj.result()) self.assertAlmostEqual(result, 0.75, 2) # 3/4 # check y_true squeeze result_t = acc_obj([[[1]], [[1]]], [[1], [0]]) result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.67, 2) # 4/6 # check with sample_weight result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]]) result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.67, 2) # 4.5/6.7
def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope('metrics', values=(regularization_losses,)): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean(name=keys.LOSS_MEAN) eval_metrics[self._accuracy_key] = metrics.Accuracy(name=keys.ACCURACY) eval_metrics[self._precision_key] = metrics.Precision(name=keys.PRECISION) eval_metrics[self._recall_key] = metrics.Recall(name=keys.RECALL) eval_metrics[self._prediction_mean_key] = metrics.Mean( name=keys.PREDICTION_MEAN) eval_metrics[self._label_mean_key] = metrics.Mean(name=keys.LABEL_MEAN) eval_metrics[self._accuracy_baseline_key] = ( metrics.Mean(name=keys.ACCURACY_BASELINE)) # The default summation_method is "interpolation" in the AUC metric. eval_metrics[self._auc_key] = metrics.AUC(name=keys.AUC) eval_metrics[self._auc_pr_key] = metrics.AUC(curve='PR', name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) eval_metrics[self._precision_keys[i]] = metrics.Precision( name=self._precision_keys[i], thresholds=threshold) eval_metrics[self._recall_keys[i]] = metrics.Recall( name=self._recall_keys[i], thresholds=threshold) return eval_metrics
def model_fn(features, labels, mode): # pylint: disable=unused-argument """model_fn which uses a single unit Dense layer.""" # You can also use the Flatten layer if you want to test a model without any # weights. layer = tf.layers.Dense(1, use_bias=True) logits = layer(features) if mode == tf.estimator.ModeKeys.PREDICT: predictions = {"logits": logits} return tf.estimator.EstimatorSpec(mode, predictions=predictions) def loss_fn(): y = tf.reshape(logits, []) - tf.constant(1.) return y * y if mode == tf.estimator.ModeKeys.EVAL: acc_obj = metrics_module.BinaryAccuracy() acc_obj.update_state(labels, labels) return tf.estimator.EstimatorSpec( mode, loss=loss_fn(), eval_metric_ops={"Accuracy": acc_obj}) assert mode == tf.estimator.ModeKeys.TRAIN global_step = tf.train.get_global_step() train_op = optimizer.minimize(loss_fn(), global_step=global_step) return tf.estimator.EstimatorSpec(mode, loss=loss_fn(), train_op=train_op)
def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope(None, 'metrics', (regularization_losses, )): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean( name=keys.LOSS_MEAN) # TODO(b/118843532): create Keras metrics # eval_metrics[self._precision_key] = metrics.Precision(name=keys.AUC) # eval_metrics[self._auc_pr_key] = metrics.Precision(name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) # TODO(b/118843532): create Keras metrics # eval_metrics[self._precision_keys[i]] = ( # metrics.PRECISION_AT_THRESHOLD( # name=self._precision_keys[i], threshold=threshold)) # eval_metrics[self._recall_keys[i]] = metrics.RECALL_AT_THRESHOLD( # name=self._recall_keys[i], threshold=threshold) for i in range(len(self._classes_for_class_based_metrics)): # TODO(b/118843532): create Keras metrics eval_metrics[self._prob_keys[i]] = metrics.Mean( name=self._prob_keys[i]) # eval_metrics[self._auc_keys[i]] = metrics.AUC(name=self._auc_keys[i]) # eval_metrics[self._auc_pr_keys[i]] = metrics.AUC_PR( # name=self._auc_pr_keys[i]) return eval_metrics
def metrics(self, regularization_losses=None): """Creates metrics. See `base_head.Head` for details.""" keys = metric_keys.MetricKeys with ops.name_scope('metrics', values=(regularization_losses, )): # Mean metric. eval_metrics = {} eval_metrics[self._loss_mean_key] = metrics.Mean( name=keys.LOSS_MEAN) eval_metrics[self._accuracy_key] = ( metrics.SparseCategoricalAccuracy(name=keys.ACCURACY)) # TODO(b/118843532): create Keras metrics # eval_metrics[self._precision_key] = metrics.Precision(name=keys.AUC) # eval_metrics[self._recall_key] = metrics.Precision(name=keys.RECALL) eval_metrics[self._prediction_mean_key] = metrics.Mean( name=keys.PREDICTION_MEAN) eval_metrics[self._label_mean_key] = metrics.Mean( name=keys.LABEL_MEAN) # TODO(b/118843532): create Keras metrics # eval_metrics[self._accuracy_baseline_key] = ( # metrics.Mean(name=keys.ACCURACY_BASELINE)) # eval_metrics[self._auc_key] = metrics.Precision(name=keys.PRECISION) # eval_metrics[self._auc_pr_key] = metrics.Precision(name=keys.AUC_PR) if regularization_losses is not None: eval_metrics[self._loss_regularization_key] = metrics.Mean( name=keys.LOSS_REGULARIZATION) for i, threshold in enumerate(self._thresholds): eval_metrics[self._accuracy_keys[i]] = metrics.BinaryAccuracy( name=self._accuracy_keys[i], threshold=threshold) # TODO(b/118843532): create Keras metrics # eval_metrics[self._precision_keys[i]] = ( # metrics.PRECISION_AT_THRESHOLD( # name=self._precision_keys[i], threshold=threshold)) # eval_metrics[self._recall_keys[i]] = metrics.RECALL_AT_THRESHOLD( # name=self._recall_keys[i], threshold=threshold) return eval_metrics
def test_binary_accuracy_threshold(self): acc_obj = metrics.BinaryAccuracy(threshold=0.7) self.evaluate(variables.variables_initializer(acc_obj.variables)) result_t = acc_obj([[1], [1], [0], [0]], [[0.9], [0.6], [0.4], [0.8]]) result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.5, 2)