def metrics_builder(): return [ keras_metrics.NumTokensCounter(masked_tokens=[pad_token]), keras_metrics.MaskedCategoricalAccuracy(name='accuracy', masked_tokens=[pad_token]), keras_metrics.MaskedCategoricalAccuracy( name='accuracy_without_out_of_vocab', masked_tokens=[pad_token] + oov_tokens), # Notice that the beginning of sentence token never appears in the # ground truth label. keras_metrics.MaskedCategoricalAccuracy( name='accuracy_without_out_of_vocab_or_end_of_sentence', masked_tokens=[pad_token, eos_token] + oov_tokens), ]
def test_update_state_with_special_character(self): metric = keras_metrics.MaskedCategoricalAccuracy(masked_tokens=[4]) metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ # A batch with 100% accruacy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], # A batch with 50% accruacy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ]) self.assertAllClose(self.evaluate(metric.result()), 5 / 7.0) metric.update_state( y_true=[[0, 4, 1, 2]], y_pred=[ # A batch with 33% accruacy. [ [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], ]) self.assertAllClose(self.evaluate(metric.result()), 6 / 10.0)
def test_constructor_no_masked_token(self): metric_name = 'my_test_metric' metric = keras_metrics.MaskedCategoricalAccuracy(name=metric_name) self.assertIsInstance(metric, tf.keras.metrics.Metric) self.assertEqual(metric.name, metric_name) self.assertAllEqual(metric.get_config()['masked_tokens'], []) self.assertEqual(self.evaluate(metric.result()), 0.0)
def test_update_state_with_no_special_character(self): metric = keras_metrics.MaskedCategoricalAccuracy() metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ # A batch with 100% accruacy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], # A batch with 50% accruacy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ]) self.assertEqual(self.evaluate(metric.result()), 6 / 8.0) metric.update_state( y_true=[[0, 4, 1, 2]], y_pred=[ # A batch with 25% accruacy. [ [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], ]) self.assertAllClose(self.evaluate(metric.result()), 8 / 12.0)
def model_fn() -> model.Model: return keras_utils.from_keras_model( keras_model=char_prediction_models.create_recurrent_model( vocab_size=VOCAB_LENGTH, sequence_length=sequence_length), loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), input_spec=task_datasets.element_type_structure, metrics=[ keras_metrics.NumTokensCounter(masked_tokens=[pad_token]), keras_metrics.MaskedCategoricalAccuracy(masked_tokens=[pad_token]) ])
def test_weighted_update_state_with_scalar_weight(self): metric = keras_metrics.MaskedCategoricalAccuracy() metric.update_state( y_true=[[1, 2, 3, 4]], y_pred=[ # A batch with 50% accuracy. [ [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], ], sample_weight=1.0) self.assertAllClose(self.evaluate(metric.result()), .5)
def test_update_state_with_all_tokens_masked(self): metric = keras_metrics.MaskedCategoricalAccuracy( masked_tokens=[1, 2, 3, 4]) metric.update_state( # All batches should be masked. y_true=[[1, 2, 3, 4], [4, 3, 2, 1]], y_pred=[ [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ]) self.assertAllClose(self.evaluate(metric.result()), 0.0)
def test_update_state_with_multiple_tokens_masked(self): metric = keras_metrics.MaskedCategoricalAccuracy( masked_tokens=[1, 2, 3, 4]) metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ [ # This batch should be masked. [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], [ # Batch with 50% accuracy [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ]) self.assertAllClose(self.evaluate(metric.result()), 0.5)
def test_weighted_update_state_special_character_rank_2_sample_weight( self): metric = keras_metrics.MaskedCategoricalAccuracy(masked_tokens=[4]) metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ # A batch with 100% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], # A batch with 50% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ], # A weight for each `y_true` scalar. sample_weight=[[1.0, 2.0, 1.0, 2.0], [1.0, 2.0, 1.0, 2.0]]) self.assertAllClose(self.evaluate(metric.result()), (6 + 2) / 10.0)
def test_weighted_update_state_no_special_character(self): metric = keras_metrics.MaskedCategoricalAccuracy() metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ # A batch with 100% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], # A batch with 50% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ], # A weight for each `y_true` scalar. sample_weight=[1.0, 2.0, 1.0, 2.0, 1.0, 2.0, 1.0, 2.0]) self.assertAllClose(self.evaluate(metric.result()), (6 + 4) / 12.0) metric.update_state( y_true=[[0, 4, 1, 2]], y_pred=[ # A batch with 25% accruacy. [ [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], ], sample_weight=[1.0, 1.0, 2.0, 2.0]) self.assertAllClose(self.evaluate(metric.result()), (6 + 4 + 2) / 18.0)
def test_weighted_update_state_with_masked_token(self): metric = keras_metrics.MaskedCategoricalAccuracy(masked_tokens=[4]) metric.update_state( y_true=[[1, 2, 3, 4], [0, 0, 0, 0]], y_pred=[ # A batch with 100% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.1, 0.1, 0.9, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], # A batch with 50% accuracy. [ [0.1, 0.9, 0.1, 0.1, 0.1], [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.9, 0.1], [0.9, 0.1, 0.1, 0.1, 0.0], ], ], # A weight for each `y_true` scalar. sample_weight=[[1.0, 2.0, 1.0, 2.0], [1.0, 2.0, 1.0, 2.0]]) self.assertAllClose(self.evaluate(metric.result()), (4 + 4) / 10.0) metric.update_state( y_true=[[0, 4, 1, 2]], y_pred=[ # A batch with 25% accruacy. [ [0.9, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], [0.1, 0.1, 0.1, 0.9, 0.1], [0.1, 0.1, 0.1, 0.1, 0.9], ], ], sample_weight=[1.0, 1.0, 2.0, 2.0]) self.assertAllClose(self.evaluate(metric.result()), (4 + 4 + 1) / 15.0)