def test_accuracy(self): metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = tf.ones((10, 1)), tf.ones((10, 1)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.binary_accuracy) metric_container = compile_utils.MetricsContainer('Accuracy') y_t, y_p = tf.ones((10, 1)), tf.ones((10, 1)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.binary_accuracy) metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = tf.ones((10, 1)), tf.ones((10, 20)) self.assertEqual(y_p.shape.as_list()[-1], 20) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.sparse_categorical_accuracy) metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = tf.ones((10, 20)), tf.ones((10, 20)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.categorical_accuracy)
def test_nested_structure(self): metric_container = compile_utils.MetricsContainer( metrics={"b": ["mse", None], "a": "mae"}, weighted_metrics={"b": [None, None], "a": "mse"}, ) y_t = { "b": [2 * tf.ones((10, 1)), tf.zeros((10, 1))], "a": tf.zeros((10, 1)), } y_p = { "b": [tf.zeros((10, 1)), tf.zeros((10, 1))], "a": tf.ones((10, 1)), } sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 3) a_mae_metric = metric_container.metrics[0] self.assertEqual(a_mae_metric.name, "a_mae") self.assertEqual(a_mae_metric.result().numpy(), 1.0) weighted_a_mae_metric = metric_container.metrics[1] self.assertEqual(weighted_a_mae_metric.name, "a_mse") self.assertEqual(weighted_a_mae_metric.result().numpy(), 1.0) b_1_mse_metric = metric_container.metrics[2] self.assertEqual(b_1_mse_metric.name, "b_1_mse") self.assertEqual(b_1_mse_metric.result().numpy(), 4.0)
def test_missing_label_with_no_metrics(self): # It's ok to exclude a label if that label has no # losses or metrics associated with it. metric_container = compile_utils.MetricsContainer(metrics={ 'output1': 'mae', 'output3': 'mse' }) y_p = { 'output1': tf.convert_to_tensor([[0], [1], [2]]), 'output2': tf.convert_to_tensor([[3], [4], [5]]), 'output3': tf.convert_to_tensor([[6], [7], [8]]) } y_t = { 'output1': tf.convert_to_tensor([[1], [2], [3]]), 'output3': tf.convert_to_tensor([[4], [5], [6]]) } metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 2) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'output1_mae') self.assertEqual(mae_metric.result().numpy(), 1.) mse_metric = metric_container.metrics[1] self.assertEqual(mse_metric.name, 'output3_mse') self.assertEqual(mse_metric.result().numpy(), 4.)
def test_metric_dict(self): metric_container = compile_utils.MetricsContainer(metrics={ 'out1': 'mse', 'out2': 'mae' }, weighted_metrics={ 'out1': 'mse', 'out2': 'mae' }) y_t = {'out1': tf.ones((10, 1)), 'out2': tf.zeros((10, 1))} y_p = {'out1': tf.ones((10, 1)), 'out2': 2 * tf.ones((10, 1))} sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, 'out1_mse') self.assertEqual(mse_metric.result().numpy(), 0.) weighted_mse_metric = metric_container.metrics[1] self.assertEqual(weighted_mse_metric.name, 'out1_weighted_mse') self.assertEqual(weighted_mse_metric.result().numpy(), 0.) mae_metric = metric_container.metrics[2] self.assertEqual(mae_metric.name, 'out2_mae') self.assertEqual(mae_metric.result().numpy(), 2.) weighted_mae_metric = metric_container.metrics[3] self.assertEqual(weighted_mae_metric.name, 'out2_weighted_mae') self.assertEqual(weighted_mae_metric.result().numpy(), 2.)
def test_metric_dict(self): metric_container = compile_utils.MetricsContainer( metrics={"out1": "mse", "out2": "mae"}, weighted_metrics={"out1": "mse", "out2": "mae"}, ) y_t = {"out1": tf.ones((10, 1)), "out2": tf.zeros((10, 1))} y_p = {"out1": tf.ones((10, 1)), "out2": 2 * tf.ones((10, 1))} sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, "out1_mse") self.assertEqual(mse_metric.result().numpy(), 0.0) weighted_mse_metric = metric_container.metrics[1] self.assertEqual(weighted_mse_metric.name, "out1_weighted_mse") self.assertEqual(weighted_mse_metric.result().numpy(), 0.0) mae_metric = metric_container.metrics[2] self.assertEqual(mae_metric.name, "out2_mae") self.assertEqual(mae_metric.result().numpy(), 2.0) weighted_mae_metric = metric_container.metrics[3] self.assertEqual(weighted_mae_metric.name, "out2_weighted_mae") self.assertEqual(weighted_mae_metric.result().numpy(), 2.0) metric_container.reset_state() self.assertEqual(mse_metric.result().numpy(), 0.0) self.assertEqual(weighted_mse_metric.result().numpy(), 0.0) self.assertEqual(mae_metric.result().numpy(), 0.0) self.assertEqual(weighted_mae_metric.result().numpy(), 0.0)
def test_duplicated_metric_instance(self): mean_obj = metrics_mod.Mean() metric = mean_obj with self.assertRaisesRegex(ValueError, 'Found duplicated metrics'): compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric) # duplicated string should be fine metric = 'acc' compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric) # complicated structure metric = [mean_obj, 'acc'] weighted_metric = {'output1': mean_obj, 'output2': 'acc'} with self.assertRaisesRegex(ValueError, 'Found duplicated metrics'): compile_utils.MetricsContainer(metrics=metric, weighted_metrics=weighted_metric)
def test_duplicated_metric_instance(self): mean_obj = metrics_mod.Mean() metric = mean_obj with self.assertRaisesRegex(ValueError, "Found duplicated metrics"): compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric) # duplicated string should be fine metric = "acc" compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric) # complicated structure metric = [mean_obj, "acc"] weighted_metric = {"output1": mean_obj, "output2": "acc"} with self.assertRaisesRegex(ValueError, "Found duplicated metrics"): compile_utils.MetricsContainer(metrics=metric, weighted_metrics=weighted_metric)
def test_reset_state_existing_metric_before_built(self): metric = metrics_mod.Mean() metric.update_state([2.0, 4.0]) self.assertEqual(metric.result().numpy(), 3.0) metric_container = compile_utils.MetricsContainer(metric) metric_container.reset_state() self.assertEqual(metric.result().numpy(), 0.0)
def test_single_metric(self): metric_container = compile_utils.MetricsContainer('mse') y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 1) metric = metric_container.metrics[0] self.assertEqual(metric.name, 'mse') self.assertEqual(metric.result().numpy(), 1.)
def test_broadcast_metrics_to_dict(self): metric_container = compile_utils.MetricsContainer(metrics=['mae']) y_p = {'output': tf.convert_to_tensor([[0], [1], [2]])} y_t = {'output': tf.convert_to_tensor([[1], [2], [3]])} metric_container.update_state(y_t, y_p) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 1.)
def test_broadcast_metrics_to_dict_with_output_names(self): metric_container = compile_utils.MetricsContainer( metrics=["mae"], output_names=["output"]) y_p = tf.convert_to_tensor([[0], [1], [2]]) y_t = {"output": tf.convert_to_tensor([[1], [2], [3]])} metric_container.update_state(y_t, y_p) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, "mae") self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_no_input_mutation(self): metric = {"a": "mae"} metric_container = compile_utils.MetricsContainer(metric) y_t = {"a": tf.zeros((10, 1))} y_p = {"a": tf.ones((10, 1)), "b": tf.zeros((10, 1))} metric_container.update_state(y_t, y_p) self.assertLen(metric, 1) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_loss_class_as_metric_with_distribution(self): distribution = tf.distribute.OneDeviceStrategy('/device:CPU:0') with distribution.scope(): metric_container = compile_utils.MetricsContainer( losses_mod.MeanSquaredError()) y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 1) metric = metric_container.metrics[0] self.assertEqual(metric.name, 'mean_squared_error') self.assertEqual(metric.result().numpy(), 1.)
def test_list_of_metrics_one_output(self): metric_container = compile_utils.MetricsContainer(['mse', 'mae']) y_t, y_p = 2 * tf.ones((10, 5)), tf.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 2) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, 'mse') self.assertEqual(mse_metric.result().numpy(), 4.) mae_metric = metric_container.metrics[1] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 2.)
def test_metric_partial_dict_with_output_names(self): metric_container = compile_utils.MetricsContainer( {'out2': 'mae'}, output_names=['out1', 'out2']) y_t = [tf.ones((10, 1)), tf.zeros((10, 1))] y_p = [tf.ones((10, 1)), tf.ones((10, 1))] sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 1) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'out2_mae') self.assertEqual(mae_metric.result().numpy(), 1.)
def test_list_of_metrics_list_of_outputs(self): metric_container = compile_utils.MetricsContainer( metrics=["mse", "mae"], # Should broadcast to both outputs. weighted_metrics=["accuracy"], ) # Should broadcast to both outputs. y_t = [tf.ones((10, 1)), tf.zeros((10, 1))] y_p = [tf.ones((10, 1)), 2 * tf.ones((10, 1))] sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 6) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, "output_1_mse") self.assertEqual(mse_metric.result().numpy(), 0.0) mse_metric = metric_container.metrics[1] self.assertEqual(mse_metric.name, "output_1_mae") self.assertEqual(mse_metric.result().numpy(), 0.0) acc_metric_1 = metric_container.metrics[2] self.assertEqual(acc_metric_1.name, "output_1_accuracy") self.assertEqual(acc_metric_1.result().numpy(), 1.0) self.assertEqual(acc_metric_1._fn, metrics_mod.binary_accuracy) mae_metric = metric_container.metrics[3] self.assertEqual(mae_metric.name, "output_2_mse") self.assertEqual(mae_metric.result().numpy(), 4.0) mae_metric = metric_container.metrics[4] self.assertEqual(mae_metric.name, "output_2_mae") self.assertEqual(mae_metric.result().numpy(), 2.0) acc_metric_2 = metric_container.metrics[5] self.assertEqual(acc_metric_2.name, "output_2_accuracy") self.assertEqual(acc_metric_2.result().numpy(), 0.0) self.assertEqual(acc_metric_2._fn, metrics_mod.binary_accuracy) weighted_metrics = metric_container.weighted_metrics self.assertLen(weighted_metrics, 2) self.assertEqual(weighted_metrics[0].name, "output_1_accuracy") self.assertEqual(weighted_metrics[1].name, "output_2_accuracy") unweighted_metrics = metric_container.unweighted_metrics self.assertLen(unweighted_metrics, 4) self.assertEqual(unweighted_metrics[0].name, "output_1_mse") self.assertEqual(unweighted_metrics[1].name, "output_1_mae") self.assertEqual(unweighted_metrics[2].name, "output_2_mse") self.assertEqual(unweighted_metrics[3].name, "output_2_mae")
def test_metric_partial_dict_with_nones(self): metric_container = compile_utils.MetricsContainer( {"out1": None, "out2": "mae"} ) y_t = {"out1": tf.ones((10, 1)), "out2": tf.zeros((10, 1))} y_p = {"out1": tf.ones((10, 1)), "out2": tf.ones((10, 1))} sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 1) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, "out2_mae") self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_custom_metric_callables(self): def custom_metric_fn(y_true, y_pred): return tf.reduce_sum(y_true - y_pred) class CustomMetricClass(object): def __call__(self, y_true, y_pred): return tf.reduce_sum(y_true - y_pred) metric_container = compile_utils.MetricsContainer( [custom_metric_fn, CustomMetricClass()]) y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0].name, 'custom_metric_fn') self.assertEqual(metric_container.metrics[1].name, 'custom_metric_class')
def test_metrics_masking(self): metrics_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mse']) y_p = tf.constant([[[1], [1]], [[0], [0]]], dtype=tf.float32) y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32) y_p._keras_mask = tf.constant([[1, 1], [0, 0]], dtype=tf.float32) metrics_container.update_state(y_t, y_p) self.assertLen(metrics_container.metrics, 2) mae_metric = metrics_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertAlmostEqual(mae_metric.result().numpy(), 0) weighted_mae_metric = metrics_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'mse') self.assertAlmostEqual(weighted_mae_metric.result().numpy(), 0)
def test_metrics_sample_weight(self): metrics_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mse']) y_p = tf.constant([[[1], [1]], [[0], [1]]], dtype=tf.float32) y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32) sw = tf.constant([[.2, .3], [.5, 0]], dtype=tf.float32) metrics_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metrics_container.metrics, 2) mae_metric = metrics_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertAlmostEqual(mae_metric.result().numpy(), .25) # 1 / 4 weighted_mae_metric = metrics_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'mse') self.assertAlmostEqual(weighted_mae_metric.result().numpy(), .5) # .5 / 1
def test_metric_weighting(self): metric_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mae']) y_t = tf.convert_to_tensor([[0], [3], [0]]) y_p = tf.convert_to_tensor([[0], [0], [0]]) sw = tf.convert_to_tensor([[1], [0], [1]]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 2) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 1.) weighted_mae_metric = metric_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'weighted_mae') self.assertEqual(weighted_mae_metric.result().numpy(), 0.)
def test_metrics_masking_sample_weight(self): metrics_container = compile_utils.MetricsContainer( metrics=["mae"], weighted_metrics=["mse"]) y_p = tf.constant([[[1], [1]], [[0], [1]]], dtype=tf.float32) y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32) sw = tf.constant([[0.3, 0.2], [0.2, 0.3]], dtype=tf.float32) y_p._keras_mask = tf.constant([[1, 0], [1, 0]], dtype=tf.float32) metrics_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metrics_container.metrics, 2) mae_metric = metrics_container.metrics[0] self.assertEqual(mae_metric.name, "mae") self.assertAlmostEqual(mae_metric.result().numpy(), 0.5) # 1 / .5 weighted_mae_metric = metrics_container.metrics[1] self.assertEqual(weighted_mae_metric.name, "mse") self.assertAlmostEqual(weighted_mae_metric.result().numpy(), 0.2 / 0.5)
def test_nested_structure(self): metric_container = compile_utils.MetricsContainer( metrics={ 'b': ['mse', None], 'a': 'mae' }, weighted_metrics={ 'b': [None, None], 'a': 'mse' }) y_t = { 'b': [2 * tf.ones((10, 1)), tf.zeros((10, 1))], 'a': tf.zeros((10, 1)) } y_p = { 'b': [tf.zeros((10, 1)), tf.zeros((10, 1))], 'a': tf.ones((10, 1)) } sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 3) a_mae_metric = metric_container.metrics[0] self.assertEqual(a_mae_metric.name, 'a_mae') self.assertEqual(a_mae_metric.result().numpy(), 1.) weighted_a_mae_metric = metric_container.metrics[1] self.assertEqual(weighted_a_mae_metric.name, 'a_mse') self.assertEqual(weighted_a_mae_metric.result().numpy(), 1.) b_1_mse_metric = metric_container.metrics[2] self.assertEqual(b_1_mse_metric.name, 'b_1_mse') self.assertEqual(b_1_mse_metric.result().numpy(), 4.)