def test_accuracy(self):
metric_container = compile_utils.MetricsContainer('accuracy')
y_t, y_p = tf.ones((10, 1)), tf.ones((10, 1))
metric_container.update_state(y_t, y_p)
self.assertEqual(metric_container.metrics[0]._fn,
metrics_mod.binary_accuracy)
metric_container = compile_utils.MetricsContainer('Accuracy')
y_t, y_p = tf.ones((10, 1)), tf.ones((10, 1))
metric_container.update_state(y_t, y_p)
self.assertEqual(metric_container.metrics[0]._fn,
metrics_mod.binary_accuracy)
metric_container = compile_utils.MetricsContainer('accuracy')
y_t, y_p = tf.ones((10, 1)), tf.ones((10, 20))
self.assertEqual(y_p.shape.as_list()[-1], 20)
metric_container.update_state(y_t, y_p)
self.assertEqual(metric_container.metrics[0]._fn,
metrics_mod.sparse_categorical_accuracy)
metric_container = compile_utils.MetricsContainer('accuracy')
y_t, y_p = tf.ones((10, 20)), tf.ones((10, 20))
metric_container.update_state(y_t, y_p)
self.assertEqual(metric_container.metrics[0]._fn,
metrics_mod.categorical_accuracy)
def test_nested_structure(self):
metric_container = compile_utils.MetricsContainer(
metrics={"b": ["mse", None], "a": "mae"},
weighted_metrics={"b": [None, None], "a": "mse"},
)
y_t = {
"b": [2 * tf.ones((10, 1)), tf.zeros((10, 1))],
"a": tf.zeros((10, 1)),
}
y_p = {
"b": [tf.zeros((10, 1)), tf.zeros((10, 1))],
"a": tf.ones((10, 1)),
}
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 3)
a_mae_metric = metric_container.metrics[0]
self.assertEqual(a_mae_metric.name, "a_mae")
self.assertEqual(a_mae_metric.result().numpy(), 1.0)
weighted_a_mae_metric = metric_container.metrics[1]
self.assertEqual(weighted_a_mae_metric.name, "a_mse")
self.assertEqual(weighted_a_mae_metric.result().numpy(), 1.0)
b_1_mse_metric = metric_container.metrics[2]
self.assertEqual(b_1_mse_metric.name, "b_1_mse")
self.assertEqual(b_1_mse_metric.result().numpy(), 4.0)
def test_missing_label_with_no_metrics(self):
# It's ok to exclude a label if that label has no
# losses or metrics associated with it.
metric_container = compile_utils.MetricsContainer(metrics={
'output1': 'mae',
'output3': 'mse'
})
y_p = {
'output1': tf.convert_to_tensor([[0], [1], [2]]),
'output2': tf.convert_to_tensor([[3], [4], [5]]),
'output3': tf.convert_to_tensor([[6], [7], [8]])
}
y_t = {
'output1': tf.convert_to_tensor([[1], [2], [3]]),
'output3': tf.convert_to_tensor([[4], [5], [6]])
}
metric_container.update_state(y_t, y_p)
self.assertLen(metric_container.metrics, 2)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, 'output1_mae')
self.assertEqual(mae_metric.result().numpy(), 1.)
mse_metric = metric_container.metrics[1]
self.assertEqual(mse_metric.name, 'output3_mse')
self.assertEqual(mse_metric.result().numpy(), 4.)
def test_metric_dict(self):
metric_container = compile_utils.MetricsContainer(metrics={
'out1': 'mse',
'out2': 'mae'
},
weighted_metrics={
'out1': 'mse',
'out2': 'mae'
})
y_t = {'out1': tf.ones((10, 1)), 'out2': tf.zeros((10, 1))}
y_p = {'out1': tf.ones((10, 1)), 'out2': 2 * tf.ones((10, 1))}
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
mse_metric = metric_container.metrics[0]
self.assertEqual(mse_metric.name, 'out1_mse')
self.assertEqual(mse_metric.result().numpy(), 0.)
weighted_mse_metric = metric_container.metrics[1]
self.assertEqual(weighted_mse_metric.name, 'out1_weighted_mse')
self.assertEqual(weighted_mse_metric.result().numpy(), 0.)
mae_metric = metric_container.metrics[2]
self.assertEqual(mae_metric.name, 'out2_mae')
self.assertEqual(mae_metric.result().numpy(), 2.)
weighted_mae_metric = metric_container.metrics[3]
self.assertEqual(weighted_mae_metric.name, 'out2_weighted_mae')
self.assertEqual(weighted_mae_metric.result().numpy(), 2.)
def test_metric_dict(self):
metric_container = compile_utils.MetricsContainer(
metrics={"out1": "mse", "out2": "mae"},
weighted_metrics={"out1": "mse", "out2": "mae"},
)
y_t = {"out1": tf.ones((10, 1)), "out2": tf.zeros((10, 1))}
y_p = {"out1": tf.ones((10, 1)), "out2": 2 * tf.ones((10, 1))}
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
mse_metric = metric_container.metrics[0]
self.assertEqual(mse_metric.name, "out1_mse")
self.assertEqual(mse_metric.result().numpy(), 0.0)
weighted_mse_metric = metric_container.metrics[1]
self.assertEqual(weighted_mse_metric.name, "out1_weighted_mse")
self.assertEqual(weighted_mse_metric.result().numpy(), 0.0)
mae_metric = metric_container.metrics[2]
self.assertEqual(mae_metric.name, "out2_mae")
self.assertEqual(mae_metric.result().numpy(), 2.0)
weighted_mae_metric = metric_container.metrics[3]
self.assertEqual(weighted_mae_metric.name, "out2_weighted_mae")
self.assertEqual(weighted_mae_metric.result().numpy(), 2.0)
metric_container.reset_state()
self.assertEqual(mse_metric.result().numpy(), 0.0)
self.assertEqual(weighted_mse_metric.result().numpy(), 0.0)
self.assertEqual(mae_metric.result().numpy(), 0.0)
self.assertEqual(weighted_mae_metric.result().numpy(), 0.0)
def test_duplicated_metric_instance(self):
mean_obj = metrics_mod.Mean()
metric = mean_obj
with self.assertRaisesRegex(ValueError, 'Found duplicated metrics'):
compile_utils.MetricsContainer(metrics=metric,
weighted_metrics=metric)
# duplicated string should be fine
metric = 'acc'
compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric)
# complicated structure
metric = [mean_obj, 'acc']
weighted_metric = {'output1': mean_obj, 'output2': 'acc'}
with self.assertRaisesRegex(ValueError, 'Found duplicated metrics'):
compile_utils.MetricsContainer(metrics=metric,
weighted_metrics=weighted_metric)
def test_duplicated_metric_instance(self):
mean_obj = metrics_mod.Mean()
metric = mean_obj
with self.assertRaisesRegex(ValueError, "Found duplicated metrics"):
compile_utils.MetricsContainer(metrics=metric,
weighted_metrics=metric)
# duplicated string should be fine
metric = "acc"
compile_utils.MetricsContainer(metrics=metric, weighted_metrics=metric)
# complicated structure
metric = [mean_obj, "acc"]
weighted_metric = {"output1": mean_obj, "output2": "acc"}
with self.assertRaisesRegex(ValueError, "Found duplicated metrics"):
compile_utils.MetricsContainer(metrics=metric,
weighted_metrics=weighted_metric)
def test_reset_state_existing_metric_before_built(self):
metric = metrics_mod.Mean()
metric.update_state([2.0, 4.0])
self.assertEqual(metric.result().numpy(), 3.0)
metric_container = compile_utils.MetricsContainer(metric)
metric_container.reset_state()
self.assertEqual(metric.result().numpy(), 0.0)
def test_single_metric(self):
metric_container = compile_utils.MetricsContainer('mse')
y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5))
metric_container.update_state(y_t, y_p)
self.assertLen(metric_container.metrics, 1)
metric = metric_container.metrics[0]
self.assertEqual(metric.name, 'mse')
self.assertEqual(metric.result().numpy(), 1.)
def test_broadcast_metrics_to_dict(self):
metric_container = compile_utils.MetricsContainer(metrics=['mae'])
y_p = {'output': tf.convert_to_tensor([[0], [1], [2]])}
y_t = {'output': tf.convert_to_tensor([[1], [2], [3]])}
metric_container.update_state(y_t, y_p)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, 'mae')
self.assertEqual(mae_metric.result().numpy(), 1.)
def test_broadcast_metrics_to_dict_with_output_names(self):
metric_container = compile_utils.MetricsContainer(
metrics=["mae"], output_names=["output"])
y_p = tf.convert_to_tensor([[0], [1], [2]])
y_t = {"output": tf.convert_to_tensor([[1], [2], [3]])}
metric_container.update_state(y_t, y_p)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, "mae")
self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_no_input_mutation(self):
metric = {"a": "mae"}
metric_container = compile_utils.MetricsContainer(metric)
y_t = {"a": tf.zeros((10, 1))}
y_p = {"a": tf.ones((10, 1)), "b": tf.zeros((10, 1))}
metric_container.update_state(y_t, y_p)
self.assertLen(metric, 1)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_loss_class_as_metric_with_distribution(self):
distribution = tf.distribute.OneDeviceStrategy('/device:CPU:0')
with distribution.scope():
metric_container = compile_utils.MetricsContainer(
losses_mod.MeanSquaredError())
y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5))
metric_container.update_state(y_t, y_p)
self.assertLen(metric_container.metrics, 1)
metric = metric_container.metrics[0]
self.assertEqual(metric.name, 'mean_squared_error')
self.assertEqual(metric.result().numpy(), 1.)
def test_list_of_metrics_one_output(self):
metric_container = compile_utils.MetricsContainer(['mse', 'mae'])
y_t, y_p = 2 * tf.ones((10, 5)), tf.zeros((10, 5))
metric_container.update_state(y_t, y_p)
self.assertLen(metric_container.metrics, 2)
mse_metric = metric_container.metrics[0]
self.assertEqual(mse_metric.name, 'mse')
self.assertEqual(mse_metric.result().numpy(), 4.)
mae_metric = metric_container.metrics[1]
self.assertEqual(mae_metric.name, 'mae')
self.assertEqual(mae_metric.result().numpy(), 2.)
def test_metric_partial_dict_with_output_names(self):
metric_container = compile_utils.MetricsContainer(
{'out2': 'mae'}, output_names=['out1', 'out2'])
y_t = [tf.ones((10, 1)), tf.zeros((10, 1))]
y_p = [tf.ones((10, 1)), tf.ones((10, 1))]
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 1)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, 'out2_mae')
self.assertEqual(mae_metric.result().numpy(), 1.)
def test_list_of_metrics_list_of_outputs(self):
metric_container = compile_utils.MetricsContainer(
metrics=["mse", "mae"], # Should broadcast to both outputs.
weighted_metrics=["accuracy"],
) # Should broadcast to both outputs.
y_t = [tf.ones((10, 1)), tf.zeros((10, 1))]
y_p = [tf.ones((10, 1)), 2 * tf.ones((10, 1))]
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 6)
mse_metric = metric_container.metrics[0]
self.assertEqual(mse_metric.name, "output_1_mse")
self.assertEqual(mse_metric.result().numpy(), 0.0)
mse_metric = metric_container.metrics[1]
self.assertEqual(mse_metric.name, "output_1_mae")
self.assertEqual(mse_metric.result().numpy(), 0.0)
acc_metric_1 = metric_container.metrics[2]
self.assertEqual(acc_metric_1.name, "output_1_accuracy")
self.assertEqual(acc_metric_1.result().numpy(), 1.0)
self.assertEqual(acc_metric_1._fn, metrics_mod.binary_accuracy)
mae_metric = metric_container.metrics[3]
self.assertEqual(mae_metric.name, "output_2_mse")
self.assertEqual(mae_metric.result().numpy(), 4.0)
mae_metric = metric_container.metrics[4]
self.assertEqual(mae_metric.name, "output_2_mae")
self.assertEqual(mae_metric.result().numpy(), 2.0)
acc_metric_2 = metric_container.metrics[5]
self.assertEqual(acc_metric_2.name, "output_2_accuracy")
self.assertEqual(acc_metric_2.result().numpy(), 0.0)
self.assertEqual(acc_metric_2._fn, metrics_mod.binary_accuracy)
weighted_metrics = metric_container.weighted_metrics
self.assertLen(weighted_metrics, 2)
self.assertEqual(weighted_metrics[0].name, "output_1_accuracy")
self.assertEqual(weighted_metrics[1].name, "output_2_accuracy")
unweighted_metrics = metric_container.unweighted_metrics
self.assertLen(unweighted_metrics, 4)
self.assertEqual(unweighted_metrics[0].name, "output_1_mse")
self.assertEqual(unweighted_metrics[1].name, "output_1_mae")
self.assertEqual(unweighted_metrics[2].name, "output_2_mse")
self.assertEqual(unweighted_metrics[3].name, "output_2_mae")
def test_metric_partial_dict_with_nones(self):
metric_container = compile_utils.MetricsContainer(
{"out1": None, "out2": "mae"}
)
y_t = {"out1": tf.ones((10, 1)), "out2": tf.zeros((10, 1))}
y_p = {"out1": tf.ones((10, 1)), "out2": tf.ones((10, 1))}
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 1)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, "out2_mae")
self.assertEqual(mae_metric.result().numpy(), 1.0)
def test_custom_metric_callables(self):
def custom_metric_fn(y_true, y_pred):
return tf.reduce_sum(y_true - y_pred)
class CustomMetricClass(object):
def __call__(self, y_true, y_pred):
return tf.reduce_sum(y_true - y_pred)
metric_container = compile_utils.MetricsContainer(
[custom_metric_fn, CustomMetricClass()])
y_t, y_p = tf.ones((10, 5)), tf.zeros((10, 5))
metric_container.update_state(y_t, y_p)
self.assertEqual(metric_container.metrics[0].name, 'custom_metric_fn')
self.assertEqual(metric_container.metrics[1].name,
'custom_metric_class')
def test_metrics_masking(self):
metrics_container = compile_utils.MetricsContainer(
metrics=['mae'], weighted_metrics=['mse'])
y_p = tf.constant([[[1], [1]], [[0], [0]]], dtype=tf.float32)
y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32)
y_p._keras_mask = tf.constant([[1, 1], [0, 0]], dtype=tf.float32)
metrics_container.update_state(y_t, y_p)
self.assertLen(metrics_container.metrics, 2)
mae_metric = metrics_container.metrics[0]
self.assertEqual(mae_metric.name, 'mae')
self.assertAlmostEqual(mae_metric.result().numpy(), 0)
weighted_mae_metric = metrics_container.metrics[1]
self.assertEqual(weighted_mae_metric.name, 'mse')
self.assertAlmostEqual(weighted_mae_metric.result().numpy(), 0)
def test_metrics_sample_weight(self):
metrics_container = compile_utils.MetricsContainer(
metrics=['mae'], weighted_metrics=['mse'])
y_p = tf.constant([[[1], [1]], [[0], [1]]], dtype=tf.float32)
y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32)
sw = tf.constant([[.2, .3], [.5, 0]], dtype=tf.float32)
metrics_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metrics_container.metrics, 2)
mae_metric = metrics_container.metrics[0]
self.assertEqual(mae_metric.name, 'mae')
self.assertAlmostEqual(mae_metric.result().numpy(), .25) # 1 / 4
weighted_mae_metric = metrics_container.metrics[1]
self.assertEqual(weighted_mae_metric.name, 'mse')
self.assertAlmostEqual(weighted_mae_metric.result().numpy(), .5) # .5 / 1
def test_metric_weighting(self):
metric_container = compile_utils.MetricsContainer(
metrics=['mae'], weighted_metrics=['mae'])
y_t = tf.convert_to_tensor([[0], [3], [0]])
y_p = tf.convert_to_tensor([[0], [0], [0]])
sw = tf.convert_to_tensor([[1], [0], [1]])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 2)
mae_metric = metric_container.metrics[0]
self.assertEqual(mae_metric.name, 'mae')
self.assertEqual(mae_metric.result().numpy(), 1.)
weighted_mae_metric = metric_container.metrics[1]
self.assertEqual(weighted_mae_metric.name, 'weighted_mae')
self.assertEqual(weighted_mae_metric.result().numpy(), 0.)
def test_metrics_masking_sample_weight(self):
metrics_container = compile_utils.MetricsContainer(
metrics=["mae"], weighted_metrics=["mse"])
y_p = tf.constant([[[1], [1]], [[0], [1]]], dtype=tf.float32)
y_t = tf.constant([[[1], [1]], [[1], [1]]], dtype=tf.float32)
sw = tf.constant([[0.3, 0.2], [0.2, 0.3]], dtype=tf.float32)
y_p._keras_mask = tf.constant([[1, 0], [1, 0]], dtype=tf.float32)
metrics_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metrics_container.metrics, 2)
mae_metric = metrics_container.metrics[0]
self.assertEqual(mae_metric.name, "mae")
self.assertAlmostEqual(mae_metric.result().numpy(), 0.5) # 1 / .5
weighted_mae_metric = metrics_container.metrics[1]
self.assertEqual(weighted_mae_metric.name, "mse")
self.assertAlmostEqual(weighted_mae_metric.result().numpy(), 0.2 / 0.5)
def test_nested_structure(self):
metric_container = compile_utils.MetricsContainer(
metrics={
'b': ['mse', None],
'a': 'mae'
},
weighted_metrics={
'b': [None, None],
'a': 'mse'
})
y_t = {
'b': [2 * tf.ones((10, 1)),
tf.zeros((10, 1))],
'a': tf.zeros((10, 1))
}
y_p = {
'b': [tf.zeros((10, 1)),
tf.zeros((10, 1))],
'a': tf.ones((10, 1))
}
sw = tf.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
metric_container.update_state(y_t, y_p, sample_weight=sw)
self.assertLen(metric_container.metrics, 3)
a_mae_metric = metric_container.metrics[0]
self.assertEqual(a_mae_metric.name, 'a_mae')
self.assertEqual(a_mae_metric.result().numpy(), 1.)
weighted_a_mae_metric = metric_container.metrics[1]
self.assertEqual(weighted_a_mae_metric.name, 'a_mse')
self.assertEqual(weighted_a_mae_metric.result().numpy(), 1.)
b_1_mse_metric = metric_container.metrics[2]
self.assertEqual(b_1_mse_metric.name, 'b_1_mse')
self.assertEqual(b_1_mse_metric.result().numpy(), 4.)
