def test_accuracy(self): metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = array_ops.ones((10, 1)), array_ops.ones((10, 1)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.binary_accuracy) metric_container = compile_utils.MetricsContainer('Accuracy') y_t, y_p = array_ops.ones((10, 1)), array_ops.ones((10, 1)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.binary_accuracy) metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = array_ops.ones((10, 1)), array_ops.ones((10, 20)) self.assertEqual(y_p.shape.as_list()[-1], 20) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.sparse_categorical_accuracy) metric_container = compile_utils.MetricsContainer('accuracy') y_t, y_p = array_ops.ones((10, 20)), array_ops.ones((10, 20)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0]._fn, metrics_mod.categorical_accuracy)
def test_missing_label_with_no_metrics(self): # It's ok to exclude a label if that label has no # losses or metrics associated with it. metric_container = compile_utils.MetricsContainer(metrics={ 'output1': 'mae', 'output3': 'mse' }) y_p = { 'output1': ops.convert_to_tensor_v2_with_dispatch([[0], [1], [2]]), 'output2': ops.convert_to_tensor_v2_with_dispatch([[3], [4], [5]]), 'output3': ops.convert_to_tensor_v2_with_dispatch([[6], [7], [8]]) } y_t = { 'output1': ops.convert_to_tensor_v2_with_dispatch([[1], [2], [3]]), 'output3': ops.convert_to_tensor_v2_with_dispatch([[4], [5], [6]]) } metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 2) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'output1_mae') self.assertEqual(mae_metric.result().numpy(), 1.) mse_metric = metric_container.metrics[1] self.assertEqual(mse_metric.name, 'output3_mse') self.assertEqual(mse_metric.result().numpy(), 4.)
def test_metric_dict(self): metric_container = compile_utils.MetricsContainer( metrics={ 'out1': 'mse', 'out2': 'mae' }, weighted_metrics={ 'out1': 'mse', 'out2': 'mae' }) y_t = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1))} y_p = {'out1': array_ops.ones((10, 1)), 'out2': 2 * array_ops.ones((10, 1))} sw = ops.convert_to_tensor_v2([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, 'out1_mse') self.assertEqual(mse_metric.result().numpy(), 0.) weighted_mse_metric = metric_container.metrics[1] self.assertEqual(weighted_mse_metric.name, 'out1_weighted_mse') self.assertEqual(weighted_mse_metric.result().numpy(), 0.) mae_metric = metric_container.metrics[2] self.assertEqual(mae_metric.name, 'out2_mae') self.assertEqual(mae_metric.result().numpy(), 2.) weighted_mae_metric = metric_container.metrics[3] self.assertEqual(weighted_mae_metric.name, 'out2_weighted_mae') self.assertEqual(weighted_mae_metric.result().numpy(), 2.)
def compile(self, optimizers, loss_fns, metrics=None): super(Model_w_self_backpropagated_branches, self).compile() assert len(optimizers)==len(loss_fns) assert len(optimizers)==self.num_models if metrics is None: user_metrics = [None]*self.num_models elif isinstance(metrics,dict): user_metrics = [None]*self.num_models for k,v in metrics.items(): user_metrics[k] = v elif isinstance(metrics,list): assert len(metrics)==self.num_models user_metrics = metrics else: raise ValueError("Metrics must be either a full-list or" " a sparse version dictionary map position to metric.") with self.distribute_strategy.scope(): self._validate_compile(optimizers, user_metrics) self.optimizers = self._get_optimizer(optimizers) self.compiled_loss = [compile_utils.LossesContainer( loss, output_names=self.output_names) for loss in loss_fns] self.compiled_metrics = [compile_utils.MetricsContainer( metric, output_names=self.output_names) for metric in user_metrics]
def test_list_of_metrics_list_of_outputs(self): metric_container = compile_utils.MetricsContainer( metrics=['mse', 'mae'], weighted_metrics=['accuracy']) # Should broadcast to both outputs. y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), 2 * array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 4) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, 'output_1_mse') self.assertEqual(mse_metric.result().numpy(), 0.) mae_metric = metric_container.metrics[1] self.assertEqual(mae_metric.name, 'output_2_mae') self.assertEqual(mae_metric.result().numpy(), 2.) acc_metric_1 = metric_container.metrics[2] self.assertEqual(acc_metric_1.name, 'output_1_accuracy') self.assertEqual(acc_metric_1.result().numpy(), 1.) self.assertEqual(acc_metric_1._fn, metrics_mod.binary_accuracy) acc_metric_2 = metric_container.metrics[3] self.assertEqual(acc_metric_2.name, 'output_2_accuracy') self.assertEqual(acc_metric_2.result().numpy(), 0.) self.assertEqual(acc_metric_2._fn, metrics_mod.binary_accuracy)
def test_reset_state_existing_metric_before_built(self): metric = metrics_mod.Mean() metric.update_state([2.0, 4.0]) self.assertEqual(metric.result().numpy(), 3.0) metric_container = compile_utils.MetricsContainer(metric) metric_container.reset_state() self.assertEqual(metric.result().numpy(), 0.0)
def test_single_metric(self): metric_container = compile_utils.MetricsContainer('mse') y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 1) metric = metric_container.metrics[0] self.assertEqual(metric.name, 'mse') self.assertEqual(metric.result().numpy(), 1.)
def test_broadcast_metrics_to_dict(self): metric_container = compile_utils.MetricsContainer(metrics=['mae']) y_p = {'output': ops.convert_to_tensor([[0], [1], [2]])} y_t = {'output': ops.convert_to_tensor([[1], [2], [3]])} metric_container.update_state(y_t, y_p) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 1.)
def compile(self, optimizer_gen='rmsprop', optimizer_disc='rmsprop', loss_gen=None, loss_disc=None, metrics_gen=None, metrics_disc=None): self.gen_optimizer = optimizers.get(optimizer_gen) self.disc_optimizer = optimizers.get(optimizer_disc) self.gen_losses_container = compile_utils.LossesContainer(loss_gen) if loss_gen else None self.disc_losses_container = compile_utils.LossesContainer(loss_disc) if loss_disc else None self.gen_metrics_container = compile_utils.MetricsContainer(metrics_gen) if metrics_gen else None self.disc_metrics_container = compile_utils.MetricsContainer(metrics_disc) if metrics_disc else None self.m_formatter = Metrics_Formatter( gen_loss_name=_get_tag_name(self.gen_losses_container), disc_loss_name=_get_tag_name(self.disc_losses_container), gen_metric_name=_get_tag_name(self.gen_metrics_container), disc_metric_name=_get_tag_name(self.disc_metrics_container), num_format='.03f' )
def test_loss_class_as_metric_with_distribution(self): distribution = one_device_strategy.OneDeviceStrategy('/device:CPU:0') with distribution.scope(): metric_container = compile_utils.MetricsContainer( losses_mod.MeanSquaredError()) y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 1) metric = metric_container.metrics[0] self.assertEqual(metric.name, 'mean_squared_error') self.assertEqual(metric.result().numpy(), 1.)
def test_list_of_metrics_one_output(self): metric_container = compile_utils.MetricsContainer(['mse', 'mae']) y_t, y_p = 2 * array_ops.ones((10, 5)), array_ops.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertLen(metric_container.metrics, 2) mse_metric = metric_container.metrics[0] self.assertEqual(mse_metric.name, 'mse') self.assertEqual(mse_metric.result().numpy(), 4.) mae_metric = metric_container.metrics[1] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 2.)
def test_metric_partial_dict_with_output_names(self): metric_container = compile_utils.MetricsContainer( {'out2': 'mae'}, output_names=['out1', 'out2']) y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 1) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'out2_mae') self.assertEqual(mae_metric.result().numpy(), 1.)
def test_custom_metric_callables(self): def custom_metric_fn(y_true, y_pred): return math_ops.reduce_sum(y_true - y_pred) class CustomMetricClass(object): def __call__(self, y_true, y_pred): return math_ops.reduce_sum(y_true - y_pred) metric_container = compile_utils.MetricsContainer( [custom_metric_fn, CustomMetricClass()]) y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) metric_container.update_state(y_t, y_p) self.assertEqual(metric_container.metrics[0].name, 'custom_metric_fn') self.assertEqual(metric_container.metrics[1].name, 'custom_metric_class')
def test_metric_partial_dict_with_nones(self): metric_container = compile_utils.MetricsContainer({ 'out1': None, 'out2': 'mae' }) y_t = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1))} y_p = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.ones((10, 1))} sw = ops.convert_to_tensor_v2_with_dispatch([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 1) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'out2_mae') self.assertEqual(mae_metric.result().numpy(), 1.)
def test_metrics_sample_weight(self): metrics_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mse']) y_p = constant_op.constant([[[1], [1]], [[0], [1]]], dtype=dtypes.float32) y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32) sw = constant_op.constant([[.2, .3], [.5, 0]], dtype=dtypes.float32) metrics_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metrics_container.metrics, 2) mae_metric = metrics_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertAlmostEqual(mae_metric.result().numpy(), .25) # 1 / 4 weighted_mae_metric = metrics_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'mse') self.assertAlmostEqual(weighted_mae_metric.result().numpy(), .5) # .5 / 1
def test_metric_weighting(self): metric_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mae']) y_t = ops.convert_to_tensor_v2_with_dispatch([[0], [3], [0]]) y_p = ops.convert_to_tensor_v2_with_dispatch([[0], [0], [0]]) sw = ops.convert_to_tensor_v2_with_dispatch([[1], [0], [1]]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 2) mae_metric = metric_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertEqual(mae_metric.result().numpy(), 1.) weighted_mae_metric = metric_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'weighted_mae') self.assertEqual(weighted_mae_metric.result().numpy(), 0.)
def test_metrics_masking(self): metrics_container = compile_utils.MetricsContainer( metrics=['mae'], weighted_metrics=['mse']) y_p = constant_op.constant([[[1], [1]], [[0], [0]]], dtype=dtypes.float32) y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32) y_p._keras_mask = constant_op.constant([[1, 1], [0, 0]], dtype=dtypes.float32) metrics_container.update_state(y_t, y_p) self.assertLen(metrics_container.metrics, 2) mae_metric = metrics_container.metrics[0] self.assertEqual(mae_metric.name, 'mae') self.assertAlmostEqual(mae_metric.result().numpy(), 0) weighted_mae_metric = metrics_container.metrics[1] self.assertEqual(weighted_mae_metric.name, 'mse') self.assertAlmostEqual(weighted_mae_metric.result().numpy(), 0)
def test_nested_structure(self): metric_container = compile_utils.MetricsContainer(metrics={ 'b': ['mse', None], 'a': 'mae' }, weighted_metrics={ 'b': [None, None], 'a': 'mse' }) y_t = { 'b': [2 * array_ops.ones((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.zeros((10, 1)) } y_p = { 'b': [array_ops.zeros((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.ones((10, 1)) } sw = ops.convert_to_tensor_v2_with_dispatch( [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) metric_container.update_state(y_t, y_p, sample_weight=sw) self.assertLen(metric_container.metrics, 3) a_mae_metric = metric_container.metrics[0] self.assertEqual(a_mae_metric.name, 'a_mae') self.assertEqual(a_mae_metric.result().numpy(), 1.) weighted_a_mae_metric = metric_container.metrics[1] self.assertEqual(weighted_a_mae_metric.name, 'a_mse') self.assertEqual(weighted_a_mae_metric.result().numpy(), 1.) b_1_mse_metric = metric_container.metrics[2] self.assertEqual(b_1_mse_metric.name, 'b_1_mse') self.assertEqual(b_1_mse_metric.result().numpy(), 4.)