def compile(self, d_optimizer, g_optimizer, g_loss_fn, d_loss_fn): super(ProgressiveGANTrainer, self).compile() self.d_optimizer = d_optimizer self.g_optimizer = g_optimizer self.g_loss_fn = compile_utils.LossesContainer(g_loss_fn) self.d_loss_fn = compile_utils.LossesContainer(d_loss_fn) if self.gradient_penalty: self.gradient_penalty_fn = compile_utils.LossesContainer(gradient_penalty)
def test_loss_dict_with_nones(self): loss_container = compile_utils.LossesContainer({ 'out1': None, 'out2': 'mae' }) y_t = { 'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1)) } y_p = { 'out1': array_ops.ones((10, 1)), 'out2': array_ops.ones((10, 1)) } sw = ops.convert_to_tensor_v2_with_dispatch( [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertEqual(total_loss.numpy(), 0.5) self.assertLen(loss_container.metrics, 2) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.5) out2_metric = loss_container.metrics[1] self.assertEqual(out2_metric.name, 'out2_loss') self.assertEqual(out2_metric.result().numpy(), 0.5)
def compile(self, optimizers, loss_fns, metrics=None): super(Model_w_self_backpropagated_branches, self).compile() assert len(optimizers)==len(loss_fns) assert len(optimizers)==self.num_models if metrics is None: user_metrics = [None]*self.num_models elif isinstance(metrics,dict): user_metrics = [None]*self.num_models for k,v in metrics.items(): user_metrics[k] = v elif isinstance(metrics,list): assert len(metrics)==self.num_models user_metrics = metrics else: raise ValueError("Metrics must be either a full-list or" " a sparse version dictionary map position to metric.") with self.distribute_strategy.scope(): self._validate_compile(optimizers, user_metrics) self.optimizers = self._get_optimizer(optimizers) self.compiled_loss = [compile_utils.LossesContainer( loss, output_names=self.output_names) for loss in loss_fns] self.compiled_metrics = [compile_utils.MetricsContainer( metric, output_names=self.output_names) for metric in user_metrics]
def _compile_losses(self, loss_functions): """ Splits the given loss_functions into the corresponding :attr:`_loss_functions' and :attr:`_loss_weights' lists. Loss functions are compiled into :class:`keras.compile_utils.LossesContainer` objects Parameters ---------- loss_functions: list A list of either a tuple of (:class:`keras.losses.Loss`, scalar weight) or just a :class:`keras.losses.Loss` function. If just the loss function is passed, then the weight is assumed to be 1.0 """ for loss_func in loss_functions: if isinstance(loss_func, tuple): assert len( loss_func ) == 2, "Tuple loss functions should contain 2 items" assert isinstance(loss_func[1], float), "weight should be a float" func, weight = loss_func else: func = loss_func weight = 1.0 self._loss_functions.append(compile_utils.LossesContainer(func)) self._loss_weights.append(weight) logger.debug("Compiled losses: (functions: %s, weights: %s", self._loss_functions, self._loss_weights)
def test_ragged_tensor_output(self): """ Ensure that ragged tensors can be passed as targets and predictions.""" def custom_loss_fn(y_true, y_pred): losses = ragged_functional_ops.map_flat_values( losses_mod.mse, y_true, y_pred) return math_ops.reduce_mean(losses) class CustomLossClass(object): def __call__(self, y_true, y_pred): losses = ragged_functional_ops.map_flat_values( losses_mod.mse, y_true, y_pred) return math_ops.reduce_mean(losses) loss_container = compile_utils.LossesContainer( [custom_loss_fn, CustomLossClass()]) v_t = constant_op.constant([[3., 4.], [1., 2.], [3., 5.]]) v_p = constant_op.constant([[3.1, 4.], [1., 2.], [3., 5.]]) y_t = array_ops.expand_dims( ragged_tensor.RaggedTensor.from_row_splits(v_t, [0, 2, 3]), 0) y_p = array_ops.expand_dims( ragged_tensor.RaggedTensor.from_row_splits(v_p, [0, 2, 3]), 0) loss_container(y_t, y_p) self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn')
def test_ragged_tensor_output(self): """Ensure that ragged tensors can be passed as targets and predictions.""" def custom_loss_fn(y_true, y_pred): """MSE supports RaggedTensors directly.""" return losses_mod.mse(y_true, y_pred) class CustomLossClass(losses_mod.Loss): """User defined loss function must implement RaggedTensor support.""" def call(self, y_true, y_pred): losses = ragged_functional_ops.map_flat_values( math_ops.squared_difference, y_true, y_pred) return math_ops.reduce_mean(losses) loss_container = compile_utils.LossesContainer( [custom_loss_fn, CustomLossClass()]) v_t = constant_op.constant([[3., 4.], [1., 2.], [3., 5.]]) v_p = constant_op.constant([[3.1, 4.], [1., 2.], [3., 5.]]) y_t = array_ops.expand_dims( ragged_tensor.RaggedTensor.from_row_splits(v_t, [0, 2, 3]), 0) y_p = array_ops.expand_dims( ragged_tensor.RaggedTensor.from_row_splits(v_p, [0, 2, 3]), 0) loss_container(y_t, y_p) self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn')
def test_missing_label_with_no_loss(self): # It's ok to exclude a label if that label has no # losses or metrics associated with it. loss_container = compile_utils.LossesContainer({ 'output1': 'mse', 'output3': 'mae' }) y_p = { 'output1': ops.convert_to_tensor_v2_with_dispatch([[0], [1], [2]]), 'output2': ops.convert_to_tensor_v2_with_dispatch([[3], [4], [5]]), 'output3': ops.convert_to_tensor_v2_with_dispatch([[6], [7], [8]]) } y_t = { 'output1': ops.convert_to_tensor_v2_with_dispatch([[1], [2], [3]]), 'output3': ops.convert_to_tensor_v2_with_dispatch([[4], [5], [6]]) } total_loss = loss_container(y_t, y_p) self.assertEqual(total_loss.numpy(), 3.) self.assertLen(loss_container.metrics, 3) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 3.) output_1_metric = loss_container.metrics[1] self.assertEqual(output_1_metric.name, 'output1_loss') self.assertEqual(output_1_metric.result().numpy(), 1.) output_3_metric = loss_container.metrics[2] self.assertEqual(output_3_metric.name, 'output3_loss') self.assertEqual(output_3_metric.result().numpy(), 2.)
def test_nested_structure(self): loss_container = compile_utils.LossesContainer( { 'b': ['mse', None], 'a': 'mae' }, loss_weights={ 'b': [0.5, 0], 'a': 1 }) y_t = { 'b': [array_ops.ones((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.zeros((10, 1)) } y_p = { 'b': [array_ops.zeros((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.ones((10, 1)) } sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss, output_losses = self.evaluate( loss_container(y_t, y_p, sample_weight=sw)) self.assertEqual(total_loss, 0.75) self.assertLen(output_losses, 2) self.assertEqual(output_losses[0], 0.5) self.assertEqual(output_losses[1], 0.25)
def test_loss_list(self): loss_container = compile_utils.LossesContainer(['mse', 'mae'], [1, 0.5]) y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor_v2_with_dispatch( [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertEqual(loss_container._output_names, ['output_1', 'output_2']) self.assertLen(loss_container._losses, 2) self.assertEqual(total_loss.numpy(), 0.25) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.25) output_1_metric = loss_container.metrics[1] self.assertEqual(output_1_metric.name, 'output_1_loss') self.assertEqual(output_1_metric.result().numpy(), 0) output_2_metric = loss_container.metrics[2] self.assertEqual(output_2_metric.name, 'output_2_loss') self.assertEqual(output_2_metric.result().numpy(), 0.5) loss_container.reset_state() self.assertEqual(loss_metric.result().numpy(), 0) self.assertEqual(output_1_metric.result().numpy(), 0) self.assertEqual(output_2_metric.result().numpy(), 0)
def test_single_loss(self): loss_container = compile_utils.LossesContainer('mse') y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) total_loss, output_losses = self.evaluate(loss_container(y_t, y_p)) self.assertTrue(loss_container._built) self.assertLen(loss_container._losses, 1) self.assertEqual(total_loss, 1.) self.assertLen(output_losses, 1)
def test_integer_dtypes(self): y_t = constant_op.constant([1, 9, 2, -5], shape=(2, 2)) y_p = constant_op.constant([4, 8, 12, 8], shape=(2, 2), dtype=dtypes.int64) def my_mae(labels, preds): self.assertEqual(labels.dtype, dtypes.int64) self.assertEqual(preds.dtype, dtypes.int64) return K.mean(math_ops.abs(preds - labels), axis=-1) loss_container = compile_utils.LossesContainer(my_mae) total_loss = loss_container(y_t, y_p) self.assertEqual(total_loss.dtype, dtypes.int64)
def compile(self, optimizer_gen='rmsprop', optimizer_disc='rmsprop', loss_gen=None, loss_disc=None, metrics_gen=None, metrics_disc=None): self.gen_optimizer = optimizers.get(optimizer_gen) self.disc_optimizer = optimizers.get(optimizer_disc) self.gen_losses_container = compile_utils.LossesContainer(loss_gen) if loss_gen else None self.disc_losses_container = compile_utils.LossesContainer(loss_disc) if loss_disc else None self.gen_metrics_container = compile_utils.MetricsContainer(metrics_gen) if metrics_gen else None self.disc_metrics_container = compile_utils.MetricsContainer(metrics_disc) if metrics_disc else None self.m_formatter = Metrics_Formatter( gen_loss_name=_get_tag_name(self.gen_losses_container), disc_loss_name=_get_tag_name(self.disc_losses_container), gen_metric_name=_get_tag_name(self.gen_metrics_container), disc_metric_name=_get_tag_name(self.disc_metrics_container), num_format='.03f' )
def test_single_loss(self): loss_container = compile_utils.LossesContainer('mse') y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) total_loss = loss_container(y_t, y_p) self.assertTrue(loss_container._built) self.assertLen(loss_container._losses, 1) self.assertEqual(total_loss.numpy(), 1.) self.assertLen(loss_container.metrics, 1) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 1.)
def test_loss_sample_weight(self): loss_container = compile_utils.LossesContainer('mae') y_p = constant_op.constant([[[1], [1]], [[0], [0]]], dtype=dtypes.float32) y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32) sw = constant_op.constant([[.2, .3], [.5, 0]], dtype=dtypes.float32) total_loss = loss_container(y_t, y_p, sample_weight=sw) # (0 * .2 + 0 * .3 + 1 * .5 + 1 * 0) / 4 self.assertAlmostEqual(total_loss.numpy(), .125) self.assertLen(loss_container.metrics, 1) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertAlmostEqual(loss_metric.result().numpy(), .125)
def test_loss_partial_dict_with_output_names(self): loss_container = compile_utils.LossesContainer( {'out2': 'mae'}, {'out2': 1.}, output_names=['out1', 'out2']) y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss, output_losses = self.evaluate( loss_container(y_t, y_p, sample_weight=sw)) self.assertEqual(total_loss, 0.5) self.assertLen(output_losses, 1) self.assertEqual(output_losses[0], 0.5)
def test_broadcast_single_loss(self): loss_container = compile_utils.LossesContainer('mse') y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss, output_losses = self.evaluate( loss_container(y_t, y_p, sample_weight=sw)) self.assertEqual(total_loss, 0.5) self.assertLen(output_losses, 2) self.assertEqual(output_losses[0], 0.) self.assertEqual(output_losses[1], 0.5)
def test_loss_masking(self): loss_container = compile_utils.LossesContainer('mae') y_p = constant_op.constant([[[1], [1]], [[0], [0]]], dtype=dtypes.float32) y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32) y_p._keras_mask = constant_op.constant([[1, 0], [1, 0]], dtype=dtypes.float32) total_loss = loss_container(y_t, y_p) self.assertAlmostEqual(total_loss.numpy(), .25) # sum over batch size self.assertLen(loss_container.metrics, 1) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertAlmostEqual(loss_metric.result().numpy(), .25)
def test_loss_list(self): loss_container = compile_utils.LossesContainer(['mse', 'mae'], [1, 0.5]) y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss, output_losses = self.evaluate( loss_container(y_t, y_p, sample_weight=sw)) self.assertLen(loss_container._losses, 2) self.assertEqual(total_loss, 0.25) self.assertLen(output_losses, 2) self.assertEqual(output_losses[0], 0) self.assertEqual(output_losses[1], 0.25)
def test_mismatched_dtypes(self): y_t = constant_op.constant([1, 9, 2, -5], shape=(2, 2)) y_p = constant_op.constant([4, 8, 12, 8], shape=(2, 2), dtype=dtypes.float32) def my_mae(labels, preds): self.assertEqual(labels.dtype, dtypes.int32) self.assertEqual(preds.dtype, dtypes.float32) labels = math_ops.cast(labels, preds.dtype) return backend.mean(math_ops.abs(preds - labels), axis=-1) loss_container = compile_utils.LossesContainer(my_mae) total_loss = loss_container(y_t, y_p) self.assertEqual(total_loss.dtype, dtypes.float32)
def test_custom_loss_callables(self): def custom_loss_fn(y_true, y_pred): return math_ops.reduce_sum(y_true - y_pred) class CustomLossClass(object): def __call__(self, y_true, y_pred): return math_ops.reduce_sum(y_true - y_pred) loss_container = compile_utils.LossesContainer( [custom_loss_fn, CustomLossClass()]) y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5)) loss_container(y_t, y_p) self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn') self.assertEqual(loss_container._losses[1].name, 'custom_loss_class')
def add_loss(self, function, weight=1.0, mask_channel=-1): """ Add the given loss function with the given weight to the loss function chain. Parameters ---------- function: :class:`keras.losses.Loss` The loss function to add to the loss chain weight: float, optional The weighting to apply to the loss function. Default: `1.0` mask_channel: int, optional The channel in the `y_true` image that the mask exists in. Set to `-1` if there is no mask for the given loss function. Default: `-1` """ logger.debug("Adding loss: (function: %s, weight: %s, mask_channel: %s)", function, weight, mask_channel) self._loss_functions.append(compile_utils.LossesContainer(function)) self._loss_weights.append(weight) self._mask_channels.append(mask_channel)
def test_loss_partial_dict_with_output_names(self): loss_container = compile_utils.LossesContainer( {'out2': 'mae'}, {'out2': 1.}, output_names=['out1', 'out2']) y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertEqual(total_loss.numpy(), 0.5) self.assertLen(loss_container.metrics, 2) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.5) out2_metric = loss_container.metrics[1] self.assertEqual(out2_metric.name, 'out2_loss') self.assertEqual(out2_metric.result().numpy(), 0.5)
def test_loss_dict(self): loss_container = compile_utils.LossesContainer( { 'out1': 'mse', 'out2': 'mae' }, { 'out1': 1, 'out2': 0.5 }) y_t = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1))} y_p = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.ones((10, 1))} sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss, output_losses = self.evaluate( loss_container(y_t, y_p, sample_weight=sw)) self.assertLen(loss_container._losses, 2) self.assertEqual(total_loss, 0.25) self.assertLen(output_losses, 2) self.assertEqual(output_losses[0], 0) self.assertEqual(output_losses[1], 0.25)
def test_nested_structure(self): loss_container = compile_utils.LossesContainer( { 'b': ['mse', None], 'a': 'mae' }, loss_weights={ 'b': [0.5, 0], 'a': 1 }) y_t = { 'b': [array_ops.ones((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.zeros((10, 1)) } y_p = { 'b': [array_ops.zeros((10, 1)), array_ops.zeros((10, 1))], 'a': array_ops.ones((10, 1)) } sw = ops.convert_to_tensor_v2_with_dispatch( [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertEqual(total_loss.numpy(), 0.75) self.assertLen(loss_container.metrics, 3) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.75) a_metric = loss_container.metrics[1] self.assertEqual(a_metric.name, 'a_loss') self.assertEqual(a_metric.result().numpy(), 0.5) b_1_metric = loss_container.metrics[2] self.assertEqual(b_1_metric.name, 'b_1_loss') self.assertEqual(b_1_metric.result().numpy(), 0.5)
def test_broadcast_single_loss(self): loss_container = compile_utils.LossesContainer('mse') y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))] y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))] sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertEqual(total_loss.numpy(), 0.5) self.assertLen(loss_container.metrics, 3) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.5) output_1_metric = loss_container.metrics[1] self.assertEqual(output_1_metric.name, 'output_1_loss') self.assertEqual(output_1_metric.result().numpy(), 0.) output_2_metric = loss_container.metrics[2] self.assertEqual(output_2_metric.name, 'output_2_loss') self.assertEqual(output_2_metric.result().numpy(), 0.5)
def test_loss_dict(self): loss_container = compile_utils.LossesContainer( { 'out1': 'mse', 'out2': 'mae' }, { 'out1': 1, 'out2': 0.5 }) y_t = { 'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1)) } y_p = { 'out1': array_ops.ones((10, 1)), 'out2': array_ops.ones((10, 1)) } sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1]) total_loss = loss_container(y_t, y_p, sample_weight=sw) self.assertLen(loss_container._losses, 2) self.assertEqual(total_loss.numpy(), 0.25) self.assertLen(loss_container.metrics, 3) loss_metric = loss_container.metrics[0] self.assertEqual(loss_metric.name, 'loss') self.assertEqual(loss_metric.result().numpy(), 0.25) out1_metric = loss_container.metrics[1] self.assertEqual(out1_metric.name, 'out1_loss') self.assertEqual(out1_metric.result().numpy(), 0) out2_metric = loss_container.metrics[2] self.assertEqual(out2_metric.name, 'out2_loss') self.assertEqual(out2_metric.result().numpy(), 0.5)
def compile(self, optimizer, loss_fn): super(ProgressiveAETrainer, self).compile() self.optimizer = optimizer self.loss_fn = compile_utils.LossesContainer(loss_fn)
def __init__(self, loss_func, mask_prop=1.0): super().__init__(name="penalized_loss") self._loss_func = compile_utils.LossesContainer(loss_func) self._mask_prop = mask_prop