def compile(self, d_optimizer, g_optimizer, g_loss_fn, d_loss_fn):
super(ProgressiveGANTrainer, self).compile()
self.d_optimizer = d_optimizer
self.g_optimizer = g_optimizer
self.g_loss_fn = compile_utils.LossesContainer(g_loss_fn)
self.d_loss_fn = compile_utils.LossesContainer(d_loss_fn)
if self.gradient_penalty:
self.gradient_penalty_fn = compile_utils.LossesContainer(gradient_penalty)
def test_loss_dict_with_nones(self):
loss_container = compile_utils.LossesContainer({
'out1': None,
'out2': 'mae'
})
y_t = {
'out1': array_ops.ones((10, 1)),
'out2': array_ops.zeros((10, 1))
}
y_p = {
'out1': array_ops.ones((10, 1)),
'out2': array_ops.ones((10, 1))
}
sw = ops.convert_to_tensor_v2_with_dispatch(
[0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertEqual(total_loss.numpy(), 0.5)
self.assertLen(loss_container.metrics, 2)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.5)
out2_metric = loss_container.metrics[1]
self.assertEqual(out2_metric.name, 'out2_loss')
self.assertEqual(out2_metric.result().numpy(), 0.5)
def compile(self, optimizers, loss_fns, metrics=None):
super(Model_w_self_backpropagated_branches, self).compile()
assert len(optimizers)==len(loss_fns)
assert len(optimizers)==self.num_models
if metrics is None:
user_metrics = [None]*self.num_models
elif isinstance(metrics,dict):
user_metrics = [None]*self.num_models
for k,v in metrics.items():
user_metrics[k] = v
elif isinstance(metrics,list):
assert len(metrics)==self.num_models
user_metrics = metrics
else:
raise ValueError("Metrics must be either a full-list or"
" a sparse version dictionary map position to metric.")
with self.distribute_strategy.scope():
self._validate_compile(optimizers, user_metrics)
self.optimizers = self._get_optimizer(optimizers)
self.compiled_loss = [compile_utils.LossesContainer(
loss, output_names=self.output_names)
for loss in loss_fns]
self.compiled_metrics = [compile_utils.MetricsContainer(
metric, output_names=self.output_names)
for metric in user_metrics]
def _compile_losses(self, loss_functions):
""" Splits the given loss_functions into the corresponding :attr:`_loss_functions' and
:attr:`_loss_weights' lists.
Loss functions are compiled into :class:`keras.compile_utils.LossesContainer` objects
Parameters
----------
loss_functions: list
A list of either a tuple of (:class:`keras.losses.Loss`, scalar weight) or just a
:class:`keras.losses.Loss` function. If just the loss function is passed, then the
weight is assumed to be 1.0 """
for loss_func in loss_functions:
if isinstance(loss_func, tuple):
assert len(
loss_func
) == 2, "Tuple loss functions should contain 2 items"
assert isinstance(loss_func[1],
float), "weight should be a float"
func, weight = loss_func
else:
func = loss_func
weight = 1.0
self._loss_functions.append(compile_utils.LossesContainer(func))
self._loss_weights.append(weight)
logger.debug("Compiled losses: (functions: %s, weights: %s",
self._loss_functions, self._loss_weights)
def test_ragged_tensor_output(self):
""" Ensure that ragged tensors can be passed as targets and predictions."""
def custom_loss_fn(y_true, y_pred):
losses = ragged_functional_ops.map_flat_values(
losses_mod.mse, y_true, y_pred)
return math_ops.reduce_mean(losses)
class CustomLossClass(object):
def __call__(self, y_true, y_pred):
losses = ragged_functional_ops.map_flat_values(
losses_mod.mse, y_true, y_pred)
return math_ops.reduce_mean(losses)
loss_container = compile_utils.LossesContainer(
[custom_loss_fn, CustomLossClass()])
v_t = constant_op.constant([[3., 4.], [1., 2.], [3., 5.]])
v_p = constant_op.constant([[3.1, 4.], [1., 2.], [3., 5.]])
y_t = array_ops.expand_dims(
ragged_tensor.RaggedTensor.from_row_splits(v_t, [0, 2, 3]), 0)
y_p = array_ops.expand_dims(
ragged_tensor.RaggedTensor.from_row_splits(v_p, [0, 2, 3]), 0)
loss_container(y_t, y_p)
self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn')
def test_ragged_tensor_output(self):
"""Ensure that ragged tensors can be passed as targets and predictions."""
def custom_loss_fn(y_true, y_pred):
"""MSE supports RaggedTensors directly."""
return losses_mod.mse(y_true, y_pred)
class CustomLossClass(losses_mod.Loss):
"""User defined loss function must implement RaggedTensor support."""
def call(self, y_true, y_pred):
losses = ragged_functional_ops.map_flat_values(
math_ops.squared_difference, y_true, y_pred)
return math_ops.reduce_mean(losses)
loss_container = compile_utils.LossesContainer(
[custom_loss_fn, CustomLossClass()])
v_t = constant_op.constant([[3., 4.], [1., 2.], [3., 5.]])
v_p = constant_op.constant([[3.1, 4.], [1., 2.], [3., 5.]])
y_t = array_ops.expand_dims(
ragged_tensor.RaggedTensor.from_row_splits(v_t, [0, 2, 3]), 0)
y_p = array_ops.expand_dims(
ragged_tensor.RaggedTensor.from_row_splits(v_p, [0, 2, 3]), 0)
loss_container(y_t, y_p)
self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn')
def test_missing_label_with_no_loss(self):
# It's ok to exclude a label if that label has no
# losses or metrics associated with it.
loss_container = compile_utils.LossesContainer({
'output1': 'mse',
'output3': 'mae'
})
y_p = {
'output1': ops.convert_to_tensor_v2_with_dispatch([[0], [1], [2]]),
'output2': ops.convert_to_tensor_v2_with_dispatch([[3], [4], [5]]),
'output3': ops.convert_to_tensor_v2_with_dispatch([[6], [7], [8]])
}
y_t = {
'output1': ops.convert_to_tensor_v2_with_dispatch([[1], [2], [3]]),
'output3': ops.convert_to_tensor_v2_with_dispatch([[4], [5], [6]])
}
total_loss = loss_container(y_t, y_p)
self.assertEqual(total_loss.numpy(), 3.)
self.assertLen(loss_container.metrics, 3)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 3.)
output_1_metric = loss_container.metrics[1]
self.assertEqual(output_1_metric.name, 'output1_loss')
self.assertEqual(output_1_metric.result().numpy(), 1.)
output_3_metric = loss_container.metrics[2]
self.assertEqual(output_3_metric.name, 'output3_loss')
self.assertEqual(output_3_metric.result().numpy(), 2.)
def test_nested_structure(self):
loss_container = compile_utils.LossesContainer(
{
'b': ['mse', None],
'a': 'mae'
}, loss_weights={
'b': [0.5, 0],
'a': 1
})
y_t = {
'b': [array_ops.ones((10, 1)),
array_ops.zeros((10, 1))],
'a': array_ops.zeros((10, 1))
}
y_p = {
'b': [array_ops.zeros((10, 1)),
array_ops.zeros((10, 1))],
'a': array_ops.ones((10, 1))
}
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss, output_losses = self.evaluate(
loss_container(y_t, y_p, sample_weight=sw))
self.assertEqual(total_loss, 0.75)
self.assertLen(output_losses, 2)
self.assertEqual(output_losses[0], 0.5)
self.assertEqual(output_losses[1], 0.25)
def test_loss_list(self):
loss_container = compile_utils.LossesContainer(['mse', 'mae'],
[1, 0.5])
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor_v2_with_dispatch(
[0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertEqual(loss_container._output_names,
['output_1', 'output_2'])
self.assertLen(loss_container._losses, 2)
self.assertEqual(total_loss.numpy(), 0.25)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.25)
output_1_metric = loss_container.metrics[1]
self.assertEqual(output_1_metric.name, 'output_1_loss')
self.assertEqual(output_1_metric.result().numpy(), 0)
output_2_metric = loss_container.metrics[2]
self.assertEqual(output_2_metric.name, 'output_2_loss')
self.assertEqual(output_2_metric.result().numpy(), 0.5)
loss_container.reset_state()
self.assertEqual(loss_metric.result().numpy(), 0)
self.assertEqual(output_1_metric.result().numpy(), 0)
self.assertEqual(output_2_metric.result().numpy(), 0)
def test_single_loss(self):
loss_container = compile_utils.LossesContainer('mse')
y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5))
total_loss, output_losses = self.evaluate(loss_container(y_t, y_p))
self.assertTrue(loss_container._built)
self.assertLen(loss_container._losses, 1)
self.assertEqual(total_loss, 1.)
self.assertLen(output_losses, 1)
def test_integer_dtypes(self):
y_t = constant_op.constant([1, 9, 2, -5], shape=(2, 2))
y_p = constant_op.constant([4, 8, 12, 8], shape=(2, 2), dtype=dtypes.int64)
def my_mae(labels, preds):
self.assertEqual(labels.dtype, dtypes.int64)
self.assertEqual(preds.dtype, dtypes.int64)
return K.mean(math_ops.abs(preds - labels), axis=-1)
loss_container = compile_utils.LossesContainer(my_mae)
total_loss = loss_container(y_t, y_p)
self.assertEqual(total_loss.dtype, dtypes.int64)
def compile(self,
optimizer_gen='rmsprop', optimizer_disc='rmsprop',
loss_gen=None, loss_disc=None,
metrics_gen=None, metrics_disc=None):
self.gen_optimizer = optimizers.get(optimizer_gen)
self.disc_optimizer = optimizers.get(optimizer_disc)
self.gen_losses_container = compile_utils.LossesContainer(loss_gen) if loss_gen else None
self.disc_losses_container = compile_utils.LossesContainer(loss_disc) if loss_disc else None
self.gen_metrics_container = compile_utils.MetricsContainer(metrics_gen) if metrics_gen else None
self.disc_metrics_container = compile_utils.MetricsContainer(metrics_disc) if metrics_disc else None
self.m_formatter = Metrics_Formatter(
gen_loss_name=_get_tag_name(self.gen_losses_container),
disc_loss_name=_get_tag_name(self.disc_losses_container),
gen_metric_name=_get_tag_name(self.gen_metrics_container),
disc_metric_name=_get_tag_name(self.disc_metrics_container),
num_format='.03f'
)
def test_single_loss(self):
loss_container = compile_utils.LossesContainer('mse')
y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5))
total_loss = loss_container(y_t, y_p)
self.assertTrue(loss_container._built)
self.assertLen(loss_container._losses, 1)
self.assertEqual(total_loss.numpy(), 1.)
self.assertLen(loss_container.metrics, 1)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 1.)
def test_loss_sample_weight(self):
loss_container = compile_utils.LossesContainer('mae')
y_p = constant_op.constant([[[1], [1]], [[0], [0]]], dtype=dtypes.float32)
y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32)
sw = constant_op.constant([[.2, .3], [.5, 0]], dtype=dtypes.float32)
total_loss = loss_container(y_t, y_p, sample_weight=sw)
# (0 * .2 + 0 * .3 + 1 * .5 + 1 * 0) / 4
self.assertAlmostEqual(total_loss.numpy(), .125)
self.assertLen(loss_container.metrics, 1)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertAlmostEqual(loss_metric.result().numpy(), .125)
def test_loss_partial_dict_with_output_names(self):
loss_container = compile_utils.LossesContainer(
{'out2': 'mae'}, {'out2': 1.}, output_names=['out1', 'out2'])
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss, output_losses = self.evaluate(
loss_container(y_t, y_p, sample_weight=sw))
self.assertEqual(total_loss, 0.5)
self.assertLen(output_losses, 1)
self.assertEqual(output_losses[0], 0.5)
def test_broadcast_single_loss(self):
loss_container = compile_utils.LossesContainer('mse')
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss, output_losses = self.evaluate(
loss_container(y_t, y_p, sample_weight=sw))
self.assertEqual(total_loss, 0.5)
self.assertLen(output_losses, 2)
self.assertEqual(output_losses[0], 0.)
self.assertEqual(output_losses[1], 0.5)
def test_loss_masking(self):
loss_container = compile_utils.LossesContainer('mae')
y_p = constant_op.constant([[[1], [1]], [[0], [0]]], dtype=dtypes.float32)
y_t = constant_op.constant([[[1], [1]], [[1], [1]]], dtype=dtypes.float32)
y_p._keras_mask = constant_op.constant([[1, 0], [1, 0]],
dtype=dtypes.float32)
total_loss = loss_container(y_t, y_p)
self.assertAlmostEqual(total_loss.numpy(), .25) # sum over batch size
self.assertLen(loss_container.metrics, 1)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertAlmostEqual(loss_metric.result().numpy(), .25)
def test_loss_list(self):
loss_container = compile_utils.LossesContainer(['mse', 'mae'], [1, 0.5])
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss, output_losses = self.evaluate(
loss_container(y_t, y_p, sample_weight=sw))
self.assertLen(loss_container._losses, 2)
self.assertEqual(total_loss, 0.25)
self.assertLen(output_losses, 2)
self.assertEqual(output_losses[0], 0)
self.assertEqual(output_losses[1], 0.25)
def test_mismatched_dtypes(self):
y_t = constant_op.constant([1, 9, 2, -5], shape=(2, 2))
y_p = constant_op.constant([4, 8, 12, 8],
shape=(2, 2),
dtype=dtypes.float32)
def my_mae(labels, preds):
self.assertEqual(labels.dtype, dtypes.int32)
self.assertEqual(preds.dtype, dtypes.float32)
labels = math_ops.cast(labels, preds.dtype)
return backend.mean(math_ops.abs(preds - labels), axis=-1)
loss_container = compile_utils.LossesContainer(my_mae)
total_loss = loss_container(y_t, y_p)
self.assertEqual(total_loss.dtype, dtypes.float32)
def test_custom_loss_callables(self):
def custom_loss_fn(y_true, y_pred):
return math_ops.reduce_sum(y_true - y_pred)
class CustomLossClass(object):
def __call__(self, y_true, y_pred):
return math_ops.reduce_sum(y_true - y_pred)
loss_container = compile_utils.LossesContainer(
[custom_loss_fn, CustomLossClass()])
y_t, y_p = array_ops.ones((10, 5)), array_ops.zeros((10, 5))
loss_container(y_t, y_p)
self.assertEqual(loss_container._losses[0].name, 'custom_loss_fn')
self.assertEqual(loss_container._losses[1].name, 'custom_loss_class')
def add_loss(self, function, weight=1.0, mask_channel=-1):
""" Add the given loss function with the given weight to the loss function chain.
Parameters
----------
function: :class:`keras.losses.Loss`
The loss function to add to the loss chain
weight: float, optional
The weighting to apply to the loss function. Default: `1.0`
mask_channel: int, optional
The channel in the `y_true` image that the mask exists in. Set to `-1` if there is no
mask for the given loss function. Default: `-1`
"""
logger.debug("Adding loss: (function: %s, weight: %s, mask_channel: %s)",
function, weight, mask_channel)
self._loss_functions.append(compile_utils.LossesContainer(function))
self._loss_weights.append(weight)
self._mask_channels.append(mask_channel)
def test_loss_partial_dict_with_output_names(self):
loss_container = compile_utils.LossesContainer(
{'out2': 'mae'}, {'out2': 1.}, output_names=['out1', 'out2'])
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertEqual(total_loss.numpy(), 0.5)
self.assertLen(loss_container.metrics, 2)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.5)
out2_metric = loss_container.metrics[1]
self.assertEqual(out2_metric.name, 'out2_loss')
self.assertEqual(out2_metric.result().numpy(), 0.5)
def test_loss_dict(self):
loss_container = compile_utils.LossesContainer(
{
'out1': 'mse',
'out2': 'mae'
}, {
'out1': 1,
'out2': 0.5
})
y_t = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.zeros((10, 1))}
y_p = {'out1': array_ops.ones((10, 1)), 'out2': array_ops.ones((10, 1))}
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss, output_losses = self.evaluate(
loss_container(y_t, y_p, sample_weight=sw))
self.assertLen(loss_container._losses, 2)
self.assertEqual(total_loss, 0.25)
self.assertLen(output_losses, 2)
self.assertEqual(output_losses[0], 0)
self.assertEqual(output_losses[1], 0.25)
def test_nested_structure(self):
loss_container = compile_utils.LossesContainer(
{
'b': ['mse', None],
'a': 'mae'
},
loss_weights={
'b': [0.5, 0],
'a': 1
})
y_t = {
'b': [array_ops.ones((10, 1)),
array_ops.zeros((10, 1))],
'a': array_ops.zeros((10, 1))
}
y_p = {
'b': [array_ops.zeros((10, 1)),
array_ops.zeros((10, 1))],
'a': array_ops.ones((10, 1))
}
sw = ops.convert_to_tensor_v2_with_dispatch(
[0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertEqual(total_loss.numpy(), 0.75)
self.assertLen(loss_container.metrics, 3)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.75)
a_metric = loss_container.metrics[1]
self.assertEqual(a_metric.name, 'a_loss')
self.assertEqual(a_metric.result().numpy(), 0.5)
b_1_metric = loss_container.metrics[2]
self.assertEqual(b_1_metric.name, 'b_1_loss')
self.assertEqual(b_1_metric.result().numpy(), 0.5)
def test_broadcast_single_loss(self):
loss_container = compile_utils.LossesContainer('mse')
y_t = [array_ops.ones((10, 1)), array_ops.zeros((10, 1))]
y_p = [array_ops.ones((10, 1)), array_ops.ones((10, 1))]
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertEqual(total_loss.numpy(), 0.5)
self.assertLen(loss_container.metrics, 3)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.5)
output_1_metric = loss_container.metrics[1]
self.assertEqual(output_1_metric.name, 'output_1_loss')
self.assertEqual(output_1_metric.result().numpy(), 0.)
output_2_metric = loss_container.metrics[2]
self.assertEqual(output_2_metric.name, 'output_2_loss')
self.assertEqual(output_2_metric.result().numpy(), 0.5)
def test_loss_dict(self):
loss_container = compile_utils.LossesContainer(
{
'out1': 'mse',
'out2': 'mae'
}, {
'out1': 1,
'out2': 0.5
})
y_t = {
'out1': array_ops.ones((10, 1)),
'out2': array_ops.zeros((10, 1))
}
y_p = {
'out1': array_ops.ones((10, 1)),
'out2': array_ops.ones((10, 1))
}
sw = ops.convert_to_tensor([0, 0, 0, 0, 0, 1, 1, 1, 1, 1])
total_loss = loss_container(y_t, y_p, sample_weight=sw)
self.assertLen(loss_container._losses, 2)
self.assertEqual(total_loss.numpy(), 0.25)
self.assertLen(loss_container.metrics, 3)
loss_metric = loss_container.metrics[0]
self.assertEqual(loss_metric.name, 'loss')
self.assertEqual(loss_metric.result().numpy(), 0.25)
out1_metric = loss_container.metrics[1]
self.assertEqual(out1_metric.name, 'out1_loss')
self.assertEqual(out1_metric.result().numpy(), 0)
out2_metric = loss_container.metrics[2]
self.assertEqual(out2_metric.name, 'out2_loss')
self.assertEqual(out2_metric.result().numpy(), 0.5)
def compile(self, optimizer, loss_fn):
super(ProgressiveAETrainer, self).compile()
self.optimizer = optimizer
self.loss_fn = compile_utils.LossesContainer(loss_fn)
def __init__(self, loss_func, mask_prop=1.0):
super().__init__(name="penalized_loss")
self._loss_func = compile_utils.LossesContainer(loss_func)
self._mask_prop = mask_prop
