def __init__(self, optimizer): # pylint: disable=super-init-not-called
self.optimizer = optimizer
with K.name_scope(self.__class__.__name__):
if context.in_graph_mode():
self.iterations = K.variable(0,
dtype='int64',
name='iterations')
def __init__(self, lr=0.01, epsilon=1e-8, decay=0., **kwargs):
super(Adagrad, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.lr = K.variable(lr, name='lr')
self.decay = K.variable(decay, name='decay')
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.epsilon = epsilon
self.initial_decay = decay
def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False, **kwargs):
super(SGD, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.momentum = K.variable(momentum, name='momentum')
self.decay = K.variable(decay, name='decay')
self.initial_decay = decay
self.nesterov = nesterov
def __init__(self, lr=0.01, momentum=0., decay=0., nesterov=False, **kwargs):
super(SGD, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.momentum = K.variable(momentum, name='momentum')
self.decay = K.variable(decay, name='decay')
self.initial_decay = decay
self.nesterov = nesterov
def __init__(self, lr=0.01, epsilon=None, decay=0., **kwargs):
super(Adagrad, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.lr = K.variable(lr, name='lr')
self.decay = K.variable(decay, name='decay')
self.iterations = K.variable(0, dtype='int64', name='iterations')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
def __init__(self, lr=1.0, rho=0.95, epsilon=None, decay=0., **kwargs):
super(Adadelta, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.lr = K.variable(lr, name='lr')
self.decay = K.variable(decay, name='decay')
self.iterations = K.variable(0, dtype='int64', name='iterations')
if epsilon is None:
epsilon = K.epsilon()
self.rho = rho
self.epsilon = epsilon
self.initial_decay = decay
def _separable_conv_block(ip,
filters,
kernel_size=(3, 3),
strides=(1, 1),
block_id=None):
"""Adds 2 blocks of [relu-separable conv-batchnorm].
Arguments:
ip: Input tensor
filters: Number of output filters per layer
kernel_size: Kernel size of separable convolutions
strides: Strided convolution for downsampling
block_id: String block_id
Returns:
A Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('separable_conv_block_%s' % block_id):
x = Activation('relu')(ip)
x = SeparableConv2D(
filters,
kernel_size,
strides=strides,
name='separable_conv_1_%s' % block_id,
padding='same',
use_bias=False,
kernel_initializer='he_normal')(
x)
x = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='separable_conv_1_bn_%s' % (block_id))(
x)
x = Activation('relu')(x)
x = SeparableConv2D(
filters,
kernel_size,
name='separable_conv_2_%s' % block_id,
padding='same',
use_bias=False,
kernel_initializer='he_normal')(
x)
x = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='separable_conv_2_bn_%s' % (block_id))(
x)
return x
def __init__(self,
lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
schedule_decay=0.004,
**kwargs):
super(Nadam, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.m_schedule = K.variable(1., name='m_schedule')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.epsilon = epsilon
self.schedule_decay = schedule_decay
def __init__(self,
lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=1e-8,
decay=0.,
**kwargs):
super(Adamax, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.decay = K.variable(decay, name='decay')
self.epsilon = epsilon
self.initial_decay = decay
def __init__(self,
lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
**kwargs):
super(Adamax, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.decay = K.variable(decay, name='decay')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
def __init__(self,
lr=0.002,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
schedule_decay=0.004,
**kwargs):
super(Nadam, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.m_schedule = K.variable(1., name='m_schedule')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.schedule_decay = schedule_decay
def __init__(self,
lr=0.001,
beta_1=0.9,
beta_2=0.999,
epsilon=None,
decay=0.,
amsgrad=False,
**kwargs):
super(Adam, self).__init__(**kwargs)
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
self.lr = K.variable(lr, name='lr')
self.beta_1 = K.variable(beta_1, name='beta_1')
self.beta_2 = K.variable(beta_2, name='beta_2')
self.decay = K.variable(decay, name='decay')
if epsilon is None:
epsilon = K.epsilon()
self.epsilon = epsilon
self.initial_decay = decay
self.amsgrad = amsgrad
def _eager_metrics_fn(model, outputs, targets):
"""Calculates the metrics for each output of the given model.
Arguments:
model: The model on which metrics are being calculated.
outputs: The outputs of the given model.
targets: The predictions or targets of the given model.
Returns:
Returns the metric names and metric results for each output of the model.
"""
metric_names = []
metric_results = []
if not isinstance(outputs, list):
outputs = [outputs]
if not isinstance(targets, list):
targets = [targets]
for i in range(len(model.outputs)):
output_metrics = model.nested_metrics[i]
for nested_output_metric in output_metrics:
metric_name, metric_fn = _get_metrics_info(
nested_output_metric, backend.int_shape(model.outputs[i]),
model.loss_functions[i])
if len(model.output_names) > 1:
metric_name = model.output_names[i] + '_' + metric_name
if metric_name not in model.metrics_names:
model.metrics_names.append(metric_name)
with backend.name_scope(metric_name):
metric_result = metric_fn(targets[i], outputs[i])
metric_names.append(metric_name)
metric_results.append(backend.mean(metric_result))
return metric_results
def _eager_metrics_fn(model, outputs, targets):
"""Calculates the metrics for each output of the given model.
Arguments:
model: The model on which metrics are being calculated.
outputs: The outputs of the given model.
targets: The predictions or targets of the given model.
Returns:
Returns the metric names and metric results for each output of the model.
"""
metric_names = []
metric_results = []
if not isinstance(outputs, list):
outputs = [outputs]
if not isinstance(targets, list):
targets = [targets]
for i in range(len(model.outputs)):
output_metrics = model.nested_metrics[i]
for nested_output_metric in output_metrics:
metric_name, metric_fn = _get_metrics_info(
nested_output_metric, backend.int_shape(model.outputs[i]),
model.loss_functions[i])
if len(model.output_names) > 1:
metric_name = model.output_names[i] + '_' + metric_name
if metric_name not in model.metrics_names:
model.metrics_names.append(metric_name)
with backend.name_scope(metric_name):
metric_result = metric_fn(outputs[i], targets[i])
metric_names.append(metric_name)
metric_results.append(backend.mean(metric_result))
return metric_results
def _eager_loss_fn(outputs, targets, loss_fn, output_name):
with backend.name_scope(output_name + '_loss'):
loss = loss_fn(targets, outputs)
return loss
def _model_loss(model, inputs, targets, sample_weights=None, training=False):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: List of input arrays.
targets: List of target arrays.
sample_weights: Optional list of sample weight arrays.
training: Whether the model should be run in inference or training mode.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
if len(inputs) == 1:
if model._expects_training_arg:
outs = model.call(inputs[0], training=training)
else:
outs = model.call(inputs[0])
else:
if model._expects_training_arg:
outs = model.call(inputs, training=training)
else:
outs = model.call(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with backend.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
if sample_weights:
weights = sample_weights[i]
else:
weights = None
# TODO(fchollet): support masking; in practice `_keras_mask` is never
# set in this context currently.
mask = outs[i]._keras_mask
weighted_masked_fn = training_utils.weighted_masked_objective(
loss_fn)
with backend.name_scope(model.output_names[i] + '_loss'):
output_loss = weighted_masked_fn(targets[i],
outs[i],
weights,
mask=mask)
# If the number of outputs is 1 then we don't append the loss metric
# associated with each model output. When there are multiple outputs
# associated with a model, each output's loss is calculated and returned
# as part of the loss_metrics.
if len(model.outputs) > 1:
loss_metrics.append(backend.mean(output_loss))
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = backend.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
def build(self, input_shape):
with K.name_scope(self.forward_layer.name):
self.forward_layer.build(input_shape)
with K.name_scope(self.backward_layer.name):
self.backward_layer.build(input_shape)
self.built = True
def _model_loss(model, inputs, targets):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: The inputs of the given model. This is typically the mini batch of
data that is fed to the model.
targets: The predictions or targets of the given model.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
if len(inputs) == 1:
outs = model.call(inputs[0])
else:
outs = model.call(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with K.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
# compute the loss
output_loss = _eager_loss_fn(outs[i], targets[i], loss_fn,
model.output_names[i])
loss_metrics.append(K.mean(output_loss))
mask = outs[i]._keras_mask
# adapted from weighted_loss_fn
if mask is not None:
# mask should have the same shape as output_loss
output_loss *= mask
# the loss per batch should be proportional
# to the number of unmasked samples.
output_loss /= K.mean(mask)
# adapted from weighted_loss_fn
# apply sample weighting
if model.sample_weights:
# reduce score_array to same ndim as weight array
ndim = K.ndim(output_loss)
weight_ndim = K.ndim(model.sample_weights)
output_loss = K.mean(output_loss, axis=list(range(weight_ndim, ndim)))
output_loss *= model.sample_weights
output_loss /= K.mean(K.cast(K.not_equal(model.sample_weights, 0),
K.floatx()))
output_loss = K.mean(output_loss)
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = K.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
def _reduction_a_cell(ip, p, filters, block_id=None):
"""Adds a Reduction cell for NASNet-A (Fig. 4 in the paper).
Arguments:
ip: Input tensor `x`
p: Input tensor `p`
filters: Number of output filters
block_id: String block_id
Returns:
A Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
with K.name_scope('reduction_A_block_%s' % block_id):
p = _adjust_block(p, ip, filters, block_id)
h = Activation('relu')(ip)
h = Conv2D(
filters, (1, 1),
strides=(1, 1),
padding='same',
name='reduction_conv_1_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(
h)
h = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='reduction_bn_1_%s' % block_id)(
h)
with K.name_scope('block_1'):
x1_1 = _separable_conv_block(
h,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_left1_%s' % block_id)
x1_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_1_%s' % block_id)
x1 = add([x1_1, x1_2], name='reduction_add_1_%s' % block_id)
with K.name_scope('block_2'):
x2_1 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left2_%s' % block_id)(
h)
x2_2 = _separable_conv_block(
p,
filters, (7, 7),
strides=(2, 2),
block_id='reduction_right2_%s' % block_id)
x2 = add([x2_1, x2_2], name='reduction_add_2_%s' % block_id)
with K.name_scope('block_3'):
x3_1 = AveragePooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_left3_%s' % block_id)(
h)
x3_2 = _separable_conv_block(
p,
filters, (5, 5),
strides=(2, 2),
block_id='reduction_right3_%s' % block_id)
x3 = add([x3_1, x3_2], name='reduction_add3_%s' % block_id)
with K.name_scope('block_4'):
x4 = AveragePooling2D(
(3, 3),
strides=(1, 1),
padding='same',
name='reduction_left4_%s' % block_id)(
x1)
x4 = add([x2, x4])
with K.name_scope('block_5'):
x5_1 = _separable_conv_block(
x1, filters, (3, 3), block_id='reduction_left4_%s' % block_id)
x5_2 = MaxPooling2D(
(3, 3),
strides=(2, 2),
padding='same',
name='reduction_right5_%s' % block_id)(
h)
x5 = add([x5_1, x5_2], name='reduction_add4_%s' % block_id)
x = concatenate(
[x2, x3, x4, x5],
axis=channel_dim,
name='reduction_concat_%s' % block_id)
return x, ip
def _adjust_block(p, ip, filters, block_id=None):
"""Adjusts the input `previous path` to match the shape of the `input`.
Used in situations where the output number of filters needs to be changed.
Arguments:
p: Input tensor which needs to be modified
ip: Input tensor whose shape needs to be matched
filters: Number of output filters to be matched
block_id: String block_id
Returns:
Adjusted Keras tensor
"""
channel_dim = 1 if K.image_data_format() == 'channels_first' else -1
img_dim = 2 if K.image_data_format() == 'channels_first' else -2
ip_shape = K.int_shape(ip)
if p is not None:
p_shape = K.int_shape(p)
with K.name_scope('adjust_block'):
if p is None:
p = ip
elif p_shape[img_dim] != ip_shape[img_dim]:
with K.name_scope('adjust_reduction_block_%s' % block_id):
p = Activation('relu', name='adjust_relu_1_%s' % block_id)(p)
p1 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_1_%s' % block_id)(
p)
p1 = Conv2D(
filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_1_%s' % block_id,
kernel_initializer='he_normal')(
p1)
p2 = ZeroPadding2D(padding=((0, 1), (0, 1)))(p)
p2 = Cropping2D(cropping=((1, 0), (1, 0)))(p2)
p2 = AveragePooling2D(
(1, 1),
strides=(2, 2),
padding='valid',
name='adjust_avg_pool_2_%s' % block_id)(
p2)
p2 = Conv2D(
filters // 2, (1, 1),
padding='same',
use_bias=False,
name='adjust_conv_2_%s' % block_id,
kernel_initializer='he_normal')(
p2)
p = concatenate([p1, p2], axis=channel_dim)
p = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(
p)
elif p_shape[channel_dim] != filters:
with K.name_scope('adjust_projection_block_%s' % block_id):
p = Activation('relu')(p)
p = Conv2D(
filters, (1, 1),
strides=(1, 1),
padding='same',
name='adjust_conv_projection_%s' % block_id,
use_bias=False,
kernel_initializer='he_normal')(
p)
p = BatchNormalization(
axis=channel_dim,
momentum=0.9997,
epsilon=1e-3,
name='adjust_bn_%s' % block_id)(
p)
return p
def __init__(self, optimizer): # pylint: disable=super-init-not-called
self.optimizer = optimizer
with K.name_scope(self.__class__.__name__):
self.iterations = K.variable(0, dtype='int64', name='iterations')
def _model_loss(model, inputs, targets, sample_weights=None, training=False):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: List of input arrays.
targets: List of target arrays.
sample_weights: Optional list of sample weight arrays.
training: Whether the model should be run in inference or training mode.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
if len(inputs) == 1:
if model._expects_training_arg:
outs = model.call(inputs[0], training=training)
else:
outs = model.call(inputs[0])
else:
if model._expects_training_arg:
outs = model.call(inputs, training=training)
else:
outs = model.call(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with backend.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
if sample_weights:
weights = sample_weights[i]
else:
weights = None
# TODO(fchollet): support masking; in practice `_keras_mask` is never
# set in this context currently.
mask = outs[i]._keras_mask
weighted_masked_fn = training_utils.weighted_masked_objective(loss_fn)
with backend.name_scope(model.output_names[i] + '_loss'):
output_loss = weighted_masked_fn(
targets[i], outs[i], weights, mask=mask)
# If the number of outputs is 1 then we don't append the loss metric
# associated with each model output. When there are multiple outputs
# associated with a model, each output's loss is calculated and returned
# as part of the loss_metrics.
if len(model.outputs) > 1:
loss_metrics.append(backend.mean(output_loss))
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = backend.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
def _model_loss(model, inputs, targets):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: The inputs of the given model. This is typically the mini batch of
data that is fed to the model.
targets: The predictions or targets of the given model.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
outs = model(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with K.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
# compute the loss
output_loss = _eager_loss_fn(outs[i], targets[i], loss_fn,
model.output_names[i])
loss_metrics.append(K.mean(output_loss))
mask = outs[i]._keras_mask
# adapted from weighted_loss_fn
if mask is not None:
# mask should have the same shape as output_loss
output_loss *= mask
# the loss per batch should be proportional
# to the number of unmasked samples.
output_loss /= K.mean(mask)
# adapted from weighted_loss_fn
# apply sample weighting
if model.sample_weights:
# reduce score_array to same ndim as weight array
ndim = K.ndim(output_loss)
weight_ndim = K.ndim(model.sample_weights)
output_loss = K.mean(output_loss,
axis=list(range(weight_ndim, ndim)))
output_loss *= model.sample_weights
output_loss /= K.mean(
K.cast(K.not_equal(model.sample_weights, 0), K.floatx()))
output_loss = K.mean(output_loss)
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = K.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
def _eager_loss_fn(outputs, targets, loss_fn, output_name):
with backend.name_scope(output_name + '_loss'):
loss = loss_fn(targets, outputs)
return loss
def _model_loss(model, inputs, targets, training=False):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: The inputs of the given model. This is typically the mini batch of
data that is fed to the model.
targets: The predictions or targets of the given model.
training: Whether the model should be run in inference or training mode.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
if len(inputs) == 1:
if model._expects_training_arg:
outs = model.call(inputs[0], training=training)
else:
outs = model.call(inputs[0])
else:
if model._expects_training_arg:
outs = model.call(inputs, training=training)
else:
outs = model.call(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with K.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
# compute the loss
output_loss = _eager_loss_fn(outs[i], targets[i], loss_fn,
model.output_names[i])
loss_metrics.append(K.mean(output_loss))
# TODO(fchollet): support masking; in practice `_keras_mask` is never
# set in this context currently.
mask = outs[i]._keras_mask
# adapted from weighted_loss_fn
if mask is not None:
# mask should have the same shape as output_loss
output_loss *= mask
# the loss per batch should be proportional
# to the number of unmasked samples.
output_loss /= K.mean(mask)
# TODO(fchollet): support sample weighting
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = K.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
def build(self, input_shape):
with K.name_scope(self.forward_layer.name):
self.forward_layer.build(input_shape)
with K.name_scope(self.backward_layer.name):
self.backward_layer.build(input_shape)
self.built = True
def _model_loss(model, inputs, targets, training=False):
"""Calculates the loss for a given model.
Arguments:
model: The model on which metrics are being calculated.
inputs: The inputs of the given model. This is typically the mini batch of
data that is fed to the model.
targets: The predictions or targets of the given model.
training: Whether the model should be run in inference or training mode.
Returns:
Returns the model output, total loss and loss value calculated using the
specified loss function. The total loss includes regularization losses and
applies masking and sample weighting to the loss value.
"""
total_loss = 0
if len(inputs) == 1:
if model._expects_training_arg:
outs = model.call(inputs[0], training=training)
else:
outs = model.call(inputs[0])
else:
if model._expects_training_arg:
outs = model.call(inputs, training=training)
else:
outs = model.call(inputs)
if not isinstance(outs, list):
outs = [outs]
if not isinstance(targets, list):
targets = [targets]
loss_metrics = []
with K.name_scope('loss'):
for i, loss_fn in enumerate(model.loss_functions):
# compute the loss
output_loss = _eager_loss_fn(outs[i], targets[i], loss_fn,
model.output_names[i])
loss_metrics.append(K.mean(output_loss))
# TODO(fchollet): support masking; in practice `_keras_mask` is never
# set in this context currently.
mask = outs[i]._keras_mask
# adapted from weighted_loss_fn
if mask is not None:
# mask should have the same shape as output_loss
output_loss *= mask
# the loss per batch should be proportional
# to the number of unmasked samples.
output_loss /= K.mean(mask)
# TODO(fchollet): support sample weighting
loss_weight = model.loss_weights_list[i]
if total_loss is None:
total_loss = loss_weight * output_loss
else:
total_loss += loss_weight * output_loss
total_loss = K.mean(total_loss)
# Add regularization losses
custom_losses = []
for layer in model.layers:
if layer.losses:
custom_losses += layer.losses
if custom_losses:
total_loss += sum(custom_losses)
return outs, total_loss, loss_metrics
