def _get_metric_object(self, metric, y_t, y_p):
"""Converts user-supplied metric to a `Metric` object.
Args:
metric: A string, function, or `Metric` object.
y_t: Sample of label.
y_p: Sample of output.
Returns:
A `Metric` object.
"""
if metric is None:
return None # Ok to have no metric for an output.
# Convenience feature for selecting b/t binary, categorical,
# and sparse categorical.
if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']:
metric_obj = metrics_mod.get(metric)
else:
y_t_rank = len(y_t.shape.as_list())
y_p_rank = len(y_p.shape.as_list())
y_t_last_dim = y_t.shape.as_list()[-1]
y_p_last_dim = y_p.shape.as_list()[-1]
is_binary = y_p_last_dim == 1
is_sparse_categorical = (y_t_rank < y_p_rank
or y_t_last_dim == 1 and y_p_last_dim > 1)
if metric in ['accuracy', 'acc']:
if is_binary:
metric_obj = metrics_mod.binary_accuracy
elif is_sparse_categorical:
metric_obj = metrics_mod.sparse_categorical_accuracy
else:
metric_obj = metrics_mod.categorical_accuracy
else:
if is_binary:
metric_obj = metrics_mod.binary_crossentropy
elif is_sparse_categorical:
metric_obj = metrics_mod.sparse_categorical_crossentropy
else:
metric_obj = metrics_mod.categorical_crossentropy
if isinstance(metric_obj, losses_mod.Loss):
metric_obj._allow_sum_over_batch_size = True # pylint: disable=protected-access
if not isinstance(metric_obj, metrics_mod.Metric):
if isinstance(metric, six.string_types):
metric_name = metric
else:
metric_name = get_custom_object_name(metric)
if metric_name is None:
raise ValueError(
'Metric should be a callable, found: {}'.format(
metric))
metric_obj = metrics_mod.MeanMetricWrapper(metric_obj,
name=metric_name)
return metric_obj
def _get_metric_object(self, metric, y_t, y_p):
"""Converts user-supplied metric to a `Metric` object.
Arguments:
metric: A string, function, or `Metric` object.
y_t: Sample of label.
y_p: Sample of output.
Returns:
A `Metric` object.
"""
if metric is None:
return None # Ok to have no metric for an output.
# Convenience feature for selecting b/t binary, categorical,
# and sparse categorical.
if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']:
metric_obj = metrics_mod.get(metric)
else:
y_t_rank = len(y_t.shape.as_list())
y_p_rank = len(y_p.shape.as_list())
y_t_last_dim = y_t.shape.as_list()[-1]
y_p_last_dim = y_p.shape.as_list()[-1]
is_binary = y_p_last_dim == 1
is_sparse_categorical = (y_t_rank < y_p_rank
or y_t_last_dim == 1 and y_p_last_dim > 1)
if metric in ['accuracy', 'acc']:
if is_binary:
metric_obj = metrics_mod.binary_accuracy
elif is_sparse_categorical:
metric_obj = metrics_mod.sparse_categorical_accuracy
else:
metric_obj = metrics_mod.categorical_accuracy
else:
if is_binary:
metric_obj = metrics_mod.binary_crossentropy
elif is_sparse_categorical:
metric_obj = metrics_mod.sparse_categorical_crossentropy
else:
metric_obj = metrics_mod.categorical_crossentropy
if isinstance(metric_obj, losses_mod.Loss):
metric_obj._allow_sum_over_batch_size = True # pylint: disable=protected-access
if not isinstance(metric_obj, metrics_mod.Metric):
if isinstance(metric, six.string_types):
metric_name = metric
elif hasattr(metric, 'name'):
metric_name = metric.name # TODO(omalleyt): Is this needed?
else:
# function was passed.
metric_name = metric.__name__
metric_obj = metrics_mod.MeanMetricWrapper(metric_obj,
name=metric_name)
return metric_obj
def _init_metrics_dict(self, metrics_dict):
if not metrics_dict:
raise ValueError(
"Evaluation metrics dictionary must not be empty.")
first_metrics = list(metrics_dict.values())[0]
if isinstance(first_metrics, dict):
self._model_have_multiple_outputs = True
self._metrics_dict = metrics_dict
else:
# When model has only one output, save it in a dict in order to
# keep the same data structure as the `metrics_dict` when model
# has multiple outputs.
self._model_have_multiple_outputs = False
self._metrics_dict = {MetricsDictKey.MODEL_OUTPUT: metrics_dict}
for output_name, metrics in self._metrics_dict.items():
for metric_name, metric in metrics.items():
if not isinstance(metric, metrics_module.Metric):
# `tf.keras.metrics.MeanMetricWrapper` wraps stateless
# functions into `tf.keras.metrics.Metric` instance.
metrics[metric_name] = metrics_module.MeanMetricWrapper(
metric, name=metric_name)
def fit_loop(model,
inputs,
targets,
sample_weights=None,
class_weight=None,
val_inputs=None,
val_targets=None,
val_sample_weights=None,
batch_size=None,
epochs=1,
verbose=1,
callbacks=None,
shuffle=True,
initial_epoch=0,
steps_per_epoch=None,
validation_steps=None):
"""Fit function for eager execution.
Arguments:
model: Instance of the model that is being executed in Eager mode.
inputs: List of input arrays.
targets: List of target arrays.
sample_weights: Optional list of sample weight arrays.
class_weight: Optional class-weight array to weight the importance of
samples in `inputs` based on the class they belong to, as conveyed by
`targets`.
val_inputs: Input data for validation.
val_targets: Target data for validation.
val_sample_weights: Sample weight data for validation.
batch_size: Integer batch size or None if unknown.
epochs: Number of times to iterate over the data
verbose: Verbosity mode, 0, 1 or 2
callbacks: List of callbacks to be called during training
shuffle: Whether to shuffle the data at the beginning of each epoch
initial_epoch: Epoch at which to start training
(useful for resuming a previous training run)
steps_per_epoch: Total number of steps (batches of samples)
before declaring one epoch finished and starting the
next epoch. Ignored with the default value of `None`.
validation_steps: Number of steps to run validation for (only if doing
validation from data tensors). Ignored with default value of `None`.
Returns:
`History` object.
Raises:
ValueError: In case of invalid argument values.
"""
# Convert training inputs to an EagerIterator
inputs, steps_per_epoch = training_utils.convert_to_iterator(
x=inputs,
y=targets,
sample_weights=sample_weights,
batch_size=batch_size,
steps_per_epoch=steps_per_epoch,
epochs=epochs,
shuffle=shuffle)
# Required for eager execution
with backend.learning_phase_scope(1):
do_validation = val_inputs is not None
callbacks = cbks.configure_callbacks(
callbacks,
model,
do_validation=do_validation,
batch_size=batch_size,
epochs=epochs,
steps_per_epoch=steps_per_epoch,
val_inputs=val_inputs,
val_targets=val_targets,
val_sample_weights=val_sample_weights,
validation_steps=validation_steps,
verbose=verbose)
# Create metric wrapper for the losses.
output_loss_metrics = []
for i in range(len(model.outputs)):
loss_fn = model.loss_functions[i]
loss_name = loss_fn.name if isinstance(
loss_fn, losses_module.Loss) else loss_fn.__name__
mean_wrapped_loss = metrics_module.MeanMetricWrapper(
loss_fn, name=loss_name)
output_loss_metrics.append(mean_wrapped_loss)
callbacks.on_train_begin()
for epoch in range(initial_epoch, epochs):
if model._is_compiled: # Model may not be compiled the first time.
# Reset stateful metrics
for m in model.metrics:
m.reset_states()
for m in output_loss_metrics:
m.reset_states()
callbacks.on_epoch_begin(epoch)
epoch_logs = {}
iterator_fit_loop(model,
inputs,
class_weight,
steps_per_epoch=steps_per_epoch,
epoch_logs=epoch_logs,
val_inputs=val_inputs,
val_targets=val_targets,
val_sample_weights=val_sample_weights,
epochs=epochs,
verbose=verbose,
callbacks=callbacks,
validation_steps=validation_steps,
do_validation=do_validation,
batch_size=batch_size,
output_loss_metrics=output_loss_metrics)
callbacks.on_epoch_end(epoch, epoch_logs)
if callbacks.model.stop_training:
break
callbacks.on_train_end()
return model.history
def iterator_test_loop(model, inputs, steps, verbose=0):
"""Test function for eager execution when input is given as dataset iterator.
Arguments:
model: Model instance that is being evaluated in Eager mode.
inputs: Input dataset iterator.
steps: Total number of steps (batches of samples) before declaring
predictions finished.
verbose: Verbosity mode.
Returns:
Scalar loss (if the model has a single output and no metrics)
or list of scalars (if the model has multiple outputs
and/or metrics). The attribute `model.metrics_names` will give you
the display labels for the scalar outputs.
Raises:
ValueError: In case of mismatch between given number of inputs and
expectations of the model.
"""
assert isinstance(inputs, iterator_ops.EagerIterator)
# make sure either x,y or x,y,sample_weights is provided
if (not isinstance(inputs.output_shapes, collections.Sequence)
or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3):
raise ValueError('Please provide either inputs and targets'
'or inputs, targets, and sample_weights')
outs = []
# Create metric wrapper for the losses.
output_loss_metrics = []
for i in range(len(model.outputs)):
loss_fn = model.loss_functions[i]
loss_name = loss_fn.name if isinstance(
loss_fn, losses_module.Loss) else loss_fn.__name__
mean_wrapped_loss = metrics_module.MeanMetricWrapper(loss_fn,
name=loss_name)
output_loss_metrics.append(mean_wrapped_loss)
num_samples = 0
if verbose == 1:
progbar = generic_utils.Progbar(target=steps)
for step_index in range(steps):
# Get data from the iterator.
try:
next_element = inputs.get_next()
except errors.OutOfRangeError:
logging.warning(
'Your dataset iterator ran out of data interrupting testing. '
'Make sure that your dataset can generate at least `steps` batches '
'(in this case, %d batches). You may need to use the repeat() '
'function when building your dataset.', steps)
break
if len(inputs.output_shapes) == 2:
x, y = next_element
sample_weights = None
else:
x, y, sample_weights = next_element
# Validate and standardize data.
x, y, sample_weights = model._standardize_user_data(
x, y, sample_weight=sample_weights)
x = training_utils.cast_if_floating_dtype(x)
y = training_utils.cast_if_floating_dtype(y)
if sample_weights:
sample_weights = [
training_utils.cast_if_floating_dtype(
ops.convert_to_tensor(val, dtype=backend.floatx()))
if val is not None else None for val in sample_weights
]
if step_index == 0:
# Get stateful metrics indices. We do not do this before the `steps` loop
# because model will be compiled only in the first iteration of this loop
# in the deferred build scenario.
if hasattr(model, '_compile_metrics'):
for m in model.metrics:
m.reset_states()
for m in output_loss_metrics:
m.reset_states()
# Calculate model output, loss values.
loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss(
model,
x,
y,
output_loss_metrics=output_loss_metrics,
sample_weights=sample_weights,
training=False)
metrics_results = _eager_metrics_fn(model,
loss_outs,
y,
sample_weights=sample_weights,
masks=masks)
batch_outs = []
for _, v in zip(model.metrics_names, [backend.mean(loss)] +
aggregated_loss_metrics + metrics_results):
batch_outs.append(tensor_util.constant_value(v))
# Get current step size.
if isinstance(x, list):
step_size = x[0].get_shape().as_list()[0]
elif isinstance(x, dict):
step_size = list(x.values())[0].get_shape().as_list()[0]
else:
step_size = x.get_shape().as_list()[0]
# Accumulate results in output array.
if not isinstance(batch_outs, list):
batch_outs = [batch_outs]
if step_index == 0:
for _ in enumerate(batch_outs):
outs.append(0.)
outs[0] += batch_outs[0] * step_size # index 0 = 'loss'
outs[1:] = batch_outs[1:]
# Calculate sample size.
num_samples += step_size
if verbose == 1:
progbar.update(step_index + 1)
outs[0] /= num_samples # index 0 = 'loss'
if len(outs) == 1:
return outs[0]
return outs
def collect_per_output_metric_info(metrics,
output_names,
output_shapes,
loss_fns,
sample_weights=None):
"""Maps metric names and functions to model outputs.
Arguments:
metrics: a list or dict of metric functions.
output_names: a list of the names (strings) of model outputs.
output_shapes: a list of the shapes (strings) of model outputs.
loss_fns: a list of the loss functions corresponding to the model outputs.
sample_weights: a list of weights to be applied on the model outputs.
Returns:
A list (one entry per model output) of dicts.
For instance, if the model has 2 outputs, and for the first output
we want to compute "binary_accuracy" and "binary_crossentropy",
and just "binary_accuracy" for the second output,
the list would look like: `[
{
'acc': (binary_accuracy(), mean_obj_1),
'ce': (binary_crossentropy(), mean_obj_2)
},
{
'acc': (binary_accuracy(), mean_obj_3)
}
]`
Raises:
TypeError: if an incorrect type is passed for the `metrics` argument.
"""
if not metrics:
return [{} for _ in output_names]
if isinstance(metrics, list):
# we then apply all metrics to all outputs.
nested_metrics = [copy.copy(metrics) for _ in output_names]
elif isinstance(metrics, dict):
nested_metrics = []
for name in output_names:
output_metrics = metrics.get(name, [])
if not isinstance(output_metrics, list):
output_metrics = [output_metrics]
nested_metrics.append(output_metrics)
else:
raise TypeError('Type of `metrics` argument not understood. '
'Expected a list or dictionary, found: ' +
str(metrics))
per_output_metrics = []
for i, metrics in enumerate(nested_metrics):
metrics_dict = OrderedDict()
for metric in metrics:
weighted = False if (sample_weights is None) else (
sample_weights[i] is not None)
metric_name = get_metric_name(metric, weighted)
metric_fn = get_metric_function(metric,
output_shape=output_shapes[i],
loss_fn=loss_fns[i])
# If the metric function is not stateful, we create a stateful version and
# return both the stateless and the stateful version together. For batch
# APIs like `train_on_batch` we will use the stateless version and for
# other APIs like `fit` we will use the stateful version.
is_stateful = isinstance(metric_fn,
base_layer.Layer) and metric_fn.stateful
stateful_fn = metric_fn
if not is_stateful:
stateful_fn = metrics_module.MeanMetricWrapper(
metric_fn, name=metric_fn.__name__)
metrics_dict[metric_name] = (metric_fn, stateful_fn)
per_output_metrics.append(metrics_dict)
return per_output_metrics
