def _get_metric_object(self, metric, y_t, y_p): """Converts user-supplied metric to a `Metric` object. Args: metric: A string, function, or `Metric` object. y_t: Sample of label. y_p: Sample of output. Returns: A `Metric` object. """ if metric is None: return None # Ok to have no metric for an output. # Convenience feature for selecting b/t binary, categorical, # and sparse categorical. if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']: metric_obj = metrics_mod.get(metric) else: y_t_rank = len(y_t.shape.as_list()) y_p_rank = len(y_p.shape.as_list()) y_t_last_dim = y_t.shape.as_list()[-1] y_p_last_dim = y_p.shape.as_list()[-1] is_binary = y_p_last_dim == 1 is_sparse_categorical = (y_t_rank < y_p_rank or y_t_last_dim == 1 and y_p_last_dim > 1) if metric in ['accuracy', 'acc']: if is_binary: metric_obj = metrics_mod.binary_accuracy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_accuracy else: metric_obj = metrics_mod.categorical_accuracy else: if is_binary: metric_obj = metrics_mod.binary_crossentropy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_crossentropy else: metric_obj = metrics_mod.categorical_crossentropy if isinstance(metric_obj, losses_mod.Loss): metric_obj._allow_sum_over_batch_size = True # pylint: disable=protected-access if not isinstance(metric_obj, metrics_mod.Metric): if isinstance(metric, six.string_types): metric_name = metric else: metric_name = get_custom_object_name(metric) if metric_name is None: raise ValueError( 'Metric should be a callable, found: {}'.format( metric)) metric_obj = metrics_mod.MeanMetricWrapper(metric_obj, name=metric_name) return metric_obj
def _get_metric_object(self, metric, y_t, y_p): """Converts user-supplied metric to a `Metric` object. Arguments: metric: A string, function, or `Metric` object. y_t: Sample of label. y_p: Sample of output. Returns: A `Metric` object. """ if metric is None: return None # Ok to have no metric for an output. # Convenience feature for selecting b/t binary, categorical, # and sparse categorical. if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']: metric_obj = metrics_mod.get(metric) else: y_t_rank = len(y_t.shape.as_list()) y_p_rank = len(y_p.shape.as_list()) y_t_last_dim = y_t.shape.as_list()[-1] y_p_last_dim = y_p.shape.as_list()[-1] is_binary = y_p_last_dim == 1 is_sparse_categorical = (y_t_rank < y_p_rank or y_t_last_dim == 1 and y_p_last_dim > 1) if metric in ['accuracy', 'acc']: if is_binary: metric_obj = metrics_mod.binary_accuracy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_accuracy else: metric_obj = metrics_mod.categorical_accuracy else: if is_binary: metric_obj = metrics_mod.binary_crossentropy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_crossentropy else: metric_obj = metrics_mod.categorical_crossentropy if isinstance(metric_obj, losses_mod.Loss): metric_obj._allow_sum_over_batch_size = True # pylint: disable=protected-access if not isinstance(metric_obj, metrics_mod.Metric): if isinstance(metric, six.string_types): metric_name = metric elif hasattr(metric, 'name'): metric_name = metric.name # TODO(omalleyt): Is this needed? else: # function was passed. metric_name = metric.__name__ metric_obj = metrics_mod.MeanMetricWrapper(metric_obj, name=metric_name) return metric_obj
def _init_metrics_dict(self, metrics_dict): if not metrics_dict: raise ValueError( "Evaluation metrics dictionary must not be empty.") first_metrics = list(metrics_dict.values())[0] if isinstance(first_metrics, dict): self._model_have_multiple_outputs = True self._metrics_dict = metrics_dict else: # When model has only one output, save it in a dict in order to # keep the same data structure as the `metrics_dict` when model # has multiple outputs. self._model_have_multiple_outputs = False self._metrics_dict = {MetricsDictKey.MODEL_OUTPUT: metrics_dict} for output_name, metrics in self._metrics_dict.items(): for metric_name, metric in metrics.items(): if not isinstance(metric, metrics_module.Metric): # `tf.keras.metrics.MeanMetricWrapper` wraps stateless # functions into `tf.keras.metrics.Metric` instance. metrics[metric_name] = metrics_module.MeanMetricWrapper( metric, name=metric_name)
def fit_loop(model, inputs, targets, sample_weights=None, class_weight=None, val_inputs=None, val_targets=None, val_sample_weights=None, batch_size=None, epochs=1, verbose=1, callbacks=None, shuffle=True, initial_epoch=0, steps_per_epoch=None, validation_steps=None): """Fit function for eager execution. Arguments: model: Instance of the model that is being executed in Eager mode. inputs: List of input arrays. targets: List of target arrays. sample_weights: Optional list of sample weight arrays. class_weight: Optional class-weight array to weight the importance of samples in `inputs` based on the class they belong to, as conveyed by `targets`. val_inputs: Input data for validation. val_targets: Target data for validation. val_sample_weights: Sample weight data for validation. batch_size: Integer batch size or None if unknown. epochs: Number of times to iterate over the data verbose: Verbosity mode, 0, 1 or 2 callbacks: List of callbacks to be called during training shuffle: Whether to shuffle the data at the beginning of each epoch initial_epoch: Epoch at which to start training (useful for resuming a previous training run) steps_per_epoch: Total number of steps (batches of samples) before declaring one epoch finished and starting the next epoch. Ignored with the default value of `None`. validation_steps: Number of steps to run validation for (only if doing validation from data tensors). Ignored with default value of `None`. Returns: `History` object. Raises: ValueError: In case of invalid argument values. """ # Convert training inputs to an EagerIterator inputs, steps_per_epoch = training_utils.convert_to_iterator( x=inputs, y=targets, sample_weights=sample_weights, batch_size=batch_size, steps_per_epoch=steps_per_epoch, epochs=epochs, shuffle=shuffle) # Required for eager execution with backend.learning_phase_scope(1): do_validation = val_inputs is not None callbacks = cbks.configure_callbacks( callbacks, model, do_validation=do_validation, batch_size=batch_size, epochs=epochs, steps_per_epoch=steps_per_epoch, val_inputs=val_inputs, val_targets=val_targets, val_sample_weights=val_sample_weights, validation_steps=validation_steps, verbose=verbose) # Create metric wrapper for the losses. output_loss_metrics = [] for i in range(len(model.outputs)): loss_fn = model.loss_functions[i] loss_name = loss_fn.name if isinstance( loss_fn, losses_module.Loss) else loss_fn.__name__ mean_wrapped_loss = metrics_module.MeanMetricWrapper( loss_fn, name=loss_name) output_loss_metrics.append(mean_wrapped_loss) callbacks.on_train_begin() for epoch in range(initial_epoch, epochs): if model._is_compiled: # Model may not be compiled the first time. # Reset stateful metrics for m in model.metrics: m.reset_states() for m in output_loss_metrics: m.reset_states() callbacks.on_epoch_begin(epoch) epoch_logs = {} iterator_fit_loop(model, inputs, class_weight, steps_per_epoch=steps_per_epoch, epoch_logs=epoch_logs, val_inputs=val_inputs, val_targets=val_targets, val_sample_weights=val_sample_weights, epochs=epochs, verbose=verbose, callbacks=callbacks, validation_steps=validation_steps, do_validation=do_validation, batch_size=batch_size, output_loss_metrics=output_loss_metrics) callbacks.on_epoch_end(epoch, epoch_logs) if callbacks.model.stop_training: break callbacks.on_train_end() return model.history
def iterator_test_loop(model, inputs, steps, verbose=0): """Test function for eager execution when input is given as dataset iterator. Arguments: model: Model instance that is being evaluated in Eager mode. inputs: Input dataset iterator. steps: Total number of steps (batches of samples) before declaring predictions finished. verbose: Verbosity mode. Returns: Scalar loss (if the model has a single output and no metrics) or list of scalars (if the model has multiple outputs and/or metrics). The attribute `model.metrics_names` will give you the display labels for the scalar outputs. Raises: ValueError: In case of mismatch between given number of inputs and expectations of the model. """ assert isinstance(inputs, iterator_ops.EagerIterator) # make sure either x,y or x,y,sample_weights is provided if (not isinstance(inputs.output_shapes, collections.Sequence) or len(inputs.output_shapes) < 2 or len(inputs.output_shapes) > 3): raise ValueError('Please provide either inputs and targets' 'or inputs, targets, and sample_weights') outs = [] # Create metric wrapper for the losses. output_loss_metrics = [] for i in range(len(model.outputs)): loss_fn = model.loss_functions[i] loss_name = loss_fn.name if isinstance( loss_fn, losses_module.Loss) else loss_fn.__name__ mean_wrapped_loss = metrics_module.MeanMetricWrapper(loss_fn, name=loss_name) output_loss_metrics.append(mean_wrapped_loss) num_samples = 0 if verbose == 1: progbar = generic_utils.Progbar(target=steps) for step_index in range(steps): # Get data from the iterator. try: next_element = inputs.get_next() except errors.OutOfRangeError: logging.warning( 'Your dataset iterator ran out of data interrupting testing. ' 'Make sure that your dataset can generate at least `steps` batches ' '(in this case, %d batches). You may need to use the repeat() ' 'function when building your dataset.', steps) break if len(inputs.output_shapes) == 2: x, y = next_element sample_weights = None else: x, y, sample_weights = next_element # Validate and standardize data. x, y, sample_weights = model._standardize_user_data( x, y, sample_weight=sample_weights) x = training_utils.cast_if_floating_dtype(x) y = training_utils.cast_if_floating_dtype(y) if sample_weights: sample_weights = [ training_utils.cast_if_floating_dtype( ops.convert_to_tensor(val, dtype=backend.floatx())) if val is not None else None for val in sample_weights ] if step_index == 0: # Get stateful metrics indices. We do not do this before the `steps` loop # because model will be compiled only in the first iteration of this loop # in the deferred build scenario. if hasattr(model, '_compile_metrics'): for m in model.metrics: m.reset_states() for m in output_loss_metrics: m.reset_states() # Calculate model output, loss values. loss_outs, loss, _, aggregated_loss_metrics, masks = _model_loss( model, x, y, output_loss_metrics=output_loss_metrics, sample_weights=sample_weights, training=False) metrics_results = _eager_metrics_fn(model, loss_outs, y, sample_weights=sample_weights, masks=masks) batch_outs = [] for _, v in zip(model.metrics_names, [backend.mean(loss)] + aggregated_loss_metrics + metrics_results): batch_outs.append(tensor_util.constant_value(v)) # Get current step size. if isinstance(x, list): step_size = x[0].get_shape().as_list()[0] elif isinstance(x, dict): step_size = list(x.values())[0].get_shape().as_list()[0] else: step_size = x.get_shape().as_list()[0] # Accumulate results in output array. if not isinstance(batch_outs, list): batch_outs = [batch_outs] if step_index == 0: for _ in enumerate(batch_outs): outs.append(0.) outs[0] += batch_outs[0] * step_size # index 0 = 'loss' outs[1:] = batch_outs[1:] # Calculate sample size. num_samples += step_size if verbose == 1: progbar.update(step_index + 1) outs[0] /= num_samples # index 0 = 'loss' if len(outs) == 1: return outs[0] return outs
def collect_per_output_metric_info(metrics, output_names, output_shapes, loss_fns, sample_weights=None): """Maps metric names and functions to model outputs. Arguments: metrics: a list or dict of metric functions. output_names: a list of the names (strings) of model outputs. output_shapes: a list of the shapes (strings) of model outputs. loss_fns: a list of the loss functions corresponding to the model outputs. sample_weights: a list of weights to be applied on the model outputs. Returns: A list (one entry per model output) of dicts. For instance, if the model has 2 outputs, and for the first output we want to compute "binary_accuracy" and "binary_crossentropy", and just "binary_accuracy" for the second output, the list would look like: `[ { 'acc': (binary_accuracy(), mean_obj_1), 'ce': (binary_crossentropy(), mean_obj_2) }, { 'acc': (binary_accuracy(), mean_obj_3) } ]` Raises: TypeError: if an incorrect type is passed for the `metrics` argument. """ if not metrics: return [{} for _ in output_names] if isinstance(metrics, list): # we then apply all metrics to all outputs. nested_metrics = [copy.copy(metrics) for _ in output_names] elif isinstance(metrics, dict): nested_metrics = [] for name in output_names: output_metrics = metrics.get(name, []) if not isinstance(output_metrics, list): output_metrics = [output_metrics] nested_metrics.append(output_metrics) else: raise TypeError('Type of `metrics` argument not understood. ' 'Expected a list or dictionary, found: ' + str(metrics)) per_output_metrics = [] for i, metrics in enumerate(nested_metrics): metrics_dict = OrderedDict() for metric in metrics: weighted = False if (sample_weights is None) else ( sample_weights[i] is not None) metric_name = get_metric_name(metric, weighted) metric_fn = get_metric_function(metric, output_shape=output_shapes[i], loss_fn=loss_fns[i]) # If the metric function is not stateful, we create a stateful version and # return both the stateless and the stateful version together. For batch # APIs like `train_on_batch` we will use the stateless version and for # other APIs like `fit` we will use the stateful version. is_stateful = isinstance(metric_fn, base_layer.Layer) and metric_fn.stateful stateful_fn = metric_fn if not is_stateful: stateful_fn = metrics_module.MeanMetricWrapper( metric_fn, name=metric_fn.__name__) metrics_dict[metric_name] = (metric_fn, stateful_fn) per_output_metrics.append(metrics_dict) return per_output_metrics