def evaluate(self, data, fn_inverse=None, horizon=1, fn_plot=None): """ Evaluate model :return: """ encoder_input_data, decoder_input_exog, y = data y_hat = self.predict(encoder_inputs=encoder_input_data, pred_steps=horizon, decoder_input_exog=decoder_input_exog) if fn_inverse is not None: y = fn_inverse(y) y_hat = fn_inverse(y_hat) y = np.float32(y) y_hat = np.float32(y_hat) if fn_plot is not None: fn_plot([y, y_hat]) results = [] for m in self.model.metrics: if isinstance(m, str): results.append(K.eval(K.mean(get(m)(y, y_hat)))) else: results.append(K.eval(K.mean(m(y, y_hat)))) return results
def _extract_metrics(self): # We want a dictionary that maps an output tensor id to a list of metrics. metrics = self._model.metrics if metrics is None: metrics = {} if not isinstance(metrics, dict): metrics = { self._output_specs[i].identifier: metrics for i in range(len(self._output_specs)) } else: # Keras stores dicts that map an output layer name to a (list of) metric(s). # We want an output tensor id. temp = {} for layer_name, metric in metrics.items(): if isinstance(metric, str) or callable(metric): # normalize to list metric = [metric] for tensor_id in self._output_layer_tensor_ids[layer_name]: temp[tensor_id] = metric metrics = temp import keras.metrics as km for tensor, metric in metrics.items(): for idx, met in enumerate(metric): metrics[tensor][idx] = km.get( met ).__name__ if met != 'accuracy' and met != 'acc' else 'acc' return metrics
def evaluate(self, inputs, fn_inverse=None, fn_plot=None): try: X, y = inputs except: X, y, _ = inputs try: X, exogenous = X except: exogenous = None y_hat = self.predict(X, exogenous) if fn_inverse is not None: y_hat = fn_inverse(y_hat) y = fn_inverse(y) if fn_plot is not None: fn_plot([y, y_hat]) results = [] for m in self.model.metrics: if isinstance(m, str): results.append(K.eval(K.mean(get(m)(y, y_hat)))) else: results.append(K.eval(K.mean(m(y, y_hat)))) return results
def _get_metric_object(self, metric, y_t, y_p): """Converts user-supplied metric to a `Metric` object. Args: metric: A string, function, or `Metric` object. y_t: Sample of label. y_p: Sample of output. Returns: A `Metric` object. """ if metric is None: return None # Ok to have no metric for an output. # Convenience feature for selecting b/t binary, categorical, # and sparse categorical. if metric not in ['accuracy', 'acc', 'crossentropy', 'ce']: metric_obj = metrics_mod.get(metric) else: y_t_rank = len(y_t.shape.as_list()) y_p_rank = len(y_p.shape.as_list()) y_t_last_dim = y_t.shape.as_list()[-1] y_p_last_dim = y_p.shape.as_list()[-1] is_binary = y_p_last_dim == 1 is_sparse_categorical = (y_t_rank < y_p_rank or y_t_last_dim == 1 and y_p_last_dim > 1) if metric in ['accuracy', 'acc']: if is_binary: metric_obj = metrics_mod.binary_accuracy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_accuracy else: metric_obj = metrics_mod.categorical_accuracy else: if is_binary: metric_obj = metrics_mod.binary_crossentropy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_crossentropy else: metric_obj = metrics_mod.categorical_crossentropy if isinstance(metric_obj, losses_mod.Loss): metric_obj._allow_sum_over_batch_size = True # pylint: disable=protected-access if not isinstance(metric_obj, metrics_mod.Metric): if isinstance(metric, six.string_types): metric_name = metric else: metric_name = get_custom_object_name(metric) if metric_name is None: raise ValueError( 'Metric should be a callable, found: {}'.format( metric)) metric_obj = metrics_mod.MeanMetricWrapper(metric_obj, name=metric_name) return metric_obj
def evaluate(self, data, fn_inverse=None, fn_plot=None): try: X, y = data y_hat = self.predict(X) except: X, X_ex, y = data y_hat = self.predict([X, X_ex]) if fn_inverse is not None: y = fn_inverse(y) y_hat = fn_inverse(y_hat) y = np.float32(y) y_hat = np.float32(y_hat) if fn_plot is not None: fn_plot([y, y_hat]) results = [] for m in self.model.metrics: if isinstance(m, str): results.append(K.eval(K.mean(get(m)(y, y_hat)))) else: results.append(K.eval(K.mean(m(y, y_hat)))) return results
def _eval(self, y, y_hat): results = [] for m in self.model.metrics: if isinstance(m, str): results.append(K.eval(K.mean(get(m)(y, y_hat)))) else: results.append(K.eval(K.mean(m(y, y_hat)))) return results
def from_config(cls, config): from keras.metrics import get # pylint: disable=g-import-not-at-top # Note that while MeanMetricWrapper itself isn't public, objects of this # class may be created and added to the model by calling model.compile. fn = config.pop('fn', None) if cls is MeanMetricWrapper: return cls(get(fn), **config) return super(MeanMetricWrapper, cls).from_config(config)
def get_metrics(identifier): '''return metrics ''' if isinstance(identifier, str): if metrics_collection.get(identifier) is not None: return metrics_collection.get(identifier) else: return get(identifier) elif callable(identifier): return identifier
def get_metric_name(name): """ Gives the keras name for a metric Parameters ---------- name : str original name of the metric Returns ------- """ if name == 'acc' or name == 'accuracy': return 'acc' try: metric_fn = metrics.get(name) return metric_fn.__name__ except: pass return name
def get_metric_name(name): """ Gives the keras name for a metric Parameters ---------- name : str original name of the metric Returns ------- """ if name == 'acc' or name == 'accuracy': return 'acc' try: metric_fn = metrics.get(name) return metric_fn.__name__ except: pass return name
def evaluate(self, inputs, fn_inverse=None, fn_plot=None): try: X, y_exog, y = inputs y_hat = self.model.predict([X, y_exog]) except: X, y = inputs y_hat = self.model.predict(X) y_hat = np.asarray(y_hat, dtype=y.dtype) if fn_inverse is not None: y_hat = fn_inverse(y_hat) y = fn_inverse(y) if fn_plot is not None: fn_plot([y, y_hat]) results = [] for m in self.model.metrics: if isinstance(m, str): results.append(K.eval(K.mean(get(m)(y, y_hat)))) else: results.append(K.eval(K.mean(m(y, y_hat)))) return results
def compile_tfrecord(self, optimizer, loss, y, metrics=None, y_val=None): """Configures the model for training. # Arguments optimizer: str (name of optimizer) or optimizer object. See [optimizers](/optimizers). loss: str (name of objective function) or objective function. See [losses](/losses). If the model has multiple outputs, you can use a different loss on each output by passing a dictionary or a list of losses. The loss value that will be minimized by the model will then be the sum of all individual losses. metrics: list of metrics to be evaluated by the model during training and testing. Typically you will use `metrics=['accuracy']`. To specify different metrics for different outputs of a multi-output model, you could also pass a dictionary, such as `metrics={'output_a': 'accuracy'}`. # Raises ValueError: In case of invalid arguments for `optimizer`, `loss`, `metrics` or `sample_weight_mode`. """ loss = loss or {} self.optimizer = optimizers.get(optimizer) self.loss = loss self.sample_weight_mode = None self.loss_weights = None self.y_val = y_val do_validation = bool(len(self.val_inputs) > 0) if do_validation and y_val is None: raise ValueError('When you use validation inputs, ' 'you should provide y_val.') # Prepare loss functions. if isinstance(loss, dict): for name in loss: if name not in self.output_names: raise ValueError('Unknown entry in loss ' 'dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(self.output_names)) loss_functions = [] for name in self.output_names: if name not in loss: warnings.warn('Output "' + name + '" missing from loss dictionary. ' 'We assume this was done on purpose, ' 'and we will not be expecting ' 'any data to be passed to "' + name + '" during training.', stacklevel=2) loss_functions.append(losses.get(loss.get(name))) elif isinstance(loss, list): if len(loss) != len(self.outputs): raise ValueError('When passing a list as loss, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(self.outputs)) + ' outputs, but you passed loss=' + str(loss)) loss_functions = [losses.get(l) for l in loss] else: loss_function = losses.get(loss) loss_functions = [loss_function for _ in range(len(self.outputs))] self.loss_functions = loss_functions # Prepare training targets of model. if isinstance(y, (list, tuple)): y = list(y) # Tensor or list of tensors. else: y = [y] self.targets = [] for i in range(len(self.outputs)): target = y[i] self.targets.append(target) # Prepare validation targets of model. if isinstance(y_val, (list, tuple)): y_val = list(y_val) # Tensor or list of tensors. else: y_val = [y_val] self.y_val = y_val self.val_targets = [] for i in range(len(self.val_outputs)): val_target = y_val[i] self.val_targets.append(val_target) # Prepare metrics. self.metrics = metrics self.metrics_names = ['loss'] self.metrics_tensors = [] self.val_metrics_names = ['loss'] self.val_metrics_tensors = [] # Compute total training loss. total_loss = None for i in range(len(self.outputs)): y_true = self.targets[i] y_pred = self.outputs[i] loss_function = loss_functions[i] val_output_loss = K.mean(loss_function(y_true, y_pred)) if len(self.outputs) > 1: self.metrics_tensors.append(val_output_loss) self.metrics_names.append(self.output_names[i] + '_loss') if total_loss is None: total_loss = val_output_loss else: total_loss += val_output_loss if total_loss is None: if not self.losses: raise RuntimeError('The model cannot be compiled ' 'because it has no loss to optimize.') else: total_loss = 0. # Compute total validation loss. val_total_loss = None for i in range(len(self.val_outputs)): y_true = self.val_targets[i] y_pred = self.val_outputs[i] loss_function = loss_functions[i] val_output_loss = K.mean(loss_function(y_true, y_pred)) if len(self.outputs) > 1: self.val_metrics_tensors.append(val_output_loss) self.val_metrics_names.append(self.output_names[i] + '_val_loss') if val_total_loss is None: val_total_loss = val_output_loss else: val_total_loss += val_output_loss if val_total_loss is None: if not self.losses and do_validation: raise RuntimeError('The model cannot be compiled ' 'because it has no loss to optimize.') else: val_total_loss = 0. # Add regularization penalties # and other layer-specific losses. for loss_tensor in self.losses: total_loss += loss_tensor val_total_loss += loss_tensor # List of same size as output_names. # contains tuples (metrics for output, names of metrics). nested_metrics = _collect_metrics(metrics, self.output_names) def append_metric(layer_num, metric_name, metric_tensor): """Helper function used in loop below.""" if len(self.output_names) > 1: metric_name = self.output_layers[ layer_num].name + '_' + metric_name self.metrics_names.append(metric_name) self.metrics_tensors.append(metric_tensor) for i in range(len(self.outputs)): y_true = self.targets[i] y_pred = self.outputs[i] output_metrics = nested_metrics[i] for metric in output_metrics: if metric == 'accuracy' or metric == 'acc': # custom handling of accuracy # (because of class mode duality) output_shape = self.internal_output_shapes[i] acc_fn = None if (output_shape[-1] == 1 or self.loss_functions[i] == losses.binary_crossentropy): # case: binary accuracy acc_fn = metrics_module.binary_accuracy elif self.loss_functions[ i] == losses.sparse_categorical_crossentropy: # case: categorical accuracy with sparse targets acc_fn = metrics_module.sparse_categorical_accuracy else: acc_fn = metrics_module.categorical_accuracy append_metric(i, 'acc', K.mean(acc_fn(y_true, y_pred))) else: metric_fn = metrics_module.get(metric) metric_result = metric_fn(y_true, y_pred) metric_result = {metric_fn.__name__: metric_result} for name, tensor in six.iteritems(metric_result): append_metric(i, name, tensor) def append_val_metric(layer_num, metric_name, metric_tensor): """Helper function used in loop below.""" if len(self.output_names) > 1: metric_name = self.output_layers[ layer_num].name + '_val_' + metric_name self.val_metrics_names.append(metric_name) self.val_metrics_tensors.append(metric_tensor) for i in range(len(self.val_outputs)): y_true = self.val_targets[i] y_pred = self.val_outputs[i] output_metrics = nested_metrics[i] for metric in output_metrics: if metric == 'accuracy' or metric == 'acc': # custom handling of accuracy # (because of class mode duality) output_shape = self.internal_output_shapes[i] acc_fn = None if (output_shape[-1] == 1 or self.loss_functions[i] == losses.binary_crossentropy): # case: binary accuracy acc_fn = metrics_module.binary_accuracy elif self.loss_functions[ i] == losses.sparse_categorical_crossentropy: # case: categorical accuracy with sparse targets acc_fn = metrics_module.sparse_categorical_accuracy else: acc_fn = metrics_module.categorical_accuracy append_val_metric(i, 'acc', K.mean(acc_fn(y_true, y_pred))) else: metric_fn = metrics_module.get(metric) metric_result = metric_fn(y_true, y_pred) metric_result = {metric_fn.__name__: metric_result} for name, tensor in six.iteritems(metric_result): append_val_metric(i, name, tensor) # Prepare gradient updates and state updates. self.total_loss = total_loss self.val_total_loss = val_total_loss # Functions for train, test and predict will # be compiled lazily when required. # This saves time when the user is not using all functions. self.train_function = None self.val_function = None self.test_function = None self.predict_function = None # Collected trainable weights and sort them deterministically. trainable_weights = self.trainable_weights # Sort weights by name. if trainable_weights: trainable_weights.sort(key=lambda x: x.name) self._collected_trainable_weights = trainable_weights
def compile(self, optimizer, loss, metrics=[], loss_weights=None, sample_weight_mode=None, **kwargs): #super(sModel, self).compile(optimizer, loss, metrics, loss_weights, # sample_weights_mode, **kwargs) self.optimizer = optimizers.get(optimizer) self.sample_weight_mode = sample_weight_mode self.loss = loss self.loss_weights = loss_weights # prepare loss weights if loss_weights is None: loss_weights_list = [1. for _ in range(len(self.outputs))] elif isinstance(loss_weights, dict): for name in loss_weights: if name not in self.output_names: raise ValueError('Unknown entry in loss_weights ' 'dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(self.output_names)) loss_weights_list = [] for name in self.output_names: loss_weights_list.append(loss_weights.get(name, 1.)) elif isinstance(loss_weights, list): if len(loss_weights) != len(self.outputs): raise ValueError('When passing a list as loss_weights, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(self.outputs)) + ' outputs, but you passed loss_weights=' + str(loss_weights)) loss_weights_list = loss_weights else: raise TypeError('Could not interpret loss_weights argument: ' + str(loss_weights) + ' - expected a list of dicts.') # prepare loss functions if isinstance(loss, dict): for name in loss: if name not in self.output_names: raise ValueError('Unknown entry in loss ' 'dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(self.output_names)) loss_functions = [] for name in self.output_names: if name not in loss: raise ValueError('Output "' + name + '" missing from loss dictionary.') loss_functions.append(objectives.get(loss[name])) elif isinstance(loss, list): if len(loss) != len(self.outputs): raise ValueError('When passing a list as loss, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(self.outputs)) + ' outputs, but you passed loss=' + str(loss)) loss_functions = [objectives.get(l) for l in loss] else: loss_function = objectives.get(loss) loss_functions = [loss_function for _ in range(len(self.outputs))] self.loss_functions = loss_functions weighted_losses = [weighted_objective(fn) for fn in loss_functions] # prepare output masks masks = self.compute_mask(self.inputs, mask=None) if masks is None: masks = [None for _ in self.outputs] if not isinstance(masks, list): masks = [masks] # prepare sample weights if isinstance(sample_weight_mode, dict): for name in sample_weight_mode: if name not in self.output_names: raise ValueError('Unknown entry in ' 'sample_weight_mode dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(self.output_names)) sample_weights = [] sample_weight_modes = [] for name in self.output_names: if name not in sample_weight_mode: raise ValueError('Output "' + name + '" missing from sample_weight_modes ' 'dictionary') if sample_weight_mode.get(name) == 'temporal': weight = K.placeholder(ndim=2, name=name + '_sample_weights') sample_weight_modes.append('temporal') else: weight = K.placeholder(ndim=1, name=name + '_sample_weights') sample_weight_modes.append(None) sample_weights.append(weight) elif isinstance(sample_weight_mode, list): if len(sample_weight_mode) != len(self.outputs): raise ValueError('When passing a list as sample_weight_mode, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(self.outputs)) + ' outputs, but you passed ' 'sample_weight_mode=' + str(sample_weight_mode)) sample_weights = [] sample_weight_modes = [] for mode, name in zip(sample_weight_mode, self.output_names): if mode == 'temporal': weight = K.placeholder(ndim=2, name=name + '_sample_weights') sample_weight_modes.append('temporal') else: weight = K.placeholder(ndim=1, name=name + '_sample_weights') sample_weight_modes.append(None) sample_weights.append(weight) else: if sample_weight_mode == 'temporal': sample_weights = [ K.placeholder(ndim=2, name=name + '_sample_weights') for name in self.output_names ] sample_weight_modes = [ 'temporal' for name in self.output_names ] else: sample_weights = [ K.placeholder(ndim=1, name=name + '_sample_weights') for name in self.output_names ] sample_weight_modes = [None for name in self.output_names] self.sample_weight_modes = sample_weight_modes # prepare targets of model self.targets = [] for i in range(len(self.outputs)): shape = self.internal_output_shapes[i] name = self.output_names[i] self.targets.append( K.placeholder(ndim=len(shape), name=name + '_target', sparse=K.is_sparse(self.outputs[i]), dtype=K.dtype(self.outputs[i]))) # prepare metrics self.metrics = metrics self.metrics_names = ['loss'] self.metrics_tensors = [] # compute total loss total_loss = None for i in range(len(self.outputs)): y_true = self.targets[i] y_pred = self.outputs[i] weighted_loss = weighted_losses[i] sample_weight = sample_weights[i] mask = masks[i] loss_weight = loss_weights_list[i] output_loss = weighted_loss(y_true, y_pred, sample_weight, mask) if len(self.outputs) > 1: self.metrics_tensors.append(output_loss) self.metrics_names.append(self.output_names[i] + '_loss') if total_loss is None: total_loss = loss_weight * output_loss else: total_loss += loss_weight * output_loss # add regularization penalties # and other layer-specific losses for loss_tensor in self.losses: total_loss += loss_tensor # list of same size as output_names. # contains tuples (metrics for output, names of metrics) nested_metrics = collect_metrics(metrics, self.output_names) def append_metric(layer_num, metric_name, metric_tensor): """Helper function, used in loop below""" if len(self.output_names) > 1: metric_name = self.output_layers[ layer_num].name + '_' + metric_name self.metrics_names.append(metric_name) self.metrics_tensors.append(metric_tensor) for i in range(len(self.outputs)): y_true = self.targets[i] y_pred = self.outputs[i] output_metrics = nested_metrics[i] for metric in output_metrics: if metric == 'accuracy' or metric == 'acc': # custom handling of accuracy # (because of class mode duality) output_shape = self.internal_output_shapes[i] acc_fn = None if output_shape[-1] == 1 or self.loss_functions[ i] == objectives.binary_crossentropy: # case: binary accuracy acc_fn = metrics_module.binary_accuracy elif self.loss_functions[ i] == objectives.sparse_categorical_crossentropy: # case: categorical accuracy with sparse targets acc_fn = metrics_module.sparse_categorical_accuracy else: acc_fn = metrics_module.categorical_accuracy append_metric(i, 'acc', acc_fn(y_true, y_pred)) else: metric_fn = metrics_module.get(metric) metric_result = metric_fn(y_true, y_pred) if not isinstance(metric_result, dict): metric_result = {metric_fn.__name__: metric_result} for name, tensor in six.iteritems(metric_result): append_metric(i, name, tensor) # prepare gradient updates and state updates self.total_loss = total_loss self.sample_weights = sample_weights # functions for train, test and predict will # be compiled lazily when required. # This saves time when the user is not using all functions. self._function_kwargs = kwargs self.train_function = None self.test_function = None self.predict_function = None # collected trainable weights and sort them deterministically. trainable_weights = self.trainable_weights # Sort weights by name if trainable_weights: if K.backend() == 'theano': trainable_weights.sort( key=lambda x: x.name if x.name else x.auto_name) else: trainable_weights.sort(key=lambda x: x.name) self._collected_trainable_weights = trainable_weights
def evaluate_keras_metric(y_true, y_pred, metric): objective_function = metrics.get(metric) objective = objective_function(y_true, y_pred) return K.eval(objective)
def compile_tfrecord(train_model, optimizer, loss, out_tensor_lst, metrics=[], loss_weights=None): train_model.build(train_model) # train_model.build() train_model.optimizer = optimizers.get(optimizer) train_model.loss = loss train_model.loss_weights = loss_weights # prepare loss weights if loss_weights is None: loss_weights_list = [1. for _ in range(len(train_model.outputs))] elif isinstance(loss_weights, dict): for name in loss_weights: if name not in train_model.output_names: raise ValueError('Unknown entry in loss_weights ' 'dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(train_model.output_names)) loss_weights_list = [] for name in train_model.output_names: loss_weights_list.append(loss_weights.get(name, 1.)) elif isinstance(loss_weights, list): if len(loss_weights) != len(train_model.outputs): raise ValueError('When passing a list as loss_weights, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(train_model.outputs)) + ' outputs, but you passed loss_weights=' + str(loss_weights)) loss_weights_list = loss_weights else: raise TypeError('Could not interpret loss_weights argument: ' + str(loss_weights) + ' - expected a list of dicts.') # prepare loss functions if isinstance(loss, dict): for name in loss: if name not in train_model.output_names: raise ValueError('Unknown entry in loss ' 'dictionary: "' + name + '". ' 'Only expected the following keys: ' + str(train_model.output_names)) loss_functions = [] for name in train_model.output_names: if name not in loss: raise ValueError('Output "' + name + '" missing from loss dictionary.') loss_functions.append(objectives.get(loss[name])) elif isinstance(loss, list): if len(loss) != len(train_model.outputs): raise ValueError('When passing a list as loss, ' 'it should have one entry per model outputs. ' 'The model has ' + str(len(train_model.outputs)) + ' outputs, but you passed loss=' + str(loss)) loss_functions = [objectives.get(l) for l in loss] else: loss_function = objectives.get(loss) loss_functions = [ loss_function for _ in range(len(train_model.outputs)) ] train_model.loss_functions = loss_functions weighted_losses = [_weighted_masked_objective(fn) for fn in loss_functions] # prepare metrics train_model.metrics = metrics train_model.metrics_names = ['loss'] train_model.metrics_tensors = [] # compute total loss total_loss = None for i in range(len(train_model.outputs)): y_true = out_tensor_lst[i] y_pred = train_model.outputs[i] _loss = loss_functions[i] # _loss = weighted_losses[i] loss_weight = loss_weights_list[i] # output_loss = _loss(y_true, y_pred, None, None) output_loss = K.mean(_loss(y_true, y_pred)) if len(train_model.outputs) > 1: train_model.metrics_tensors.append(output_loss) train_model.metrics_names.append(train_model.output_names[i] + '_loss') if total_loss is None: total_loss = loss_weight * output_loss else: total_loss += loss_weight * output_loss # add regularization penalties # and other layer-specific losses for loss_tensor in train_model.losses: total_loss += loss_tensor # list of same size as output_names. # contains tuples (metrics for output, names of metrics) nested_metrics = _collect_metrics(metrics, train_model.output_names) def append_metric(layer_num, metric_name, metric_tensor): """Helper function, used in loop below""" if len(train_model.output_names) > 1: metric_name = train_model.output_layers[ layer_num].name + '_' + metric_name train_model.metrics_names.append(metric_name) train_model.metrics_tensors.append(metric_tensor) for i in range(len(train_model.outputs)): y_true = out_tensor_lst[i] y_pred = train_model.outputs[i] output_metrics = nested_metrics[i] for metric in output_metrics: if metric == 'accuracy' or metric == 'acc': # custom handling of accuracy # (because of class mode duality) output_shape = train_model.internal_output_shapes[i] acc_fn = None if output_shape[-1] == 1 or train_model.loss_functions[ i] == objectives.binary_crossentropy: # case: binary accuracy acc_fn = metrics_module.binary_accuracy elif train_model.loss_functions[ i] == objectives.sparse_categorical_crossentropy: # case: categorical accuracy with sparse targets acc_fn = metrics_module.sparse_categorical_accuracy else: acc_fn = metrics_module.categorical_accuracy append_metric(i, 'acc', acc_fn(y_true, y_pred)) else: metric_fn = metrics_module.get(metric) metric_result = metric_fn(y_true, y_pred) if not isinstance(metric_result, dict): metric_result = {metric_fn.__name__: metric_result} for name, tensor in six.iteritems(metric_result): append_metric(i, name, tensor) # prepare gradient updates and state updates train_model.optimizer = optimizers.get(optimizer) train_model.total_loss = total_loss train_model.train_function = None train_model.test_function = None train_model.predict_function = None # collected trainable weights and sort them deterministically. trainable_weights = train_model.trainable_weights # Sort weights by name trainable_weights.sort(key=lambda x: x.name) train_model._collected_trainable_weights = trainable_weights
def evaluate_keras_metric(y_true, y_pred, metric): objective_function = metrics.get(metric) objective = objective_function(y_true, y_pred) return K.eval(objective)
def test_invalid_get(): with pytest.raises(ValueError): metrics.get(5)
def test_invalid_get(): with pytest.raises(ValueError): metrics.get(5)
def _get_metric_object(self, metric, y_t, y_p): """Converts user-supplied metric to a `Metric` object. Args: metric: A string, function, or `Metric` object. y_t: Sample of label. y_p: Sample of output. Returns: A `Metric` object. """ if metric is None: return None # Ok to have no metric for an output. # Convenience feature for selecting b/t binary, categorical, # and sparse categorical. if str(metric).lower() not in [ "accuracy", "acc", "crossentropy", "ce" ]: metric_obj = metrics_mod.get(metric) else: y_t_rank = len(y_t.shape.as_list()) y_p_rank = len(y_p.shape.as_list()) y_t_last_dim = y_t.shape.as_list()[-1] y_p_last_dim = y_p.shape.as_list()[-1] is_binary = y_p_last_dim == 1 is_sparse_categorical = (y_t_rank < y_p_rank or y_t_last_dim == 1 and y_p_last_dim > 1) if str(metric).lower() in ["accuracy", "acc"]: if is_binary: metric_obj = metrics_mod.binary_accuracy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_accuracy else: metric_obj = metrics_mod.categorical_accuracy else: if is_binary: metric_obj = metrics_mod.binary_crossentropy elif is_sparse_categorical: metric_obj = metrics_mod.sparse_categorical_crossentropy else: metric_obj = metrics_mod.categorical_crossentropy if isinstance(metric_obj, losses_mod.Loss): metric_obj._allow_sum_over_batch_size = True if not isinstance(metric_obj, metrics_mod.Metric): if isinstance(metric, str): metric_name = metric else: metric_name = get_custom_object_name(metric) if metric_name is None: raise ValueError( f"Metric should be a callable, received: {metric}") metric_obj = metrics_mod.MeanMetricWrapper(metric_obj, name=metric_name) return metric_obj