def _evaluate_ops(self, features): """Add ops for evaluation (aka filtering) to the graph.""" mode = estimator_lib.ModeKeys.EVAL with variable_scope.variable_scope("model", use_resource=True): model_outputs = self.create_loss(features, mode) metrics = {} # Just output in-sample predictions for the last chunk seen for prediction_key, prediction_value in model_outputs.predictions.items( ): metrics[prediction_key] = _identity_metric_single( prediction_key, prediction_value) metrics[feature_keys.FilteringResults.TIMES] = _identity_metric_single( feature_keys.FilteringResults.TIMES, model_outputs.prediction_times) metrics[feature_keys.FilteringResults.STATE_TUPLE] = ( _identity_metric_nested(feature_keys.FilteringResults.STATE_TUPLE, model_outputs.end_state)) metrics[metric_keys.MetricKeys.LOSS_MEAN] = metrics_impl.mean( model_outputs.loss, name="average_loss") return estimator_lib.EstimatorSpec( loss=model_outputs.loss, mode=mode, eval_metric_ops=metrics, # needed for custom metrics. predictions=model_outputs.predictions)
def my_model_fn(features, mode): loss = math_ops.reduce_max(features) eval_metric_ops = { "feature_mean": metrics_impl.mean(features), } return model_fn_lib.EstimatorSpec(mode, loss=loss, eval_metric_ops=eval_metric_ops)
def top_5_accuracy(labels, predictions, weights=None, metrics_collections=None, updates_collections=None, name=None): """Calculates how often `predictions` matches `labels`. The `accuracy` function creates two local variables, `total` and `count` that are used to compute the frequency with which `predictions` matches `labels`. This frequency is ultimately returned as `accuracy`: an idempotent operation that simply divides `total` by `count`. For estimation of the metric over a stream of data, the function creates an `update_op` operation that updates these variables and returns the `accuracy`. Internally, an `is_correct` operation computes a `Tensor` with elements 1.0 where the corresponding elements of `predictions` and `labels` match and 0.0 otherwise. Then `update_op` increments `total` with the reduced sum of the product of `weights` and `is_correct`, and it increments `count` with the reduced sum of `weights`. If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. Args: labels: The ground truth values, a `Tensor` whose shape matches `predictions`. predictions: The predicted values, a `Tensor` of any shape. weights: Optional `Tensor` whose rank is either 0, or the same rank as `labels`, and must be broadcastable to `labels` (i.e., all dimensions must be either `1`, or the same as the corresponding `labels` dimension). metrics_collections: An optional list of collections that `accuracy` should be added to. updates_collections: An optional list of collections that `update_op` should be added to. name: An optional variable_scope name. Returns: accuracy: A `Tensor` representing the accuracy, the value of `total` divided by `count`. update_op: An operation that increments the `total` and `count` variables appropriately and whose value matches `accuracy`. Raises: ValueError: If `predictions` and `labels` have mismatched shapes, or if `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): raise RuntimeError('tf.metrics.accuracy is not supported when eager ' 'execution is enabled.') predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights ) predictions.get_shape().assert_is_compatible_with(labels.get_shape()) if labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) is_correct = math_ops.to_float(math_ops.equal(predictions, labels)) is_correct = math_ops.reduce_prod(is_correct, axis=-2) is_correct = math_ops.reduce_max(is_correct, axis=-1) return mean(is_correct, weights, metrics_collections, updates_collections, name or 'accuracy')
def mean_absolute_percentage_error( labels, predictions, weights=None, metrics_collections=None, updates_collections=None, name=None): predictions, labels, weights = _remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) absolute_percentage_errors = math_ops.abs(math_ops.div(predictions-labels, labels)) return mean(absolute_percentage_errors, weights, metrics_collections, updates_collections, name or 'mean_absolute_percentage_error')
def rmspe(labels, predictions, weights=None): if context.executing_eagerly(): raise RuntimeError('rmspe is not supported ' 'when eager execution is enabled.') predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) # The target has been take log1p, so take expm1 back labels, predictions = math_ops.expm1(labels), math_ops.expm1( predictions) mspe, update_op = metrics_impl.mean( math_ops.square((labels - predictions) / labels), weights) rmspe = math_ops.sqrt(mspe) rmspe_update_op = math_ops.sqrt(update_op) return rmspe, rmspe_update_op
def my_model_fn(features, labels, mode): loss = features + labels # Make different graphs for train and eval if mode == model_fn.ModeKeys.TRAIN: train_op = array_ops.identity(loss) return model_fn.EstimatorSpec(mode=mode, loss=loss, train_op=train_op) elif mode == model_fn.ModeKeys.EVAL: eval_metric_ops = { "metric": metrics_impl.mean(features * labels) } return model_fn.EstimatorSpec(mode=mode, loss=loss, eval_metric_ops=eval_metric_ops) else: raise NotImplementedError(mode)
def _evaluate_ops(self, features): """Add ops for evaluation (aka filtering) to the graph.""" mode = estimator_lib.ModeKeys.EVAL with variable_scope.variable_scope("model", use_resource=True): model_outputs = self.create_loss(features, mode) metrics = {} # Just output in-sample predictions for the last chunk seen for prediction_key, prediction_value in model_outputs.predictions.items(): metrics[prediction_key] = _identity_metric_single(prediction_key, prediction_value) metrics[feature_keys.FilteringResults.TIMES] = _identity_metric_single( feature_keys.FilteringResults.TIMES, model_outputs.prediction_times) metrics[feature_keys.FilteringResults.STATE_TUPLE] = ( _identity_metric_nested(feature_keys.FilteringResults.STATE_TUPLE, model_outputs.end_state)) metrics[metric_keys.MetricKeys.LOSS_MEAN] = metrics_impl.mean( model_outputs.loss, name="average_loss") return estimator_lib.EstimatorSpec( loss=model_outputs.loss, mode=mode, eval_metric_ops=metrics, # needed for custom metrics. predictions=model_outputs.predictions)
def mean_absolute_percentage_error(labels, predictions, weights=None, metrics_collections=None, updates_collections=None, name=None): """Computes the mean absolute percentage error between the labels and predictions. The `mean_absolute_percentage_error` function creates two local variables, `total` and `count` that are used to compute the mean absolute percentage error. This average is weighted by `weights`, and it is ultimately returned as `mean_absolute_percentage_error`: an idempotent operation that simply divides `total` by `count`. For estimation of the metric over a stream of data, the function creates an `update_op` operation that updates these variables and returns the `mean_absolute_percentage_error`. Internally, an `absolute_percentage_errors` operation computes the absolute value of the percentage differences between `predictions` and `labels`. Then `update_op` increments `total` with the reduced sum of the product of `weights` and `absolute_percentage_errors`, and it increments `count` with the reduced sum of `weights` If `weights` is `None`, weights default to 1. Use weights of 0 to mask values. Args: labels: A `Tensor` of the same shape as `predictions`. predictions: A `Tensor` of arbitrary shape. weights: Optional `Tensor` whose rank is either 0, or the same rank as `labels`, and must be broadcastable to `labels` (i.e., all dimensions must be either `1`, or the same as the corresponding `labels` dimension). metrics_collections: An optional list of collections that `mean_absolute_percentage_error` should be added to. updates_collections: An optional list of collections that `update_op` should be added to. name: An optional variable_scope name. Returns: mean_absolute_percentage_error: A `Tensor` representing the current mean, the value of `total` divided by `count`. update_op: An operation that increments the `total` and `count` variables appropriately and whose value matches `mean_absolute_percentage_error`. Raises: ValueError: If `predictions` and `labels` have mismatched shapes, or if `weights` is not `None` and its shape doesn't match `predictions`, or if either `metrics_collections` or `updates_collections` are not a list or tuple. RuntimeError: If eager execution is enabled. """ if context.executing_eagerly(): raise RuntimeError( 'tf.metrics.mean_absolute_percentage_error is not supported ' 'when eager execution is enabled.') if predictions.dtype in (dtypes.float16, dtypes.float32, dtypes.float64) \ and labels.dtype != predictions.dtype: labels = math_ops.cast(labels, predictions.dtype) elif labels.dtype in (dtypes.float16, dtypes.float32, dtypes.float64) \ and labels.dtype != predictions.dtype: predictions = math_ops.cast(predictions, labels.dtype) else: labels = math_ops.cast(labels, dtypes.float32) predictions = math_ops.cast(predictions, dtypes.float32) predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( predictions=predictions, labels=labels, weights=weights) min_value = constant_op.constant(EPSILON, dtype=dtypes.float32) max_value = constant_op.constant(float('Inf'), dtype=dtypes.float32) percentage_absolute_errors = 100 * math_ops.abs( (predictions - labels) / math_ops.abs( clip_ops.clip_by_value(math_ops.abs(labels), min_value, max_value))) return metrics_impl.mean(percentage_absolute_errors, weights, metrics_collections, updates_collections, name or 'mape')
def eval_metrics_fn(global_step, loss): return { "global_step_observed": metrics_impl.mean(global_step), "loss": loss, }
def eval_metrics_fn(partial, squared): return { "mean": metrics_impl.mean(partial), "loss": squared, }
def eval_metrics_fn(prediction): return {"prediction": metrics_impl.mean(prediction)}
def _streaming_mean_absolute_persentace_error(predictions, labels): absolute_errors = math_ops.abs((predictions - labels) / labels) mean_t, update_op = metrics_impl.mean(absolute_errors, None, None, None, 'mean_absolute_persentace_error') return tf.multiply(mean_t, 100.), update_op