def _get_eval_ops(self, features, targets, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. targets: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metric ops to run. If None, the default metric functions are used; if {}, no metrics are used. If model has one output (i.e., returning single predction), keys are `str`, e.g. `'accuracy'` - just a name of the metric that will show up in the logs / summaries. Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')` - name of the metric and name of `Tensor` in the predictions to run this metric on. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in ../../../../metrics/python/metrics/ops/streaming_metrics.py. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `targets`. """ predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL) result = {"loss": metrics_lib.streaming_mean(loss)} weights = self._get_weight_tensor(features) metrics = metrics or {} if isinstance(targets, dict) and len(targets) == 1: # Unpack single target into just tensor. targets = targets[list(targets.keys())[0]] for name, metric in six.iteritems(metrics): if isinstance(name, tuple): # Multi-head metrics. if not isinstance(predictions, dict): raise ValueError( "Metrics passed provide (name, prediction), " "but predictions are not dict. " "Metrics: %s, Predictions: %s." % (metrics, predictions) ) # Here are two options: targets are single Tensor or a dict. if isinstance(targets, dict) and name[1] in targets: # If targets are dict and the prediction name is in it, apply metric. result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets[name[1]], weights) else: # Otherwise pass the targets to the metric. result[name[0]] = metrics_lib.run_metric(metric, predictions[name[1]], targets, weights) else: # Single head metrics. if isinstance(predictions, dict): raise ValueError( "Metrics passed provide only name, no prediction, " "but predictions are dict. " "Metrics: %s, Targets: %s." % (metrics, targets) ) result[name] = metrics_lib.run_metric(metric, predictions, targets, weights) return result
def _run_metrics(predictions, targets, metrics, weights): result = {} targets = math_ops.cast(targets, predictions.dtype) for name, metric in six.iteritems(metrics or {}): result[name] = metrics_lib.run_metric( metric, predictions, targets, weights=weights) return result
def _get_eval_ops(self, features, targets, metrics): """Method that builds model graph and returns evaluation ops. Expected to be overriden by sub-classes that require custom support. This implementation uses `model_fn` passed as parameter to constructor to build model. Args: features: `Tensor` or `dict` of `Tensor` objects. targets: `Tensor` or `dict` of `Tensor` objects. metrics: Dict of metric ops to run. If None, the default metric functions are used; if {}, no metrics are used. If model has one output (i.e., returning single predction), keys are `str`, e.g. `'accuracy'` - just a name of the metric that will show up in the logs / summaries. Otherwise, keys are tuple of two `str`, e.g. `('accuracy', 'classes')` - name of the metric and name of `Tensor` in the predictions to run this metric on. Metric ops should support streaming, e.g., returning update_op and value tensors. See more details in ../../../../metrics/python/metrics/ops/streaming_metrics.py. Returns: metrics: `dict` of `Tensor` objects. Raises: ValueError: if `metrics` don't match `targets`. """ predictions, loss, _ = self._call_model_fn(features, targets, ModeKeys.EVAL) result = {'loss': metrics_lib.streaming_mean(loss)} weights = self._get_weight_tensor(features) metrics = metrics or {} if isinstance(targets, dict) and len(targets) == 1: # Unpack single target into just tensor. targets = targets[list(targets.keys())[0]] for name, metric in six.iteritems(metrics): if isinstance(name, tuple): # Multi-head metrics. if not isinstance(predictions, dict): raise ValueError( 'Metrics passed provide (name, prediction), ' 'but predictions are not dict. ' 'Metrics: %s, Predictions: %s.' % (metrics, predictions)) # Here are two options: targets are single Tensor or a dict. if isinstance(targets, dict) and name[1] in targets: # If targets are dict and the prediction name is in it, apply metric. result[name[0]] = metrics_lib.run_metric( metric, predictions[name[1]], targets[name[1]], weights) else: # Otherwise pass the targets to the metric. result[name[0]] = metrics_lib.run_metric( metric, predictions[name[1]], targets, weights) else: # Single head metrics. if isinstance(predictions, dict): raise ValueError( 'Metrics passed provide only name, no prediction, ' 'but predictions are dict. ' 'Metrics: %s, Targets: %s.' % (metrics, targets)) result[name] = metrics_lib.run_metric( metric, predictions, targets, weights) return result