def _eval_metric_ops( self, labels, probabilities, weights, unreduced_loss, regularization_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( None, 'metrics', [labels, probabilities, weights, unreduced_loss, regularization_loss]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( values=unreduced_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, name=keys.AUC), head_lib._summary_key(self._name, keys.AUC_PR): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, curve='PR', name=keys.AUC_PR), } if regularization_loss is not None: loss_regularization_key = head_lib._summary_key( # pylint:disable=protected-access self._name, keys.LOSS_REGULARIZATION) metric_ops[loss_regularization_key] = ( metrics_lib.mean( values=regularization_loss, name=keys.LOSS_REGULARIZATION)) for threshold in self._thresholds: accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, accuracy_key)] = ( # pylint:disable=protected-access head_lib._accuracy_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=accuracy_key)) # Precision for positive examples. precision_key = keys.PRECISION_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, precision_key)] = ( # pylint:disable=protected-access head_lib._precision_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=precision_key)) # Recall for positive examples. recall_key = keys.RECALL_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, recall_key)] = ( # pylint:disable=protected-access head_lib._recall_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=recall_key)) return metric_ops
def _eval_metric_ops(self, labels, probabilities, weights, weighted_sum_loss, example_weight_sum): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope(None, 'metrics', [ labels, probabilities, weights, weighted_sum_loss, example_weight_sum ]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( # Both values and weights here are reduced, scalar Tensors. # values is the actual mean we want, but we pass the scalar # example_weight_sum in order to return the correct update_op # alongside the value_op for streaming metrics. values=(weighted_sum_loss / example_weight_sum), weights=example_weight_sum, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, name=keys.AUC), head_lib._summary_key(self._name, keys.AUC_PR): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, curve='PR', name=keys.AUC_PR), } for threshold in self._thresholds: accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, accuracy_key)] = ( # pylint:disable=protected-access head_lib._accuracy_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=accuracy_key)) # Precision for positive examples. precision_key = keys.PRECISION_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key( self._name, precision_key)] = ( # pylint:disable=protected-access head_lib._precision_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=precision_key)) # Recall for positive examples. recall_key = keys.RECALL_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, recall_key)] = ( # pylint:disable=protected-access head_lib._recall_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=recall_key)) return metric_ops
def _eval_metric_ops(self, labels, probabilities, weights, weighted_sum_loss, example_weight_sum): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( None, 'metrics', [labels, probabilities, weights, weighted_sum_loss, example_weight_sum ]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( # Both values and weights here are reduced, scalar Tensors. # values is the actual mean we want, but we pass the scalar # example_weight_sum in order to return the correct update_op # alongside the value_op for streaming metrics. values=(weighted_sum_loss / example_weight_sum), weights=example_weight_sum, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, name=keys.AUC), head_lib._summary_key(self._name, keys.AUC_PR): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, curve='PR', name=keys.AUC_PR), } for threshold in self._thresholds: accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, accuracy_key)] = ( # pylint:disable=protected-access head_lib._accuracy_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=accuracy_key)) # Precision for positive examples. precision_key = keys.PRECISION_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, precision_key)] = ( # pylint:disable=protected-access head_lib._precision_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=precision_key)) # Recall for positive examples. recall_key = keys.RECALL_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, recall_key)] = ( # pylint:disable=protected-access head_lib._recall_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=recall_key)) return metric_ops
def _eval_metric_ops( self, labels, probabilities, weights, unreduced_loss, regularization_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope( None, 'metrics', [labels, probabilities, weights, unreduced_loss, regularization_loss]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( values=unreduced_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, name=keys.AUC), head_lib._summary_key(self._name, keys.AUC_PR): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, curve='PR', name=keys.AUC_PR), } if regularization_loss is not None: loss_regularization_key = head_lib._summary_key( # pylint:disable=protected-access self._name, keys.LOSS_REGULARIZATION) metric_ops[loss_regularization_key] = ( metrics_lib.mean( values=regularization_loss, name=keys.LOSS_REGULARIZATION)) for threshold in self._thresholds: accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, accuracy_key)] = ( # pylint:disable=protected-access head_lib._accuracy_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=accuracy_key)) # Precision for positive examples. precision_key = keys.PRECISION_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, precision_key)] = ( # pylint:disable=protected-access head_lib._precision_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=precision_key)) # Recall for positive examples. recall_key = keys.RECALL_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, recall_key)] = ( # pylint:disable=protected-access head_lib._recall_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=recall_key)) for class_id in self._classes_for_class_based_metrics: batch_rank = array_ops.rank(probabilities) - 1 begin = array_ops.concat( [array_ops.zeros([batch_rank], dtype=dtypes.int32), [class_id]], axis=0) size = array_ops.concat( [-1 * array_ops.ones([batch_rank], dtype=dtypes.int32), [1]], axis=0) class_probabilities = array_ops.slice( probabilities, begin=begin, size=size) class_labels = array_ops.slice(labels, begin=begin, size=size) prob_key = keys.PROBABILITY_MEAN_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, prob_key)] = ( # pylint:disable=protected-access head_lib._predictions_mean( # pylint:disable=protected-access predictions=class_probabilities, weights=weights, name=prob_key)) auc_key = keys.AUC_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, auc_key)] = ( # pylint:disable=protected-access head_lib._auc( # pylint:disable=protected-access labels=class_labels, predictions=class_probabilities, weights=weights, name=auc_key)) auc_pr_key = keys.AUC_PR_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, auc_pr_key)] = ( # pylint:disable=protected-access head_lib._auc( # pylint:disable=protected-access labels=class_labels, predictions=class_probabilities, weights=weights, curve='PR', name=auc_pr_key)) return metric_ops
def _eval_metric_ops(self, labels, probabilities, weights, unreduced_loss, regularization_loss): """Returns a dict of metrics for eval_metric_ops.""" with ops.name_scope(None, 'metrics', [ labels, probabilities, weights, unreduced_loss, regularization_loss ]): keys = metric_keys.MetricKeys metric_ops = { # Estimator already adds a metric for loss. head_lib._summary_key(self._name, keys.LOSS_MEAN): # pylint:disable=protected-access metrics_lib.mean( values=unreduced_loss, weights=weights, name=keys.LOSS_MEAN), head_lib._summary_key(self._name, keys.AUC): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, name=keys.AUC), head_lib._summary_key(self._name, keys.AUC_PR): # pylint:disable=protected-access metrics_lib.auc(labels=labels, predictions=probabilities, weights=weights, curve='PR', name=keys.AUC_PR), } if regularization_loss is not None: loss_regularization_key = head_lib._summary_key( # pylint:disable=protected-access self._name, keys.LOSS_REGULARIZATION) metric_ops[loss_regularization_key] = (metrics_lib.mean( values=regularization_loss, name=keys.LOSS_REGULARIZATION)) for threshold in self._thresholds: accuracy_key = keys.ACCURACY_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, accuracy_key)] = ( # pylint:disable=protected-access head_lib._accuracy_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=accuracy_key)) # Precision for positive examples. precision_key = keys.PRECISION_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key( self._name, precision_key)] = ( # pylint:disable=protected-access head_lib._precision_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=precision_key)) # Recall for positive examples. recall_key = keys.RECALL_AT_THRESHOLD % threshold metric_ops[head_lib._summary_key(self._name, recall_key)] = ( # pylint:disable=protected-access head_lib._recall_at_threshold( # pylint:disable=protected-access labels=labels, predictions=probabilities, weights=weights, threshold=threshold, name=recall_key)) for class_id in self._classes_for_class_based_metrics: batch_rank = array_ops.rank(probabilities) - 1 begin = array_ops.concat([ array_ops.zeros([batch_rank], dtype=dtypes.int32), [class_id] ], axis=0) size = array_ops.concat([ -1 * array_ops.ones([batch_rank], dtype=dtypes.int32), [1] ], axis=0) class_probabilities = array_ops.slice(probabilities, begin=begin, size=size) class_labels = array_ops.slice(labels, begin=begin, size=size) prob_key = keys.PROBABILITY_MEAN_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, prob_key)] = ( # pylint:disable=protected-access head_lib._predictions_mean( # pylint:disable=protected-access predictions=class_probabilities, weights=weights, name=prob_key)) auc_key = keys.AUC_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, auc_key)] = ( # pylint:disable=protected-access head_lib._auc( # pylint:disable=protected-access labels=class_labels, predictions=class_probabilities, weights=weights, name=auc_key)) auc_pr_key = keys.AUC_PR_AT_CLASS % class_id metric_ops[head_lib._summary_key(self._name, auc_pr_key)] = ( # pylint:disable=protected-access head_lib._auc( # pylint:disable=protected-access labels=class_labels, predictions=class_probabilities, weights=weights, curve='PR', name=auc_pr_key)) return metric_ops