def _confusion_matrix_metric_ops(
features_dict,
predictions_dict,
labels_dict,
example_weight_key,
thresholds,
):
"""Metric ops for computing confusion matrix at the given thresholds.
This is factored out because it's common to AucPlots and
ConfusionMatrixAtThresholds.
Args:
features_dict: Features dict.
predictions_dict: Predictions dict.
labels_dict: Labels dict.
example_weight_key: Example weight key (into features_dict).
thresholds: List of thresholds to compute the confusion matrix at.
Returns:
(value_op, update_op) for the metric. Note that the value_op produces a
matrix as described in the comments below.
"""
# Note that we have to squeeze predictions, labels, weights so they are all
# N element vectors (otherwise some of them might be N x 1 tensors, and
# multiplying a N element vector with a N x 1 tensor uses matrix
# multiplication rather than element-wise multiplication).
squeezed_weights = None
if example_weight_key:
squeezed_weights = tf.squeeze(features_dict[example_weight_key])
prediction_tensor = tf.cast(_get_prediction_tensor(predictions_dict),
tf.float64)
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
tf.squeeze(labels_dict), tf.squeeze(prediction_tensor), thresholds,
squeezed_weights)
values['precision'] = values['tp'] / (values['tp'] + values['fp'])
values['recall'] = values['tp'] / (values['tp'] + values['fn'])
# The final matrix will look like the following:
#
# [ fn@threshold_0 tn@threshold_0 ... recall@threshold_0 ]
# [ fn@threshold_1 tn@threshold_1 ... recall@threshold_1 ]
# [ : : ... : ]
# [ : : ... : ]
# [ fn@threshold_k tn@threshold_k ... recall@threshold_k ]
#
value_op = tf.transpose(
tf.stack([
values['fn'], values['tn'], values['fp'], values['tp'],
values['precision'], values['recall']
]))
update_op = tf.group(update_ops['fn'], update_ops['tn'], update_ops['fp'],
update_ops['tp'])
return (value_op, update_op)
def f1_score(labels, predictions, weights=None, num_thresholds=200,
metrics_collections=None, updates_collections=None, name=None):
with variable_scope.variable_scope(
name, 'f1', (labels, predictions, weights)):
predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access
predictions=predictions, labels=labels, weights=weights)
# To account for floating point imprecisions / avoid division by zero.
epsilon = 1e-7
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [0.0 - epsilon] + thresholds + [1.0 + epsilon]
thresholds_tensor = tf.constant(thresholds)
# Confusion matrix.
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
labels, predictions, thresholds, weights, includes=('tp', 'fp', 'fn'))
# Compute precision and recall at various thresholds.
def compute_best_f1_score(tp, fp, fn, name):
precision_at_t = math_ops.div(tp, epsilon + tp + fp,
name='precision_' + name)
recall_at_t = math_ops.div(tp, epsilon + tp + fn, name='recall_' + name)
# Compute F1 score.
f1_at_thresholds = (
2.0 * precision_at_t * recall_at_t /
(precision_at_t + recall_at_t + epsilon))
best_f1 = math_ops.reduce_max(f1_at_thresholds)
best_f1_index = tf.math.argmax(f1_at_thresholds)
precision = precision_at_t[best_f1_index]
recall = recall_at_t[best_f1_index]
threshold = thresholds_tensor[best_f1_index]
return best_f1, precision, recall, threshold
def f1_across_replicas(_, values):
best_f1, precision, recall, threshold = compute_best_f1_score(tp=values['tp'], fp=values['fp'],
fn=values['fn'], name='value')
if metrics_collections:
ops.add_to_collections(metrics_collections, best_f1, precision, recall, threshold)
return best_f1, precision, recall, threshold
best_f1, precision, recall, threshold = distribution_strategy_context.get_replica_context().merge_call(
f1_across_replicas, args=(values,))
update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'],
fn=update_ops['fn'], name='update')
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
# return (best_f1, precision, recall, threshold), update_op
return (best_f1, update_op), (precision, update_op), (recall, update_op), (threshold, update_op)
def get_metric_ops(self, features_dict, predictions_dict, labels_dict):
# Note that we have to squeeze predictions, labels, weights so they are all
# N element vectors (otherwise some of them might be N x 1 tensors, and
# multiplying a N element vector with a N x 1 tensor uses matrix
# multiplication rather than element-wise multiplication).
squeezed_weights = None
if self._example_weight_key:
squeezed_weights = tf.squeeze(
features_dict[self._example_weight_key])
thresholds = [
i * 1.0 / self._HISTOGRAM_NUM_BUCKETS
for i in range(0, self._HISTOGRAM_NUM_BUCKETS + 1)
]
thresholds = [-1e-6] + thresholds
prediction_tensor = tf.cast(_get_prediction_tensor(predictions_dict),
tf.float64)
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
tf.squeeze(labels_dict), tf.squeeze(prediction_tensor), thresholds,
squeezed_weights)
values['precision'] = values['tp'] / (values['tp'] + values['fp'])
values['recall'] = values['tp'] / (values['tp'] + values['fn'])
# The final matrix will look like the following:
#
# [ fn@threshold_0 tn@threshold_0 ... recall@threshold_0 ]
# [ fn@threshold_1 tn@threshold_1 ... recall@threshold_1 ]
# [ : : ... : ]
# [ : : ... : ]
# [ fn@threshold_k tn@threshold_k ... recall@threshold_k ]
#
value_op = tf.transpose(
tf.stack([
values['fn'], values['tn'], values['fp'], values['tp'],
values['precision'], values['recall']
]))
update_op = tf.group(update_ops['fn'], update_ops['tn'],
update_ops['fp'], update_ops['tp'])
return {
metric_keys.AUC_PLOTS_MATRICES: (value_op, update_op),
metric_keys.AUC_PLOTS_THRESHOLDS:
(tf.identity(thresholds), tf.no_op()),
}
def confusion_matrix_metric_ops(
self,
features_dict,
predictions_dict,
labels_dict,
):
"""Metric ops for computing confusion matrix at the given thresholds.
This is factored out because it's common to AucPlots and
ConfusionMatrixAtThresholds.
Args:
features_dict: Features dict.
predictions_dict: Predictions dict.
labels_dict: Labels dict.
Returns:
(value_ops, update_ops) for the confusion matrix.
"""
# Note that we have to squeeze predictions, labels, weights so they are all
# N element vectors (otherwise some of them might be N x 1 tensors, and
# multiplying a N element vector with a N x 1 tensor uses matrix
# multiplication rather than element-wise multiplication).
squeezed_weights = None
if self._example_weight_key:
squeezed_weights = tf.squeeze(
features_dict[self._example_weight_key])
prediction_tensor = tf.cast(_get_prediction_tensor(predictions_dict),
tf.float64)
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
tf.squeeze(labels_dict), tf.squeeze(prediction_tensor),
self._thresholds, squeezed_weights)
values['precision'] = values['tp'] / (values['tp'] + values['fp'])
values['recall'] = values['tp'] / (values['tp'] + values['fn'])
return (values, update_ops) # pytype: disable=bad-return-type
def f1_score(labels, predictions, weights=None, num_thresholds=200,
metrics_collections=None, updates_collections=None, name=None):
"""Computes the approximately best F1-score across different thresholds.
The f1_score function applies a range of thresholds to the predictions to
convert them from [0, 1] to bool. Precision and recall are computed by
comparing them to the labels. The F1-Score is then defined as
2 * precision * recall / (precision + recall). The best one across the
thresholds is returned.
Disclaimer: In practice it may be desirable to choose the best threshold on
the validation set and evaluate the F1 score with this threshold on a
separate code set. Or it may be desirable to use a fixed threshold (e.g. 0.5).
This function internally creates four local variables, `true_positives`,
`true_negatives`, `false_positives` and `false_negatives` that are used to
compute the pairs of recall and precision values for a linearly spaced set of
thresholds from which the best f1-score is derived.
This value is ultimately returned as `f1-score`, an idempotent operation that
computes the F1-score (computed using the aforementioned variables). The
`num_thresholds` variable controls the degree of discretization with larger
numbers of thresholds more closely approximating the true best F1-score.
For estimation of the metric over a stream of data, the function creates an
`update_op` operation that updates these variables and returns the F1-score.
Example usage with a custom estimator:
def model_fn(features, labels, mode):
predictions = make_predictions(features)
loss = make_loss(predictions, labels)
train_op = tf.contrib.training.create_train_op(
total_loss=loss,
optimizer='Adam')
eval_metric_ops = {'f1': f1_score(labels, predictions)}
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
export_outputs=export_outputs)
estimator = tf.estimator.Estimator(model_fn=model_fn)
If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
Args:
labels: A `Tensor` whose shape matches `predictions`. Will be cast to
`bool`.
predictions: A floating point `Tensor` of arbitrary shape and whose values
are in the range `[0, 1]`.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`labels`, and must be broadcastable to `labels` (i.e., all dimensions must
be either `1`, or the same as the corresponding `labels` dimension).
num_thresholds: The number of thresholds to use when discretizing the roc
curve.
metrics_collections: An optional list of collections that `f1_score` should
be added to.
updates_collections: An optional list of collections that `update_op` should
be added to.
name: An optional variable_scope name.
Returns:
f1_score: A scalar `Tensor` representing the current best f1-score across
different thresholds.
update_op: An operation that increments the `true_positives`,
`true_negatives`, `false_positives` and `false_negatives` variables
appropriately and whose value matches the `f1_score`.
Raises:
ValueError: If `predictions` and `labels` have mismatched shapes, or if
`weights` is not `None` and its shape doesn't match `predictions`, or if
either `metrics_collections` or `updates_collections` are not a list or
tuple.
"""
with variable_scope.variable_scope(
name, 'f1', (labels, predictions, weights)):
predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access
predictions=predictions, labels=labels, weights=weights)
# To account for floating point imprecisions / avoid division by zero.
epsilon = 1e-7
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [0.0 - epsilon] + thresholds + [1.0 + epsilon]
# Confusion matrix.
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
labels, predictions, thresholds, weights, includes=('tp', 'fp', 'fn'))
# Compute precision and recall at various thresholds.
def compute_best_f1_score(tp, fp, fn, name):
precision_at_t = math_ops.div(tp, epsilon + tp + fp,
name='precision_' + name)
recall_at_t = math_ops.div(tp, epsilon + tp + fn, name='recall_' + name)
# Compute F1 score.
f1_at_thresholds = (
2.0 * precision_at_t * recall_at_t /
(precision_at_t + recall_at_t + epsilon))
return math_ops.reduce_max(f1_at_thresholds)
def f1_across_towers(_, values):
best_f1 = compute_best_f1_score(tp=values['tp'], fp=values['fp'],
fn=values['fn'], name='value')
if metrics_collections:
ops.add_to_collections(metrics_collections, best_f1)
return best_f1
best_f1 = distribution_strategy_context.get_tower_context().merge_call(
f1_across_towers, values)
update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'],
fn=update_ops['fn'], name='update')
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return best_f1, update_op
def _auc(labels,
predictions,
weights=None,
num_thresholds=200,
metrics_collections=None,
updates_collections=None,
curve='ROC',
name=None,
summation_method='trapezoidal'):
"""Computes the approximate AUC via a Riemann sum.
Modified version of tf.metrics.auc. Add support for AUC computation
of the recall curve.
"""
with tf.variable_scope(name, 'auc', (labels, predictions, weights)):
if curve != 'ROC' and curve != 'PR' and curve != 'R':
raise ValueError('curve must be either ROC, PR or R, %s unknown' %
(curve))
kepsilon = 1e-7 # to account for floating point imprecisions
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon]
values, update_ops = _confusion_matrix_at_thresholds(
labels, predictions, thresholds, weights)
# Add epsilons to avoid dividing by 0.
epsilon = 1.0e-6
def compute_auc(tp, fn, tn, fp, name):
"""Computes the roc-auc or pr-auc based on confusion counts."""
rec = tf.div(tp + epsilon, tp + fn + epsilon)
if curve == 'ROC':
fp_rate = tf.div(fp, fp + tn + epsilon)
x = fp_rate
y = rec
elif curve == 'R': # recall auc
x = tf.linspace(1., 0., num_thresholds)
y = rec
else: # curve == 'PR'.
prec = tf.div(tp + epsilon, tp + fp + epsilon)
x = rec
y = prec
if summation_method == 'trapezoidal':
return tf.reduce_sum(tf.multiply(
x[:num_thresholds - 1] - x[1:],
(y[:num_thresholds - 1] + y[1:]) / 2.),
name=name)
elif summation_method == 'minoring':
return tf.reduce_sum(tf.multiply(
x[:num_thresholds - 1] - x[1:],
tf.minimum(y[:num_thresholds - 1], y[1:])),
name=name)
elif summation_method == 'majoring':
return tf.reduce_sum(tf.multiply(
x[:num_thresholds - 1] - x[1:],
tf.maximum(y[:num_thresholds - 1], y[1:])),
name=name)
else:
raise ValueError('Invalid summation_method: %s' %
summation_method)
# sum up the areas of all the trapeziums
auc_value = compute_auc(values['tp'], values['fn'], values['tn'],
values['fp'], 'value')
update_op = compute_auc(update_ops['tp'], update_ops['fn'],
update_ops['tn'], update_ops['fp'],
'update_op')
if metrics_collections:
ops.add_to_collections(metrics_collections, auc_value)
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return auc_value, update_op
def f1_score(labels, predictions, weights=None, num_thresholds=200,
metrics_collections=None, updates_collections=None, name=None):
"""Computes the approximately best F1-score across different thresholds.
The f1_score function applies a range of thresholds to the predictions to
convert them from [0, 1] to bool. Precision and recall are computed by
comparing them to the labels. The F1-Score is then defined as
2 * precision * recall / (precision + recall). The best one across the
thresholds is returned.
Disclaimer: In practice it may be desirable to choose the best threshold on
the validation set and evaluate the F1 score with this threshold on a
separate test set. Or it may be desirable to use a fixed threshold (e.g. 0.5).
This function internally creates four local variables, `true_positives`,
`true_negatives`, `false_positives` and `false_negatives` that are used to
compute the pairs of recall and precision values for a linearly spaced set of
thresholds from which the best f1-score is derived.
This value is ultimately returned as `f1-score`, an idempotent operation that
computes the F1-score (computed using the aforementioned variables). The
`num_thresholds` variable controls the degree of discretization with larger
numbers of thresholds more closely approximating the true best F1-score.
For estimation of the metric over a stream of data, the function creates an
`update_op` operation that updates these variables and returns the F1-score.
Example usage with a custom estimator:
def model_fn(features, labels, mode):
predictions = make_predictions(features)
loss = make_loss(predictions, labels)
train_op = tf.contrib.training.create_train_op(
total_loss=loss,
optimizer='Adam')
eval_metric_ops = {'f1': f1_score(labels, predictions)}
return tf.estimator.EstimatorSpec(
mode=mode,
predictions=predictions,
loss=loss,
train_op=train_op,
eval_metric_ops=eval_metric_ops,
export_outputs=export_outputs)
estimator = tf.estimator.Estimator(model_fn=model_fn)
If `weights` is `None`, weights default to 1. Use weights of 0 to mask values.
Args:
labels: A `Tensor` whose shape matches `predictions`. Will be cast to
`bool`.
predictions: A floating point `Tensor` of arbitrary shape and whose values
are in the range `[0, 1]`.
weights: Optional `Tensor` whose rank is either 0, or the same rank as
`labels`, and must be broadcastable to `labels` (i.e., all dimensions must
be either `1`, or the same as the corresponding `labels` dimension).
num_thresholds: The number of thresholds to use when discretizing the roc
curve.
metrics_collections: An optional list of collections that `f1_score` should
be added to.
updates_collections: An optional list of collections that `update_op` should
be added to.
name: An optional variable_scope name.
Returns:
f1_score: A scalar `Tensor` representing the current best f1-score across
different thresholds.
update_op: An operation that increments the `true_positives`,
`true_negatives`, `false_positives` and `false_negatives` variables
appropriately and whose value matches the `f1_score`.
Raises:
ValueError: If `predictions` and `labels` have mismatched shapes, or if
`weights` is not `None` and its shape doesn't match `predictions`, or if
either `metrics_collections` or `updates_collections` are not a list or
tuple.
"""
with variable_scope.variable_scope(
name, 'f1', (labels, predictions, weights)):
predictions, labels, weights = metrics_impl._remove_squeezable_dimensions( # pylint: disable=protected-access
predictions=predictions, labels=labels, weights=weights)
# To account for floating point imprecisions / avoid division by zero.
epsilon = 1e-7
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [0.0 - epsilon] + thresholds + [1.0 + epsilon]
# Confusion matrix.
values, update_ops = metrics_impl._confusion_matrix_at_thresholds( # pylint: disable=protected-access
labels, predictions, thresholds, weights, includes=('tp', 'fp', 'fn'))
# Compute precision and recall at various thresholds.
def compute_best_f1_score(tp, fp, fn, name):
precision_at_t = math_ops.div(tp, epsilon + tp + fp,
name='precision_' + name)
recall_at_t = math_ops.div(tp, epsilon + tp + fn, name='recall_' + name)
# Compute F1 score.
f1_at_thresholds = (
2.0 * precision_at_t * recall_at_t /
(precision_at_t + recall_at_t + epsilon))
return math_ops.reduce_max(f1_at_thresholds)
def f1_across_replicas(_, values):
best_f1 = compute_best_f1_score(tp=values['tp'], fp=values['fp'],
fn=values['fn'], name='value')
if metrics_collections:
ops.add_to_collections(metrics_collections, best_f1)
return best_f1
best_f1 = distribution_strategy_context.get_replica_context().merge_call(
f1_across_replicas, values)
update_op = compute_best_f1_score(tp=update_ops['tp'], fp=update_ops['fp'],
fn=update_ops['fn'], name='update')
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return best_f1, update_op
def _auc(labels, predictions, weights=None, num_thresholds=200,
metrics_collections=None, updates_collections=None,
curve='ROC', name=None, summation_method='trapezoidal'):
"""Computes the approximate AUC via a Riemann sum.
Modified version of tf.metrics.auc. Add support for AUC computation
of the recall curve.
"""
with tf.variable_scope(
name, 'auc', (labels, predictions, weights)):
if curve != 'ROC' and curve != 'PR' and curve != 'R':
raise ValueError('curve must be either ROC, PR or R, %s unknown' %
(curve))
kepsilon = 1e-7 # to account for floating point imprecisions
thresholds = [(i + 1) * 1.0 / (num_thresholds - 1)
for i in range(num_thresholds - 2)]
thresholds = [0.0 - kepsilon] + thresholds + [1.0 + kepsilon]
values, update_ops = _confusion_matrix_at_thresholds(
labels, predictions, thresholds, weights)
# Add epsilons to avoid dividing by 0.
epsilon = 1.0e-6
def compute_auc(tp, fn, tn, fp, name):
"""Computes the roc-auc or pr-auc based on confusion counts."""
rec = tf.div(tp + epsilon, tp + fn + epsilon)
if curve == 'ROC':
fp_rate = tf.div(fp, fp + tn + epsilon)
x = fp_rate
y = rec
elif curve == 'R': # recall auc
x = tf.linspace(1., 0., num_thresholds)
y = rec
else: # curve == 'PR'.
prec = tf.div(tp + epsilon, tp + fp + epsilon)
x = rec
y = prec
if summation_method == 'trapezoidal':
return tf.reduce_sum(
tf.multiply(x[:num_thresholds - 1] - x[1:],
(y[:num_thresholds - 1] + y[1:]) / 2.),
name=name)
elif summation_method == 'minoring':
return tf.reduce_sum(
tf.multiply(x[:num_thresholds - 1] - x[1:],
tf.minimum(y[:num_thresholds - 1], y[1:])),
name=name)
elif summation_method == 'majoring':
return tf.reduce_sum(
tf.multiply(x[:num_thresholds - 1] - x[1:],
tf.maximum(y[:num_thresholds - 1], y[1:])),
name=name)
else:
raise ValueError('Invalid summation_method: %s' % summation_method)
# sum up the areas of all the trapeziums
auc_value = compute_auc(
values['tp'], values['fn'], values['tn'], values['fp'], 'value')
update_op = compute_auc(
update_ops['tp'], update_ops['fn'], update_ops['tn'], update_ops['fp'],
'update_op')
if metrics_collections:
ops.add_to_collections(metrics_collections, auc_value)
if updates_collections:
ops.add_to_collections(updates_collections, update_op)
return auc_value, update_op