def test_cross_entropy_loss(self):
"""Checks that our cross-entropy loss matches TensorFlow's."""
loss_function = loss.SoftmaxCrossEntropyLoss()
predictions = tf.constant(self._penalty_predictions, dtype=tf.float32)
labels = tf.cast(self._penalty_labels > 0.0, dtype=tf.float32)
weights = tf.constant(self._penalty_weights, dtype=tf.float32)
context = subsettable_context.rate_context(
predictions=lambda: predictions,
labels=lambda: labels,
weights=lambda: weights)
with self.wrapped_session() as session:
# We don't use a split context, so we only care about at the "penalty"
# portion.
expected_losses = tf.nn.sigmoid_cross_entropy_with_logits(
labels=labels, logits=predictions)
expected_loss_numerator = tf.reduce_mean(weights * expected_losses)
expected_loss_denominator = tf.reduce_mean(weights)
expected_loss = session.run(expected_loss_numerator /
expected_loss_denominator)
actual_expression = binary_rates.error_rate(
context, penalty_loss=loss_function, constraint_loss=loss_function)
self._check_rates(expected_loss, expected_loss, actual_expression)
def test_hinge_loss(self):
"""Checks that our hinge loss matches TensorFlow's."""
loss_function = loss.HingeLoss()
predictions = tf.constant(self._penalty_predictions, dtype=tf.float32)
labels = tf.cast(self._penalty_labels > 0.0, dtype=tf.float32)
weights = tf.constant(self._penalty_weights, dtype=tf.float32)
context = subsettable_context.rate_context(
predictions=lambda: predictions,
labels=lambda: labels,
weights=lambda: weights)
with self.wrapped_session() as session:
# We don't use a split context, so we only care about at the "penalty"
# portion.
expected_loss_numerator = tf.reduce_mean(
tf.compat.v1.losses.hinge_loss(
labels=labels,
logits=predictions,
weights=weights,
reduction=tf.compat.v1.losses.Reduction.NONE))
expected_loss_denominator = tf.reduce_mean(weights)
expected_loss = session.run(expected_loss_numerator /
expected_loss_denominator)
actual_expression = binary_rates.error_rate(
context, penalty_loss=loss_function, constraint_loss=loss_function)
self._check_rates(expected_loss, expected_loss, actual_expression)
def _context(self):
"""Creates a new non-split and non-subsetted context."""
# We can't create the context in __init__, since it would then wind up in
# the wrong TensorFlow graph.
return subsettable_context.rate_context(
predictions=tf.constant(self._penalty_predictions,
dtype=tf.float32),
labels=tf.constant(self._penalty_labels, dtype=tf.float32),
weights=tf.constant(self._penalty_weights, dtype=tf.float32))
def contexts(self):
# We can't create the contexts in __init__, since they would then wind up in
# the wrong TensorFlow graph.
result = []
for index in xrange(self._num_datasets):
predictions = tf.constant(self._predictions[index], dtype=tf.float32)
weights = tf.constant(self._weights[index], dtype=tf.float32)
result.append(
subsettable_context.rate_context(
predictions=predictions, weights=weights))
return result
def problem_fn(logits, labels, features, weight_column=None):
# Returns a `RateMinimizationProblem` minimizing error rate subject to
# recall >= recall_target.
del features
del weight_column
context = subsettable_context.rate_context(logits, labels)
objective = binary_rates.error_rate(context)
constraints = [
binary_rates.true_positive_rate(context) >= recall_target
]
return rate_minimization_problem.RateMinimizationProblem(
objective, constraints)
def create_contexts():
"""Returns a pair of `SubsettableContext`s to use in tests.
We'll refer to the two contexts as "context1" and "context2". Both are subsets
of the same parent context, which has:
penalty_predicate = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0]
constraint_predicate = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
context1 is subsetted from the parent context using:
penalty_predicate1 = [0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
constraint_predicate1 = [1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
while context2 is subsetted deom the parent context using:
penalty_predicate2 = [0, 0, 0, 0, 1, 1, 1, 1, 0, 0]
constraint_predicate2 = [1, 1, 1, 1, 0, 0, 0, 0, 1, 1]
Returns:
The pair (context1, context2).
"""
predictions = tf.constant(0.0, dtype=tf.float32, shape=(1, ))
# The predictions are passed as a lambda to support eager mode (in graph mode,
# either a Tensor or a function are fine, but in eager mode, the predictions
# *must* be a function).
context = subsettable_context.rate_context(lambda: predictions)
penalty_predicate = [
True, False, True, False, True, False, True, False, True, False
],
constraint_predicate = [
False, True, False, True, False, True, False, True, False, True
],
context = context.subset(penalty_predicate, constraint_predicate)
penalty_predicate1 = [
False, False, True, True, True, True, False, False, False, False
],
constraint_predicate1 = [
True, True, False, False, False, False, True, True, True, True
],
penalty_predicate2 = [
False, False, False, False, True, True, True, True, False, False
],
constraint_predicate2 = [
True, True, True, True, False, False, False, False, True, True
],
context1 = context.subset(penalty_predicate1, constraint_predicate1)
context2 = context.subset(penalty_predicate2, constraint_predicate2)
return context1, context2
def test_soft_hinge_loss(self):
"""Checks that our soft hinge loss matches TensorFlow's."""
loss_function = loss.HingeLoss()
predictions = tf.constant(self._penalty_predictions, dtype=tf.float32)
labels = tf.cast(self._soft_penalty_labels, dtype=tf.float32)
weights = tf.constant(self._penalty_weights, dtype=tf.float32)
context = subsettable_context.rate_context(
predictions=lambda: predictions,
labels=lambda: labels,
weights=lambda: weights,
labels_are_probabilities=True)
with self.wrapped_session() as session:
# This library tries to find tighter bounds for losses by eliminating
# constants, so we need to adjust the label weights appropriately.
# For example, if we have a 0.1 probability of being positive, and a 0.9
# probability of being negative, then we're guaranteed to be wrong with
# a probability of at least 0.1, so we can just treat that as a constant,
# and only use a loss on what's left.
positive_labels = labels
negative_labels = 1.0 - labels
constants = tf.minimum(positive_labels, negative_labels)
positive_labels -= constants
negative_labels -= constants
# We don't use a split context, so we only care about at the "penalty"
# portion.
one_labels = tf.ones_like(labels)
zero_labels = tf.zeros_like(labels)
expected_loss_numerator = tf.reduce_mean(
weights * constants + tf.compat.v1.losses.hinge_loss(
labels=one_labels,
logits=predictions,
weights=weights * positive_labels,
reduction=tf.compat.v1.losses.Reduction.NONE) +
tf.compat.v1.losses.hinge_loss(
labels=zero_labels,
logits=predictions,
weights=weights * negative_labels,
reduction=tf.compat.v1.losses.Reduction.NONE))
expected_loss_denominator = tf.reduce_mean(weights)
expected_loss = session.run(expected_loss_numerator /
expected_loss_denominator)
actual_expression = binary_rates.error_rate(
context, penalty_loss=loss_function, constraint_loss=loss_function)
self._check_rates(expected_loss, expected_loss, actual_expression)
def create_contexts():
"""Returns a pair of `SubsettableContext`s to use in tests.
We'll refer to the two contexts as "context1" and "context2". Both are subsets
of the same parent context, which has:
penalty_predicate = [1, 0, 1, 0, 1, 0, 1, 0, 1, 0]
constraint_predicate = [0, 1, 0, 1, 0, 1, 0, 1, 0, 1]
context1 is subsetted from the parent context using:
penalty_predicate1 = [0, 0, 1, 1, 1, 1, 0, 0, 0, 0]
constraint_predicate1 = [1, 1, 0, 0, 0, 0, 1, 1, 1, 1]
while context2 is subsetted deom the parent context using:
penalty_predicate2 = [0, 0, 0, 0, 1, 1, 1, 1, 0, 0]
constraint_predicate2 = [1, 1, 1, 1, 0, 0, 0, 0, 1, 1]
Returns:
The pair (context1, context2).
"""
predictions = tf.constant(0.0, dtype=tf.float32, shape=(1,))
context = subsettable_context.rate_context(predictions)
penalty_predicate = tf.constant(
[True, False, True, False, True, False, True, False, True, False],
dtype=tf.bool)
constraint_predicate = tf.constant(
[False, True, False, True, False, True, False, True, False, True],
dtype=tf.bool)
context = context.subset(penalty_predicate, constraint_predicate)
penalty_predicate1 = tf.constant(
[False, False, True, True, True, True, False, False, False, False],
dtype=tf.bool)
constraint_predicate1 = tf.constant(
[True, True, False, False, False, False, True, True, True, True],
dtype=tf.bool)
penalty_predicate2 = tf.constant(
[False, False, False, False, True, True, True, True, False, False],
dtype=tf.bool)
constraint_predicate2 = tf.constant(
[True, True, True, True, False, False, False, False, True, True],
dtype=tf.bool)
context1 = context.subset(penalty_predicate1, constraint_predicate1)
context2 = context.subset(penalty_predicate2, constraint_predicate2)
return context1, context2
def _contexts(self):
# We can't create the contexts in __init__, since they would then wind up in
# the wrong TensorFlow graph.
result = []
for index in xrange(self._num_datasets):
# To support eager mode, the predictions should be a nullary function
# returning a Tensor, but we need to make sure that it has its own copy of
# "index".
def predictions(index=index):
return tf.constant(self._predictions[index], dtype=tf.float32)
# In support eager mode, the weights should be a nullary function
# returning a Tensor, but we need to make sure that it has its own copy of
# "index".
def weights(index=index):
return tf.constant(self._weights[index], dtype=tf.float32)
result.append(
subsettable_context.rate_context(predictions=predictions,
weights=weights))
return result
