def _convert_to_binary_classification_predictions(predictions):
"""Converts a `Tensor` into a set of binary classification predictions.
This function checks that the given `Tensor` is floating-point, and that it is
trivially convertible to rank-1, i.e. has only one "nontrivial" dimension
(e.g. the shapes [1000] and [1, 1, None, 1] are allowed, but [None, 1, None]
and [50, 10] are not). If it satisfies these conditions, then it is reshaped
to be rank-1 (if necessary) and returned.
Args:
predictions: a rank-1 floating-point `Tensor` of predictions.
Returns:
The predictions `Tensor`, reshaped to be rank-1, if necessary.
Raises:
TypeError: if "predictions" is not a floating-point `Tensor`.
ValueError: if "predictions" is not trivially convertible to rank-1.
"""
if not tf.contrib.framework.is_tensor(predictions):
raise TypeError("predictions must be a tensor")
if not predictions.dtype.is_floating:
raise TypeError("predictions must be floating-point")
return helpers.convert_to_1d_tensor(predictions, name="predictions")
def test_convert_to_1d_tensor(self):
"""Tests the "convert_to_1d_tensor" function."""
self.assertFalse(tf.executing_eagerly())
# Trying to make a rank-1 Tensor from a 0d Tensor should succeed.
expected = [2.7]
actual = helpers.convert_to_1d_tensor(2.7)
with self.session() as session:
self.assertAllClose(expected,
session.run(actual),
rtol=0,
atol=1e-6)
# Trying to make a rank-1 Tensor from a rank-1 Tensor should succeed.
expected = [-6.3, 1.0, 5.1]
actual = helpers.convert_to_1d_tensor(expected)
with self.session() as session:
self.assertAllClose(expected,
session.run(actual),
rtol=0,
atol=1e-6)
# Trying to make a rank-1 Tensor from a shape-(1,2,1) Tensor should succeed
# (only one of the dimensions is nontrivial).
expected = [3.6, -1.7]
actual = helpers.convert_to_1d_tensor([[[3.6], [-1.7]]])
with self.session() as session:
self.assertAllClose(expected,
session.run(actual),
rtol=0,
atol=1e-6)
# Trying to make a rank-1 Tensor from a shape-(1,None,1) Tensor should
# succeed (only one of the dimensions is nontrivial).
expected = [0.2, -2.4, 0.0]
placeholder = tf.compat.v1.placeholder(tf.float32, shape=(1, None, 1))
actual = helpers.convert_to_1d_tensor(placeholder)
with self.session() as session:
self.assertAllClose(
expected,
session.run(actual,
feed_dict={placeholder: [[[0.2], [-2.4], [0.0]]]}),
rtol=0,
atol=1e-6)
# Trying to make a rank-1 Tensor from a rank-2 Tensor should fail.
with self.assertRaises(ValueError):
_ = helpers.convert_to_1d_tensor([[1, 2], [3, 4]])
# Trying to make a rank-1 Tensor from a shape-(None,2) Tensor should fail.
placeholder = tf.compat.v1.placeholder(tf.float32, shape=(None, 2))
with self.assertRaises(ValueError):
_ = helpers.convert_to_1d_tensor(placeholder)
def update_ops_fn(running_averages_variable, structure_memoizer,
value_memoizer):
"""Updates the running sums before each call to the train_op."""
weights, denominator_predicate = denominator
weights = helpers.convert_to_1d_tensor(weights(
structure_memoizer, value_memoizer),
name="weights")
dtype = weights.dtype.base_dtype
if not dtype.is_floating:
raise TypeError("weights must be floating-point")
update_ops = []
update_ops.append(
tf.debugging.assert_non_negative(
weights, message="weights must be non-negative"))
denominator_weights = weights * tf.cast(
denominator_predicate.tensor(structure_memoizer,
value_memoizer),
dtype=dtype)
# We take convex combinations (with parameter running_proportion) to
# make sure that both running_average_sum and running_average_count
# are divided by the number of minibatches, as explained below.
running_proportion = 1.0 / (tf.maximum(
tf.cast(structure_memoizer[defaults.GLOBAL_STEP_KEY],
dtype=running_dtype), 0.0) + 1.0)
running_average_sum = (
running_averages_variable[0] * (1.0 - running_proportion) +
tf.cast(tf.reduce_sum(denominator_weights),
dtype=running_dtype) * running_proportion)
running_average_count = (
running_averages_variable[1] * (1.0 - running_proportion) +
tf.cast(tf.size(denominator_weights),
dtype=running_dtype) * running_proportion)
update_ops.append(
running_averages_variable.assign(
[running_average_sum, running_average_count]))
return update_ops
def value_fn(weights_value, predicate_value):
"""Returns the numerator `Tensor`.
Args:
weights_value: `Tensor` of example weights.
predicate_value: indicator `Tensor` representing whether each example
should be included in the ratio's numerator.
Returns:
A `Tensor` containing the element-wise product of the two arguments.
Raises:
TypeError: if "weights" is not floating-point.
ValueError: if "weights" cannot be converted to a rank-1 `Tensor`.
"""
value = helpers.convert_to_1d_tensor(weights_value, name="weights")
dtype = value.dtype.base_dtype
if not dtype.is_floating:
raise TypeError("weights must be floating-point")
value *= tf.cast(predicate_value, dtype=dtype)
return value
def ratio(cls, weights, numerator_predicate, denominator_predicate):
"""Creates a new `_RatioWeights` representing the weights for a ratio.
This method is used to create the weights for a single ratio, for which
"numerator_predicate" indicates which examples should be included in the
numerator of the ratio, and "denominator_predicate" which should be included
in the denominator.
The numerator and denominator predicates can be arbitrary indicator
`Tensor`s, but the numerator subset will be intersected with the
denominator's before the rate is calculated.
Args:
weights: `Tensor` of example weights.
numerator_predicate: boolean indicator `Tensor` representing whether each
example should be included in the ratio's numerator.
denominator_predicate: boolean indicator `Tensor` representing whether
each example should be included in the ratio's denominator.
Returns:
A new `_RatioWeights` representing the ratio.
Raises:
TypeError: if "weights" is not floating-point.
ValueError: if "weights" cannot be converted to a rank-1 `Tensor`.
"""
key = (weights, denominator_predicate)
value = helpers.convert_to_1d_tensor(weights, name="weights")
dtype = value.dtype.base_dtype
if not dtype.is_floating:
raise TypeError("weights must be floating-point")
# Notice that we force the set of examples included in the numerator to be a
# subset of those in the denominator. It'll actually work fine either way,
# but this should be closer to what users expect.
value *= tf.cast(
(numerator_predicate & denominator_predicate).predicate,
dtype=dtype)
return _RatioWeights(dtype, {key: value})
def convert_and_clip_fn(arg):
"""Converts the given object to a rank-one float32 `Tensor` in [0,1]."""
return tf.clip_by_value(
tf.cast(
helpers.convert_to_1d_tensor(arg, "predicate"), dtype=tf.float32),
0.0, 1.0)
def evaluate_denominator(self, denominator):
"""Evaluates the denominator portion of a ratio.
Recall that a `_RatioWeights` object is responsible for computing:
ratio_weights[j] = weights[j] 1{j in numerator_subset}
/ (mean_i weights[i] 1{i in denominator_subset})
This method returns (an approximation of) the denominator portion of this
ratio. The numerator is calculated in the `_RatioWeights`.evaluate method.
The implementation is complicated by the fact that, although the
denominators of our ratios should evaluate to "the average weight of the
examples included in the ratio's denominator", we don't have access to the
entire dataset (instead, we will typically just get a sequence of
minibatches). Hence, we can't compute the average weight across the entire
dataset directly. Instead, we keep running sums of the total weight of
examples included in the denominator, and the number of examples seen, and
update them before each minibatch (in the set of `Operation`s returned by
this method).
Args:
denominator: (`Tensor`, `Predicate`) pair, the first being the example
weights, and the second the predicate indicating which examples are
included in the denominator.
Returns:
A (`Tensor`, set, set) tuple containing the (approximate) denominator, a
set of `Operation`s that should be executed before each training step
(to update the internal state upon which the denominator depends), and a
set of `Operation`s that can be executed to re-initialize this state.
Raises:
TypeError: if "weights" is not floating-point.
ValueError: if "weights" cannot be converted to a rank-1 `Tensor`.
"""
if denominator not in self._denominators:
weights, denominator_predicate = denominator
weights = helpers.convert_to_1d_tensor(weights, name="weights")
dtype = weights.dtype.base_dtype
if not dtype.is_floating:
raise TypeError("weights must be floating-point")
pre_train_ops = set()
pre_train_ops.add(
tf.assert_non_negative(
weights, message="weights must be non-negative"))
denominator_weights = weights * tf.cast(
denominator_predicate.predicate, dtype=dtype)
# The running_average_sum variable will contain the sum of the weights
# included in the denominator that we've seen so far, divided by the
# number of minibatches that we've seen so far. Similarly,
# running_average_count will contain the average size of the minibatches
# we've seen so far. Their ratio will therefore be the sum of the
# weights included in the denominator, divided by the number of examples
# we've seen so far. The reason for dividing both quantities by the
# number of minibatches is to prevent them from growing without bound
# during training.
#
# We use double precision arithmetic for the running sums because we
# don't want numerical errors to ruin our estimates if we perform a very
# large number of iterations.
running_dtype = tf.float64
running_average_sum = tf.Variable(1.0,
trainable=False,
dtype=running_dtype,
name="running_average_sum")
running_average_count = tf.Variable(
1.0,
trainable=False,
dtype=running_dtype,
name="running_average_count")
# To restart the denominator calculations, we set the two running
# averages to their initial values.
restart_ops = set([
tf.assign(running_average_sum, 1.0),
tf.assign(running_average_count, 1.0)
])
# We take convex combinations (with parameter running_proportion) to
# make sure that both running_average_sum and running_average_count are
# divided by the number of minibatches, as explained above.
running_proportion = 1.0 / (tf.maximum(
tf.cast(self._global_step, dtype=running_dtype), 0.0) +
1.0)
pre_train_ops.add(
tf.assign(
running_average_sum,
running_average_sum * (1.0 - running_proportion) +
tf.cast(tf.reduce_sum(denominator_weights),
dtype=running_dtype) * running_proportion))
pre_train_ops.add(
tf.assign(
running_average_count,
running_average_count * (1.0 - running_proportion) +
tf.cast(tf.size(denominator_weights),
dtype=running_dtype) * running_proportion))
# This code calculates max(denominator_lower_bound, running_average_sum
# / running_average_count) safely, even when running_average_count is
# zero (including when running_average_sum is also zero, in which case
# the result will be denominator_lower_bound). We use a tf.cond to make
# sure that we only perform the division if we know that it will result
# in a quantity larger than denominator_lower_bound.
running_denominator_lower_bound = tf.cast(
self._denominator_lower_bound, dtype=running_dtype)
average_denominator_weight = tf.cond(
running_average_count * running_denominator_lower_bound <
running_average_sum,
true_fn=lambda: running_average_sum /
running_average_count,
false_fn=lambda: running_denominator_lower_bound)
self._denominators[denominator] = (average_denominator_weight,
pre_train_ops, restart_ops)
return self._denominators[denominator]