def upper_bound(expressions):
"""Creates an `Expression` upper bounding the given expressions.
This function introduces a slack variable, and adds constraints forcing this
variable to upper bound all elements of the given expression list. It then
returns the slack variable.
If you're going to be upper-bounding or minimizing the result of this
function, then you can think of it as taking the `max` of its arguments. You
should *never* lower-bound or maximize the result, however, since the
consequence would be to increase the value of the slack variable, without
affecting the contents of the expressions list.
Args:
expressions: list of `Expression`s, the quantities to upper-bound.
Returns:
An `Expression` representing an upper bound on the given expressions.
Raises:
ValueError: if the expressions list is empty.
TypeError: if the expressions list contains a non-`Expression`.
"""
if not expressions:
raise ValueError("upper_bound cannot be given an empty expression list")
if not all(isinstance(ee, expression.Expression) for ee in expressions):
raise TypeError(
"upper_bound expects a list of rate Expressions (perhaps you need to "
"call wrap_rate() to create an Expression from a Tensor?)")
# Ideally the slack variable would have the same dtype as the predictions, but
# we might not know their dtype (e.g. in eager mode), so instead we always use
# float32 with auto_cast=True.
bound = deferred_tensor.DeferredVariable(
0.0,
trainable=True,
name="tfco_upper_bound",
dtype=tf.float32,
auto_cast=True)
bound_basic_expression = basic_expression.BasicExpression(
terms=[], tensor=bound)
bound_expression = expression.Expression(
penalty_expression=bound_basic_expression,
constraint_expression=bound_basic_expression,
extra_variables=[bound])
extra_constraints = [ee <= bound_expression for ee in expressions]
return expression.Expression(
penalty_expression=bound_basic_expression,
constraint_expression=bound_basic_expression,
extra_variables=[bound],
extra_constraints=extra_constraints)
def test_arithmetic(self):
"""Tests `Expression`'s arithmetic operators."""
memoizer = {
defaults.DENOMINATOR_LOWER_BOUND_KEY: 0.0,
defaults.GLOBAL_STEP_KEY: tf.compat.v2.Variable(0, dtype=tf.int32)
}
penalty_values = [-3.6, 1.5, 0.4]
constraint_values = [-0.2, -0.5, 2.3]
# Create three expressions containing the constants in "penalty_values" in
# their penalty_expressions, and "constraint_values" in their
# constraint_expressions.
expression_objects = []
for penalty_value, constraint_value in zip(penalty_values,
constraint_values):
expression_object = expression.Expression(
basic_expression.BasicExpression(
[],
deferred_tensor.DeferredTensor(
tf.constant(penalty_value, dtype=tf.float32))),
basic_expression.BasicExpression(
[],
deferred_tensor.DeferredTensor(
tf.constant(constraint_value))))
expression_objects.append(expression_object)
# This expression exercises all of the operators.
expression_object = (
0.3 - (expression_objects[0] / 2.3 + 0.7 * expression_objects[1]) -
(1.2 + expression_objects[2] - 0.1) * 0.6 + 0.8)
actual_penalty_value, penalty_variables = (
expression_object.penalty_expression.evaluate(memoizer))
actual_constraint_value, constraint_variables = (
expression_object.constraint_expression.evaluate(memoizer))
# We need to explicitly create the variables before creating the wrapped
# session.
variables = deferred_tensor.DeferredVariableList(penalty_variables +
constraint_variables)
for variable in variables:
variable.create(memoizer)
# This is the same expression as above, applied directly to the python
# floats.
expected_penalty_value = (
0.3 - (penalty_values[0] / 2.3 + 0.7 * penalty_values[1]) -
(1.2 + penalty_values[2] - 0.1) * 0.6 + 0.8)
expected_constraint_value = (
0.3 - (constraint_values[0] / 2.3 + 0.7 * constraint_values[1]) -
(1.2 + constraint_values[2] - 0.1) * 0.6 + 0.8)
with self.wrapped_session() as session:
self.assertNear(expected_penalty_value,
session.run(actual_penalty_value(memoizer)),
err=1e-6)
self.assertNear(expected_constraint_value,
session.run(actual_constraint_value(memoizer)),
err=1e-6)
def lower_bound(expressions):
"""Creates an `Expression` lower bounding the given expressions.
This function introduces a slack variable, and adds constraints forcing this
variable to lower bound all elements of the given expression list. It then
returns the slack variable.
If you're going to be lower-bounding or maximizing the result of this
function, then you can think of it as taking the `min` of its arguments. It's
different from `min` if you're going to be upper-bounding or minimizing the
result, however, since the consequence would be to decrease the value of the
slack variable, without affecting the contents of the expressions list.
Args:
expressions: list of `Expression`s, the quantities to lower-bound.
Returns:
An `Expression` representing an lower bound on the given expressions.
Raises:
ValueError: if the expressions list is empty.
TypeError: if the expressions list contains a non-`Expression`, or if any
`Expression` has a different dtype.
"""
bound = _create_slack_variable(expressions, "lower_bound")
bound_expression = basic_expression.BasicExpression(terms=[], tensor=bound)
extra_constraints = set(ee >= bound for ee in set(expressions))
return expression.Expression(
penalty_expression=bound_expression,
constraint_expression=bound_expression,
extra_constraints=extra_constraints)
def test_arithmetic(self):
"""Tests `Expression`'s arithmetic operators."""
denominator_lower_bound = 0.0
global_step = tf.Variable(0, dtype=tf.int32)
evaluation_context = basic_expression.BasicExpression.EvaluationContext(
denominator_lower_bound, global_step)
penalty_values = [-3.6, 1.5, 0.4]
constraint_values = [-0.2, -0.5, 2.3]
# Create three expressions containing the constants in "penalty_values" in
# their penalty_expressions, and "constraint_values" in their
# constraint_expressions.
expression_objects = []
for penalty_value, constraint_value in zip(penalty_values,
constraint_values):
expression_object = expression.Expression(
basic_expression.BasicExpression([],
tf.constant(
penalty_value,
dtype=tf.float32)),
basic_expression.BasicExpression([], tf.constant(constraint_value)))
expression_objects.append(expression_object)
# This expression exercises all of the operators.
expression_object = (
0.3 - (expression_objects[0] / 2.3 + 0.7 * expression_objects[1]) -
(1.2 + expression_objects[2] - 0.1) * 0.6 + 0.8)
actual_penalty_value, _, _ = expression_object.penalty_expression.evaluate(
evaluation_context)
actual_constraint_value, _, _ = (
expression_object.constraint_expression.evaluate(evaluation_context))
# This is the same expression as above, applied directly to the python
# floats.
expected_penalty_value = (
0.3 - (penalty_values[0] / 2.3 + 0.7 * penalty_values[1]) -
(1.2 + penalty_values[2] - 0.1) * 0.6 + 0.8)
expected_constraint_value = (
0.3 - (constraint_values[0] / 2.3 + 0.7 * constraint_values[1]) -
(1.2 + constraint_values[2] - 0.1) * 0.6 + 0.8)
with self.session() as session:
session.run(
[tf.global_variables_initializer(),
tf.local_variables_initializer()])
self.assertNear(
expected_penalty_value, session.run(actual_penalty_value), err=1e-6)
self.assertNear(
expected_constraint_value,
session.run(actual_constraint_value),
err=1e-6)
def wrap_rate(penalty_tensor, constraint_tensor=None):
"""Creates an `Expression` representing the given `Tensor`(s).
The reason an `Expression` contains two `BasicExpression`s is that the
"penalty" `BasicExpression` will be differentiable, while the "constraint"
`BasicExpression` need not be. During optimization, the former will be used
whenever we need to take gradients, and the latter otherwise.
Args:
penalty_tensor: scalar `Tensor`, the quantity to store in the "penalty"
portion of the result (and also the "constraint" portion, if
constraint_tensor is not provided).
constraint_tensor: scalar `Tensor`, the quantity to store in the
"constraint" portion of the result.
Returns:
An `Expression` wrapping the given `Tensor`(s).
Raises:
TypeError: if wrap_rate() is called on an `Expression`.
"""
# Ideally, we'd check that "penalty_tensor" and "constraint_tensor" are scalar
# Tensors, or are types that can be converted to a scalar Tensor.
# Unfortunately, this includes a lot of possible types, so the easiest
# solution would be to actually perform the conversion, and then check that
# the resulting Tensor has only one element. This, however, would add a dummy
# element to the Tensorflow graph, and wouldn't work for a Tensor with an
# unknown size. Hence, we only check that "penalty_tensor" and
# "constraint_tensor" are not types that we know for certain are disallowed:
# objects internal to this library.
if (isinstance(penalty_tensor, helpers.RateObject) or
isinstance(constraint_tensor, helpers.RateObject)):
raise TypeError("you cannot wrap an object that has already been wrapped")
penalty_basic_expression = basic_expression.BasicExpression(
terms=[], tensor=deferred_tensor.DeferredTensor(penalty_tensor))
if constraint_tensor is None:
constraint_basic_expression = penalty_basic_expression
else:
constraint_basic_expression = basic_expression.BasicExpression(
terms=[], tensor=deferred_tensor.DeferredTensor(constraint_tensor))
return expression.Expression(penalty_basic_expression,
constraint_basic_expression)
def _binary_classification_rate(positive_coefficient=0.0,
negative_coefficient=0.0,
numerator_context=None,
denominator_context=None,
penalty_loss=_DEFAULT_PENALTY_LOSS,
constraint_loss=_DEFAULT_CONSTRAINT_LOSS):
"""Creates an `Expression` representing positive and negative rates.
The result of this function represents:
total_rate := (positive_coefficient * positive_rate +
negative_coefficient * negative_rate)
where:
positive_rate := sum_i{w_i * c_i * d_i * 1{z_i > 0}} / sum_i{w_i * d_i}
negative_rate := sum_i{w_i * c_i * d_i * 1{z_i <= 0}} / sum_i{w_i * d_i}
where z_i and w_i are the given predictions and weights, and c_i and d_i are
indicators for which examples to include the numerator and denominator (all
four of z, w, c and d are in the contexts).
The resulting `Expression` contains *two* different approximations to
"total_rate". The "penalty" `BasicExpression` is an approximation to using
penalty_loss, while the "constraint" `BasicExpression` is based on
constraint_loss (if constraint_loss is the zero-one loss, which is the
default, then the "constraint" expression will be exactly total_rate as
defined above, with no approximation).
The reason an `Expression` contains two `BasicExpression`s is that the
"penalty" `BasicExpression` will be differentiable, while the "constraint"
`BasicExpression` need not be. During optimization, the former will be used
whenever we need to take gradients, and the latter otherwise.
Args:
positive_coefficient: float, scalar coefficient on the positive prediction
rate.
negative_coefficient: float, scalar coefficient on the negative prediction
rate.
numerator_context: `SubsettableContext`, the block of data to use when
calculating the numerators of the rates.
denominator_context: `SubsettableContext`, the block of data to use when
calculating the denominators of the rates.
penalty_loss: `BinaryClassificationLoss`, the (differentiable) loss function
to use when calculating the "penalty" approximation to the rates.
constraint_loss: `BinaryClassificationLoss`, the (not necessarily
differentiable) loss function to use when calculating the "constraint"
approximation to the rates.
Returns:
An `Expression` representing total_rate (as defined above).
Raises:
TypeError: if either context is not a SubsettableContext, or either loss is
not a BinaryClassificationLoss.
ValueError: if either context is not provided, or the two contexts are
incompatible (have different predictions, labels or weights).
"""
if numerator_context is None or denominator_context is None:
raise ValueError("both numerator_context and denominator_context must be "
"provided")
if not (
isinstance(numerator_context, subsettable_context.SubsettableContext) and
isinstance(denominator_context, subsettable_context.SubsettableContext)):
raise TypeError("numerator and denominator contexts must be "
"SubsettableContexts")
raw_context = numerator_context.raw_context
if denominator_context.raw_context != raw_context:
raise ValueError("numerator and denominator contexts must be compatible")
if not (isinstance(penalty_loss, loss.BinaryClassificationLoss) and
isinstance(constraint_loss, loss.BinaryClassificationLoss)):
raise TypeError("penalty and constraint losses must be "
"BinaryClassificationLosses")
penalty_term = term.BinaryClassificationTerm.ratio(
positive_coefficient, negative_coefficient,
raw_context.penalty_predictions, raw_context.penalty_weights,
numerator_context.penalty_predicate,
denominator_context.penalty_predicate, penalty_loss)
constraint_term = term.BinaryClassificationTerm.ratio(
positive_coefficient, negative_coefficient,
raw_context.constraint_predictions, raw_context.constraint_weights,
numerator_context.constraint_predicate,
denominator_context.constraint_predicate, constraint_loss)
return expression.Expression(
basic_expression.BasicExpression([penalty_term]),
basic_expression.BasicExpression([constraint_term]))
def _roc_auc(context,
bins,
lower_bound=False,
upper_bound=False,
penalty_loss=_DEFAULT_PENALTY_LOSS,
constraint_loss=_DEFAULT_CONSTRAINT_LOSS):
"""Creates an `Expression` representing an approximate ROC AUC.
The result of this function represents a Riemann approximation to the area
under the ROC curve (false positive rate on the horizontal axis, true positive
rate on the vertical axis), using the constraint-based method proposed by:
> Eban, Schain, Mackey, Gordon, Rifkin and Elidan. "Scalable Learning of
> Non-Decomposable Objectives". AISTATS 2017.
If you're going to be lower-bounding or maximizing the result of this
function, then need to set the lower_bound parameter to `True`. Likewise, if
you're going to be upper-bounding or minimizing the result of this function
(which normally wouldn't make much sense for ROC AUC), then the upper_bound
parameter must be `True`. At least one of these parameters *must* be `True`,
and it's permitted for both of them to be `True` (but we recommend against
this, since it would result in equality constraints, which might cause
problems during optimization and/or post-processing).
Args:
context: `SubsettableContext`, the block of data to use when calculating the
rate. This context *must* contain labels.
bins: positive integer, the number of "rectangles" to use for the Riemann
approximation to ROC AUC.
lower_bound: bool, `True` if you want the result of this function to
lower-bound the approximate ROC AUC.
upper_bound: bool, `True` if you want the result of this function to
upper-bound the approximate ROC AUC.
penalty_loss: `BinaryClassificationLoss`, the (differentiable) loss function
to use when calculating the "penalty" approximation to the rate.
constraint_loss: `BinaryClassificationLoss`, the (not necessarily
differentiable) loss function to use when calculating the "constraint"
approximation to the rate. This loss must be "normalized" (see
`BinaryClassificationLoss.is_normalized`).
Returns:
An `Expression` representing a Riemann approximation to ROC AUC.
Raises:
TypeError: if the context is not a SubsettableContext, the number of bins is
not an integer, or either loss is not a BinaryClassificationLoss.
ValueError: if the context doesn't contain labels, the number of bins is
nonpositive, both lower_bound and upper_bound are `False`, or the
constraint_loss is not normalized.
"""
if not isinstance(context, subsettable_context.SubsettableContext):
raise TypeError("context must be a SubsettableContext")
raw_context = context.raw_context
if (raw_context.penalty_labels is None
or raw_context.constraint_labels is None):
raise ValueError("roc_auc_lower_bound requires a context with labels")
if not isinstance(bins, numbers.Integral):
raise TypeError(
"number of roc_auc_lower_bound bins must be an integer")
if bins <= 0:
raise ValueError("number of roc_auc_lower_bound bins must be strictly "
"positive")
# One could set both lower_bound and upper_bound to True, in which case the
# result of this function could be treated as the Riemann approximation to ROC
# AUC itself (instead of a {lower,upper} bound of it). However, this would
# come with some drawbacks: it would of course make optimization more
# difficult, but more importantly, it would potentially cause post-processing
# for feasibility (e.g. using "shrinking") to fail to find a feasible
# solution.
if not (lower_bound or upper_bound):
raise ValueError(
"at least one of lower_bound or upper_bound must be True")
if not (isinstance(penalty_loss, loss.BinaryClassificationLoss)
and isinstance(constraint_loss, loss.BinaryClassificationLoss)):
raise TypeError("penalty and constraint losses must be "
"BinaryClassificationLosses")
# For the constraints on the false positive rates to make sense, it would be
# best to be using a normalized loss. The reason for this is that, if both
# lower_bound and upper_bound are True (or if one imposes constraints
# including separate lower and upper bounds), our constraints on the false
# positive rates will be equality constraints, which could be infeasible for
# an unnormalized loss. This could be changed to a warning, however.
if not constraint_loss.is_normalized:
raise ValueError(
"roc_auc_lower_bound can only be used with a normalized "
"constraint_loss (e.g. zero/one, sigmoid or ramp)")
dtype = raw_context.penalty_predictions.dtype.real_dtype
if dtype != raw_context.constraint_predictions.dtype.real_dtype:
raise ValueError(
"penalty and constraint predictions must have the same "
"dtype")
# We use a lambda to initialize the thresholds so that, if this function call
# is inside the scope of a tf.control_dependencies() block, the dependencies
# will not be applied to the initializer.
thresholds = tf.Variable(lambda: tf.zeros((bins, )),
dtype=dtype,
name="roc_auc_thresholds")
positive_context = context.subset(raw_context.penalty_labels > 0,
raw_context.constraint_labels > 0)
negative_context = context.subset(raw_context.penalty_labels <= 0,
raw_context.constraint_labels <= 0)
penalty_average_tpr_terms = []
constraint_average_tpr_terms = []
extra_constraints = set()
for bin_index in xrange(bins):
threshold = thresholds[bin_index]
# It's tempting to wrap tf.stop_gradient() around the threshold, so that
# only the model parameters (and not the thresholds) will be adjusted to
# increase the average true positive rate. However, this would prevent the
# one-sided constraint, as described below, from working, since we need
# something to be "pushing against" the constraint.
penalty_tpr_term = term.BinaryClassificationTerm.ratio(
1.0, 0.0, raw_context.penalty_predictions - threshold,
raw_context.penalty_weights, positive_context.penalty_predicate,
positive_context.penalty_predicate, penalty_loss)
constraint_tpr_term = term.BinaryClassificationTerm.ratio(
1.0, 0.0, raw_context.constraint_predictions - threshold,
raw_context.constraint_weights,
positive_context.constraint_predicate,
positive_context.constraint_predicate, constraint_loss)
penalty_average_tpr_terms.append(penalty_tpr_term / bins)
constraint_average_tpr_terms.append(constraint_tpr_term / bins)
# We wrap tf.stop_gradient() around the predictions because we want to
# adjust the thresholds, and only the thresholds, to satisfy the false
# positive rate constraints.
penalty_fpr_term = term.BinaryClassificationTerm.ratio(
1.0, 0.0,
tf.stop_gradient(raw_context.penalty_predictions) - threshold,
raw_context.penalty_weights, negative_context.penalty_predicate,
negative_context.penalty_predicate, penalty_loss)
constraint_fpr_term = term.BinaryClassificationTerm.ratio(
1.0, 0.0,
tf.stop_gradient(raw_context.constraint_predictions) - threshold,
raw_context.constraint_weights,
negative_context.constraint_predicate,
negative_context.constraint_predicate, constraint_loss)
fpr_expression = expression.Expression(
basic_expression.BasicExpression([penalty_fpr_term]),
basic_expression.BasicExpression([constraint_fpr_term]))
target_fpr = (bin_index + 0.5) / bins
# Ideally fpr_expression would equal target_fpr, but we prefer to only
# impose a one-sided constraint (when exactly one of lower_bound or
# upper_bound is True) since using an equality constraint would come with
# drawbacks: it would of course make optimization more difficult, but more
# importantly, it would potentially cause post-processing for feasibility
# (e.g. using "shrinking") to fail to find a feasible solution.
#
# The reason why a <= constraint results in a lower bound, and a >=
# constraint results in an upper bound, is that, in the lower-bound case
# (the upper-bound case is similar), adjusting the threshold to increase the
# FPR will increase the corresponding TPR, and therefore the ROC AUC
# estimate. In other words, the objective (increasing ROC AUC, and therefore
# the FPR of each bin) will be "pushing against" the constraint.
if lower_bound:
extra_constraints.add(fpr_expression <= target_fpr)
if upper_bound:
extra_constraints.add(fpr_expression >= target_fpr)
return expression.Expression(
basic_expression.BasicExpression(penalty_average_tpr_terms),
basic_expression.BasicExpression(constraint_average_tpr_terms),
extra_constraints)
def create_dummy_expression(extra_variables=None):
"""Creates an empty `Expression` with the given extra variables."""
return expression.Expression(basic_expression.BasicExpression([]),
basic_expression.BasicExpression([]),
extra_variables=extra_variables)
def create_dummy_expression(extra_constraints=None):
"""Creates an empty `Expression` with the given extra constraints."""
return expression.Expression(basic_expression.BasicExpression([]),
basic_expression.BasicExpression([]),
extra_constraints=extra_constraints)