def test_roc_auc(self):
"""Tests that roc_auc's constraints give correct thresholds."""
# We don't check roc_auc_upper_bound since most of the code is shared, and
# the test is too slow already.
bins = 3
bisection_epsilon = 1e-6
structure_memoizer = {
defaults.DENOMINATOR_LOWER_BOUND_KEY: 0.0,
defaults.GLOBAL_STEP_KEY: tf.Variable(0, dtype=tf.int32),
defaults.VARIABLE_FN_KEY: tf.Variable
}
expression = binary_rates.roc_auc(self._context, bins)
# Extract the the constraints and the associated variables.
constraint_list = []
variables = deferred_tensor.DeferredVariableList()
for constraint in expression.extra_constraints:
constraint_value = constraint.expression.constraint_expression.evaluate(
structure_memoizer)
constraint_list.append(constraint_value)
variables += constraint_value.variables
variables = variables.list
self.assertEqual(bins, len(constraint_list))
constraints = deferred_tensor.DeferredTensor.apply(
lambda *args: tf.stack(args), *constraint_list)
# We need to explicitly create all variables included in the expression
# before we can try to extract the roc_auc_thresholds.
for variable in variables:
variable.create(structure_memoizer)
# The find_zeros_of_functions() helper will perform a bisection search over
# the roc_auc_thresholds, so we need to extract the Tensor containing them
# from the graph.
roc_auc_thresholds = None
for variable in variables:
tensor = variable(structure_memoizer)
if tensor.name.startswith("tfco_roc_auc_thresholds"):
self.assertIsNone(roc_auc_thresholds)
roc_auc_thresholds = tensor
self.assertIsNotNone(roc_auc_thresholds)
def update_ops_fn():
update_ops = []
for variable in variables:
update_ops += variable.update_ops(structure_memoizer)
return update_ops
with self.wrapped_session() as session:
session.run_ops(update_ops_fn)
def evaluate_fn(values):
"""Assigns the variables and evaluates the constraints."""
session.run_ops(lambda: roc_auc_thresholds.assign(values))
return session.run(constraints(structure_memoizer))
actual_thresholds = test_helpers.find_zeros_of_functions(
bins, evaluate_fn, epsilon=bisection_epsilon)
expected_thresholds = self._find_roc_auc_thresholds(bins)
self.assertAllClose(
expected_thresholds, actual_thresholds, rtol=0, atol=bisection_epsilon)
def test_precision(self):
"""Checks `precision`."""
bisection_epsilon = 1e-6
structure_memoizer = {
defaults.DENOMINATOR_LOWER_BOUND_KEY: 0.0,
defaults.GLOBAL_STEP_KEY: tf.Variable(0, dtype=tf.int32),
defaults.VARIABLE_FN_KEY: tf.Variable
}
expression = general_rates.precision(self._context)
# Extract the the constraints and the associated variables.
constraint_list = []
variables = deferred_tensor.DeferredVariableList()
for constraint in expression.extra_constraints:
constraint_value = constraint.expression.constraint_expression.evaluate(
structure_memoizer)
constraint_list.append(constraint_value)
variables += constraint_value.variables
variables = variables.list
self.assertEqual(2, len(constraint_list))
constraints = deferred_tensor.DeferredTensor.apply(
lambda *args: tf.stack(args), *constraint_list)
# We need to explicitly create all variables included in the expression
# before we can try to extract the ratio_bounds.
for variable in variables:
variable.create(structure_memoizer)
# The find_zeros_of_functions() helper will perform a bisection search over
# the ratio_bounds, so we need to extract the Tensor containing them from
# the graph.
ratio_bounds = None
for variable in variables:
tensor = variable(structure_memoizer)
if tensor.name.startswith("tfco_ratio_bounds"):
self.assertIsNone(ratio_bounds)
ratio_bounds = tensor
self.assertIsNotNone(ratio_bounds)
def update_ops_fn():
update_ops = []
for variable in variables:
update_ops += variable.update_ops(structure_memoizer)
return update_ops
with self.wrapped_session() as session:
session.run_ops(update_ops_fn)
def evaluate_fn(values):
"""Assigns the variables and evaluates the constraints."""
session.run_ops(lambda: ratio_bounds.assign(values))
return session.run(constraints(structure_memoizer))
actual_ratio_bounds = test_helpers.find_zeros_of_functions(
2, evaluate_fn, epsilon=bisection_epsilon)
actual_numerator = actual_ratio_bounds[0]
actual_denominator = actual_ratio_bounds[1]
expected_numerator = (np.sum(
(0.5 * (1.0 + np.sign(self._predictions))) *
(self._labels > 0.0) * self._weights) / np.sum(self._weights))
expected_denominator = (np.sum(
(0.5 * (1.0 + np.sign(self._predictions))) * self._weights) /
np.sum(self._weights))
self.assertAllClose(expected_numerator,
actual_numerator,
rtol=0,
atol=bisection_epsilon)
self.assertAllClose(expected_denominator,
actual_denominator,
rtol=0,
atol=bisection_epsilon)
def test_f_score(self):
"""Checks `f_score`."""
beta = 1.6
bisection_epsilon = 1e-6
structure_memoizer = {
defaults.DENOMINATOR_LOWER_BOUND_KEY: 0.0,
defaults.GLOBAL_STEP_KEY: tf.Variable(0, dtype=tf.int32),
defaults.VARIABLE_FN_KEY: tf.Variable
}
expression = general_rates.f_score(self._context, beta)
# Extract the the constraints and the associated variables.
constraint_list = []
variables = deferred_tensor.DeferredVariableList()
for constraint in expression.extra_constraints:
constraint_value = constraint.expression.constraint_expression.evaluate(
structure_memoizer)
constraint_list.append(constraint_value)
variables += constraint_value.variables
variables = variables.list
self.assertEqual(1, len(constraint_list))
constraints = deferred_tensor.DeferredTensor.apply(
lambda *args: tf.stack(args), *constraint_list)
# We need to explicitly create all variables included in the expression
# before we can try to extract the denominator_bound.
for variable in variables:
variable.create(structure_memoizer)
# The find_zeros_of_functions() helper will perform a bisection search over
# the denominator_bound, so we need to extract the Tensor containing them
# from the graph.
denominator_bound = None
for variable in variables:
tensor = variable(structure_memoizer)
if tensor.name.startswith("tfco_denominator_bound"):
self.assertIsNone(denominator_bound)
denominator_bound = tensor
self.assertIsNotNone(denominator_bound)
def update_ops_fn():
update_ops = []
for variable in variables:
update_ops += variable.update_ops(structure_memoizer)
return update_ops
with self.wrapped_session() as session:
session.run_ops(update_ops_fn)
def evaluate_fn(values):
"""Assigns the variables and evaluates the constraints."""
# We need to extract the first element from the one-element "values"
# Tensor, since the denominator_bound has shape (), not (1,).
session.run_ops(lambda: denominator_bound.assign(values[0]))
return session.run(constraints(structure_memoizer))
# We need to extract the first element here, for the same reason as above.
actual_denominator_bound = test_helpers.find_zeros_of_functions(
1, evaluate_fn, epsilon=bisection_epsilon)[0]
expected_denominator = (((1.0 + beta * beta) * np.sum(
(0.5 * (1.0 + np.sign(self._predictions))) *
(self._labels > 0.0) * self._weights) + (beta * beta) * np.sum(
(0.5 * (1.0 - np.sign(self._predictions))) *
(self._labels > 0.0) * self._weights) + np.sum(
(0.5 * (1.0 + np.sign(self._predictions))) *
(self._labels <= 0.0) * self._weights)) /
np.sum(self._weights))
self.assertAllClose(expected_denominator,
actual_denominator_bound,
rtol=0,
atol=bisection_epsilon)