def test_local_variable(self):
with self.cached_session() as sess:
self.assertEquals([], variables_lib.local_variables())
value0 = 42
variables_lib2.local_variable(value0)
value1 = 43
variables_lib2.local_variable(value1)
variables = variables_lib.local_variables()
self.assertEquals(2, len(variables))
self.assertRaises(errors_impl.OpError, sess.run, variables)
variables_lib.variables_initializer(variables).run()
self.assertAllEqual(set([value0, value1]), set(sess.run(variables)))
def test_div_by_zero(self): r_obj = metrics.Recall() y_pred = constant_op.constant([0, 0, 0, 0]) y_true = constant_op.constant([0, 0, 0, 0]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertEqual(0, self.evaluate(result))
def test_unweighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result))
def test_save_restore(self):
checkpoint_directory = self.get_temp_dir()
checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
m = metrics.Mean()
checkpoint = checkpointable_utils.Checkpoint(mean=m)
self.evaluate(variables.variables_initializer(m.variables))
# update state
self.evaluate(m(100.))
self.evaluate(m(200.))
# save checkpoint and then add an update
save_path = checkpoint.save(checkpoint_prefix)
self.evaluate(m(1000.))
# restore to the same checkpoint mean object
checkpoint.restore(save_path).assert_consumed().run_restore_ops()
self.evaluate(m(300.))
self.assertEqual(200., self.evaluate(m.result()))
# restore to a different checkpoint mean object
restore_mean = metrics.Mean()
restore_checkpoint = checkpointable_utils.Checkpoint(mean=restore_mean)
status = restore_checkpoint.restore(save_path)
restore_update = restore_mean(300.)
status.assert_consumed().run_restore_ops()
self.evaluate(restore_update)
self.assertEqual(200., self.evaluate(restore_mean.result()))
self.assertEqual(3, self.evaluate(restore_mean.count))
def test_placeholder_with_default_fed(self):
with self.test_session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(4.0)
ph = array_ops.placeholder_with_default(v, shape=[])
out = ph * 2
sess.run(variables.variables_initializer([v]))
self.assertEqual(2.0, sess.run(out, {ph: 1.0}))
def test_example(self):
with self.test_session() as session:
for tower_id in range(3):
self.create_tower_metrics(tower_id)
session.run(
variables.variables_initializer(
ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES)))
session.run(
replicate_model_fn._reduce_metric_variables(number_of_towers=3))
# 1st tower = 1.3, 2.3, [3.3, 3.5, 3.7]
# 2nd tower = 2.6, 4.6, [6.6, 7.0, 7.4]
# 3rd tower = 3.9, 6.9, [9.9, 10.5, 11.1]
# Reduced = 7.8, 13.8, [19.8, 21.0, 22.2]
# Towers are accumulated in the first tower.
local_metrics = session.run(
ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES))
self.assertNear(7.8, local_metrics[0], 0.01)
self.assertNear(13.8, local_metrics[1], 0.01)
self.assertAllClose([19.8, 21., 22.1], local_metrics[2], 0.01)
self.assertNear(0.0, local_metrics[3], 0.01)
self.assertNear(0.0, local_metrics[4], 0.01)
self.assertAllClose([0.0, 0.0, 0.0], local_metrics[5], 0.01)
self.assertNear(0.0, local_metrics[6], 0.01)
self.assertNear(0.0, local_metrics[7], 0.01)
self.assertAllClose([0.0, 0.0, 0.0], local_metrics[8], 0.01)
def testConvertVariablesToConstsWithFunctions(self):
@function.Defun(dtypes.float32)
def plus_one(x):
return x + 1.0
with ops.Graph().as_default():
variable_node = variables.Variable(1.0, name="variable_node")
_ = variables.Variable(1.0, name="unused_variable_node")
defun_node = plus_one(variable_node)
output_node = math_ops_lib.multiply(
defun_node, 2.0, name="output_node")
with session.Session() as sess:
init = variables.variables_initializer([variable_node])
sess.run(init)
output = sess.run(output_node)
self.assertNear(4.0, output, 0.00001)
variable_graph_def = sess.graph.as_graph_def()
# First get the constant_graph_def when variable_names_whitelist is set,
# note that if variable_names_whitelist is not set an error will be
# thrown because unused_variable_node is not initialized.
constant_graph_def = graph_util.convert_variables_to_constants(
sess,
variable_graph_def, ["output_node"],
variable_names_whitelist=set(["variable_node"]))
self.assertEqual(variable_graph_def.library,
constant_graph_def.library)
def test_reduce_is_idempotent(self):
with self.test_session() as session:
for tower_id in range(3):
self.create_tower_metrics(tower_id)
session.run(
variables.variables_initializer(
ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES)))
for _ in range(20):
session.run(
replicate_model_fn._reduce_metric_variables(number_of_towers=3))
local_metrics = session.run(
ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES))
self.assertNear(7.8, local_metrics[0], 0.01)
self.assertNear(13.8, local_metrics[1], 0.01)
self.assertAllClose([19.8, 21., 22.1], local_metrics[2], 0.01)
self.assertNear(0.0, local_metrics[3], 0.01)
self.assertNear(0.0, local_metrics[4], 0.01)
self.assertAllClose([0.0, 0.0, 0.0], local_metrics[5], 0.01)
self.assertNear(0.0, local_metrics[6], 0.01)
self.assertNear(0.0, local_metrics[7], 0.01)
self.assertAllClose([0.0, 0.0, 0.0], local_metrics[8], 0.01)
def test_unweighted_top_k(self): p_obj = metrics.Precision(top_k=3) y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) self.evaluate(variables.variables_initializer(p_obj.variables)) result = p_obj(y_true, y_pred) self.assertAlmostEqual(1. / 3, self.evaluate(result))
def test_unweighted_with_threshold(self): r_obj = metrics.Recall(thresholds=[0.5, 0.7]) y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
def test_binary_accuracy(self):
acc_obj = metrics.BinaryAccuracy(name='my acc')
# check config
self.assertEqual(acc_obj.name, 'my acc')
self.assertTrue(acc_obj.stateful)
self.assertEqual(len(acc_obj.variables), 2)
self.assertEqual(acc_obj.dtype, dtypes.float32)
self.evaluate(variables.variables_initializer(acc_obj.variables))
# verify that correct value is returned
update_op = acc_obj.update_state([[1], [0]], [[1], [0]])
self.evaluate(update_op)
result = self.evaluate(acc_obj.result())
self.assertEqual(result, 1) # 2/2
# check y_pred squeeze
update_op = acc_obj.update_state([[1], [1]], [[[1]], [[0]]])
self.evaluate(update_op)
result = self.evaluate(acc_obj.result())
self.assertAlmostEqual(result, 0.75, 2) # 3/4
# check y_true squeeze
result_t = acc_obj([[[1]], [[1]]], [[1], [0]])
result = self.evaluate(result_t)
self.assertAlmostEqual(result, 0.67, 2) # 4/6
# check with sample_weight
result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]])
result = self.evaluate(result_t)
self.assertAlmostEqual(result, 0.67, 2) # 4.5/6.7
def test_mean(self):
m = metrics.Mean(name='my_mean')
# check config
self.assertEqual(m.name, 'my_mean')
self.assertTrue(m.stateful)
self.assertEqual(m.dtype, dtypes.float32)
self.assertEqual(len(m.variables), 2)
self.evaluate(variables.variables_initializer(m.variables))
# check initial state
self.assertEqual(self.evaluate(m.total), 0)
self.assertEqual(self.evaluate(m.count), 0)
# check __call__()
self.assertEqual(self.evaluate(m(100)), 100)
self.assertEqual(self.evaluate(m.total), 100)
self.assertEqual(self.evaluate(m.count), 1)
# check update_state() and result() + state accumulation + tensor input
update_op = m.update_state(ops.convert_n_to_tensor([1, 5]))
self.evaluate(update_op)
self.assertAlmostEqual(self.evaluate(m.result()), 106 / 3, 2)
self.assertEqual(self.evaluate(m.total), 106) # 100 + 1 + 5
self.assertEqual(self.evaluate(m.count), 3)
# check reset_states()
m.reset_states()
self.assertEqual(self.evaluate(m.total), 0)
self.assertEqual(self.evaluate(m.count), 0)
def test_placeholder_with_default_default(self):
with self.cached_session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(4.0)
ph = array_ops.placeholder_with_default(v, shape=[])
out = ph * 2
sess.run(variables.variables_initializer([v]))
self.assertEqual(8.0, self.evaluate(out))
def create_checkpoint_from_values(self,
var_names_to_values,
checkpoint_dir,
global_step=None):
"""Creates a checkpoint from a mapping of name to values in model_dir.
Args:
var_names_to_values: a map from variable names to values.
checkpoint_dir: the directory where the checkpoint will be saved.
global_step: the global step used to save the checkpoint.
Returns:
the model_path to the checkpoint.
"""
var_list = []
with session.Session('', graph=ops.Graph()) as sess:
# Create a set of variables to save in the checkpoint.
for var_name in var_names_to_values:
var_value = var_names_to_values[var_name]
var_list.append(variables_lib.Variable(var_value, name=var_name))
saver = saver_lib.Saver(var_list)
init_op = variables_lib.variables_initializer(var_list)
sess.run(init_op)
# Save the initialized values in the file at 'checkpoint_dir'
return saver.save(sess, checkpoint_dir, global_step=global_step)
def initialize_op(self):
"""Returns an op for initializing tensorflow variables."""
all_vars = self._row_factors + self._col_factors
all_vars.extend([self._row_gramian, self._col_gramian])
if self._row_weights is not None:
assert self._col_weights is not None
all_vars.extend(self._row_weights + self._col_weights)
return variables.variables_initializer(all_vars)
def test_unweighted_all_correct(self): s_obj = metrics.SensitivityAtSpecificity(0.7) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs, dtype=dtypes.float32) y_true = constant_op.constant(inputs) self.evaluate(variables.variables_initializer(s_obj.variables)) result = s_obj(y_true, y_pred) self.assertAlmostEqual(1, self.evaluate(result))
def testSparseRead0DIndices(self):
for dtype in self.numeric_types:
init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype)
with self.test_session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(init)
sess.run(variables.variables_initializer([v]))
x = v.sparse_read(2)
self.assertAllClose(np.array([8, 9, 10, 11], dtype=dtype), sess.run(x))
def test_unweighted_all_incorrect(self): r_obj = metrics.Recall(thresholds=[0.5]) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs) y_true = constant_op.constant(1 - inputs) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0, self.evaluate(result))
def test_extreme_thresholds(self):
r_obj = metrics.Recall(thresholds=[-1.0, 2.0]) # beyond values range
y_pred = math_ops.cast(
constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32)
y_true = math_ops.cast(
constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32)
self.evaluate(variables.variables_initializer(r_obj.variables))
result = r_obj(y_true, y_pred)
self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
def test_layer_output(self, attention_cls):
attention = attention_cls(self.units, self.memory)
score = attention([self.query, self.state])
self.evaluate(variables.variables_initializer(attention.variables))
score_val = self.evaluate(score)
self.assertLen(score_val, 2)
self.assertEqual(score_val[0].shape, (self.batch, self.timestep))
self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
def test_weighted(self):
tp_obj = metrics.TruePositives()
self.evaluate(variables.variables_initializer(tp_obj.variables))
y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
(1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
(0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
result = tp_obj(y_true, y_pred, sample_weight=sample_weight)
self.assertAllClose([12.], self.evaluate(result))
def test_unweighted_top_k_and_threshold(self):
r_obj = metrics.Recall(thresholds=.7, top_k=2)
self.evaluate(variables.variables_initializer(r_obj.variables))
y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5))
y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5))
result = r_obj(y_true, y_pred)
self.assertAlmostEqual(0.25, self.evaluate(result))
self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives))
self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
def test_weighted(self):
cosine_obj = metrics.CosineProximity()
self.evaluate(variables.variables_initializer(cosine_obj.variables))
y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
(1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
(0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
sample_weight = constant_op.constant((1., 1.5, 2., 2.5))
result = cosine_obj(y_true, y_pred, sample_weight=sample_weight)
self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5)
def test_unweighted_low_specificity(self):
s_obj = metrics.SensitivityAtSpecificity(0.4)
pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
y_true = constant_op.constant(label_values)
self.evaluate(variables.variables_initializer(s_obj.variables))
result = s_obj(y_true, y_pred)
self.assertAlmostEqual(0.6, self.evaluate(result))
def test_weighted_with_thresholds(self):
tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(tp_obj.variables))
y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
(0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
(1, 1, 1, 1)))
result = tp_obj(y_true, y_pred, sample_weight=37.)
self.assertAllClose([222., 111., 37.], self.evaluate(result))
def _testAddUpdate(self, scope):
with scope:
layer_with_update = LayerWithUpdate(dtype=dtypes.int32)
model = testing_utils.get_model_from_layers([layer_with_update],
input_shape=(3,),
input_dtype=dtypes.int32)
if testing_utils.get_model_type() == 'subclass':
model._set_inputs(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32))
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir, save_format='tf')
loaded = keras_load.load(saved_model_dir)
loaded_layer = loaded.layers[-1]
self.evaluate(variables.variables_initializer(loaded.variables))
self.assertEqual(self.evaluate(loaded_layer.v), 0)
loaded.predict(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32),
steps=1)
self.assertEqual(self.evaluate(loaded_layer.v), 6)
def _testAddUpdate(self, scope):
with scope:
layer_with_update = LayerWithUpdate()
model = testing_utils.get_model_from_layers([layer_with_update],
input_shape=(3,))
x = np.ones((10, 3))
if testing_utils.get_model_type() == 'subclass':
model.predict(x, batch_size=10)
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir, save_format='tf')
loaded = keras_load.load(saved_model_dir)
loaded_layer = loaded.layers[-1]
self.evaluate(variables.variables_initializer(loaded.variables))
self.assertEqual(self.evaluate(loaded_layer.v), 0.)
loaded.compile('sgd', 'mse')
loaded.fit(x, x, batch_size=10)
self.assertEqual(self.evaluate(loaded_layer.v), 1.)
def testSparseRead2DIndices(self):
for dtype in self.numeric_types:
init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype)
with self.test_session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(init)
sess.run(variables.variables_initializer([v]))
x = v.sparse_read([[2, 1], [0, 2]])
self.assertAllClose(
np.array(
[[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2, 3], [8, 9, 10,
11]]],
dtype=dtype), sess.run(x))
def test_weighted_with_thresholds(self):
tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(tp_obj.variables))
y_pred = constant_op.constant(
((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3),
(0, 1, 0.7, 0.3)))
y_true = constant_op.constant(
((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1)))
result = tp_obj(y_true, y_pred, sample_weight=37.)
self.assertAllClose([222., 111., 37.], self.evaluate(result))
def test_weighted(self, label_dtype):
s_obj = metrics.SensitivityAtSpecificity(0.4)
pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
y_true = math_ops.cast(label_values, dtype=label_dtype)
weights = constant_op.constant(weight_values)
self.evaluate(variables.variables_initializer(s_obj.variables))
result = s_obj(y_true, y_pred, sample_weight=weights)
self.assertAlmostEqual(0.675, self.evaluate(result))
def testBatchNormUpdates(self):
model = keras.models.Sequential(
keras.layers.BatchNormalization(input_shape=(1,)))
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
model.save(saved_model_dir, save_format='tf')
loaded = keras_load.load(saved_model_dir)
self.evaluate(variables.variables_initializer(loaded.variables))
input_arr = array_ops.constant([[11], [12], [13]], dtype=dtypes.float32)
input_arr2 = array_ops.constant([[14], [15], [16]], dtype=dtypes.float32)
self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0])
self.evaluate(loaded(input_arr, training=True))
if not context.executing_eagerly():
self.evaluate(loaded.get_updates_for(input_arr))
self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0.12])
self.evaluate(loaded(input_arr2, training=False))
if not context.executing_eagerly():
self.evaluate(loaded.get_updates_for(input_arr2))
self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0.12])
def test_extreme_thresholds(self):
p_obj = metrics.Precision(thresholds=[-1.0,
2.0]) # beyond values range
y_pred = math_ops.cast(constant_op.constant([1, 0, 1, 0],
shape=(1, 4)),
dtype=dtypes.float32)
y_true = math_ops.cast(constant_op.constant([0, 1, 1, 1],
shape=(1, 4)),
dtype=dtypes.float32)
self.evaluate(variables.variables_initializer(p_obj.variables))
result = p_obj(y_true, y_pred)
self.assertArrayNear([0.75, 0.], self.evaluate(result), 0)
def test_weighted_with_thresholds(self):
tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(tn_obj.variables))
y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
(0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
(1, 1, 1, 1)))
sample_weight = ((0.0, 2.0, 3.0, 5.0),)
result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
self.assertAllClose([5., 15., 23.], self.evaluate(result))
def test_weighted(self, label_dtype):
s_obj = metrics.SensitivityAtSpecificity(0.4)
pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26]
label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]
weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]
y_pred = constant_op.constant(pred_values, dtype=dtypes.float32)
y_true = math_ops.cast(label_values, dtype=label_dtype)
weights = constant_op.constant(weight_values)
self.evaluate(variables.variables_initializer(s_obj.variables))
result = s_obj(y_true, y_pred, sample_weight=weights)
self.assertAlmostEqual(0.675, self.evaluate(result))
def test_unweighted(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
self.evaluate(variables.variables_initializer(auc_obj.variables))
result = auc_obj(self.y_true, self.y_pred)
# tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2]
# recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0]
# fp_rate = [2/2, 0, 0] = [1, 0, 0]
# heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.75 * 1 + 0.25 * 0)
self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
def test_weighted(self):
r_obj = metrics.Recall()
y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]])
y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
self.evaluate(variables.variables_initializer(r_obj.variables))
result = r_obj(
y_true,
y_pred,
sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
weighted_tp = 3.0 + 1.0
weighted_t = (2.0 + 3.0) + (4.0 + 1.0)
expected_recall = weighted_tp / weighted_t
self.assertAlmostEqual(expected_recall, self.evaluate(result))
def testSaveWithRaggedInputs(self):
class EmbeddingMerger(keras.layers.Layer):
def __init__(self, list_features, **kwargs):
super().__init__(**kwargs)
self._supports_ragged_inputs = True
self.embeddings = {
feature: keras.layers.Embedding(10, 3)
for feature in list_features
}
self.mean = keras.layers.Lambda(math_ops.reduce_mean,
arguments=dict(axis=1))
def call(self, inputs):
tensors = [self.embeddings[col](inputs[col]) for col in inputs]
tensors = [self.mean(inp) for inp in tensors]
return keras.layers.Add()(tensors)
list_features = ['feature_1', 'feature_2']
feature_1 = ragged_factory_ops.constant([[0.], [1, 3]])
feature_2 = ragged_factory_ops.constant([[1., 2], [4]])
f = {'feature_1': feature_1, 'feature_2': feature_2}
f_inputs = {
'feature_1': keras.Input(shape=(None, ),
name='feature_1',
ragged=True),
'feature_2': keras.Input(shape=(None, ),
name='feature_2',
ragged=True)
}
out = EmbeddingMerger(list_features)(f_inputs)
model = keras.Model(f_inputs, out)
self.evaluate(variables.variables_initializer(model.variables))
saved_model_dir = self._save_model_dir()
tf_save.save(model, saved_model_dir)
loaded = keras_load.load(saved_model_dir)
self.evaluate(variables.variables_initializer(loaded.variables))
self.assertAllClose(model.predict(f), loaded.predict(f))
def test_weighted_with_thresholds(self):
tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(tn_obj.variables))
y_pred = constant_op.constant(
((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3),
(0, 1, 0.7, 0.3)))
y_true = constant_op.constant(
((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1)))
sample_weight = ((0.0, 2.0, 3.0, 5.0), )
result = tn_obj(y_true, y_pred, sample_weight=sample_weight)
self.assertAllClose([5., 15., 23.], self.evaluate(result))
def test_correctness(self):
a_obj = metrics.SparseTopKCategoricalAccuracy()
self.evaluate(variables.variables_initializer(a_obj.variables))
y_true = constant_op.constant([2, 1])
y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]])
result = a_obj(y_true, y_pred)
self.assertEqual(1, self.evaluate(result)) # both the samples match
# With `k` < 5.
a_obj = metrics.SparseTopKCategoricalAccuracy(k=1)
self.evaluate(variables.variables_initializer(a_obj.variables))
result = a_obj(y_true, y_pred)
self.assertEqual(0.5, self.evaluate(result)) # only sample #2 matches
# With `k` > 5.
y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4],
[0.05, 0.95, 0, 0, 0, 0, 0]])
a_obj = metrics.SparseTopKCategoricalAccuracy(k=6)
self.evaluate(variables.variables_initializer(a_obj.variables))
result = a_obj(y_true, y_pred)
self.assertEqual(0.5, self.evaluate(result)) # only 1 sample matches.
def test_doesnt_accept_uneven_number_of_variables(self):
with self.test_session() as session:
for tower_id in range(3):
self.create_tower_metrics(tower_id)
self.create_metric_variable(-1.0, 'oddball')
session.run(
variables.variables_initializer(
ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES)))
with self.assertRaisesRegexp(ValueError, ''):
session.run(
replicate_model_fn._reduce_metric_variables(number_of_towers=3))
def test_weighted_roc_interpolation(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=self.num_thresholds)
self.evaluate(variables.variables_initializer(auc_obj.variables))
result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight)
# tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3]
# recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0]
# fp_rate = [3/3, 0, 0] = [1, 0, 0]
# heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855]
# widths = [(1 - 0), (0 - 0)] = [1, 0]
expected_result = (0.7855 * 1 + 0.2855 * 0)
self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
def test_unweighted(self):
fn_obj = metrics.FalseNegatives()
self.evaluate(variables.variables_initializer(fn_obj.variables))
y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
(1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
(0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
update_op = fn_obj.update_state(y_true, y_pred)
self.evaluate(update_op)
result = fn_obj.result()
self.assertAllClose(3., result)
def test_weighted(self):
p_obj = metrics.Precision()
y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]])
y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]])
self.evaluate(variables.variables_initializer(p_obj.variables))
result = p_obj(
y_true,
y_pred,
sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]]))
weighted_tp = 3.0 + 4.0
weighted_positives = (1.0 + 3.0) + (4.0 + 2.0)
expected_precision = weighted_tp / weighted_positives
self.assertAlmostEqual(expected_precision, self.evaluate(result))
def test_unweighted(self):
cosine_obj = metrics.CosineProximity()
self.evaluate(variables.variables_initializer(cosine_obj.variables))
y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
(1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
(0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
update_op = cosine_obj.update_state(y_true, y_pred)
self.evaluate(update_op)
result = cosine_obj.result()
self.assertAllClose(-0.60723, result, atol=1e-5)
def test_weighted(self):
self.setup()
k_obj = metrics.KullbackLeiblerDivergence()
self.evaluate(variables.variables_initializer(k_obj.variables))
sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
result = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight)
sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape(
(2, 3))
expected_result = np.multiply(self.expected_results, sample_weight)
expected_result = np.sum(expected_result) / (1.2 + 3.4)
self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
def test_unweighted_with_thresholds(self):
tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(tp_obj.variables))
y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
(0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
(1, 1, 1, 1)))
update_op = tp_obj.update_state(y_true, y_pred)
self.evaluate(update_op)
result = tp_obj.result()
self.assertAllClose([6., 3., 1.], result)
def test_unweighted(self):
tp_obj = metrics.TruePositives()
self.evaluate(variables.variables_initializer(tp_obj.variables))
y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1),
(1, 1, 1, 1, 0), (0, 0, 0, 0, 1)))
y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1),
(0, 1, 0, 1, 0), (1, 1, 1, 1, 1)))
update_op = tp_obj.update_state(y_true, y_pred)
self.evaluate(update_op)
result = tp_obj.result()
self.assertAllClose([7.], result)
def test_weighted_with_thresholds(self):
fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85])
self.evaluate(variables.variables_initializer(fp_obj.variables))
y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6),
(0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3)))
y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0),
(1, 1, 1, 1)))
sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0),
(19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0))
result = fp_obj(y_true, y_pred, sample_weight=sample_weight)
self.assertAllClose([125., 42., 12.], self.evaluate(result))
def testStatelessParameterizedTruncatedNormalHasGrads(self):
mean = variables.Variable(0.01)
stddev = variables.Variable(1.)
minval = variables.Variable(-1.)
maxval = variables.Variable(1.)
with self.cached_session() as sess:
with backprop.GradientTape(persistent=True) as tape:
samples = stateless.stateless_parameterized_truncated_normal(
[1], [1, 2], mean, stddev, minval, maxval)
sess.run(
variables.variables_initializer([mean, stddev, minval,
maxval]))
[mean_grad,
std_grad], mean_actual_grad, std_actual_grad = sess.run([
tape.gradient(samples, [mean, stddev]),
array_ops.ones_like(mean), (samples - mean) / stddev
])
self.assertAllClose(mean_grad, mean_actual_grad)
self.assertAllClose(std_grad, std_actual_grad[0])
try:
import scipy.stats # pylint:disable=g-import-not-at-top
truncnorm = scipy.stats.truncnorm(a=-1.,
b=1.,
loc=0.,
scale=1.)
samples_np, [minval_grad, maxval_grad] = sess.run(
[samples,
tape.gradient(samples, [minval, maxval])])
sample_cdf = truncnorm.cdf(samples_np)
# These come from the implicit reparameterization trick.
scipy_maxval_grad = np.exp(0.5 * (samples_np**2 -
((1. - 0.01) / 1.)**2) +
np.log(sample_cdf))
scipy_minval_grad = np.exp(0.5 * (samples_np**2 -
((-1. - 0.01) / 1.)**2) +
np.log1p(-sample_cdf))
self.assertAllClose(minval_grad,
scipy_minval_grad[0],
rtol=1e-2)
self.assertAllClose(maxval_grad,
scipy_maxval_grad[0],
rtol=1e-2)
except ImportError as e:
tf_logging.warn("Cannot test truncated normal op: %s" % str(e))
def test_save_and_load(self):
mfn = function.ModelFunction.from_function(_model_fn)
out = mfn.train(constant_op.constant(3), constant_op.constant(5))
self.evaluate(variables.variables_initializer(mfn.variables.values()))
self.evaluate(out['predictions'])
for _ in range(2):
out = mfn.train(constant_op.constant(3), constant_op.constant(5))
self.evaluate(out['predictions'])
self.assertEqual(
3, self.evaluate(mfn._variable_holder.variables['global_step']))
mfn.evaluate(constant_op.constant(7), constant_op.constant(9))
mfn.predict(constant_op.constant(10))
save_dir = os.path.join(self.get_temp_dir(), 'model_function')
save.save(mfn, save_dir)
obj = load.load(save_dir)
variables_by_name = obj._variables_by_name
self.evaluate(
variables.variables_initializer(
variables_by_name._unconditional_dependency_names.values()))
self.assertEqual(3, self.evaluate(variables_by_name.global_step))
out = obj._functions['train'](constant_op.constant(3),
constant_op.constant(5))
self.assertEqual(15, self.evaluate(out['predictions']))
self.assertEqual(4, self.evaluate(variables_by_name.global_step))
out = obj._functions['eval'](constant_op.constant(7),
constant_op.constant(9))
self.assertEqual(16, self.evaluate(out['predictions']))
out = obj._functions['infer'](constant_op.constant(10))
self.assertEqual(11, self.evaluate(out['predictions']))
def test_value_is_idempotent(self):
self.setup()
auc_obj = metrics.AUC(num_thresholds=3)
self.evaluate(variables.variables_initializer(auc_obj.variables))
# Run several updates.
update_op = auc_obj.update_state(self.y_true, self.y_pred)
for _ in range(10):
self.evaluate(update_op)
# Then verify idempotency.
initial_auc = self.evaluate(auc_obj.result())
for _ in range(10):
self.assertAllClose(initial_auc, self.evaluate(auc_obj.result()), 1e-3)
def testOneWriteOneOutput(self):
# Regression test for a bug where computations with one non-constant
# output and one variable update were mishandled.
for dtype in self.numeric_types:
init = np.array([[1, 2j], [3, 4]]).astype(dtype)
with self.session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(init)
sess.run(variables.variables_initializer([v]))
p = array_ops.placeholder(dtype)
x = v.assign_add(p)
with ops.control_dependencies([x]):
y = v.read_value()
self.assertAllClose(
np.array([[2, 1 + 2j], [4, 5]]).astype(dtype), sess.run(y, {p: 1}))
def testSparseRead2DIndices3DTensor(self):
for dtype in self.numeric_types:
init = np.array([[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]],
[[20, 21, 22], [23, 24j, 25]],
[[30, 31, 32], [33, 34, 35]]]).astype(dtype)
with self.session() as sess, self.test_scope():
v = resource_variable_ops.ResourceVariable(init)
sess.run(variables.variables_initializer([v]))
x = v.sparse_read([[2, 1], [3, 0]])
self.assertAllClose(
np.array(
[[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]],
[[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]]
],).astype(dtype), self.evaluate(x))
def test_loading_without_providing_class_fails(self):
input_data = keras.Input(shape=(1,))
layer = AddingPreprocessingLayer()
output = layer(input_data)
model = keras.Model(input_data, output)
if not context.executing_eagerly():
self.evaluate(variables.variables_initializer(model.variables))
output_path = os.path.join(self.get_temp_dir(), "tf_keras_saved_model")
model.save(output_path, save_format="tf")
with self.assertRaisesRegex(RuntimeError, "Unable to restore a layer of"):
_ = keras.models.load_model(output_path)
def test_multiple_updates(self):
r_obj = F1Binary()
y_true = tf.constant([[0, 1], [1, 0]], shape=(2, 2))
y_pred = tf.constant([[[0.9], [0.1]], [[0.9], [0.1]]],
shape=(2, 2, 1),
dtype=tf.float32)
weights = tf.constant([[1, 4], [3, 2]], shape=(2, 2), dtype=tf.float32)
self.evaluate(variables.variables_initializer(r_obj.variables))
update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights)
for _ in range(2):
self.evaluate(update_op)
self.assertAlmostEqual(0.5454545, self.evaluate(r_obj.result()))
def test_weighted(self):
self.setup()
poisson_obj = metrics.Poisson()
self.evaluate(variables.variables_initializer(poisson_obj.variables))
sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1))
result = poisson_obj(self.y_true,
self.y_pred,
sample_weight=sample_weight)
sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape(
(2, 3))
expected_result = np.multiply(self.expected_results, sample_weight)
expected_result = np.sum(expected_result) / np.sum(sample_weight)
self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
def test_unweighted(self):
np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32)
np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32)
expected_error = np.mean(
np.divide(np.absolute(np_y_pred - np_y_true), np_y_true))
y_pred = constant_op.constant(np_y_pred, shape=(1, 4), dtype=dtypes.float32)
y_true = constant_op.constant(np_y_true, shape=(1, 4))
mre_obj = metrics.MeanRelativeError(normalizer=y_true)
self.evaluate(variables.variables_initializer(mre_obj.variables))
result = mre_obj(y_true, y_pred)
self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)
