def test_local_variable(self): with self.cached_session() as sess: self.assertEquals([], variables_lib.local_variables()) value0 = 42 variables_lib2.local_variable(value0) value1 = 43 variables_lib2.local_variable(value1) variables = variables_lib.local_variables() self.assertEquals(2, len(variables)) self.assertRaises(errors_impl.OpError, sess.run, variables) variables_lib.variables_initializer(variables).run() self.assertAllEqual(set([value0, value1]), set(sess.run(variables)))
def test_div_by_zero(self): r_obj = metrics.Recall() y_pred = constant_op.constant([0, 0, 0, 0]) y_true = constant_op.constant([0, 0, 0, 0]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertEqual(0, self.evaluate(result))
def test_unweighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([1, 0, 1, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.5, self.evaluate(result))
def test_save_restore(self): checkpoint_directory = self.get_temp_dir() checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt') m = metrics.Mean() checkpoint = checkpointable_utils.Checkpoint(mean=m) self.evaluate(variables.variables_initializer(m.variables)) # update state self.evaluate(m(100.)) self.evaluate(m(200.)) # save checkpoint and then add an update save_path = checkpoint.save(checkpoint_prefix) self.evaluate(m(1000.)) # restore to the same checkpoint mean object checkpoint.restore(save_path).assert_consumed().run_restore_ops() self.evaluate(m(300.)) self.assertEqual(200., self.evaluate(m.result())) # restore to a different checkpoint mean object restore_mean = metrics.Mean() restore_checkpoint = checkpointable_utils.Checkpoint(mean=restore_mean) status = restore_checkpoint.restore(save_path) restore_update = restore_mean(300.) status.assert_consumed().run_restore_ops() self.evaluate(restore_update) self.assertEqual(200., self.evaluate(restore_mean.result())) self.assertEqual(3, self.evaluate(restore_mean.count))
def test_placeholder_with_default_fed(self): with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(4.0) ph = array_ops.placeholder_with_default(v, shape=[]) out = ph * 2 sess.run(variables.variables_initializer([v])) self.assertEqual(2.0, sess.run(out, {ph: 1.0}))
def test_example(self): with self.test_session() as session: for tower_id in range(3): self.create_tower_metrics(tower_id) session.run( variables.variables_initializer( ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES))) session.run( replicate_model_fn._reduce_metric_variables(number_of_towers=3)) # 1st tower = 1.3, 2.3, [3.3, 3.5, 3.7] # 2nd tower = 2.6, 4.6, [6.6, 7.0, 7.4] # 3rd tower = 3.9, 6.9, [9.9, 10.5, 11.1] # Reduced = 7.8, 13.8, [19.8, 21.0, 22.2] # Towers are accumulated in the first tower. local_metrics = session.run( ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES)) self.assertNear(7.8, local_metrics[0], 0.01) self.assertNear(13.8, local_metrics[1], 0.01) self.assertAllClose([19.8, 21., 22.1], local_metrics[2], 0.01) self.assertNear(0.0, local_metrics[3], 0.01) self.assertNear(0.0, local_metrics[4], 0.01) self.assertAllClose([0.0, 0.0, 0.0], local_metrics[5], 0.01) self.assertNear(0.0, local_metrics[6], 0.01) self.assertNear(0.0, local_metrics[7], 0.01) self.assertAllClose([0.0, 0.0, 0.0], local_metrics[8], 0.01)
def testConvertVariablesToConstsWithFunctions(self): @function.Defun(dtypes.float32) def plus_one(x): return x + 1.0 with ops.Graph().as_default(): variable_node = variables.Variable(1.0, name="variable_node") _ = variables.Variable(1.0, name="unused_variable_node") defun_node = plus_one(variable_node) output_node = math_ops_lib.multiply( defun_node, 2.0, name="output_node") with session.Session() as sess: init = variables.variables_initializer([variable_node]) sess.run(init) output = sess.run(output_node) self.assertNear(4.0, output, 0.00001) variable_graph_def = sess.graph.as_graph_def() # First get the constant_graph_def when variable_names_whitelist is set, # note that if variable_names_whitelist is not set an error will be # thrown because unused_variable_node is not initialized. constant_graph_def = graph_util.convert_variables_to_constants( sess, variable_graph_def, ["output_node"], variable_names_whitelist=set(["variable_node"])) self.assertEqual(variable_graph_def.library, constant_graph_def.library)
def test_reduce_is_idempotent(self): with self.test_session() as session: for tower_id in range(3): self.create_tower_metrics(tower_id) session.run( variables.variables_initializer( ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES))) for _ in range(20): session.run( replicate_model_fn._reduce_metric_variables(number_of_towers=3)) local_metrics = session.run( ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES)) self.assertNear(7.8, local_metrics[0], 0.01) self.assertNear(13.8, local_metrics[1], 0.01) self.assertAllClose([19.8, 21., 22.1], local_metrics[2], 0.01) self.assertNear(0.0, local_metrics[3], 0.01) self.assertNear(0.0, local_metrics[4], 0.01) self.assertAllClose([0.0, 0.0, 0.0], local_metrics[5], 0.01) self.assertNear(0.0, local_metrics[6], 0.01) self.assertNear(0.0, local_metrics[7], 0.01) self.assertAllClose([0.0, 0.0, 0.0], local_metrics[8], 0.01)
def test_unweighted_top_k(self): p_obj = metrics.Precision(top_k=3) y_pred = constant_op.constant([0.2, 0.1, 0.5, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([0, 1, 1, 0, 0], shape=(1, 5)) self.evaluate(variables.variables_initializer(p_obj.variables)) result = p_obj(y_true, y_pred) self.assertAlmostEqual(1. / 3, self.evaluate(result))
def test_unweighted_with_threshold(self): r_obj = metrics.Recall(thresholds=[0.5, 0.7]) y_pred = constant_op.constant([1, 0, 0.6, 0], shape=(1, 4)) y_true = constant_op.constant([0, 1, 1, 0], shape=(1, 4)) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([0.5, 0.], self.evaluate(result), 0)
def test_binary_accuracy(self): acc_obj = metrics.BinaryAccuracy(name='my acc') # check config self.assertEqual(acc_obj.name, 'my acc') self.assertTrue(acc_obj.stateful) self.assertEqual(len(acc_obj.variables), 2) self.assertEqual(acc_obj.dtype, dtypes.float32) self.evaluate(variables.variables_initializer(acc_obj.variables)) # verify that correct value is returned update_op = acc_obj.update_state([[1], [0]], [[1], [0]]) self.evaluate(update_op) result = self.evaluate(acc_obj.result()) self.assertEqual(result, 1) # 2/2 # check y_pred squeeze update_op = acc_obj.update_state([[1], [1]], [[[1]], [[0]]]) self.evaluate(update_op) result = self.evaluate(acc_obj.result()) self.assertAlmostEqual(result, 0.75, 2) # 3/4 # check y_true squeeze result_t = acc_obj([[[1]], [[1]]], [[1], [0]]) result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.67, 2) # 4/6 # check with sample_weight result_t = acc_obj([[1], [1]], [[1], [0]], [[0.5], [0.2]]) result = self.evaluate(result_t) self.assertAlmostEqual(result, 0.67, 2) # 4.5/6.7
def test_mean(self): m = metrics.Mean(name='my_mean') # check config self.assertEqual(m.name, 'my_mean') self.assertTrue(m.stateful) self.assertEqual(m.dtype, dtypes.float32) self.assertEqual(len(m.variables), 2) self.evaluate(variables.variables_initializer(m.variables)) # check initial state self.assertEqual(self.evaluate(m.total), 0) self.assertEqual(self.evaluate(m.count), 0) # check __call__() self.assertEqual(self.evaluate(m(100)), 100) self.assertEqual(self.evaluate(m.total), 100) self.assertEqual(self.evaluate(m.count), 1) # check update_state() and result() + state accumulation + tensor input update_op = m.update_state(ops.convert_n_to_tensor([1, 5])) self.evaluate(update_op) self.assertAlmostEqual(self.evaluate(m.result()), 106 / 3, 2) self.assertEqual(self.evaluate(m.total), 106) # 100 + 1 + 5 self.assertEqual(self.evaluate(m.count), 3) # check reset_states() m.reset_states() self.assertEqual(self.evaluate(m.total), 0) self.assertEqual(self.evaluate(m.count), 0)
def test_placeholder_with_default_default(self): with self.cached_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(4.0) ph = array_ops.placeholder_with_default(v, shape=[]) out = ph * 2 sess.run(variables.variables_initializer([v])) self.assertEqual(8.0, self.evaluate(out))
def create_checkpoint_from_values(self, var_names_to_values, checkpoint_dir, global_step=None): """Creates a checkpoint from a mapping of name to values in model_dir. Args: var_names_to_values: a map from variable names to values. checkpoint_dir: the directory where the checkpoint will be saved. global_step: the global step used to save the checkpoint. Returns: the model_path to the checkpoint. """ var_list = [] with session.Session('', graph=ops.Graph()) as sess: # Create a set of variables to save in the checkpoint. for var_name in var_names_to_values: var_value = var_names_to_values[var_name] var_list.append(variables_lib.Variable(var_value, name=var_name)) saver = saver_lib.Saver(var_list) init_op = variables_lib.variables_initializer(var_list) sess.run(init_op) # Save the initialized values in the file at 'checkpoint_dir' return saver.save(sess, checkpoint_dir, global_step=global_step)
def initialize_op(self): """Returns an op for initializing tensorflow variables.""" all_vars = self._row_factors + self._col_factors all_vars.extend([self._row_gramian, self._col_gramian]) if self._row_weights is not None: assert self._col_weights is not None all_vars.extend(self._row_weights + self._col_weights) return variables.variables_initializer(all_vars)
def test_unweighted_all_correct(self): s_obj = metrics.SensitivityAtSpecificity(0.7) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs, dtype=dtypes.float32) y_true = constant_op.constant(inputs) self.evaluate(variables.variables_initializer(s_obj.variables)) result = s_obj(y_true, y_pred) self.assertAlmostEqual(1, self.evaluate(result))
def testSparseRead0DIndices(self): for dtype in self.numeric_types: init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read(2) self.assertAllClose(np.array([8, 9, 10, 11], dtype=dtype), sess.run(x))
def test_unweighted_all_incorrect(self): r_obj = metrics.Recall(thresholds=[0.5]) inputs = np.random.randint(0, 2, size=(100, 1)) y_pred = constant_op.constant(inputs) y_true = constant_op.constant(1 - inputs) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0, self.evaluate(result))
def test_extreme_thresholds(self): r_obj = metrics.Recall(thresholds=[-1.0, 2.0]) # beyond values range y_pred = math_ops.cast( constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32) y_true = math_ops.cast( constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj(y_true, y_pred) self.assertArrayNear([1.0, 0.], self.evaluate(result), 0)
def test_layer_output(self, attention_cls): attention = attention_cls(self.units, self.memory) score = attention([self.query, self.state]) self.evaluate(variables.variables_initializer(attention.variables)) score_val = self.evaluate(score) self.assertLen(score_val, 2) self.assertEqual(score_val[0].shape, (self.batch, self.timestep)) self.assertEqual(score_val[1].shape, (self.batch, self.timestep))
def test_weighted(self): tp_obj = metrics.TruePositives() self.evaluate(variables.variables_initializer(tp_obj.variables)) y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) result = tp_obj(y_true, y_pred, sample_weight=sample_weight) self.assertAllClose([12.], self.evaluate(result))
def test_unweighted_top_k_and_threshold(self): r_obj = metrics.Recall(thresholds=.7, top_k=2) self.evaluate(variables.variables_initializer(r_obj.variables)) y_pred = constant_op.constant([0.2, 0.8, 0.6, 0, 0.2], shape=(1, 5)) y_true = constant_op.constant([1, 1, 1, 0, 1], shape=(1, 5)) result = r_obj(y_true, y_pred) self.assertAlmostEqual(0.25, self.evaluate(result)) self.assertAlmostEqual(1, self.evaluate(r_obj.true_positives)) self.assertAlmostEqual(3, self.evaluate(r_obj.false_negatives))
def test_weighted(self): cosine_obj = metrics.CosineProximity() self.evaluate(variables.variables_initializer(cosine_obj.variables)) y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) sample_weight = constant_op.constant((1., 1.5, 2., 2.5)) result = cosine_obj(y_true, y_pred, sample_weight=sample_weight) self.assertAllClose(-0.59916, self.evaluate(result), atol=1e-5)
def test_unweighted_low_specificity(self): s_obj = metrics.SensitivityAtSpecificity(0.4) pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) y_true = constant_op.constant(label_values) self.evaluate(variables.variables_initializer(s_obj.variables)) result = s_obj(y_true, y_pred) self.assertAlmostEqual(0.6, self.evaluate(result))
def test_weighted_with_thresholds(self): tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(tp_obj.variables)) y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) result = tp_obj(y_true, y_pred, sample_weight=37.) self.assertAllClose([222., 111., 37.], self.evaluate(result))
def _testAddUpdate(self, scope): with scope: layer_with_update = LayerWithUpdate(dtype=dtypes.int32) model = testing_utils.get_model_from_layers([layer_with_update], input_shape=(3,), input_dtype=dtypes.int32) if testing_utils.get_model_type() == 'subclass': model._set_inputs(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32)) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') loaded = keras_load.load(saved_model_dir) loaded_layer = loaded.layers[-1] self.evaluate(variables.variables_initializer(loaded.variables)) self.assertEqual(self.evaluate(loaded_layer.v), 0) loaded.predict(constant_op.constant([[1, 2, 3]], dtype=dtypes.int32), steps=1) self.assertEqual(self.evaluate(loaded_layer.v), 6)
def _testAddUpdate(self, scope): with scope: layer_with_update = LayerWithUpdate() model = testing_utils.get_model_from_layers([layer_with_update], input_shape=(3,)) x = np.ones((10, 3)) if testing_utils.get_model_type() == 'subclass': model.predict(x, batch_size=10) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') loaded = keras_load.load(saved_model_dir) loaded_layer = loaded.layers[-1] self.evaluate(variables.variables_initializer(loaded.variables)) self.assertEqual(self.evaluate(loaded_layer.v), 0.) loaded.compile('sgd', 'mse') loaded.fit(x, x, batch_size=10) self.assertEqual(self.evaluate(loaded_layer.v), 1.)
def testSparseRead2DIndices(self): for dtype in self.numeric_types: init = np.array([[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], dtype=dtype) with self.test_session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [0, 2]]) self.assertAllClose( np.array( [[[8, 9, 10, 11], [4, 5, 6, 7]], [[0, 1, 2, 3], [8, 9, 10, 11]]], dtype=dtype), sess.run(x))
def test_weighted_with_thresholds(self): tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(tp_obj.variables)) y_pred = constant_op.constant( ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant( ((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) result = tp_obj(y_true, y_pred, sample_weight=37.) self.assertAllClose([222., 111., 37.], self.evaluate(result))
def test_weighted(self, label_dtype): s_obj = metrics.SensitivityAtSpecificity(0.4) pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) y_true = math_ops.cast(label_values, dtype=label_dtype) weights = constant_op.constant(weight_values) self.evaluate(variables.variables_initializer(s_obj.variables)) result = s_obj(y_true, y_pred, sample_weight=weights) self.assertAlmostEqual(0.675, self.evaluate(result))
def testBatchNormUpdates(self): model = keras.models.Sequential( keras.layers.BatchNormalization(input_shape=(1,))) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() model.save(saved_model_dir, save_format='tf') loaded = keras_load.load(saved_model_dir) self.evaluate(variables.variables_initializer(loaded.variables)) input_arr = array_ops.constant([[11], [12], [13]], dtype=dtypes.float32) input_arr2 = array_ops.constant([[14], [15], [16]], dtype=dtypes.float32) self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0]) self.evaluate(loaded(input_arr, training=True)) if not context.executing_eagerly(): self.evaluate(loaded.get_updates_for(input_arr)) self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0.12]) self.evaluate(loaded(input_arr2, training=False)) if not context.executing_eagerly(): self.evaluate(loaded.get_updates_for(input_arr2)) self.assertAllClose(self.evaluate(loaded.layers[-1].moving_mean), [0.12])
def test_extreme_thresholds(self): p_obj = metrics.Precision(thresholds=[-1.0, 2.0]) # beyond values range y_pred = math_ops.cast(constant_op.constant([1, 0, 1, 0], shape=(1, 4)), dtype=dtypes.float32) y_true = math_ops.cast(constant_op.constant([0, 1, 1, 1], shape=(1, 4)), dtype=dtypes.float32) self.evaluate(variables.variables_initializer(p_obj.variables)) result = p_obj(y_true, y_pred) self.assertArrayNear([0.75, 0.], self.evaluate(result), 0)
def test_weighted_with_thresholds(self): tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(tn_obj.variables)) y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) sample_weight = ((0.0, 2.0, 3.0, 5.0),) result = tn_obj(y_true, y_pred, sample_weight=sample_weight) self.assertAllClose([5., 15., 23.], self.evaluate(result))
def test_weighted(self, label_dtype): s_obj = metrics.SensitivityAtSpecificity(0.4) pred_values = [0.0, 0.1, 0.2, 0.3, 0.4, 0.01, 0.02, 0.25, 0.26, 0.26] label_values = [0, 0, 0, 0, 0, 1, 1, 1, 1, 1] weight_values = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] y_pred = constant_op.constant(pred_values, dtype=dtypes.float32) y_true = math_ops.cast(label_values, dtype=label_dtype) weights = constant_op.constant(weight_values) self.evaluate(variables.variables_initializer(s_obj.variables)) result = s_obj(y_true, y_pred, sample_weight=weights) self.assertAlmostEqual(0.675, self.evaluate(result))
def test_unweighted(self): self.setup() auc_obj = metrics.AUC(num_thresholds=self.num_thresholds) self.evaluate(variables.variables_initializer(auc_obj.variables)) result = auc_obj(self.y_true, self.y_pred) # tp = [2, 1, 0], fp = [2, 0, 0], fn = [0, 1, 2], tn = [0, 2, 2] # recall = [2/2, 1/(1+1), 0] = [1, 0.5, 0] # fp_rate = [2/2, 0, 0] = [1, 0, 0] # heights = [(1 + 0.5)/2, (0.5 + 0)/2] = [0.75, 0.25] # widths = [(1 - 0), (0 - 0)] = [1, 0] expected_result = (0.75 * 1 + 0.25 * 0) self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
def test_weighted(self): r_obj = metrics.Recall() y_pred = constant_op.constant([[1, 0, 1, 0], [0, 1, 0, 1]]) y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) self.evaluate(variables.variables_initializer(r_obj.variables)) result = r_obj( y_true, y_pred, sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) weighted_tp = 3.0 + 1.0 weighted_t = (2.0 + 3.0) + (4.0 + 1.0) expected_recall = weighted_tp / weighted_t self.assertAlmostEqual(expected_recall, self.evaluate(result))
def testSaveWithRaggedInputs(self): class EmbeddingMerger(keras.layers.Layer): def __init__(self, list_features, **kwargs): super().__init__(**kwargs) self._supports_ragged_inputs = True self.embeddings = { feature: keras.layers.Embedding(10, 3) for feature in list_features } self.mean = keras.layers.Lambda(math_ops.reduce_mean, arguments=dict(axis=1)) def call(self, inputs): tensors = [self.embeddings[col](inputs[col]) for col in inputs] tensors = [self.mean(inp) for inp in tensors] return keras.layers.Add()(tensors) list_features = ['feature_1', 'feature_2'] feature_1 = ragged_factory_ops.constant([[0.], [1, 3]]) feature_2 = ragged_factory_ops.constant([[1., 2], [4]]) f = {'feature_1': feature_1, 'feature_2': feature_2} f_inputs = { 'feature_1': keras.Input(shape=(None, ), name='feature_1', ragged=True), 'feature_2': keras.Input(shape=(None, ), name='feature_2', ragged=True) } out = EmbeddingMerger(list_features)(f_inputs) model = keras.Model(f_inputs, out) self.evaluate(variables.variables_initializer(model.variables)) saved_model_dir = self._save_model_dir() tf_save.save(model, saved_model_dir) loaded = keras_load.load(saved_model_dir) self.evaluate(variables.variables_initializer(loaded.variables)) self.assertAllClose(model.predict(f), loaded.predict(f))
def test_weighted_with_thresholds(self): tn_obj = metrics.TrueNegatives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(tn_obj.variables)) y_pred = constant_op.constant( ((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant( ((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) sample_weight = ((0.0, 2.0, 3.0, 5.0), ) result = tn_obj(y_true, y_pred, sample_weight=sample_weight) self.assertAllClose([5., 15., 23.], self.evaluate(result))
def test_correctness(self): a_obj = metrics.SparseTopKCategoricalAccuracy() self.evaluate(variables.variables_initializer(a_obj.variables)) y_true = constant_op.constant([2, 1]) y_pred = constant_op.constant([[0.1, 0.9, 0.8], [0.05, 0.95, 0]]) result = a_obj(y_true, y_pred) self.assertEqual(1, self.evaluate(result)) # both the samples match # With `k` < 5. a_obj = metrics.SparseTopKCategoricalAccuracy(k=1) self.evaluate(variables.variables_initializer(a_obj.variables)) result = a_obj(y_true, y_pred) self.assertEqual(0.5, self.evaluate(result)) # only sample #2 matches # With `k` > 5. y_pred = constant_op.constant([[0.5, 0.9, 0.1, 0.7, 0.6, 0.5, 0.4], [0.05, 0.95, 0, 0, 0, 0, 0]]) a_obj = metrics.SparseTopKCategoricalAccuracy(k=6) self.evaluate(variables.variables_initializer(a_obj.variables)) result = a_obj(y_true, y_pred) self.assertEqual(0.5, self.evaluate(result)) # only 1 sample matches.
def test_doesnt_accept_uneven_number_of_variables(self): with self.test_session() as session: for tower_id in range(3): self.create_tower_metrics(tower_id) self.create_metric_variable(-1.0, 'oddball') session.run( variables.variables_initializer( ops_lib.get_collection(ops_lib.GraphKeys.METRIC_VARIABLES))) with self.assertRaisesRegexp(ValueError, ''): session.run( replicate_model_fn._reduce_metric_variables(number_of_towers=3))
def test_weighted_roc_interpolation(self): self.setup() auc_obj = metrics.AUC(num_thresholds=self.num_thresholds) self.evaluate(variables.variables_initializer(auc_obj.variables)) result = auc_obj(self.y_true, self.y_pred, sample_weight=self.sample_weight) # tp = [7, 4, 0], fp = [3, 0, 0], fn = [0, 3, 7], tn = [0, 3, 3] # recall = [7/7, 4/(4+3), 0] = [1, 0.571, 0] # fp_rate = [3/3, 0, 0] = [1, 0, 0] # heights = [(1 + 0.571)/2, (0.571 + 0)/2] = [0.7855, 0.2855] # widths = [(1 - 0), (0 - 0)] = [1, 0] expected_result = (0.7855 * 1 + 0.2855 * 0) self.assertAllClose(self.evaluate(result), expected_result, 1e-3)
def test_unweighted(self): fn_obj = metrics.FalseNegatives() self.evaluate(variables.variables_initializer(fn_obj.variables)) y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) update_op = fn_obj.update_state(y_true, y_pred) self.evaluate(update_op) result = fn_obj.result() self.assertAllClose(3., result)
def test_weighted(self): p_obj = metrics.Precision() y_pred = constant_op.constant([[1, 0, 1, 0], [1, 0, 1, 0]]) y_true = constant_op.constant([[0, 1, 1, 0], [1, 0, 0, 1]]) self.evaluate(variables.variables_initializer(p_obj.variables)) result = p_obj( y_true, y_pred, sample_weight=constant_op.constant([[1, 2, 3, 4], [4, 3, 2, 1]])) weighted_tp = 3.0 + 4.0 weighted_positives = (1.0 + 3.0) + (4.0 + 2.0) expected_precision = weighted_tp / weighted_positives self.assertAlmostEqual(expected_precision, self.evaluate(result))
def test_unweighted(self): cosine_obj = metrics.CosineProximity() self.evaluate(variables.variables_initializer(cosine_obj.variables)) y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) update_op = cosine_obj.update_state(y_true, y_pred) self.evaluate(update_op) result = cosine_obj.result() self.assertAllClose(-0.60723, result, atol=1e-5)
def test_weighted(self): self.setup() k_obj = metrics.KullbackLeiblerDivergence() self.evaluate(variables.variables_initializer(k_obj.variables)) sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) result = k_obj(self.y_true, self.y_pred, sample_weight=sample_weight) sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape( (2, 3)) expected_result = np.multiply(self.expected_results, sample_weight) expected_result = np.sum(expected_result) / (1.2 + 3.4) self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
def test_unweighted_with_thresholds(self): tp_obj = metrics.TruePositives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(tp_obj.variables)) y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) update_op = tp_obj.update_state(y_true, y_pred) self.evaluate(update_op) result = tp_obj.result() self.assertAllClose([6., 3., 1.], result)
def test_unweighted(self): tp_obj = metrics.TruePositives() self.evaluate(variables.variables_initializer(tp_obj.variables)) y_true = constant_op.constant(((0, 1, 0, 1, 0), (0, 0, 1, 1, 1), (1, 1, 1, 1, 0), (0, 0, 0, 0, 1))) y_pred = constant_op.constant(((0, 0, 1, 1, 0), (1, 1, 1, 1, 1), (0, 1, 0, 1, 0), (1, 1, 1, 1, 1))) update_op = tp_obj.update_state(y_true, y_pred) self.evaluate(update_op) result = tp_obj.result() self.assertAllClose([7.], result)
def test_weighted_with_thresholds(self): fp_obj = metrics.FalsePositives(thresholds=[0.15, 0.5, 0.85]) self.evaluate(variables.variables_initializer(fp_obj.variables)) y_pred = constant_op.constant(((0.9, 0.2, 0.8, 0.1), (0.2, 0.9, 0.7, 0.6), (0.1, 0.2, 0.4, 0.3), (0, 1, 0.7, 0.3))) y_true = constant_op.constant(((0, 1, 1, 0), (1, 0, 0, 0), (0, 0, 0, 0), (1, 1, 1, 1))) sample_weight = ((1.0, 2.0, 3.0, 5.0), (7.0, 11.0, 13.0, 17.0), (19.0, 23.0, 29.0, 31.0), (5.0, 15.0, 10.0, 0)) result = fp_obj(y_true, y_pred, sample_weight=sample_weight) self.assertAllClose([125., 42., 12.], self.evaluate(result))
def testStatelessParameterizedTruncatedNormalHasGrads(self): mean = variables.Variable(0.01) stddev = variables.Variable(1.) minval = variables.Variable(-1.) maxval = variables.Variable(1.) with self.cached_session() as sess: with backprop.GradientTape(persistent=True) as tape: samples = stateless.stateless_parameterized_truncated_normal( [1], [1, 2], mean, stddev, minval, maxval) sess.run( variables.variables_initializer([mean, stddev, minval, maxval])) [mean_grad, std_grad], mean_actual_grad, std_actual_grad = sess.run([ tape.gradient(samples, [mean, stddev]), array_ops.ones_like(mean), (samples - mean) / stddev ]) self.assertAllClose(mean_grad, mean_actual_grad) self.assertAllClose(std_grad, std_actual_grad[0]) try: import scipy.stats # pylint:disable=g-import-not-at-top truncnorm = scipy.stats.truncnorm(a=-1., b=1., loc=0., scale=1.) samples_np, [minval_grad, maxval_grad] = sess.run( [samples, tape.gradient(samples, [minval, maxval])]) sample_cdf = truncnorm.cdf(samples_np) # These come from the implicit reparameterization trick. scipy_maxval_grad = np.exp(0.5 * (samples_np**2 - ((1. - 0.01) / 1.)**2) + np.log(sample_cdf)) scipy_minval_grad = np.exp(0.5 * (samples_np**2 - ((-1. - 0.01) / 1.)**2) + np.log1p(-sample_cdf)) self.assertAllClose(minval_grad, scipy_minval_grad[0], rtol=1e-2) self.assertAllClose(maxval_grad, scipy_maxval_grad[0], rtol=1e-2) except ImportError as e: tf_logging.warn("Cannot test truncated normal op: %s" % str(e))
def test_save_and_load(self): mfn = function.ModelFunction.from_function(_model_fn) out = mfn.train(constant_op.constant(3), constant_op.constant(5)) self.evaluate(variables.variables_initializer(mfn.variables.values())) self.evaluate(out['predictions']) for _ in range(2): out = mfn.train(constant_op.constant(3), constant_op.constant(5)) self.evaluate(out['predictions']) self.assertEqual( 3, self.evaluate(mfn._variable_holder.variables['global_step'])) mfn.evaluate(constant_op.constant(7), constant_op.constant(9)) mfn.predict(constant_op.constant(10)) save_dir = os.path.join(self.get_temp_dir(), 'model_function') save.save(mfn, save_dir) obj = load.load(save_dir) variables_by_name = obj._variables_by_name self.evaluate( variables.variables_initializer( variables_by_name._unconditional_dependency_names.values())) self.assertEqual(3, self.evaluate(variables_by_name.global_step)) out = obj._functions['train'](constant_op.constant(3), constant_op.constant(5)) self.assertEqual(15, self.evaluate(out['predictions'])) self.assertEqual(4, self.evaluate(variables_by_name.global_step)) out = obj._functions['eval'](constant_op.constant(7), constant_op.constant(9)) self.assertEqual(16, self.evaluate(out['predictions'])) out = obj._functions['infer'](constant_op.constant(10)) self.assertEqual(11, self.evaluate(out['predictions']))
def test_value_is_idempotent(self): self.setup() auc_obj = metrics.AUC(num_thresholds=3) self.evaluate(variables.variables_initializer(auc_obj.variables)) # Run several updates. update_op = auc_obj.update_state(self.y_true, self.y_pred) for _ in range(10): self.evaluate(update_op) # Then verify idempotency. initial_auc = self.evaluate(auc_obj.result()) for _ in range(10): self.assertAllClose(initial_auc, self.evaluate(auc_obj.result()), 1e-3)
def testOneWriteOneOutput(self): # Regression test for a bug where computations with one non-constant # output and one variable update were mishandled. for dtype in self.numeric_types: init = np.array([[1, 2j], [3, 4]]).astype(dtype) with self.session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) p = array_ops.placeholder(dtype) x = v.assign_add(p) with ops.control_dependencies([x]): y = v.read_value() self.assertAllClose( np.array([[2, 1 + 2j], [4, 5]]).astype(dtype), sess.run(y, {p: 1}))
def testSparseRead2DIndices3DTensor(self): for dtype in self.numeric_types: init = np.array([[[0, 1, 2], [3, 4, 5]], [[10, 11, 12], [13, 14, 15]], [[20, 21, 22], [23, 24j, 25]], [[30, 31, 32], [33, 34, 35]]]).astype(dtype) with self.session() as sess, self.test_scope(): v = resource_variable_ops.ResourceVariable(init) sess.run(variables.variables_initializer([v])) x = v.sparse_read([[2, 1], [3, 0]]) self.assertAllClose( np.array( [[[[20, 21, 22], [23, 24j, 25]], [[10, 11, 12], [13, 14, 15]]], [[[30, 31, 32], [33, 34, 35]], [[0, 1, 2], [3, 4, 5]]] ],).astype(dtype), self.evaluate(x))
def test_loading_without_providing_class_fails(self): input_data = keras.Input(shape=(1,)) layer = AddingPreprocessingLayer() output = layer(input_data) model = keras.Model(input_data, output) if not context.executing_eagerly(): self.evaluate(variables.variables_initializer(model.variables)) output_path = os.path.join(self.get_temp_dir(), "tf_keras_saved_model") model.save(output_path, save_format="tf") with self.assertRaisesRegex(RuntimeError, "Unable to restore a layer of"): _ = keras.models.load_model(output_path)
def test_multiple_updates(self): r_obj = F1Binary() y_true = tf.constant([[0, 1], [1, 0]], shape=(2, 2)) y_pred = tf.constant([[[0.9], [0.1]], [[0.9], [0.1]]], shape=(2, 2, 1), dtype=tf.float32) weights = tf.constant([[1, 4], [3, 2]], shape=(2, 2), dtype=tf.float32) self.evaluate(variables.variables_initializer(r_obj.variables)) update_op = r_obj.update_state(y_true, y_pred, sample_weight=weights) for _ in range(2): self.evaluate(update_op) self.assertAlmostEqual(0.5454545, self.evaluate(r_obj.result()))
def test_weighted(self): self.setup() poisson_obj = metrics.Poisson() self.evaluate(variables.variables_initializer(poisson_obj.variables)) sample_weight = constant_op.constant([1.2, 3.4], shape=(2, 1)) result = poisson_obj(self.y_true, self.y_pred, sample_weight=sample_weight) sample_weight = np.asarray([1.2, 1.2, 1.2, 3.4, 3.4, 3.4]).reshape( (2, 3)) expected_result = np.multiply(self.expected_results, sample_weight) expected_result = np.sum(expected_result) / np.sum(sample_weight) self.assertAllClose(self.evaluate(result), expected_result, atol=1e-3)
def test_unweighted(self): np_y_pred = np.asarray([2, 4, 6, 8], dtype=np.float32) np_y_true = np.asarray([1, 3, 2, 3], dtype=np.float32) expected_error = np.mean( np.divide(np.absolute(np_y_pred - np_y_true), np_y_true)) y_pred = constant_op.constant(np_y_pred, shape=(1, 4), dtype=dtypes.float32) y_true = constant_op.constant(np_y_true, shape=(1, 4)) mre_obj = metrics.MeanRelativeError(normalizer=y_true) self.evaluate(variables.variables_initializer(mre_obj.variables)) result = mre_obj(y_true, y_pred) self.assertAllClose(self.evaluate(result), expected_error, atol=1e-3)