def test_call_sparse(self): layer = ToNumber(out_type=tf.int32, default_value=-1) input = tf.SparseTensor( indices=[[0, 2], [2, 1], [2, 3], [5, 4]], values=tf.constant(["123", "", "456", "-789"], tf.string), dense_shape=[6, 5], ) output = layer.call(input) expected_output = tf.SparseTensor( indices=[[0, 2], [2, 1], [2, 3], [5, 4]], values=tf.constant([123, -1, 456, -789], tf.int32), dense_shape=[6, 5], ) self.assertTrue(sparse_tensor_equal(output, expected_output)) layer = ToNumber(out_type=tf.float32, default_value=0.0) input = tf.SparseTensor( indices=[[0, 2], [2, 1], [2, 3], [5, 4]], values=tf.constant(["123.1", "", "456", "-789.987"], tf.string), dense_shape=[6, 5], ) output = layer.call(input) expected_output = tf.SparseTensor( indices=[[0, 2], [2, 1], [2, 3], [5, 4]], values=tf.constant([123.1, 0.0, 456.0, -789.987], tf.float32), dense_shape=[6, 5], ) self.assertTrue(sparse_tensor_equal(output, expected_output))
def test_concatenate_with_offset(self): tensor_1 = tf.constant([[1], [1], [1]]) tensor_2 = tf.constant([[2], [2], [2]]) offsets = [0, 10] concat_layer = ConcatenateWithOffset(offsets=offsets, axis=1) output = concat_layer([tensor_1, tensor_2]) expected_out = np.array([[1, 12], [1, 12], [1, 12]]) self.assertTrue(np.array_equal(output.numpy(), expected_out)) ragged_tensor_1 = tf.ragged.constant([[1], [], [1]], dtype=tf.int64) ragged_tensor_2 = tf.ragged.constant([[2], [2], []], dtype=tf.int64) output = concat_layer([ragged_tensor_1, ragged_tensor_2]) expected_out = tf.ragged.constant([[1, 12], [12], [1]], dtype=tf.int64) self.assertTrue(ragged_tensor_equal(output, expected_out)) sparse_tensor_1 = ragged_tensor_1.to_sparse() sparse_tensor_2 = ragged_tensor_2.to_sparse() output = concat_layer([sparse_tensor_1, sparse_tensor_2]) expected_out = tf.SparseTensor( indices=np.array([[0, 0], [0, 1], [1, 1], [2, 0]]), values=np.array([1, 12, 12, 1]), dense_shape=(3, 2), ) self.assertTrue(sparse_tensor_equal(output, expected_out))
def test_to_sparse(self): layer = ToSparse() inp = tf.constant([["A", ""], ["B", "C"]], tf.string) output = layer.call(inp) expected_out = tf.SparseTensor( indices=np.array([[0, 0], [1, 0], [1, 1]]), values=np.array(["A", "B", "C"]), dense_shape=(2, 2), ) self.assertTrue(sparse_tensor_equal(output, expected_out)) layer = ToSparse() inp = tf.constant([[12, -1], [45, 78]], tf.int64) output = layer.call(inp) expected_out = tf.SparseTensor( indices=np.array([[0, 0], [1, 0], [1, 1]]), values=np.array([12, 45, 78]), dense_shape=(2, 2), ) self.assertTrue(sparse_tensor_equal(output, expected_out))
def test_model_with_to_sparse(self): inputs = tf.keras.Input(shape=(1, ), dtype=tf.int32) sparse_inputs = ToSparse(ignore_value=-1)(inputs) model = tf.keras.Model(inputs=inputs, outputs=sparse_inputs) out = model.call(tf.constant([[1], [-1], [2], [3]])) expect_out = tf.SparseTensor( indices=tf.constant([[0, 0], [2, 0], [3, 0]], dtype=tf.int64), values=tf.constant([1, 2, 3], dtype=tf.int32), dense_shape=(4, 1), ) self.assertTrue(sparse_tensor_equal(out, expect_out))
def test_hashing(self): hash_layer = Hashing(num_bins=3) inp = np.asarray([["A"], ["B"], ["C"], ["D"], ["E"]]) hash_out = hash_layer(inp) expected_out = np.array([[1], [0], [1], [1], [2]]) self.assertTrue(np.array_equal(hash_out.numpy(), expected_out)) ragged_in = tf.ragged.constant([["A", "B"], ["C", "D"], ["E"], []]) hash_out = hash_layer(ragged_in) expected_ragged_out = tf.ragged.constant([[1, 0], [1, 1], [2], []], dtype=tf.int64) self.assertTrue(ragged_tensor_equal(hash_out, expected_ragged_out)) sparse_in = ragged_in.to_sparse() hash_out = hash_layer(sparse_in) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue(sparse_tensor_equal(hash_out, expected_sparse_out))
def _check_lookup(self, lookup_layer): dense_input = tf.constant([["A"], ["B"], ["C"], ["D"], ["E"]]) output = lookup_layer(dense_input) expected_out = np.array([[0], [1], [2], [3], [3]]) self.assertTrue(np.array_equal(output.numpy(), expected_out)) ragged_input = tf.ragged.constant([["A", "B", "C"], ["D", "E"]]) ragged_output = lookup_layer(ragged_input) expected_ragged_out = tf.ragged.constant([[0, 1, 2], [3, 3]], dtype=tf.int64) self.assertTrue(ragged_tensor_equal(ragged_output, expected_ragged_out)) sparse_input = ragged_input.to_sparse() sparse_output = lookup_layer(sparse_input) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue(sparse_tensor_equal(sparse_output, expected_sparse_out))
def test_discretize(self): discretize_layer = Discretization(bins=[1, 5, 10]) dense_in = tf.constant([[0.2], [1.6], [4.2], [6.1], [10.9]]) dense_out = discretize_layer(dense_in) expected_out = np.array([[0], [1], [1], [2], [3]]) self.assertTrue(np.array_equal(dense_out.numpy(), expected_out)) ragged_input = tf.ragged.constant([[0.2, 1.6, 4.2], [6.1], [10.9]]) ragged_output = discretize_layer(ragged_input) expected_ragged_out = tf.ragged.constant([[0, 1, 1], [2], [3]], dtype=tf.int64) self.assertTrue(ragged_tensor_equal(ragged_output, expected_ragged_out)) sparse_input = ragged_input.to_sparse() sparse_output = discretize_layer(sparse_input) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue(sparse_tensor_equal(sparse_output, expected_sparse_out))
def test_round_indentity(self): round_identity = RoundIdentity(num_buckets=10) dense_input = tf.constant([[1.2], [1.6], [0.2], [3.1], [4.9]]) output = round_identity(dense_input) expected_out = np.array([[1], [2], [0], [3], [5]]) self.assertTrue(np.array_equal(output.numpy(), expected_out)) ragged_input = tf.ragged.constant([[1.1, 3.4], [0.5]]) ragged_output = round_identity(ragged_input) expected_ragged_out = tf.ragged.constant([[1, 3], [0]], dtype=tf.int64) self.assertTrue( ragged_tensor_equal(ragged_output, expected_ragged_out) ) sparse_input = ragged_input.to_sparse() sparse_output = round_identity(sparse_input) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue( sparse_tensor_equal(sparse_output, expected_sparse_out) )
def test_normalizer(self): normalizer = Normalizer(1.0, 2.0) dense_input = tf.constant([[5.0], [7.0], [9.0], [11.0], [13.0]]) output = normalizer(dense_input) expected_out = np.array([[2.0], [3.0], [4.0], [5.0], [6.0]]) self.assertTrue(np.array_equal(output.numpy(), expected_out)) ragged_input = tf.ragged.constant([[5.0, 7.0], [9.0]]) ragged_output = normalizer(ragged_input) expected_ragged_out = tf.ragged.constant([[2.0, 3.0], [4.0]], dtype=tf.float32) self.assertTrue(ragged_tensor_equal(ragged_output, expected_ragged_out)) sparse_input = ragged_input.to_sparse() sparse_output = normalizer(sparse_input) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue(sparse_tensor_equal(sparse_output, expected_sparse_out))
def test_round_indentity(self): log_round = LogRound(num_bins=20, base=2) dense_input = tf.constant([[0.0], [2.6], [0.2], [3.1], [1024]]) output = log_round(dense_input) expected_out = np.array([[0], [1], [0], [2], [10]]) self.assertTrue(np.array_equal(output.numpy(), expected_out)) ragged_input = tf.ragged.constant([[1.1, 3.4], [1025]]) ragged_output = log_round(ragged_input) expected_ragged_out = tf.ragged.constant([[0, 2], [10]], dtype=tf.int64) self.assertTrue(ragged_tensor_equal(ragged_output, expected_ragged_out)) sparse_input = ragged_input.to_sparse() sparse_output = log_round(sparse_input) expected_sparse_out = expected_ragged_out.to_sparse() self.assertTrue(sparse_tensor_equal(sparse_output, expected_sparse_out))