def test_concatenate_with_offset(self):
tensor_1 = tf.constant([[1], [1], [1]])
tensor_2 = tf.constant([[2], [2], [2]])
offsets = [0, 10]
concat_layer = ConcatenateWithOffset(offsets=offsets, axis=1)
output = concat_layer([tensor_1, tensor_2])
expected_out = np.array([[1, 12], [1, 12], [1, 12]])
self.assertTrue(np.array_equal(output.numpy(), expected_out))
ragged_tensor_1 = tf.ragged.constant([[1], [], [1]], dtype=tf.int64)
ragged_tensor_2 = tf.ragged.constant([[2], [2], []], dtype=tf.int64)
output = concat_layer([ragged_tensor_1, ragged_tensor_2])
expected_out = tf.ragged.constant([[1, 12], [12], [1]], dtype=tf.int64)
self.assertTrue(ragged_tensor_equal(output, expected_out))
sparse_tensor_1 = ragged_tensor_1.to_sparse()
sparse_tensor_2 = ragged_tensor_2.to_sparse()
output = concat_layer([sparse_tensor_1, sparse_tensor_2])
expected_out = tf.SparseTensor(
indices=np.array([[0, 0], [0, 1], [1, 1], [2, 0]]),
values=np.array([1, 12, 12, 1]),
dense_shape=(3, 2),
)
self.assertTrue(sparse_tensor_equal(output, expected_out))
def test_hashing(self):
hash_layer = Hashing(num_bins=3)
inp = np.asarray([["A"], ["B"], ["C"], ["D"], ["E"]])
hash_out = hash_layer(inp)
expected_out = np.array([[1], [0], [1], [1], [2]])
self.assertTrue(np.array_equal(hash_out.numpy(), expected_out))
ragged_in = tf.ragged.constant([["A", "B"], ["C", "D"], ["E"], []])
hash_out = hash_layer(ragged_in)
expected_ragged_out = tf.ragged.constant([[1, 0], [1, 1], [2], []],
dtype=tf.int64)
self.assertTrue(ragged_tensor_equal(hash_out, expected_ragged_out))
sparse_in = ragged_in.to_sparse()
hash_out = hash_layer(sparse_in)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(sparse_tensor_equal(hash_out, expected_sparse_out))
def _check_lookup(self, lookup_layer):
dense_input = tf.constant([["A"], ["B"], ["C"], ["D"], ["E"]])
output = lookup_layer(dense_input)
expected_out = np.array([[0], [1], [2], [3], [3]])
self.assertTrue(np.array_equal(output.numpy(), expected_out))
ragged_input = tf.ragged.constant([["A", "B", "C"], ["D", "E"]])
ragged_output = lookup_layer(ragged_input)
expected_ragged_out = tf.ragged.constant([[0, 1, 2], [3, 3]],
dtype=tf.int64)
self.assertTrue(ragged_tensor_equal(ragged_output,
expected_ragged_out))
sparse_input = ragged_input.to_sparse()
sparse_output = lookup_layer(sparse_input)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(sparse_tensor_equal(sparse_output,
expected_sparse_out))
def test_discretize(self):
discretize_layer = Discretization(bins=[1, 5, 10])
dense_in = tf.constant([[0.2], [1.6], [4.2], [6.1], [10.9]])
dense_out = discretize_layer(dense_in)
expected_out = np.array([[0], [1], [1], [2], [3]])
self.assertTrue(np.array_equal(dense_out.numpy(), expected_out))
ragged_input = tf.ragged.constant([[0.2, 1.6, 4.2], [6.1], [10.9]])
ragged_output = discretize_layer(ragged_input)
expected_ragged_out = tf.ragged.constant([[0, 1, 1], [2], [3]],
dtype=tf.int64)
self.assertTrue(ragged_tensor_equal(ragged_output,
expected_ragged_out))
sparse_input = ragged_input.to_sparse()
sparse_output = discretize_layer(sparse_input)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(sparse_tensor_equal(sparse_output,
expected_sparse_out))
def test_round_indentity(self):
round_identity = RoundIdentity(num_buckets=10)
dense_input = tf.constant([[1.2], [1.6], [0.2], [3.1], [4.9]])
output = round_identity(dense_input)
expected_out = np.array([[1], [2], [0], [3], [5]])
self.assertTrue(np.array_equal(output.numpy(), expected_out))
ragged_input = tf.ragged.constant([[1.1, 3.4], [0.5]])
ragged_output = round_identity(ragged_input)
expected_ragged_out = tf.ragged.constant([[1, 3], [0]], dtype=tf.int64)
self.assertTrue(
ragged_tensor_equal(ragged_output, expected_ragged_out)
)
sparse_input = ragged_input.to_sparse()
sparse_output = round_identity(sparse_input)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(
sparse_tensor_equal(sparse_output, expected_sparse_out)
)
def test_normalizer(self):
normalizer = Normalizer(1.0, 2.0)
dense_input = tf.constant([[5.0], [7.0], [9.0], [11.0], [13.0]])
output = normalizer(dense_input)
expected_out = np.array([[2.0], [3.0], [4.0], [5.0], [6.0]])
self.assertTrue(np.array_equal(output.numpy(), expected_out))
ragged_input = tf.ragged.constant([[5.0, 7.0], [9.0]])
ragged_output = normalizer(ragged_input)
expected_ragged_out = tf.ragged.constant([[2.0, 3.0], [4.0]],
dtype=tf.float32)
self.assertTrue(ragged_tensor_equal(ragged_output,
expected_ragged_out))
sparse_input = ragged_input.to_sparse()
sparse_output = normalizer(sparse_input)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(sparse_tensor_equal(sparse_output,
expected_sparse_out))
def test_round_indentity(self):
log_round = LogRound(num_bins=20, base=2)
dense_input = tf.constant([[0.0], [2.6], [0.2], [3.1], [1024]])
output = log_round(dense_input)
expected_out = np.array([[0], [1], [0], [2], [10]])
self.assertTrue(np.array_equal(output.numpy(), expected_out))
ragged_input = tf.ragged.constant([[1.1, 3.4], [1025]])
ragged_output = log_round(ragged_input)
expected_ragged_out = tf.ragged.constant([[0, 2], [10]],
dtype=tf.int64)
self.assertTrue(ragged_tensor_equal(ragged_output,
expected_ragged_out))
sparse_input = ragged_input.to_sparse()
sparse_output = log_round(sparse_input)
expected_sparse_out = expected_ragged_out.to_sparse()
self.assertTrue(sparse_tensor_equal(sparse_output,
expected_sparse_out))
def test_dense_to_ragged(self):
layer = ToRagged()
input_data = tf.constant([[1], [-1], [4]], tf.int64)
out = layer(input_data)
expected_out = tf.ragged.constant([[1], [], [4]], tf.int64)
self.assertTrue(ragged_tensor_equal(out, expected_out))
def test_string_split_to_ragged(self):
layer = ToRagged()
input_data = tf.ragged.constant([["1", "2", "3"], ["4", "5"], [""]])
out = layer(input_data)
expected_out = tf.ragged.constant([["1", "2", "3"], ["4", "5"], []])
self.assertTrue(ragged_tensor_equal(out, expected_out))