def test_config_with_custom_name(self):
vocabulary_list = ['A', 'B', 'C', 'D', 'E']
layer = categorical.CategoryLookup(
vocabulary=vocabulary_list, num_oov_tokens=0, name='lookup')
config = layer.get_config()
layer_1 = categorical.CategoryLookup.from_config(config)
self.assertEqual(layer_1.name, layer.name)
def test_vocab_list_invalid_input(self):
vocabulary_list = ['A', 'B', 'C', 'D', 'E']
layer = categorical.CategoryLookup(vocabulary=vocabulary_list,
num_oov_tokens=0)
inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']])
output = layer(inp)
self.assertAllClose(np.asarray([[0, -1], [4, 2], [3, 0]]), output)
def test_vocab_file_compute_output_signature(self):
input_shape = tensor_shape.TensorShape([2, 3])
input_spec = tensor_spec.TensorSpec(input_shape, dtypes.string)
layer = categorical.CategoryLookup(
vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0)
output_spec = layer.compute_output_signature(input_spec)
self.assertEqual(output_spec.shape.dims, input_shape.dims)
self.assertEqual(output_spec.dtype, dtypes.int64)
def test_vocab_list_compute_output_signature(self):
input_shape = tensor_shape.TensorShape([2, 3])
input_spec = tensor_spec.TensorSpec(input_shape, dtypes.string)
vocabulary_list = ['A', 'B', 'C', 'D', 'E']
layer = categorical.CategoryLookup(vocabulary=vocabulary_list,
num_oov_tokens=0)
output_spec = layer.compute_output_signature(input_spec)
self.assertEqual(output_spec.shape.dims, input_shape.dims)
self.assertEqual(output_spec.dtype, dtypes.int64)
def test_vocab_list_sparse_input(self):
layer = categorical.CategoryLookup(
vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0)
inp = np.asarray([['omar', ''], ['stringer', 'marlo'], ['marlo', 'omar']])
indices = array_ops.where_v2(math_ops.not_equal(inp, ''))
sp_inp = sparse_tensor.SparseTensor(
indices,
array_ops.gather_nd_v2(inp, indices),
dense_shape=array_ops.shape_v2(inp, out_type=dtypes.int64))
output = layer(sp_inp)
self.assertIsInstance(output, sparse_tensor.SparseTensor)
self.assertAllClose(
np.asarray([[0, 0], [1, 0], [1, 1], [2, 0], [2, 1]]), output.indices)
self.assertAllClose(np.asarray([0, 1, 2, 2, 0]), output.values)
def test_vocab_list_sparse_input(self):
vocabulary_list = ['A', 'B', 'C', 'D', 'E']
layer = categorical.CategoryLookup(
vocabulary=vocabulary_list, num_oov_tokens=0)
inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']])
indices = array_ops.where_v2(math_ops.not_equal(inp, ''))
sp_inp = sparse_tensor.SparseTensor(
indices,
array_ops.gather_nd_v2(inp, indices),
dense_shape=array_ops.shape_v2(inp, out_type=dtypes.int64))
output = layer(sp_inp)
self.assertIsInstance(output, sparse_tensor.SparseTensor)
self.assertAllClose(
np.asarray([[0, 0], [1, 0], [1, 1], [2, 0], [2, 1]]), output.indices)
self.assertAllClose(np.asarray([0, 4, 2, 3, 0]), output.values)
def test_vocab_file_basic(self):
layer = categorical.CategoryLookup(
vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0)
inp = np.asarray([['marlo', 'omar'], ['stringer', 'omar']])
output = layer(inp)
self.assertAllClose(np.asarray([[2, 0], [1, 0]]), output)
def test_vocab_list_unknown_input(self):
vocabulary_list = ['A', 'B', 'C', 'D', 'E']
layer = categorical.CategoryLookup(vocabulary=vocabulary_list)
inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']])
output = layer(inp)
self.assertAllClose(np.asarray([[0, 5], [4, 2], [3, 0]]), output)
def test_vocab_file_unknown_input(self):
layer = categorical.CategoryLookup(
vocabulary=self._wire_vocabulary_file_name)
inp = np.asarray([['marlo', 'omar'], ['skywalker', 'omar']])
output = layer(inp)
self.assertAllClose(np.asarray([[2, 0], [3, 0]]), output)
