def test_config_with_custom_name(self): vocabulary_list = ['A', 'B', 'C', 'D', 'E'] layer = categorical.CategoryLookup( vocabulary=vocabulary_list, num_oov_tokens=0, name='lookup') config = layer.get_config() layer_1 = categorical.CategoryLookup.from_config(config) self.assertEqual(layer_1.name, layer.name)
def test_vocab_list_invalid_input(self): vocabulary_list = ['A', 'B', 'C', 'D', 'E'] layer = categorical.CategoryLookup(vocabulary=vocabulary_list, num_oov_tokens=0) inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']]) output = layer(inp) self.assertAllClose(np.asarray([[0, -1], [4, 2], [3, 0]]), output)
def test_vocab_file_compute_output_signature(self): input_shape = tensor_shape.TensorShape([2, 3]) input_spec = tensor_spec.TensorSpec(input_shape, dtypes.string) layer = categorical.CategoryLookup( vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0) output_spec = layer.compute_output_signature(input_spec) self.assertEqual(output_spec.shape.dims, input_shape.dims) self.assertEqual(output_spec.dtype, dtypes.int64)
def test_vocab_list_compute_output_signature(self): input_shape = tensor_shape.TensorShape([2, 3]) input_spec = tensor_spec.TensorSpec(input_shape, dtypes.string) vocabulary_list = ['A', 'B', 'C', 'D', 'E'] layer = categorical.CategoryLookup(vocabulary=vocabulary_list, num_oov_tokens=0) output_spec = layer.compute_output_signature(input_spec) self.assertEqual(output_spec.shape.dims, input_shape.dims) self.assertEqual(output_spec.dtype, dtypes.int64)
def test_vocab_list_sparse_input(self): layer = categorical.CategoryLookup( vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0) inp = np.asarray([['omar', ''], ['stringer', 'marlo'], ['marlo', 'omar']]) indices = array_ops.where_v2(math_ops.not_equal(inp, '')) sp_inp = sparse_tensor.SparseTensor( indices, array_ops.gather_nd_v2(inp, indices), dense_shape=array_ops.shape_v2(inp, out_type=dtypes.int64)) output = layer(sp_inp) self.assertIsInstance(output, sparse_tensor.SparseTensor) self.assertAllClose( np.asarray([[0, 0], [1, 0], [1, 1], [2, 0], [2, 1]]), output.indices) self.assertAllClose(np.asarray([0, 1, 2, 2, 0]), output.values)
def test_vocab_list_sparse_input(self): vocabulary_list = ['A', 'B', 'C', 'D', 'E'] layer = categorical.CategoryLookup( vocabulary=vocabulary_list, num_oov_tokens=0) inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']]) indices = array_ops.where_v2(math_ops.not_equal(inp, '')) sp_inp = sparse_tensor.SparseTensor( indices, array_ops.gather_nd_v2(inp, indices), dense_shape=array_ops.shape_v2(inp, out_type=dtypes.int64)) output = layer(sp_inp) self.assertIsInstance(output, sparse_tensor.SparseTensor) self.assertAllClose( np.asarray([[0, 0], [1, 0], [1, 1], [2, 0], [2, 1]]), output.indices) self.assertAllClose(np.asarray([0, 4, 2, 3, 0]), output.values)
def test_vocab_file_basic(self): layer = categorical.CategoryLookup( vocabulary=self._wire_vocabulary_file_name, num_oov_tokens=0) inp = np.asarray([['marlo', 'omar'], ['stringer', 'omar']]) output = layer(inp) self.assertAllClose(np.asarray([[2, 0], [1, 0]]), output)
def test_vocab_list_unknown_input(self): vocabulary_list = ['A', 'B', 'C', 'D', 'E'] layer = categorical.CategoryLookup(vocabulary=vocabulary_list) inp = np.asarray([['A', ''], ['E', 'C'], ['D', 'A']]) output = layer(inp) self.assertAllClose(np.asarray([[0, 5], [4, 2], [3, 0]]), output)
def test_vocab_file_unknown_input(self): layer = categorical.CategoryLookup( vocabulary=self._wire_vocabulary_file_name) inp = np.asarray([['marlo', 'omar'], ['skywalker', 'omar']]) output = layer(inp) self.assertAllClose(np.asarray([[2, 0], [3, 0]]), output)