def __init__(self, hub_module_handle):
"""Initializes a new HubModuleTokenizer instance.
Args:
hub_module_handle: A string handle accepted by hub.load(). Supported
cases include (1) a local path to a directory containing a module, and
(2) a handle to a module uploaded to e.g., https://tfhub.dev
"""
super(HubModuleTokenizer, self).__init__()
self._splitter = hub_module_splitter.HubModuleSplitter(
hub_module_handle)
def testSplit(self, text_input, expected_pieces, expected_starts,
expected_ends):
hub_module_handle = ("tensorflow_text/python/ops/test_data/"
"segmenter_hub_module")
splitter = hub_module_splitter.HubModuleSplitter(hub_module_handle)
pieces, starts, ends = splitter.split_with_offsets(text_input)
pieces_no_offset = splitter.split(text_input)
self.evaluate(lookup_ops.tables_initializer())
self.evaluate(variables_lib.global_variables_initializer())
self.assertAllEqual(expected_pieces, pieces)
self.assertAllEqual(expected_starts, starts)
self.assertAllEqual(expected_ends, ends)
self.assertAllEqual(expected_pieces, pieces_no_offset)
def exportSavedModel(self):
hub_module_handle = ("tensorflow_text/python/ops/test_data/"
"segmenter_hub_module")
splitter = hub_module_splitter.HubModuleSplitter(hub_module_handle)
save.save(splitter, "ram://saved_model")
self.assertEqual(file_io.file_exists_v2("ram://saved_model"), True)