def create_jieba(config: Optional[Dict] = None) -> JiebaTokenizer:
config = config if config else {}
return JiebaTokenizer.create(
{
**JiebaTokenizer.get_default_config(),
**config
}, None, None, None)
def test_jieba_load_and_persist_dictionary(
tmp_path_factory: TempPathFactory,
default_model_storage: ModelStorage,
default_execution_context: ExecutionContext,
caplog: LogCaptureFixture,
):
dictionary_directory = tmp_path_factory.mktemp("dictionaries")
dictionary_path = dictionary_directory / "dictionary_1"
dictionary_contents = """
创新办 3 i
云计算 5
凱特琳 nz
台中
"""
dictionary_path.write_text(dictionary_contents, encoding="utf-8")
component_config = {"dictionary_path": dictionary_directory}
resource = Resource("jieba")
tk = JiebaTokenizer.create(
{
**JiebaTokenizer.get_default_config(),
**component_config
},
default_model_storage,
resource,
default_execution_context,
)
tk.process_training_data(TrainingData([Message(data={TEXT: ""})]))
# The dictionary has not been persisted yet.
with caplog.at_level(logging.DEBUG):
JiebaTokenizer.load(
{
**JiebaTokenizer.get_default_config(),
**component_config
},
default_model_storage,
resource,
default_execution_context,
)
assert any(
"Failed to load JiebaTokenizer from model storage." in message
for message in caplog.messages)
tk.persist()
# Check the persisted dictionary matches the original file.
with default_model_storage.read_from(resource) as resource_dir:
contents = (resource_dir / "dictionary_1").read_text(encoding="utf-8")
assert contents == dictionary_contents
# Delete original files to show that we read from the model storage.
dictionary_path.unlink()
dictionary_directory.rmdir()
JiebaTokenizer.load(
{
**JiebaTokenizer.get_default_config(),
**component_config
},
default_model_storage,
resource,
default_execution_context,
)
tk.process([Message(data={TEXT: ""})])