def create_jieba(config: Optional[Dict] = None) -> JiebaTokenizer: config = config if config else {} return JiebaTokenizer.create( { **JiebaTokenizer.get_default_config(), **config }, None, None, None)
def test_jieba(text, expected_tokens, expected_indices): tk = JiebaTokenizer() tokens = tk.tokenize(Message(text), attribute=TEXT_ATTRIBUTE) assert [t.text for t in tokens] == expected_tokens assert [t.start for t in tokens] == [i[0] for i in expected_indices] assert [t.end for t in tokens] == [i[1] for i in expected_indices]
def test_jieba_load_dictionary(tmpdir_factory): dictionary_path = tmpdir_factory.mktemp("jieba_custom_dictionary").strpath component_config = {"dictionary_path": dictionary_path} with patch.object(JiebaTokenizer, "load_custom_dictionary", return_value=None) as mock_method: tk = JiebaTokenizer(component_config) tk.tokenize(Message(""), attribute=TEXT_ATTRIBUTE) mock_method.assert_called_once_with(dictionary_path)
def test_jieba_load_dictionary(tmp_path: Path): dictionary_path = str(tmp_path) component_config = {"dictionary_path": dictionary_path} with patch.object(JiebaTokenizer, "load_custom_dictionary", return_value=None) as mock_method: tk = JiebaTokenizer(component_config) tk.tokenize(Message(data={TEXT: ""}), attribute=TEXT) mock_method.assert_called_once_with(dictionary_path)
def test_jieba_load_dictionary(tmpdir_factory): from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer dictionary_path = tmpdir_factory.mktemp("jieba_custom_dictionary").strpath component_config = {"dictionary_path": dictionary_path} with patch.object(JiebaTokenizer, "load_custom_dictionary", return_value=None) as mock_method: tk = JiebaTokenizer(component_config) tk.tokenize("") mock_method.assert_called_once_with(dictionary_path)
def test_custom_intent_symbol(text, expected_tokens): component_config = { "intent_tokenization_flag": True, "intent_split_symbol": "+" } tk = JiebaTokenizer(component_config) message = Message(text) message.set(INTENT_ATTRIBUTE, text) tk.train(TrainingData([message])) assert [t.text for t in message.get(TOKENS_NAMES[INTENT_ATTRIBUTE]) ] == expected_tokens
def test_jieba_add_cls_token(): from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer component_config = {"use_cls_token": True} tk = JiebaTokenizer(component_config) assert [t.text for t in tk.tokenize("Micheal你好吗?")] == [ "Micheal", "你好", "吗", "?", CLS_TOKEN, ] assert [t.offset for t in tk.tokenize("Micheal你好吗?")] == [0, 7, 9, 10, 12]
def __init__(self, component_config: Optional[Dict[Text, Any]] = None) -> None: super(HFTransformersNLP, self).__init__(component_config) self._load_model() self.whitespace_tokenizer = WhitespaceTokenizer() self.jieba_tokenizer = None if 'semi-lm' in component_config and component_config['semi-lm'] == 1: self.jieba_tokenizer = JiebaTokenizer(component_config) self.tokenizer.do_basic_tokenize = False
def test_jieba(): from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer tk = JiebaTokenizer() assert [t.text for t in tk.tokenize("我想去吃兰州拉面")] == ["我", "想", "去", "吃", "兰州", "拉面"] assert [t.offset for t in tk.tokenize("我想去吃兰州拉面")] == [0, 1, 2, 3, 4, 6] assert [t.text for t in tk.tokenize("Micheal你好吗?")] == ["Micheal", "你好", "吗", "?"] assert [t.offset for t in tk.tokenize("Micheal你好吗?")] == [0, 7, 9, 10]
def test_jieba(): from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer component_config = {"use_cls_token": False} tk = JiebaTokenizer(component_config) assert [t.text for t in tk.tokenize("我想去吃兰州拉面")] == ["我", "想", "去", "吃", "兰州", "拉面"] assert [t.offset for t in tk.tokenize("我想去吃兰州拉面")] == [0, 1, 2, 3, 4, 6] assert [t.text for t in tk.tokenize("Micheal你好吗?")] == ["Micheal", "你好", "吗", "?"] assert [t.offset for t in tk.tokenize("Micheal你好吗?")] == [0, 7, 9, 10]
def test_jieba(): from rasa.nlu.tokenizers.jieba_tokenizer import JiebaTokenizer tk = JiebaTokenizer() assert [t.text for t in tk.tokenize("我想去吃兰州拉面")] == \ ['我', '想', '去', '吃', '兰州', '拉面'] assert [t.offset for t in tk.tokenize("我想去吃兰州拉面")] == \ [0, 1, 2, 3, 4, 6] assert [t.text for t in tk.tokenize("Micheal你好吗?")] == \ ['Micheal', '你好', '吗', '?'] assert [t.offset for t in tk.tokenize("Micheal你好吗?")] == \ [0, 7, 9, 10]
def test_jieba_load_and_persist_dictionary( tmp_path_factory: TempPathFactory, default_model_storage: ModelStorage, default_execution_context: ExecutionContext, caplog: LogCaptureFixture, ): dictionary_directory = tmp_path_factory.mktemp("dictionaries") dictionary_path = dictionary_directory / "dictionary_1" dictionary_contents = """ 创新办 3 i 云计算 5 凱特琳 nz 台中 """ dictionary_path.write_text(dictionary_contents, encoding="utf-8") component_config = {"dictionary_path": dictionary_directory} resource = Resource("jieba") tk = JiebaTokenizer.create( { **JiebaTokenizer.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) tk.process_training_data(TrainingData([Message(data={TEXT: ""})])) # The dictionary has not been persisted yet. with caplog.at_level(logging.DEBUG): JiebaTokenizer.load( { **JiebaTokenizer.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) assert any( "Failed to load JiebaTokenizer from model storage." in message for message in caplog.messages) tk.persist() # Check the persisted dictionary matches the original file. with default_model_storage.read_from(resource) as resource_dir: contents = (resource_dir / "dictionary_1").read_text(encoding="utf-8") assert contents == dictionary_contents # Delete original files to show that we read from the model storage. dictionary_path.unlink() dictionary_directory.rmdir() JiebaTokenizer.load( { **JiebaTokenizer.get_default_config(), **component_config }, default_model_storage, resource, default_execution_context, ) tk.process([Message(data={TEXT: ""})])
context = {} time = datetime.datetime.now() default_output_attributes = { "intent": { "name": None, "confidence": 0.0 }, "entities": [] } message = Message(text, data=default_output_attributes, time=time) model_dir = './models/link/model_20190517-113416' model_metadata = Metadata.load(model_dir) jieba_meta = model_metadata.for_component(index=0) jie = JiebaTokenizer.load(meta=jieba_meta, model_dir=model_dir, model_metadata=Metadata.load(model_dir)) pprint.pprint(message.data) jie.process(message) ltp_meta = model_metadata.for_component(index=5) ltp = LtpHelper.load(meta=ltp_meta, model_dir=model_dir, model_metadata=Metadata.load(model_dir)) pprint.pprint(message.data) ltp.process(message) pprint.pprint(message.data)