def tokenize(self, text: str): """Tokenize.""" result = [] for token in self.tokenizer.tokenize(text, self.mode): _token = Token(token.surface()) if self.with_postag: _token.postag = token.part_of_speech()[0] result.append(_token) return result
def tokenize(self, text: str): """Tokenize""" return_result = [] parse_result = self.mecab.parse(text) if self.with_postag: for elem in parse_result.split("\n")[:-1]: surface, feature = elem.split() postag = feature.split(",")[0] return_result.append(Token(surface=surface, postag=postag)) else: for surface in parse_result.split(" "): return_result.append(Token(surface=surface)) return return_result
def tokenize(self, text: str): return_result = [] if self.with_postag: response = self.kytea.getTagsToString(text) response = response.replace(" ", " <SPACE>") # FIXME for elem in response.split(" ")[:-1]: surface, postag, _ = elem.split("/") surface = surface.replace("<SPACE>", " ") return_result.append(Token(surface=surface, postag=postag)) else: for surface in list(self.kytea.getWS(text)): return_result.append(Token(surface=surface)) return return_result
def test_word_tokenize_with_character(self): """Test Character tokenizer.""" tokenizer1 = WordTokenizer(tokenizer="Character") tokenizer2 = WordTokenizer(tokenizer="character") # assert tokenizer1 == tokenizer2 expect = [Token(surface=w) for w in "吾 輩 は 猫 で あ る".split(" ")] # NOQA result1 = tokenizer1.tokenize(SENTENCE1) result2 = tokenizer2.tokenize(SENTENCE1) assert expect == result1 # NOQA assert result1 == result2
def test_word_tokenize_with_sudachi_mode_c(self): """Test Sudachi tokenizer.""" try: tokenizer = WordTokenizer(tokenizer="Sudachi", mode="C") except ModuleNotFoundError: pytest.skip("skip sudachi") expect = [Token(surface=w) for w in "医薬品安全管理責任者".split(" ")] result = tokenizer.tokenize(SENTENCE2) self.assertEqual(expect, result)
def test_word_tokenize_with_mecab(self): """Test MeCab tokenizer.""" try: tokenizer1 = WordTokenizer(tokenizer="MeCab") tokenizer2 = WordTokenizer(tokenizer="mecab") except ModuleNotFoundError: pytest.skip("skip mecab") expect = [Token(surface=w) for w in "吾輩 は 猫 で ある".split(" ")] # NOQA result1 = tokenizer1.tokenize(SENTENCE1) result2 = tokenizer2.tokenize(SENTENCE1) assert expect == result1 # NOQA assert result1 == result2
def test_word_tokenize_with_mecab(self): """Test MeCab tokenizer.""" try: tokenizer = WordTokenizer(tokenizer="mecab", with_postag=True) except ModuleNotFoundError: pytest.skip("skip mecab") words = "吾輩 は 猫 で ある".split(" ") # NOQA postags = "名詞 助詞 名詞 助動詞 助動詞".split(" ") expect = [Token(surface=w, postag=p) for w, p in zip(words, postags)] result = tokenizer.tokenize(SENTENCE1) assert expect == result
def test_token_with_postag2(self): token = Token( surface="大崎", postag="名詞", postag2="固有名詞,人名,姓", conj_type="*", conj_form="*", origin_form="大崎", yomi="オオサキ", pron="オーサキ") self.assertEqual( "名詞,固有名詞,人名,姓,*,*,大崎,オオサキ,オーサキ", token.feature)
def test_word_tokenize_with_sentencepiece(self): """Test Sentencepiece tokenizer.""" try: tokenizer1 = WordTokenizer(tokenizer="Sentencepiece", model_path="data/model.spm") tokenizer2 = WordTokenizer(tokenizer="Sentencepiece", model_path="data/model.spm") except ModuleNotFoundError: pytest.skip("skip sentencepiece") expect = [Token(surface=w) for w in "▁ 吾 輩 は 猫 である".split(" ")] # NOQA result1 = tokenizer1.tokenize(SENTENCE1) result2 = tokenizer2.tokenize(SENTENCE1) assert expect == result1 # NOQA assert result1 == result2
def test_word_tokenize_with_sudachi_mode_a(self): """Test Sudachi tokenizer.""" try: tokenizer = WordTokenizer(tokenizer="sudachi", mode="A", with_postag=True) except ModuleNotFoundError: pytest.skip("skip sudachi") words = "医薬 品 安全 管理 責任 者".split(" ") # NOQA postags = "名詞 接尾辞 名詞 名詞 名詞 接尾辞".split(" ") expect = [Token(surface=w, postag=p) for w, p in zip(words, postags)] result = tokenizer.tokenize(SENTENCE2) self.assertEqual(expect, result)
def test_token_without_feature(self): token = Token(surface="大崎") self.assertEqual("大崎", token.surface) self.assertEqual("", token.feature)
def test_token_with_postag(self): token = Token(surface="大崎", postag="名詞") self.assertEqual("大崎", token.surface) self.assertEqual("名詞", token.feature)
def tokenize(self, text: str): result = [] for subword in self.tokenizer.EncodeAsPieces(text): token = Token(surface=subword) result.append(token) return result
def tokenize(self, text: str): return [Token(surface=char) for char in list(text)]