def test_tokenizer_convert_url(self): tokenizer = TokenizerJP() self.assertIsNotNone(tokenizer) tokenizer.set_configuration_before_concatenation_rule('.*[ -~]') tokenizer.set_configuration_after_concatenation_rule('[ -~].*') words = tokenizer.texts_to_words("http://192.168.1.10/index.html") self.assertEqual(["http://192.168.1.10/index.html"], words) self.assertEqual("http://192.168.1.10/index.html", tokenizer.words_to_texts(words)) words_en = tokenizer._texts_to_words_en( "http://192.168.1.10/index.html") self.assertEqual(["http://192.168.1.10/index.html"], words_en) self.assertEqual("http://192.168.1.10/index.html", tokenizer.words_to_texts(words_en)) words_jp = tokenizer._texts_to_words_jp( "http://192.168.1.10/index.html") self.assertEqual([ "http", "://", "192", ".", "168", ".", "1", ".", "10", "/", "index", ".", "html" ], words_jp) self.assertEqual("http :// 192 . 168 . 1 . 10 / index . html", tokenizer.words_to_texts(words_jp)) words_mix = tokenizer.texts_to_words( "URLはhttp://192.168.1.10/index.html") self.assertEqual(["URL", "は", "http://192.168.1.10/index.html"], words_mix) self.assertEqual("URLはhttp://192.168.1.10/index.html", tokenizer.words_to_texts(words_mix))
def test_tokenizer_no_test(self): tokenizer = TokenizerJP() self.assertIsNotNone(tokenizer) words = tokenizer._texts_to_words_en(None) self.assertEqual(0, len(words)) words = tokenizer._texts_to_words_en('') self.assertEqual(0, len(words)) words = tokenizer._texts_to_words_jp(None) self.assertEqual(0, len(words)) words = tokenizer._texts_to_words_jp('') self.assertEqual(0, len(words)) words = tokenizer._template_texts_to_words_jp(None) self.assertEqual(0, len(words)) words = tokenizer._template_texts_to_words_jp('') self.assertEqual(0, len(words)) texts = tokenizer._words_to_texts(None) self.assertEqual("", texts) words = tokenizer._words_to_texts('') self.assertEqual("", texts)