Exemple #1
0
 def test_word_tokenize_deepcut(self):
     self.assertEqual(tokenize_deepcut.segment(None), [])
     self.assertEqual(tokenize_deepcut.segment(""), [])
     self.assertIsNotNone(
         tokenize_deepcut.segment("ทดสอบ", DEFAULT_DICT_TRIE))
     self.assertIsNotNone(tokenize_deepcut.segment("ทดสอบ", ["ทด", "สอบ"]))
     self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut"))
 def test_deepcut(self):
     self.assertEqual(deepcut.segment(None), [])
     self.assertEqual(deepcut.segment(""), [])
     self.assertIsNotNone(deepcut.segment("ทดสอบ", DEFAULT_WORD_DICT_TRIE))
     self.assertIsNotNone(deepcut.segment("ทดสอบ", ["ทด", "สอบ"]))
     self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut"))
     self.assertIsNotNone(
         word_tokenize("ทดสอบ",
                       engine="deepcut",
                       custom_dict=DEFAULT_WORD_DICT_TRIE))
Exemple #3
0
 def test_word_tokenize_deepcut(self):
     self.assertEqual(tokenize_deepcut.segment(None), [])
     self.assertEqual(tokenize_deepcut.segment(""), [])
     self.assertIsNotNone(tokenize_deepcut.segment("ทดสอบ", DEFAULT_DICT_TRIE))
     self.assertIsNotNone(tokenize_deepcut.segment("ทดสอบ", ["ทด", "สอบ"]))
     self.assertIsNotNone(word_tokenize("ทดสอบ", engine="deepcut"))