Esempi in Python per segment

Esempio n. 1

File: test_tokenize.py Progetto: hopedataannotations/pythainlp

 def test_icu(self):
     self.assertEqual(pyicu.segment(None), [])
     self.assertEqual(pyicu.segment(""), [])
     self.assertEqual(
         word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"),
         ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
     )

Esempio n. 2

File: __init__.py Progetto: wannaphongcom/pythainlp

 def test_word_tokenize_icu(self):
     self.assertEqual(tokenize_pyicu.segment(None), [])
     self.assertEqual(tokenize_pyicu.segment(""), [])
     self.assertEqual(
         word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"),
         ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],
     )