def test_corpus(self): self.assertIsInstance(thai_negations(), frozenset) self.assertIsInstance(thai_stopwords(), frozenset) self.assertIsInstance(thai_syllables(), frozenset) self.assertIsInstance(thai_words(), frozenset) self.assertIsInstance(countries(), frozenset) self.assertIsInstance(provinces(), frozenset) self.assertIsInstance(thai_female_names(), frozenset) self.assertIsInstance(thai_male_names(), frozenset) self.assertEqual(get_corpus_db_detail("XXX"), {}) # corpus does not exist self.assertTrue(download("test")) # download the first time self.assertTrue(download(name="test", force=True)) # force download self.assertTrue(download(name="test")) # try download existing self.assertFalse(download(name="test", url="wrongurl")) # URL not exist self.assertFalse( download(name="XxxXXxxx817d37sf")) # corpus name not exist self.assertIsNotNone(get_corpus_db_detail("test")) # corpus exists self.assertTrue(remove("test")) # remove existing self.assertFalse(remove("test")) # remove non-existing self.assertTrue(download(name="test", version="0.1")) self.assertTrue(remove("test"))
def test_corpus(self): self.assertIsInstance(thai_negations(), frozenset) self.assertIsInstance(thai_stopwords(), frozenset) self.assertIsInstance(thai_syllables(), frozenset) self.assertIsInstance(thai_words(), frozenset) self.assertIsInstance(countries(), frozenset) self.assertIsInstance(provinces(), frozenset) self.assertIsInstance(provinces(details=True), list) self.assertEqual(len(provinces(details=False)), len(provinces(details=True))) self.assertIsInstance(thai_family_names(), frozenset) self.assertIsInstance(list(thai_family_names())[0], str) self.assertIsInstance(thai_female_names(), frozenset) self.assertIsInstance(thai_male_names(), frozenset) self.assertIsInstance( get_corpus_db("https://example.com/XXXXXX0lkjasd/SXfmskdjKKXXX"), Response, ) # URL does not exist, should get 404 response self.assertIsNone(get_corpus_db("XXXlkja3sfdXX")) # Invalid URL self.assertEqual(get_corpus_db_detail("XXXmx3KSXX"), {}) # corpus does not exist self.assertEqual(get_corpus_db_detail("XXXmx3KSXX", version="0.2"), {}) # corpus does not exist self.assertTrue(download("test")) # download the first time self.assertTrue(download(name="test", force=True)) # force download self.assertTrue(download(name="test")) # try download existing self.assertFalse(download(name="test", url="wrongurl")) # URL not exist self.assertFalse( download(name="XxxXXxxx817d37sf")) # corpus name not exist self.assertIsNotNone(get_corpus_db_detail("test")) # corpus exists self.assertIsNotNone(get_corpus_path("test")) # corpus exists self.assertTrue(remove("test")) # remove existing self.assertFalse(remove("test")) # remove non-existing self.assertIsNone(get_corpus_path("XXXkdjfBzc")) # query non-existing self.assertFalse(download(name="test", version="0.0")) self.assertFalse(download(name="test", version="0.0.0")) self.assertFalse(download(name="test", version="0.0.1")) self.assertFalse(download(name="test", version="0.0.2")) self.assertFalse(download(name="test", version="0.0.3")) self.assertFalse(download(name="test", version="0.0.4")) self.assertIsNotNone(download(name="test", version="0.0.5")) self.assertTrue(download("test")) self.assertIsNotNone(remove("test")) # remove existing self.assertIsNotNone(download(name="test", version="0.0.6")) self.assertIsNotNone(download(name="test", version="0.0.7")) self.assertIsNotNone(download(name="test", version="0.0.8")) self.assertIsNotNone(download(name="test", version="0.0.9")) self.assertIsNotNone(download(name="test", version="0.0.10")) with self.assertRaises(Exception) as context: self.assertIsNotNone(download(name="test", version="0.0.11")) self.assertTrue( "Hash does not match expected." in str(context.exception)) self.assertIsNotNone(download(name="test", version="0.1")) self.assertIsNotNone(remove("test"))
def test_corpus(self): self.assertIsNotNone(countries()) self.assertIsNotNone(provinces()) self.assertIsNotNone(thai_negations()) self.assertIsNotNone(thai_stopwords()) self.assertIsNotNone(thai_syllables()) self.assertIsNotNone(thai_words()) download("test") self.assertIsNotNone(remove("test")) self.assertIsNotNone(remove("tnc_freq"))
def test_corpus(self): self.assertIsNotNone(countries()) self.assertIsNotNone(provinces()) self.assertIsNotNone(thai_negations()) self.assertIsNotNone(thai_stopwords()) self.assertIsNotNone(thai_syllables()) self.assertIsNotNone(thai_words()) download("test") self.assertIsNotNone(remove("test")) self.assertIsNotNone(remove("tnc_freq"))
def test_corpus(self): self.assertIsNotNone(countries()) self.assertIsNotNone(provinces()) self.assertIsNotNone(thai_negations()) self.assertIsNotNone(thai_stopwords()) self.assertIsNotNone(thai_syllables()) self.assertIsNotNone(thai_words()) self.assertIsNotNone(thai_female_names()) self.assertIsNotNone(thai_male_names()) self.assertEqual(get_corpus_db_detail("XXX"), {}) self.assertIsNone(download("test")) self.assertIsNone(download("test", force=True)) self.assertIsNotNone(get_corpus_db_detail("test")) self.assertIsNotNone(remove("test")) self.assertFalse(remove("test"))
def syllable_tokenize(text: str, engine: str = "default") -> List[str]: """ This function is to tokenize text into syllable (Thai: พยางค์), a unit of pronunciation having one vowel sound. For example, the word 'รถไฟ' contains two syallbles including 'รถ', and 'ไฟ'. Under the hood, this function uses :func:`pythainlp.tokenize.word_tokenize` with *newmm* as a tokenizer. The function tokenize the text with the dictionary of Thai words from :func:`pythainlp.corpus.common.thai_words` and then dictionary of Thai syllable from :func:`pythainlp.corpus.common.thai_syllables`. As a result, only syllables are obtained. :param str text: input string to be tokenized :param str engine: name of the syllable tokenizer :return: list of syllables where whitespaces in the text **are included** :rtype: list[str] **Options for engine** * *default* * *ssg* - CRF syllable segmenter for Thai. :Example:: :: from pythainlp.tokenize import syllable_tokenize text = 'รถไฟสมัยใหม่จะใช้กำลังจากหัวรถจักรดีเซล หรือจากไฟฟ้า' syllable_tokenize(text) ['รถ', 'ไฟ', 'สมัย', 'ใหม่', 'ใช้', 'กำ', 'ลัง', 'จาก', 'หัว', 'รถ', 'จักร', 'ดี', 'เซล', ' ', 'หรือ', 'จาก', 'ไฟ', 'ฟ้า'] """ if not text or not isinstance(text, str): return [] tokens = [] if engine == "default": words = word_tokenize(text) trie = dict_trie(dict_source=thai_syllables()) for word in words: tokens.extend(word_tokenize(text=word, custom_dict=trie)) else: from .ssg import segment tokens = segment(text) return tokens
def syllable_tokenize(text: str) -> List[str]: """ :param str text: input string to be tokenized :return: list of syllables """ if not text or not isinstance(text, str): return [] tokens = [] if text: words = word_tokenize(text) trie = dict_trie(dict_source=thai_syllables()) for word in words: tokens.extend(word_tokenize(text=word, custom_dict=trie)) return tokens
def syllable_tokenize(text: str) -> List[str]: """ :param str text: input string to be tokenized :return: returns list of strings of syllables """ if not text or not isinstance(text, str): return [] tokens = [] if text: words = word_tokenize(text) trie = dict_trie(dict_source=thai_syllables()) for word in words: tokens.extend(word_tokenize(text=word, custom_dict=trie)) return tokens
"sent_tokenize", "subword_tokenize", "syllable_tokenize", "word_tokenize", ] from pythainlp.corpus import thai_syllables, thai_words from pythainlp.util.trie import Trie DEFAULT_WORD_TOKENIZE_ENGINE = "newmm" DEFAULT_SENT_TOKENIZE_ENGINE = "crfcut" DEFAULT_SUBWORD_TOKENIZE_ENGINE = "tcc" DEFAULT_SYLLABLE_TOKENIZE_ENGINE = "dict" DEFAULT_WORD_DICT_TRIE = Trie(thai_words()) DEFAULT_SYLLABLE_DICT_TRIE = Trie(thai_syllables()) DEFAULT_DICT_TRIE = DEFAULT_WORD_DICT_TRIE from pythainlp.tokenize.core import ( Tokenizer, sent_tokenize, subword_tokenize, syllable_tokenize, word_tokenize, ) from pythainlp.corpus import get_corpus as _get_corpus THAI2FIT_TOKENIZER = Tokenizer( custom_dict=_get_corpus("words_th_thai2fit_201810.txt"), engine="newmm")
ADDITIONAL_SPECIAL_TOKENS_EXCLUDE_SPACE_TOKEN = \ [e for e in ADDITIONAL_SPECIAL_TOKENS if e != SPACE_TOKEN] SET_ADDITIONAL_SPECIAL_TOKENS = frozenset(ADDITIONAL_SPECIAL_TOKENS) PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { "th-roberta-base": 514, } # Store pre tokenizer function (text cutter) PRE_TOKENIZERS_MAP = {'newmm': partial( word_tokenize, custom_dict=Trie(frozenset(set(thai_words()).union(set(ADDITIONAL_SPECIAL_TOKENS)))) ), 'syllable': partial( word_tokenize, custom_dict=Trie(frozenset(set(thai_syllables()).union(set(ADDITIONAL_SPECIAL_TOKENS)))) ), } _nb_cores = multiprocessing.cpu_count() def split_additional_special_token(texts): """ Split list of text by additional special exclude space token. Args: texts: list of text. Returns: list_of_pre_cut_texts: list of list of pre cut text.
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = { "th-roberta-base": 514, } # Store pre tokenizer function (text cutter) PRE_TOKENIZERS_MAP = { 'newmm': partial(word_tokenize, custom_dict=Trie( frozenset( set(thai_words()).union(set(ADDITIONAL_SPECIAL_TOKENS))))), 'syllable': partial(word_tokenize, custom_dict=Trie( frozenset( set(thai_syllables()).union( set(ADDITIONAL_SPECIAL_TOKENS))))), } _nb_cores = multiprocessing.cpu_count() def split_additional_special_token(texts): """ Split list of text by additional special exclude space token. Args: texts: list of text. Returns: list_of_pre_cut_texts: list of list of pre cut text.