def text_to_item(self, raw_text): res = {} text = self.replace_word(raw_text) words = dict_word_tokenize(text, self.keyword_dict) new_w = [] for w in words: if w in self.keyword: new_w.append(w) else: word = word_tokenize(w) [new_w.append(i) for i in word] words = new_w print(words) items_translated_2 = self.translate2(words) # items = self.split_item(words) # items_translated_1 = [] # for item in items: # item = self.translate1(item) # items_translated_1.append(item) # if items_translated_2 != items_translated_1: # print(raw_text) # print(items_translated_1) # print(items_translated_2) if (items_translated_2 == []): res["status"] = "fail to processing \"{}\".".format(raw_text) else: res["status"] = "process word \"{}\" complete.".format(raw_text) res["item"] = items_translated_2 return res
def word_tokenize_to_g2p(text): wordall = dict_word_tokenize(text, custom_dict_trie=DEFAULT_DICT_TRIE) list = [] for a in wordall: try: list.append(data[a]) #romanization(a,engine='pyicu')) except: word_list_icu = word_tokenize(a, engine="icu") for b in word_list_icu: list.append(romanization(b, engine='pyicu')) return '|'.join(list)
def word_tokenize_to_g2p(text): wordall=dict_word_tokenize(text, custom_dict=DEFAULT_DICT_TRIE) list=[] for a in wordall: try: list.append(data[a])#romanization(a,engine='pyicu')) except: word_list_icu=word_tokenize(a) for b in word_list_icu: list.append(romanization(b)) return '|'.join(list).replace("-","|").split('|')
def test_dict_word_tokenize(self): self.assertEqual(dict_word_tokenize("", custom_dict=FROZEN_DICT_TRIE), []) self.assertIsNotNone( dict_word_tokenize("รถไฟฟ้ากรุงเทพBTSหูว์ค์", custom_dict=FROZEN_DICT_TRIE) ) self.assertIsNotNone(dict_trie(())) self.assertIsNotNone( dict_word_tokenize( "รถไฟฟ้ากรุงเทพBTSหูว์ค์", custom_dict=FROZEN_DICT_TRIE, engine="newmm" ) ) self.assertIsNotNone( dict_word_tokenize( "รถไฟฟ้ากรุงเทพBTSหูว์ค์", custom_dict=FROZEN_DICT_TRIE, engine="longest", ) ) self.assertIsNotNone( dict_word_tokenize( "รถไฟฟ้ากรุงเทพBTSหูว์ค์", custom_dict=FROZEN_DICT_TRIE, engine="mm" ) ) self.assertIsNotNone( dict_word_tokenize( "รถไฟฟ้ากรุงเทพBTSหูว์ค์", custom_dict=FROZEN_DICT_TRIE, engine="XX" ) )
def test_word_tokenize(self): self.assertEqual(word_tokenize(""), []) self.assertEqual( word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย"), ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"], ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="newmm") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="mm") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="longest") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="icu") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="deepcut") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="attacut") ) self.assertIsNotNone( word_tokenize("หมอนทองตากลมหูว์MBK39", engine="XX") ) # XX engine is not existed self.assertIsNotNone(dict_trie(())) self.assertIsNotNone(dict_trie(("ทดสอบ", "สร้าง", "Trie"))) self.assertIsNotNone(dict_trie(["ทดสอบ", "สร้าง", "Trie"])) self.assertIsNotNone(dict_trie({"ทดสอบ", "สร้าง", "Trie"})) self.assertIsNotNone(dict_trie(thai_words())) self.assertIsNotNone(dict_trie(DEFAULT_DICT_TRIE)) self.assertIsNotNone( dict_trie(os.path.join(_CORPUS_PATH, _THAI_WORDS_FILENAME)) ) self.assertTrue( "ไฟ" in word_tokenize("รถไฟฟ้า", custom_dict=dict_trie(["ไฟ"])) ) # Commented out until this unittest bug get fixed: # https://bugs.python.org/issue29620 # with self.assertWarns(DeprecationWarning): # dict_word_tokenize("เลิกใช้แล้ว", custom_dict=DEFAULT_DICT_TRIE) self.assertEqual( word_tokenize("รถไฟฟ้า", custom_dict=dict_trie(["ไฟ"])), dict_word_tokenize("รถไฟฟ้า", custom_dict=dict_trie(["ไฟ"])), )
def run(self, text): self.sound = None self.text_cut = dict_word_tokenize(text, Trie(self.word_list)) self.i = 0 # ประกาศตัวแปร i เพื่อใช้ในการลูป self.num_sound = 0 # ประกาศตัวแปร num_sound สำหรับเก็บจำนวนเสียงที่ถูกรวม while self.i < len(self.text_cut): if self.text_cut[ self. i] in self.word_list and self.num_sound == 0: # ถ้ามีคำนี้ในระบบและเป็นคำแรกที่รวม self.sound = AudioSegment.from_wav( "data/" + self.data_file[self.text_cut[self.i]] + ".wav") # ให้ self.sound แทนไฟล์เสียงที่ถูกตัดไฟล์แรก self.num_sound += 1 elif self.text_cut[self.i] in self.word_list: self.sound += AudioSegment.from_wav( "data/" + self.data_file[self.text_cut[self.i]] + ".wav") # ทำการรวมไฟล์เสียงเข้าไป self.num_sound += 1 self.i += 1 self.sound.export(self.file, format="wav")
# -*- coding: utf-8 -*- from pythainlp.tokenize import dict_word_tokenize, create_custom_dict_trie data = create_custom_dict_trie("wordlist.txt") while True: text = input("text : ") print(dict_word_tokenize(text, custom_dict_trie=data, engine="newmm")) print("\r\n")
อย่างหนึ่ง""".split("\n") # หน้า 64 http://www.arts.chula.ac.th/~ling/thesis/2556MA-LING-Nalinee.pdf with codecs.open("corpus.txt", 'r',encoding='utf8') as f: lines1 = list(set(normalize(f.read()).splitlines())) f.close() test=True#False#True##เปิด/ปิดการ test #''' with codecs.open("thai.txt", "r",encoding="utf8") as f: lines2 = f.read().splitlines()#''' ''' from pythainlp.corpus.thaiword import get_data lines2 =get_data()''' data_all=[] thaiword=create_custom_dict_trie(list(set(ccc+lines2+stopwords+conjunctions))) print("จำนวนประโยค : "+str(len(lines1))) for lines in lines1: text=dict_word_tokenize(lines,thaiword) #text=word_tokenize(lines,thai_tokenize) data_all.append(text) sents=data_all tokens = [] boundaries = set() offset = 0 def check_punctuation(text): for i in text: if i in list(set(punctuation)): return True return False def num_there(s): return any(i.isdigit() for i in s) for sent in sents: tokens.extend(sent)
def segment(txt): return dict_word_tokenize(text=txt, data=get_data(filename) + get_data(filename2), data_type="list", engine="newmm")
# -*- coding: utf-8 -*- import sqlite3 connection = sqlite3.connect('db.sqlite3') cursor = connection.execute('select word from word') wordlist=[i[0] for i in cursor.fetchall()] #print('\n'.join(wordlist)) print("จำนวนคำ : "+str(len(wordlist))) connection.close() from pythainlp.tokenize import dict_word_tokenize,create_custom_dict_trie,word_tokenize dictthai=create_custom_dict_trie(wordlist) while True: text=input("text : ") if text=="exit": break print("ผลจาก dict : \t"+'|'.join(dict_word_tokenize(text,dictthai))) print("ผลจาก PyThaiNLP : \t"+'|'.join(word_tokenize(text)))
def test_dict_word_tokenize(self): self.assertEqual(dict_word_tokenize(""), [])
อย่างหนึ่ง""".split("\n") # หน้า 64 http://www.arts.chula.ac.th/~ling/thesis/2556MA-LING-Nalinee.pdf with codecs.open("corpus.txt", 'r',encoding='utf8') as f: lines1 = list(set(normalize(f.read()).splitlines())) f.close() test=False#True##เปิด/ปิดการ test #''' with codecs.open("thai.txt", "r",encoding="utf8") as f: lines2 = f.read().splitlines()#''' ''' from pythainlp.corpus.thaiword import get_data lines2 =get_data()''' data_all=[] thaiword=create_custom_dict_trie(list(set(ccc+lines2+stopwords+conjunctions))) print("จำนวนประโยค : "+str(len(lines1))) for lines in lines1: text=dict_word_tokenize(lines,thaiword) #text=word_tokenize(lines,thai_tokenize) data_all.append(text) sents=data_all tokens = [] boundaries = set() offset = 0 def check_punctuation(text): for i in text: if i in list(set(punctuation)): return True return False def num_there(s): return any(i.isdigit() for i in s) for sent in sents: tokens.extend(sent)
def test_dict_word_tokenize(self): self.assertEqual(dict_word_tokenize(""), [])