def tokenize(filename='thairath1.tsv'):
"""
tokenize headline (line[1]) & description (line[2])
save as txt with whitespace
"""
path = '/Users/Nozomi/files/metonymy/'
open_name = path + filename
save_name = open_name.rsplit('.tsv')[0]
open_file = open(open_name, 'r', encoding='utf-8')
title_file = open(save_name + '_title.tsv', 'w', encoding='utf-8')
description_file = open(save_name + '_description.tsv', 'w', encoding='utf-8')
articles = csv.reader(open_file, delimiter='\t')
write1 = csv.writer(title_file, lineterminator='\n', delimiter='\t')
write2 = csv.writer(description_file, lineterminator='\n', delimiter='\t')
for article in articles:
if article[2] != '':
ID = [article[0]]
title = word_tokenize(article[1])
description = word_tokenize(article[2])
write1.writerow(ID + title)
write2.writerow(ID + description)
open_file.close()
title_file.close()
description_file.close()
def sentence_similarity(sentence1, sentence2):
""" compute the sentence similarity using Wordnet """
# Tokenize and tag
sentence1 = pos_tag(word_tokenize(sentence1), 'artagger')
sentence2 = pos_tag(word_tokenize(sentence2), 'artagger')
# Get the synsets for the tagged words
synsets1 = [tagged_to_synset(*tagged_word) for tagged_word in sentence1]
synsets2 = [tagged_to_synset(*tagged_word) for tagged_word in sentence2]
# Filter out the Nones
synsets1 = [ss for ss in synsets1 if ss]
synsets2 = [ss for ss in synsets2 if ss]
score, count = 0.0, 0
# For each word in the first sentence
for synset in synsets1:
# Get the similarity value of the most similar word in the other sentence
best_score = max([synset.path_similarity(ss) for ss in synsets2])
# Check that the similarity could have been computed
if best_score is not None:
score += best_score
count += 1
# Average the values
if count != 0:
score /= count
return score
def change_word_tokenize(self, name):
if self.dictlist == []:
self.words = WordList(
pythainlp.word_tokenize(self.text, engine=name))
else:
self.words = WordList(pythainlp.word_tokenize(
self.text, self.dict))
self.tags = pythainlp.pos_tag(self.words)
def compare(self, statement, other_statement):
"""
Return the calculated similarity of two
statements based on the Jaccard index.
"""
from nltk.corpus import wordnet
from pythainlp.tag import pos_tag
from pythainlp.tokenize import word_tokenize
import nltk
import string
a = statement.text.lower()
b = other_statement.text.lower()
# Get default English stopwords and extend with punctuation
from pythainlp.corpus import stopwords
stopwords = stopwords.words('thai') + nltk.corpus.stopwords.words(
'english')
stopwords.extend(string.punctuation)
stopwords.append('')
lemmatizer = nltk.stem.wordnet.WordNetLemmatizer()
def get_wordnet_pos(pos_tag):
if pos_tag[1].startswith('J'):
return (pos_tag[0], wordnet.ADJ)
elif pos_tag[1].startswith('V'):
return (pos_tag[0], wordnet.VERB)
elif pos_tag[1].startswith('N'):
return (pos_tag[0], wordnet.NOUN)
elif pos_tag[1].startswith('R'):
return (pos_tag[0], wordnet.ADV)
else:
return (pos_tag[0], wordnet.NOUN)
ratio = 0
pos_a = map(get_wordnet_pos, pos_tag(word_tokenize(a), 'artagger'))
pos_b = map(get_wordnet_pos, pos_tag(word_tokenize(b), 'artagger'))
lemma_a = [
lemmatizer.lemmatize(token.strip(string.punctuation), pos)
for token, pos in pos_a if pos == wordnet.NOUN
and token.strip(string.punctuation) not in stopwords
]
lemma_b = [
lemmatizer.lemmatize(token.strip(string.punctuation), pos)
for token, pos in pos_b if pos == wordnet.NOUN
and token.strip(string.punctuation) not in stopwords
]
# Calculate Jaccard similarity
try:
numerator = len(set(lemma_a).intersection(lemma_b))
denominator = float(len(set(lemma_a).union(lemma_b)))
ratio = numerator / denominator
except Exception as e:
print('Error', e)
return ratio >= self.SIMILARITY_THRESHOLD
def __init__(self, text, dictlist=[]):
self.text = text
self.dictlist = dictlist
if self.dictlist == []:
self.words = WordList(pythainlp.word_tokenize(self.text))
else:
self.dict = pythainlp.tokenize.dict_trie(self.dictlist)
self.words = WordList(pythainlp.word_tokenize(
self.text, self.dict))
self.tags = pythainlp.pos_tag(self.words)
self.romanize = [romanize_pythainlp(i) for i in self.words]
self.word_counts = Counter(self.words)
def compare(self, statement, other_statement):
"""
Compare the two input statements.
:return: The percent of similarity between the closest synset distance.
:rtype: float
.. _wordnet: http://www.nltk.org/howto/wordnet.html
.. _NLTK: http://www.nltk.org/
"""
from nltk.corpus import wordnet
from pythainlp.tokenize import word_tokenize
from chatterbot import utils
import itertools
tokens1 = word_tokenize(statement.text.lower())
tokens2 = word_tokenize(other_statement.text.lower())
# Remove all stop words from the list of word tokens
tokens1 = utils.remove_stopwords(tokens1, language='thai')
tokens2 = utils.remove_stopwords(tokens2, language='thai')
# The maximum possible similarity is an exact match
# Because path_similarity returns a value between 0 and 1,
# max_possible_similarity is the number of words in the longer
# of the two input statements.
max_possible_similarity = max(len(statement.text.split()),
len(other_statement.text.split()))
max_similarity = 0.0
# Get the highest matching value for each possible combination of words
for combination in itertools.product(*[tokens1, tokens2]):
synset1 = wordnet.synsets(combination[0], lang="tha")
synset2 = wordnet.synsets(combination[1], lang="tha")
if synset1 and synset2:
# Get the highest similarity for each combination of synsets
for synset in itertools.product(*[synset1, synset2]):
similarity = synset[0].path_similarity(synset[1])
if similarity and (similarity > max_similarity):
max_similarity = similarity
if max_possible_similarity == 0:
return 0
return max_similarity / max_possible_similarity
def selectLibraryFile(self, TextList):
if self.LibraryNumber == 'glex':
Key = {'text': TextList, 'useDict': self.Dictname}
TextResponse = requests.get('http://127.0.0.1:8080/glex/segment',
Key)
cutWords = TextResponse.json()['results']
elif self.LibraryNumber == 'deepcut':
cutWords = deepcut.tokenize(TextList)
elif self.LibraryNumber == 'tltk':
cutWords = tltk.nlp.word_segment(TextList).split('|')
elif self.LibraryNumber == 'newmm':
cutWords = word_tokenize(TextList, engine='newmm')
elif self.LibraryNumber == 'mm':
cutWords = word_tokenize(TextList, engine='mm')
return cutWords
def process(lines, core, acc_result):
print('core {} started'.format(core))
pattern_count = {}
for idx, line in enumerate(lines):
sentence = clean.fixing(line)
result = pyt.word_tokenize(sentence, engine='newmm')
crit0 = [
bool(char_repeat_pattern.search(token))
and bool(thai_pattern.search(token)) for token in result
]
res0 = [(result[idx - 1:idx + 1]) for idx, log in enumerate(crit0)
if log]
if any(crit0):
for patt in res0:
if len(patt) == 0 or patt[0] == ' ':
continue
patt = '+'.join(patt)
if patt in pattern_count:
pattern_count[patt] += 1
else:
pattern_count[patt] = 1
acc_result.append((core, pattern_count))
print('core {} finished'.format(core))
def extract_word_from_message(data) -> List[str]:
result: List = list()
for sentence in data.message:
words = word_tokenize(sentence, engine='newmm', keep_whitespace=False)
striped_word = [ word.strip() for word in words if word.strip() != '']
result.extend(striped_word)
return result
def tokenizer(self, text: str) -> List[str]:
"""
:meth: tokenize text with a frozen newmm engine
:param str text: text to tokenize
:return: tokenized text
"""
return word_tokenize(text, engine="ulmfit")
def split_word(text):
tokens = word_tokenize(text, engine='newmm')
# # Remove stop words ภาษาไทย และภาษาอังกฤษ
# tokens = [i for i in tokens if not i in th_stop and not i in en_stop]
# หารากศัพท์ภาษาไทย และภาษาอังกฤษ
# English
tokens = [p_stemmer.stem(i) for i in tokens]
# Thai
tokens_temp = []
for i in tokens:
w_syn = wordnet.synsets(i)
if (len(w_syn) > 0) and (len(w_syn[0].lemma_names('tha')) > 0):
tokens_temp.append(w_syn[0].lemma_names('tha')[0])
else:
tokens_temp.append(i)
tokens = tokens_temp
# ลบตัวเลข
tokens = [i for i in tokens if not i.isnumeric()]
# ลบช่องว่าง
tokens = [i for i in tokens if not ' ' in i]
# tokens_list = [split_word(txt) for txt in text_list]
return tokens
def _train_on_text(self, text):
n = 3
self.grams = {}
gram_buffer = []
text = ' '.join(word_tokenize(text))
for letter in text:
# letter = letter.lower()
if letter not in ['\n', ' ']:
gram_buffer.append(letter)
else:
gram_buffer.append(MarkovWordGenerator.stopword)
if len(gram_buffer) >= n:
as_tuple = tuple(gram_buffer)
if as_tuple not in self.grams:
self.grams[as_tuple] = 0
self.grams[as_tuple] += 1
gram_buffer = gram_buffer[1:]
if letter in ['\n', ' ', '*']:
gram_buffer.clear()
gram_buffer.append(MarkovWordGenerator.startword)
def crawler():
count = 0
for status in tweepy.Cursor(api.user_timeline,
screen_name=name,
tweet_mode="extended").items():
if count > 4:
break
messages = status.full_text.splitlines()
proc = ''
matching = []
noti = ''
time = ''
for i in messages:
proc = word_tokenize(i, engine='newmm')
matching = [
s for s in proc if ('ขัดข้อง' in s) or ('ขออภัย' in s) or (
'ขณะนี้' in s) or ('ความไม่สะดวก' in s) or ('ตามปกติ' in s)
]
print(matching)
if len(matching) != 0:
noti = status.full_text
time = status.created_at
# break
if noti != '':
break
# f.write(f'index:{count} message:{status.full_text}')
count += 1
print(noti)
print(time)
return noti, time
async def get_count_by_list(data: list,
return_value: str = 'wordcount') -> dict:
wordcount = {}
hashtag = {}
mention = {}
for row in data:
word_list = word_tokenize(row['message'], keep_whitespace=False)
for word in get_word_or_hashtag(word_list):
if word[0] == '@' and return_value == 'mention':
if word in mention.keys():
mention[word] += 1
else:
mention[word] = 1
elif word[0] == '#' and return_value == 'hashtag':
if word in hashtag.keys():
hashtag[word] += 1
else:
hashtag[word] = 1
else:
if word in wordcount.keys():
wordcount[word] += 1
else:
wordcount[word] = 1
if return_value == 'mention':
return mention
if return_value == 'hashtag':
return hashtag
return wordcount
def predict_ensemble(test_df):
"""
Predict input ``test_df`` with columns ``review`` and ``rating``
"""
instances = [
wongnai_predictor._dataset_reader.text_to_instance(
word_tokenize(review)) for review in list(test_df.review)
]
model_paths = glob('output_*/model.tar.gz')
all_predicted_labels = []
for model_path in model_paths:
archive = load_archive(model_path) # load trained model
wongnai_predictor = Predictor.from_archive(archive,
'wongnai_predictor')
predicted_labels = [
int(
wongnai_predictor.predict_instance(instance)
['predicted_label']) for instance in instances
]
all_predicted_labels.append(predicted_labels)
all_predicted_labels = np.array(all_predicted_labels)
predicted_labels_vote = mode(np.array(all_predicted_labels).T,
axis=-1).mode.ravel()
test_df['rating'] = predicted_labels_vote
return test_df.drop('review', axis=1)
def classify(self, title_text: str) -> str:
feat_dic = {word: 1
for word in word_tokenize(title_text)
} # make feature dictionary of one title
result = self.model.predict(self.DV.transform(
[feat_dic])) # predict with model
return result[0] # result = array(['คุณภาพชีวิต'], dtype='<U12')
def classify(self, title_text: str) -> str:
"""
tokenize title_text, get vectors of all words and append to the list
if there are only UNK, returns zero vector
title_text = 'ไปกินอาหาร'
> tokenized_title = ['ไป', 'กิน', 'อาหาร']
> vecs = [[11,2,5...], [3,-1,4...], [6,2,9...]]
> np.mean(vecs, axis=0) = [2,1,5,...]
"""
tokenized_title = word_tokenize(title_text)
vecs = [] # make the list of each word vector
for word in tokenized_title:
if word in self.wv.vocab: # append vector iff the word is in vocab
vecs.append(self.wv[word])
if vecs == []: # if there is no vector in list, return [0,0,...0]
mean = np.zeros((300))
else:
mean = np.mean(np.array(vecs),
axis=0) # calculate mean along column
result = self.model.predict_proba(np.array(
[mean])) # predict by model, result is 12 dim vector
return self.i_to_l[np.argmax(
result
)] # get the index of the most largest element, and convert to str
def tokenizer(text: str) -> List[str]:
"""
:meth: tokenize text with a frozen newmm engine
:param str text: text to tokenize
:return: tokenized text
"""
return word_tokenize(text, engine="ulmfit")
def find_place(text):
proc = word_tokenize(text.lower(), keep_whitespace=False)
listToStr = ' '.join(map(str, proc))
keyplace = keyword_processor2.extract_keywords(listToStr)
if (keyplace == []):
keyplace.append("-")
return keyplace[0]
def _json_to_instance(self, json_dict: JsonDict) -> Instance:
comment = json_dict['comment']
comment = comment.strip().replace('-', ' ')
comment = ' '.join(comment.split())
tokenized_comment = word_tokenize(comment)
instance = self._dataset_reader.text_to_instance(tokenized_comment=tokenized_comment)
return instance
def split_word(text):
# ตัดคำโดยใช้ dict ใน corpus ที่ผม edit ไป มันจะตัดเฉพาะเมนูอาหารที่ผมใส่ไปใน words.th.txt
tokens = word_tokenize(text, engine='dict')
# Remove stop words ภาษาไทย และภาษาอังกฤษ
tokens = [i for i in tokens if not i in th_stop and not i in en_stop]
# หารากศัพท์ภาษาไทย และภาษาอังกฤษ
# English
tokens = [p_stemmer.stem(i) for i in tokens]
# Thai
tokens_temp = []
for i in tokens:
w_syn = wordnet.synsets(i)
if (len(w_syn) > 0) and (len(w_syn[0].lemma_names('tha')) > 0):
tokens_temp.append(w_syn[0].lemma_names('tha')[0])
else:
tokens_temp.append(i)
tokens = tokens_temp
# ลบตัวเลข
tokens = [i for i in tokens if not i.isnumeric()]
# ลบช่องว่าง
tokens = [i for i in tokens if not ' ' in i]
return tokens
def tokenize(self, corpus, tokenizer=None):
tokenized_corpus = []
for sentence in corpus:
sentence = self.text_preprocess(sentence)
text = word_tokenize(sentence, keep_whitespace=False)
tokenized_corpus.append(list(text))
return tokenized_corpus
def predict(storyId, sentence):
tokenizedSentence = word_tokenize(sentence)
taggedToken = posTagger(sentence)
tagger = pycrfsuite.Tagger()
tagger.open("{}/{}.model".format(app.config["MODELS_DIR"], storyId))
predictedLabels = tagger.tag(sentToFeatures(taggedToken))
extractedEntities = extractEntities(zip(tokenizedSentence,
predictedLabels))
return extractedEntities
def sentence_vectorizer(ss,dim=300,use_mean=True):
s = word_tokenize(ss)
vec = np.zeros((1,dim))
for word in s:
if word in model.wv.index2word:
vec+= model.wv.word_vec(word)
else: pass
if use_mean: vec /= len(s)
return vec
def make_word_tokenize(datas):
list_data = []
for data in datas:
word_tokenized = word_tokenize(data, engine='deepcut')
list_data.append(word_tokenized)
return list_data
def stemmingSingle(self, text):
result = word_tokenize(text)
temp = result.copy()
t = ''
for i in result:
if (i == 'จังหวัด' or i == 'ที่'):
temp.remove(i)
return temp[0]
def get_intention(sentence):
data = word_tokenize(sentence)
data = to_index(data)
data = data[:49] + [0]*(49 - len(data))
data = np.array([data])
intention = intent.predict(data)
# print(intention)
intention = intention[0].argmax()
# print(intention)
return intention
def tokenize(self, message: Message, attribute: Text) -> List[Token]:
text = message.get(attribute)
if not self.case_sensitive:
text = text.lower()
words = word_tokenize(text)
if not words:
words = [text]
return self._convert_words_to_tokens(words, text)
def reply(intent, text, reply_token, id, disname, req):
if intent == 'รุ่นที่1_Akimoto Manatsu':
cut = []
cut = word_tokenize(text, keep_whitespace=False)
print(cut)
print(len(cut))
members_docs = db.collection(u'DataMembers').document(
u'tYybUVgsllwd5U2QliNE')
docs = members_docs.get().to_dict()
print(docs)
bday = members_docs.get(field_paths={'Bday'}).to_dict().get('Bday')
blood = members_docs.get(
field_paths={'BloodType'}).to_dict().get('BloodType')
gen = members_docs.get(field_paths={'Gen'}).to_dict().get('Gen')
height = members_docs.get(
field_paths={'Height'}).to_dict().get('Height')
zodiac = members_docs.get(
field_paths={'Zodiac'}).to_dict().get('Zodiac')
print(bday, blood, gen, height, zodiac)
for i in cut:
if (i == "เกิด" or i == "วันเกิด"):
text_message = TextSendMessage(
text='อะคิโมโตะ มานัตสึ รุ่นที่1 เกิดวันที่{}'.format(
bday))
elif (i == "เลือด" or i == "หมู่โลหิต"):
text_message = TextSendMessage(
text='อะคิโมโตะ มานัตสึ รุ่นที่1 กรุ๊ปเลือด{}'.format(
blood))
elif (i == "สูง" or i == "ส่วนสูง"):
text_message = TextSendMessage(
text='อะคิโมโตะ มานัตสึ รุ่นที่1 ส่วนสูง{}'.format(height))
elif (i == "ราศี"):
text_message = TextSendMessage(
text='อะคิโมโตะ มานัตสึ รุ่นที่1 ราศี{}'.format(zodiac))
#text_message = TextSendMessage(text='Akimoto Manatsu \nเกิดวันที่ {} ราศี {} กรุ๊ปเลือด {} '.format(bday, zodiac, blood))
line_bot_api.reply_message(reply_token, text_message)
if intent == 'A_Test':
# Note: Use of CollectionRef stream() is prefered to get()
#song_ref = db.collection(u'DataSongs').where(u'Sname', u'==', u'Guru Guru Curtain').stream()
doc_ref = db.collection(u'DataSongs').document(u'00Test')
doc = doc_ref.get().to_dict()
print(doc)
songName = doc['Sname']
albumName = doc['Aname']
linkSong = doc['SLink']
text_message = TextSendMessage(
text=
'ฟังเพลง Guru Guru Curtain จากอัลบั้ม{}\n คลิกเพื่อฟังเพลงที่Spotify{} '
.format(albumName, linkSong))
line_bot_api.reply_message(reply_token, text_message)
def dsegth():
m = Frame.m
txt = m.get()
seg = ptn.word_tokenize(txt, engine='deepcut')
print(seg)
pyperclip.copy(" ".join(seg))
root12 = tk.Tk()
root12.title('Result(DeepCutTH)')
label12 = tk.Label(root12, text=seg, font=16)
label12.pack(fill="x")
root12.mainloop()
def handle_message(event):
text = event.message.text
proc = word_tokenize(text, engine='newmm')
matching = [s for s in proc if ('กิน' in s) or (
'อาหาร' in s) or ('อะไร' in s)]
if len(matching) != 0:
i = randint(0, len(food_list)-1)
line_bot_api.reply_message(
event.reply_token, TextSendMessage(text=food_list[i]))
else:
line_bot_api.reply_message(event.reply_token, TextSendMessage(
text='ต้องการสุ่มอาหารหรือเปล่า หากต้องการสุ่ม พิมพ์ กินอะไรดี'))
def romanize(text: str) -> str:
"""
Rendering Thai words in the Latin alphabet or "romanization",
using the Royal Thai General System of Transcription (RTGS),
which is the official system published by the Royal Institute of Thailand.
ถอดเสียงภาษาไทยเป็นอักษรละติน
:param str text: Thai text to be romanized
:return: A string of Thai words rendered in the Latin alphabet.
"""
words = word_tokenize(text)
romanized_words = [_romanize(word) for word in words]
return "".join(romanized_words)
def candidate_words(self, stripped_input):
import pythainlp
tokens = pythainlp.word_tokenize(stripped_input)
return tokens
