class Stemmer:
def __init__(self):
self.stemmer()
self.stopword()
def stemmer(self):
self.factory = StemmerFactory()
self.stemmer = self.factory.create_stemmer()
def stopword(self):
stop_factory = StopWordRemoverFactory().get_stop_words()
more_stopword = ['diatur', 'perjodohan', 'dengan', 'ia', 'bahwa', 'oleh', 'nya']
data = stop_factory + more_stopword
stop_factory = StopWordRemoverFactory()
dictionary = ArrayDictionary(data)
self.stopword = StopWordRemover(dictionary)
def stem(self, sentence = None):
sentence = self.stemmer.stem(sentence)
return sentence
def remove(self, sentence = None):
sentence = self.stopword.remove(sentence)
return sentence
def removeStopWord(query):
factory = StopWordRemoverFactory().get_stop_words()
more_stopword = ['!', '.', ',', '?']
data = factory + more_stopword
dic = ArrayDictionary(data)
stopword = StopWordRemover(dic)
return stopword.remove(query)
def stop_stem_remover(kalimat):
"""
membersihkan stop word dan melakukan seemming
input :
kalimat : kalimat di dalam corpus
return
kalimat
"""
#buang kata tidak terlalu penting
#factory = StopWordRemoverFactory()
stop_factory = StopWordRemoverFactory().get_stop_words()
add_stop_word = ['dkk', 'et', 'al', 'all'] #tambah manual stopwords
stop = stop_factory + add_stop_word
dicts = ArrayDictionary(stop)
all_stop = StopWordRemover(dicts)
kalimat = all_stop.remove(kalimat)
#stemming (menjadi kata dasar)
stemmerFactory = StemmerFactory()
stemmer = stemmerFactory.create_stemmer()
kalimat = stemmer.stem(kalimat)
return kalimat
def _processTweet(self, tweet):
punctuations = '''!()-![]{};:+'"\,<>./?@#$%^&*_~'''
tweet = tweet.lower() # convert text to lower-case
tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))', '',
tweet) # remove URLs
tweet = re.sub('@[^\s]+', '', tweet) # remove usernames
tweet = re.sub(r'#([^\s]+)', r'\1', tweet) # remove the # in #hashtag
tweet = "".join(
(char for char in tweet if char not in string.punctuation))
tweet = re.sub('\s+', ' ', tweet).strip()
tweet = re.sub(r"\d", "", tweet)
# Ambil Stopword bawaan
stop_factory = StopWordRemoverFactory().get_stop_words()
more_stopword = open("stopword.txt", "r").read().split()
# Merge stopword
data = stop_factory + more_stopword
dictionary = ArrayDictionary(data)
str = StopWordRemover(dictionary)
factory1 = StemmerFactory() #stemming factory
stemmer = factory1.create_stemmer() #buat stemming
#
tweet = str.remove(tweet)
# tweet = stemmer.stem(tweet) # stemming tweet
tweet = word_tokenize(
tweet) # remove repeated characters (helloooooooo into hello)
# return [word for word in tweet if word not in self._stopwords]
return tweet
def Stopword(doc):
stop_factory = StopWordRemoverFactory().get_stop_words()
more_stopword = ['ini', 'itu', 'the']
data = stop_factory + more_stopword
dictionary = ArrayDictionary(data)
data_str = StopWordRemover(dictionary)
dokumen = data_str.remove(doc)
return dokumen
def generateStopWords(pat, txt):
# Ambil Stopword bawaan
stop_factory = StopWordRemoverFactory().get_stop_words()
more_stopwords = [' ?', '?', ' .', '.', ' ,', ',']
# Merge stopword
data = stop_factory + more_stopwords
dictionary = ArrayDictionary(data)
str = StopWordRemover(dictionary)
temppat = str.remove(pat)
if (temppat == '' or temppat == None):
temppat = pat
temptxt = str.remove(txt)
if (temptxt == '' or temptxt == None):
temptxt = txt
return temppat, temptxt
def __stopward_removal(self, tokens):
stop_factory = StopWordRemoverFactory().get_stop_words()
more_stopword = ['dong', 'atuh', 'plis']
data = stop_factory + more_stopword
dictionary = ArrayDictionary(data)
str_remove = StopWordRemover(dictionary)
tokens = word_tokenize(str_remove.remove(' '.join(tokens)))
return tokens
def stopword(text):
# Ambil Stopword bawaan
stop_factory = StopWordRemoverFactory().get_stop_words()
print(stop_factory)
more_stopword = ['diatur', 'perjodohan']
# Merge stopword
data = stop_factory + more_stopword
dictionary = ArrayDictionary(data)
str = StopWordRemover(dictionary)
hasil = str.remove(text)
# print(hasil)
return hasil
def preprocess_sentence(self, q=""):
#tokenize, lower, stopword,stem
default_stopwords = StopWordRemoverFactory().get_stop_words()
additional_stopwords = [
"(", ")", "senin", "selasa", "rabu", "kamis", "jumat", "sabtu",
"minggu"
]
dictionary = ArrayDictionary(default_stopwords + additional_stopwords)
stopword = StopWordRemover(dictionary)
factory = StemmerFactory()
stemmer = factory.create_stemmer()
tokenizer = RegexpTokenizer(r'\w+')
res = " ".join(tokenizer.tokenize(q))
res = res.lower()
res = stopword.remove(res)
res = factory = stemmer.stem(res)
return res
def pre_processing(text):
stopwords = pd.read_csv('stopwordbahasa.csv', names=['stopword'])['stopword'].tolist()
stem = StemmerFactory()
stemmer = stem.create_stemmer()
factory = StopWordRemoverFactory()
stopword = StopWordRemover(ArrayDictionary(factory.get_stop_words() + stopwords))
clean_str = text.lower() # lowercase
clean_str = re.sub(r"(?:\@|#|https?\://)\S+", " ", clean_str) # eliminate username, url, hashtags
clean_str = re.sub(r'&', '', clean_str) # remove & as it equals &
clean_str = re.sub(r'[^\w\s]',' ', clean_str) # remove punctuation
clean_str = re.sub('[\s\n\t\r]+', ' ', clean_str) # remove extra space
clean_str = clean_str.strip() # trim
clean_str = " ".join([stemmer.stem(word) for word in clean_str.split()]) # stem
clean_str = stopword.remove(clean_str) # remove stopwords
return clean_str
def remove_stopwords(self,
csv_src="",
csv_dest="",
cols_to_clean=["KOMPETENSI"],
sep=";"):
#factory = StopWordRemoverFactory()
default_stopwords = StopWordRemoverFactory().get_stop_words()
additional_stopwords = [
"(", ")", "senin", "selasa", "rabu", "kamis", "jumat", "sabtu",
"minggu"
]
dictionary = ArrayDictionary(default_stopwords + additional_stopwords)
stopword = StopWordRemover(
dictionary
) #factory.create_stop_word_remover(dictionary = dictionary)
tokenizer = RegexpTokenizer(r'\w+')
df = pd.read_csv(csv_src, sep=sep)
for c in cols_to_clean:
df[c] = df[c].map(lambda x: " ".join(tokenizer.tokenize(x))
) #get only words without symbols
df[c] = df[c].map(lambda x: stopword.remove(x)) #remove stop words
df.to_csv(csv_dest, sep=sep, index=None)
print("lower %d rows" % len(df))
precision = precision_m(y_true, y_pred)
recall = recall_m(y_true, y_pred)
return 2*((precision*recall)/(precision+recall+K.epsilon()))
, f1_m, precision_m, recall_m
label = to_categorical(label)
print(label)
print(label.shape)
data2 = []
for x in data:
# casefolding
x2 = x.lower()
# punctuation proccess
bersih = x2.translate(str.maketrans("","",string.punctuation))
# stoppword use PYsastrawi
stop = stopword.remove(bersih)
# steeming use PYsastrawi
katadasar = stemmer.stem(stop)
# insert to another or new list with name data2
data2.append(katadasar)
# ready to proccess
X_train, X_test, y_train, y_test = train_test_split(
data2,
label,
test_size=0.2,
random_state=1)
# fit hanya berdasarkan data train
tokenizer.fit_on_texts(X_train)
# konversi train
except:
clean = stripped
letters_only = re.sub("[^a-zA-Z]", " ", clean)
lower_case = letters_only.lower()
words = tok.tokenize(lower_case)
#filtered_words = [stop_word.remove(words)]
return (" ".join(words)).strip()
clean_tweet_texts = []
for i in range(len(df_t)):
clean_tweet_texts.append(tweet_cleaner(df_t[i]))
clean_tweet = []
for i in range(len(clean_tweet_texts)):
clean_tweet.append(swr.remove(clean_tweet_texts[i]))
#load vocabulary
df['text'] = clean_tweet
v_pkl = 'df_text.pkl'
v_open = open(v_pkl, 'rb')
vocab = pickle.load(v_open)
#IMport TFid Vectorizer
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
Tfidf = TfidfVectorizer()
#vector vocabulary
vector = Tfidf.fit_transform(vocab)
#vector prediski
"Kepada, Yth. Pemerintah Provinsi DKI Jakarta, PT.Metaliska yang telah mendirikan bangunan di atas tanah bantaran kali (melanggar GSK) 300 m dari terminal bus Pulogadung arah ke Timur, mohon ditindak Terima kasih"
]
# kalimat = "Kepada, Yth. Pemerintah Provinsi DKI Jakarta, PT. Metaliska yang telah mendirikan bangunan di atas tanah bantaran kali (melanggar GSK) 300 m dari terminal bus Pulogadung arah ke Timur, mohon ditindak Terima kasih"
kalimat = input("Masukkan Kalimat : ")
print('==================================')
print('Kalimat awal/asli : ')
print(str(kalimat) + '\n')
print("1. hasil proses case folding :")
print(kalimat.lower())
print('\n')
print('2. hasil proses menghilangkan tanda baca/filtering :')
hasil = kalimat.lower().translate(str.maketrans("", "", string.punctuation))
print(hasil)
print('\n')
print("3. hasil proses stopping word : ")
stop = stopword.remove(hasil)
print(stop)
print('\n')
print('4. hasil proses steeming : ')
katadasar = stemmer.stem(stop)
print(katadasar)
print('\n')
print('5. hasil proses tokenisasi : ')
token = [katadasar]
tokenizer.fit_on_texts(token)
seq = tokenizer.texts_to_sequences(token)
print(tokenizer.word_index)
print(seq)
# print(kalimat.lower())
class TweetProcessing():
user_pattern = r'(?<=^|(?<=[^a-zA-Z0-9-_\.]))@([A-Za-z]+[A-Za-z0-9-_]+)'
url_pattern = r'(https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9]\.[^\s]{2,})'
url_pattern2 = r'https://t.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]\.[^\s]{2,}|www\.[a-zA-Z0-9]\.[^\s]{2,}'
#url_pattern = '^(http:\/\/www\.|https:\/\/www\.|http:\/\/|https:\/\/)?[a-z0-9]+([\-\.]{1}[a-z0-9]+)*\.[a-z]{2,5}(:[0-9]{1,5})?(\/.*)?$'
digit_pattern = r'^\d+\s|\s\d+\s|\s\d+$'
rt_pattern = r'RT\s:*'
additional_stopwords = [
'cc ', 'cc:', 'cc.', 'a', 'd', 'g', 'e', 'y', 'ga', 'gmn', 'tdk',
'nah', 'sih', 'blm', 'ni', 'di', 'sy', 'sya', 'rt', 'jl', 'jl.', 'jln',
'jln.', 'no', 'no.', 'dlm', 'tx', 'thx', 'he', 'd', 'k', 'sm'
]
def __init__(self, tweet):
self.tweet = tweet
stop_factory = StopWordRemoverFactory().get_stop_words()
stop_factory = stop_factory + self.additional_stopwords
dictionary = ArrayDictionary(stop_factory)
self.strword = StopWordRemover(dictionary)
def set_tweet(self, tweet):
self.tweet = tweet
def get_tweet(self, tweet):
return self.tweet
def clean_up_tweet_usernames(self):
return re.sub(self.user_pattern, '', self.tweet)
def clean_up_tweet_url(self):
self.tweet = re.sub(self.url_pattern, '', self.tweet)
self.tweet = self.tweet.replace("https://t.?", '')
self.tweet = self.tweet.replace("https://t?", '')
self.tweet = self.tweet.replace("https://?", '')
return re.sub(self.url_pattern2, '', self.tweet)
def clean_up_tweet_rt(self):
return re.sub(self.rt_pattern, '', self.tweet)
def clean_up_tweet_digits(self):
self.tweet = ''.join([i for i in str(self.tweet) if not i.isdigit()])
return self.tweet
#return re.sub(self.digit_pattern,'', self.tweet)
def remove_stop_words(self):
self.tweet = self.strword.remove(self.tweet)
return self.tweet
def stemming_tweet(self):
factory = StemmerFactory()
stemmer = factory.create_stemmer()
self.tweet = stemmer.stem(self.tweet)
return self.tweet
def clean_up_tweet(self):
self.tweet = self.tweet.lower()
#self.tweet = self.clean_up_tweet_usernames()
self.tweet = self.clean_up_tweet_url()
self.tweet = self.clean_up_tweet_rt()
self.tweet = self.clean_up_tweet_digits()
#self.tweet = self.tweet.replace('.',' ')
self.tweet = self.tweet.replace(',', ' ')
self.tweet = self.tweet.replace('?', '')
self.tweet = self.tweet.replace(' ', ' ')
self.tweet = self.stemming_tweet()
self.tweet = self.remove_stop_words()
self.tweet = self.tweet.translate(string.punctuation)
if self.tweet.startswith('"') and self.tweet.endswith('"'):
self.tweet = self.tweet[1:-1]
return self.tweet
class TextSummarization():
def __init__(self):
with open('./stopwords.txt') as f:
more_stopword=f.read().split('\n')
SWfactory = StopWordRemoverFactory()
stopword_data = ArrayDictionary(more_stopword+SWfactory.get_stop_words())
self.stopword = StopWordRemover(stopword_data)
def Preprocessing(self, text):
clean = re.sub("#[^\W]+|@[^\W]+|http[^*\s]+|<[^>]*>|[0-9]", '', text) #cleansing data
emoticons = re.findall('(?::|;|=)()(?:-)?(?:\)|\(|D|P)', clean)
text = (re.sub('[\W]+', ' ', clean.lower()) + #Case folding
' '.join(emoticons).replace('-', ''))
result=''
for kata in text.split():
stop = self.stopword.remove(kata) #Stopword
result += f"{stop} " if stop else ''
return result
def Summary(self, doc, preprocess=False):
doc_tokenizer = PunktSentenceTokenizer()
sentences_list = doc_tokenizer.tokenize(doc)
clean_sentences_list=[]
for sentence in sentences_list:
clean_sentences_list.append(self.Preprocessing(sentence))
cv = CountVectorizer()
cv_matrix = cv.fit_transform(clean_sentences_list if preprocess else sentences_list)
normal_matrix = TfidfTransformer().fit_transform(cv_matrix)
tfidf=normal_matrix.toarray()
res_graph = normal_matrix * normal_matrix.T # similaritas /adjacency matrix
nx_graph= from_scipy_sparse_matrix(res_graph)
pageranks = pagerank(nx_graph)
sentence_array = sorted(((pageranks[i], s) for i, s in enumerate(sentences_list)), reverse=True)
sentence_array = np.asarray(sentence_array)
rank_max = float(sentence_array[0][0])
rank_min = float(sentence_array[len(sentence_array) - 1][0])
temp_array = []
# Jika semua rank sama
# taking any sentence will give the summary, say the first sentence
flag = 0
if rank_max - rank_min == 0:
temp_array.append(0)
flag = 1
# If the sentence has different ranks
if flag != 1:
for i in range(0, len(sentence_array)):
temp_array.append((float(sentence_array[i][0]) - rank_min) / (rank_max - rank_min))
# Calculation of threshold:
# We take the mean value of normalized scores
# any sentence with the normalized score 0.2 more than the mean value is considered to be
threshold = (sum(temp_array) / len(temp_array))# + 0.2
# Separate out the sentences that satiasfy the criteria of having a score above the threshold
sentence_list = []
if len(temp_array) > 1:
for i in range(0, len(temp_array)):
if temp_array[i] > threshold:
sentence_list.append(sentence_array[i][1])
else:
sentence_list.append(sentence_array[0][1])
summary = " ".join(str(x) for x in sentence_list)
return summary
def api_echo():
if request.method == 'POST':
# create stemmer
factory = StemmerFactory()
stemmer = factory.create_stemmer()
factory = StopWordRemoverFactory()
more_stopword = []
# add stopword
with open('dataset/stopword.csv') as csvfile:
readCSV = csv.reader(csvfile, delimiter=',')
for row in readCSV:
more_stopword.append(row[0])
dictionary = ArrayDictionary(more_stopword)
str = StopWordRemover(dictionary)
newsTrainer = Trainer(tokenizer)
kesehatan = []
konsultasi = []
marketing = []
with open("dataset/kesehatan.txt", "r") as ins:
for line in ins:
kesehatan.append({
'text': line.rstrip(),
'category': 'kesehatan'
})
with open("dataset/konsultasi.txt", "r") as ins:
for line in ins:
konsultasi.append({
'text': line.rstrip(),
'category': 'konsultasi'
})
with open("dataset/marketing.txt", "r") as ins:
for line in ins:
marketing.append({
'text': line.rstrip(),
'category': 'marketing'
})
# You need to train the system passing each text one by one to the trainer module.
newsSet = kesehatan + konsultasi + marketing
for news in newsSet:
newsTrainer.train(news['text'], news['category'])
# When you have sufficient trained data, you are almost done and can start to use
# a classifier.
newsClassifier = Classifier(newsTrainer.data, tokenizer)
query = request.form['query'].encode("utf8")
#query = "Apa saja level bonus yang didapat bagi seorang agen?"
# stemming and remove stop word on Query
out = stemmer.stem(query)
out = str.remove(out)
classification = newsClassifier.classify(out)
# the classification variable holds the detected categories sorted
#return classification[0][0]
return jsonify(classification)