def Get_root_word(self, body):
ArListem = ArabicLightStemmer()
word = body.split(u" ")
word_stem = list()
for w in word:
w_stem = ArListem.light_stem(w)
word_stem.append(ArListem.get_root())
body = " ".join(word_stem)
return body
def search_engine(search_id):
print("Input query: ", search_id)
hasilQuery = preprocessing_query(search_id)
print("Preprocessing query: ", hasilQuery[-1])
query = query_translation(hasilQuery)
print("Query translation: ", query)
ArListem = ArabicLightStemmer()
stem = ArListem.light_stem(query)
hasil = ArListem.get_root()
print("Stem: ", hasil)
exquery = request.POST.get('exquery', None)
print(exquery)
# Query Expansion
if(exquery=='Iya'):
print("Pakai Ekspansi Query")
# pass
token = wordpunct_tokenize(hasil)
query = []
for word in token:
pq = PredictorConfig.modelFT.wv.most_similar(word)
print(pq)
words = []
for i in range(4):
words.append(pq[i][0])
words.append(word)
print(words)
query.append(' '.join(words))
queries = []
queries.append(' '.join(query))
print("Query Expansion: ", queries)
hasil = queries[0]
query_vec = PredictorConfig.tfidf_vectorizer.transform([hasil])
print(query_vec)
results = cosine_similarity(PredictorConfig.tfidf_matrix,query_vec).reshape((-1,))
list_object = []
list_id = results.argsort()[-10:][::-1]
list_id = [x+1 for x in list_id]
for x in list_id:
list_object.append(Kitabs.objects.filter(id=x))
return list_object
def test_rooter(dataframe_result):
"""
"""
from pyarabic.arabrepr import arepr
#test with tashaphyne
from tashaphyne.stemming import ArabicLightStemmer
import rootslibclass
asl = ArabicLightStemmer()
rooter = rootslibclass.rootDict(algos=['rhyzome'])
# debug in rhyzome rooter
rooter.rhyzome_rooter.debug = True
#~ rooter = rootslibclass.rootDict()
df = dataframe_result
# avoid null roots
#~ total = df.size
total = len(df.index)
cpt = 0
for word, root in zip(df["word"], df["root"]):
root_list = root.split(';')
print((u"**********%s*********" % word).encode('utf8'))
asl.light_stem(word)
print((u"Start Word : %s" % asl.get_starword()).encode('utf8'))
word = re.sub(u"[%s]" % (araby.ALEF_MADDA), araby.HAMZA + araby.ALEF,
word)
asl.segment(word)
print(asl.get_segment_list())
seg_list = asl.get_segment_list()
starstem_list = []
affixa_list = asl.get_affix_list()
# stems prints
stems = [d['stem'] for d in affixa_list]
print("Stems: " + u' '.join(stems).encode('utf8'))
roots = [d['root'] for d in affixa_list]
print((u"Dafault roots: [%s] a %s" %
(asl.get_root(), u' '.join(roots))).encode('utf8'))
#~ root_result = rooter.choose_wazn_root(affixa_list, debug=True)
root_result = rooter.choose_root(word, affixa_list, debug=True)
#~ print(u"Test root",root_result.encode('utf8'), u"found root",root_result.encode('utf8'), root_result == root)
print((u" ".join([
u"Test root", root, u"found root", root_result,
str(root_result in root_list)
])).encode('utf8'))
if root_result in root_list:
cpt += 1
print("***** Percent %.2f%% [%d/%d]" % (cpt * 100.0 / total, cpt, total))
def stemmingـprocess(word):
# Initialize Arabic stemmer
arepr = pyarabic.arabrepr.ArabicRepr()
repr = arepr.repr
ArListem = ArabicLightStemmer()
if word in stem_not:
wordRoot = word
elif len(word) <= 3:
wordRoot = word
else:
# Stemming word
stem = ArListem.light_stem(word)
# Extract root
wordRoot = ArListem.get_root()
return wordRoot
def getStemmedText(self, text):
stemmedText = []
if self.lang == 1:
stemmer = nltk.stem.snowball.FrenchStemmer()
stemmedText = [
stemmer.stem(word) for word in text if word.isalpha()
]
else:
from tashaphyne.stemming import ArabicLightStemmer
ArListem = ArabicLightStemmer()
for word in text:
if word.isalpha():
stem = ArListem.light_stem(word)
root = ArListem.get_root()
stemmedText.append(root)
return stemmedText
def test_rooter_matrix(dataframe_result):
"""
"""
from pyarabic.arabrepr import arepr
#test with tashaphyne
from tashaphyne.stemming import ArabicLightStemmer
import rootslibclass
asl = ArabicLightStemmer()
rooter = rootslibclass.rootDict()
df = dataframe_result
total = df.size
cpt = 0
for word, root in zip(df["word"], df["root"]):
print((u"**********%s*********"%word).encode('utf8'))
asl.light_stem(word)
root_list = root.split(';')
print((u"Start Word : %s"%asl.get_starword()).encode('utf8'))
asl.segment(word)
print(asl.get_segment_list() )
seg_list = asl.get_segment_list()
starstem_list =[]
affixa_list = asl.get_affix_list()
# stems prints
stems = [ d['stem'] for d in affixa_list]
print("Stems: "+u' '.join(stems).encode('utf8'))
roots = [ d['root'] for d in affixa_list]
print((u"Dafault roots: [%s] a %s"%(asl.get_root(),u' '.join(roots))).encode('utf8'))
#~ root_result = rooter.choose_wazn_root(affixa_list, debug=True)
root_result = rooter.choose_root_matrix(word, affixa_list, debug=True)
#~ print(u"Test root",root_result.encode('utf8'), u"found root",root_result.encode('utf8'), root_result == root)
#~ print((u" ".join([u"Test root", root, u"found root",
#~ root_result, str(root_result == root)])).encode('utf8'))
#~ if root_result == root:
#~ cpt += 1
print((u" ".join([u"Test root", root, u"found root",
root_result, str(root_result in root_list)])).encode('utf8'))
if root_result in root_list:
cpt += 1
#~ print("***** Percent %.2f%%"%(cpt*100/total))
print("***** Percent %.2f%% [%d/%d]"%(cpt*100.0/total, cpt, total))
def test1(args):
word = u"لعلهم"
print(is_root(word))
word = u"علم"
print(is_root(word))
#test with tashaphyne
from tashaphyne.stemming import ArabicLightStemmer
asl = ArabicLightStemmer()
words = [
u'أفتضاربانني',
u'بأبأ',
u'يريدون',
u'يستطعن',
u'كتاب',
u"بالميدان",
u"بالأسيهم",
]
ext = extend_root(u"رم")
print("extende")
print(repr(ext).decode('unicode-escape').encode('utf8'))
for word in words:
print(u"**********%s*********" % word)
asl.light_stem(word)
asl.segment(word)
print(asl.get_segment_list())
seg_list = asl.get_segment_list()
starstem_list = []
for seg in seg_list:
left, right = seg
starstem_list.append(asl.get_starstem(left, right))
print("star stems")
print(u"\t".join(starstem_list)).encode('utf8')
filtered_starstem_list = filter(valid_starstem, starstem_list)
print("filtred star stem")
print(u"\t".join(filtered_starstem_list)).encode('utf8')
for st in starstem_list:
print(st, u"\t".join(valid_starstem(st)).encode('utf8'))
affixation_list = asl.get_affix_list()
stems = [d['stem'] for d in affixation_list]
print("Candidats stems%s" % u'\t'.join(stems))
for st in stems:
print(st, u"\t".join(valid_starstem(st)).encode('utf8'))
print(
repr(affixation_list).replace(
'},', '},\n').decode('unicode-escape').encode('utf8'))
print("reduce")
#~ affixation_list = filter(verify_affix, affixation_list)
print(
repr(affixation_list).replace(
'},', '},\n').decode('unicode-escape').encode('utf8'))
roots = [normalize_root(d['root']) for d in affixation_list]
print("Candidats %s" % u'\t'.join(roots))
# get uniq root
accepted = set(filter(is_root, roots))
print("accepted %s" % u'\t'.join(accepted))
if not accepted:
# try to extend roots
extended_roots = []
for x in roots:
extended_roots.extend(extend_root(x))
print("Candidats extended %s" % u'\t'.join(extended_roots))
accepted = set(filter(is_root, extended_roots))
print("accepted level2 %s" % u'\t'.join(accepted))
print('root %s' % asl.get_root())
#~ print repr(STAMP_DICT).replace('},','},\n').decode('unicode-escape').encode('utf8')
return 0
from tinydb import TinyDB, where
from tashaphyne.stemming import ArabicLightStemmer
ArListem = ArabicLightStemmer()
db = TinyDB('/json.json')
while True:
x = input('Input to search or "q" to quit:\n>>> ')
if x == 'q':
break
ArListem.light_stem(x)
x = ArListem.get_root()
data = db.search(where('name').matches('.*%s.*' % x))
for line in data:
print(line['name'] + ': ', end='')
print(line['value'])
print()
if not data:
print('Not found result')
data1.append(tweet.strip())
#print(data1[:10])
#tashfeen
data2 = []
import pyarabic.arabrepr
arepr = pyarabic.arabrepr.ArabicRepr()
repr = arepr.repr
from tashaphyne.stemming import ArabicLightStemmer
ArListem = ArabicLightStemmer()
for tx in texts:
tweet = ""
for a in word_tokenize(tx):
stem = ArListem.light_stem(a)
#tweet = tweet + ArListem.get_stem()+ " "
tweet = tweet + ArListem.get_root() + " "
data2.append(tweet.strip())
#print(data2[:10])
# create a dataframe using texts and lables
trainDF = pandas.DataFrame()
trainDF['tweet'] = texts
trainDF['class'] = labels
# split the dataset into training and validation datasets
train_x, valid_x, train_y, valid_y = model_selection.train_test_split(
trainDF['tweet'], trainDF['class'], test_size=0.2)
# create a count vectorizer object
count_vect = CountVectorizer(analyzer='word', token_pattern=r'\w{1,}')
count_vect.fit(trainDF['tweet'])
''' Created on 15 juin 2019 @author: KHALID-RAMI ''' # coding=utf8 import pyarabic.arabrepr from tashaphyne.stemming import ArabicLightStemmer arepr = pyarabic.arabrepr.ArabicRepr() repr = arepr.repr ArListem = ArabicLightStemmer() word = u'قال' stem = ArListem.light_stem(word) print(ArListem.get_stem()) print(ArListem.get_root()) print(ArListem.get_left()) print(ArListem.get_prefix(2)) print(ArListem.get_right()) print(ArListem.get_unvocalized())
from tinydb import TinyDB, Query
from tashaphyne.stemming import ArabicLightStemmer
import os
ArListem = ArabicLightStemmer()
dir_path = os.path.dirname(os.path.realpath(__file__))
sw = tuple(open('/home/ahmd/Downloads/python/stopword.txt').read().split())
text = open('/home/ahmd/Downloads/code/out.txt')
count = {}
xx = 1
for line in text:
if xx == 1000000:
break
for word in line.split():
print(xx)
xx += 1
if xx == 1000000:
break
if not word in sw:
ArListem.light_stem(word)
count[ArListem.get_root()] = count.get(ArListem.get_root(), 0) + 1
lis = []
for key, value in count.items():
lis.append({'name': key, 'value': value})
db = TinyDB('/home/ahmd/Downloads/code/bows.json')
db.purge_tables()
db.insert_multiple(lis)