def similar_word(model,sentence,word,named_entities,central_words): # find and print the most similar terms to a word val_sim_central_words=0.7 val_sim=0.7 if RepresentsInt(word) : an= number.ArNumbers() elif stopwords_classified.this_is_stop_word(word): return '' #Remove Stop words elif word in named_entities: return word #Option6--> fixé les entités nommées elif word in central_words: val_sim=val_sim_central_words; try: word_compose=word origin_word=word print origin_word word_compose=word_compose.split() if len(word_compose)>1: return word word=clean_str(word) most_similar = model.wv.most_similar( araby.normalize_ligature(word)) lst=[x for x,y in most_similar if y>val_sim] lst=[''.join(c for c in x if c not in string.punctuation) for x in lst] if len(lst)==0: return origin_word lst=[seg.stem(x) for x in lst] #option10 lst.insert(0,word) #insertion de mots source est obligatoire lst=set_list(lst) if len(lst)==1: return origin_word return '(' +(u' | '.join(lst))+')' #Option8--> similarwords to others using word2vec except Exception as e: return origin_word
def preprocess(sentence): sentence = araby.strip_tatweel(sentence) sentence = sentence.replace( araby.SMALL_ALEF+araby.ALEF_MAKSURA, araby.ALEF_MAKSURA) sentence = sentence.replace( araby.ALEF_MAKSURA+araby.SMALL_ALEF, araby.ALEF_MAKSURA) sentence = re.sub(ALEFAT_PATTERN, araby.ALEF, sentence) sentence = araby.normalize_ligature(sentence) sentence = araby.normalize_teh(sentence) sentence = araby.strip_tashkeel(sentence) sentence = re.sub(r'[^\d\w]', r' ', sentence) sentence = re.sub(r'( ){2,}', r'\1', sentence) return sentence
def create_arabic_node(self, cluster_name, label, **kwargs): """ Checks that label is an arabic string, removes tatweel and normalizes ligatures. Adds unvocalized_label. """ label = araby.normalize_ligature(araby.strip_tatweel(label)) label = label.replace(araby.SMALL_ALEF, "") if not araby.is_arabicstring(label): raise RuntimeError("'%s' is not an Arabic string" % label) if "unvocalized_label" not in kwargs: kwargs["unvocalized_label"] = araby.strip_tashkeel(label) return self.create_node(cluster_name, label, **kwargs)
def create_arabic_node(self, cluster_name, label, **kwargs): """ Checks that label is an arabic string, removes tatweel and normalizes ligatures. Adds unvocalized_label. """ label = araby.normalize_ligature(araby.strip_tatweel(label)) label = label.replace(araby.SMALL_ALEF, "") if not araby.is_arabicstring(label): raise RuntimeError("'%s' is not an Arabic string" % label) if "unvocalized_label" not in kwargs: kwargs["unvocalized_label"] = araby.strip_tashkeel(label) return self.create_node(cluster_name, label, **kwargs)
def clean_str(text): #Option1--> normalizing search = [u"أ",u"إ",u"آ",u"ة",u"_",u"-",u"/",u".",u"،",u" و ",u" يا ",u'"',u"ـ",u"'",u"ى",u"\\",u'\n', u'\t',u'"',u'?',u'؟',u'!'] replace = [u"ا",u"ا",u"ا",u"ه",u" ",u" ",u"",u"",u"",u" و",u" يا",u' " ',u"",u"",u"ي",u"",u' ', u' ',u' ',u' ',u' ',u' ! '] text=araby.normalize_ligature(text) text=unicodedata.normalize('NFKD',text) text=araby.strip_tashkeel(text)#remove tashkeel p_longation = re.compile(r'(.)\1+')#remove longation subst = r"\1\1" text = re.sub(p_longation, subst, text) text = text.replace(u'وو', u'و') text = text.replace(u'يي', u'ي') text = text.replace(u'اا', u'ا') for i in range(0, len(search)): text = text.replace(unicodedata.normalize('NFKD',search[i]), unicodedata.normalize('NFKD',replace[i])) #trim text = text.replace(u'ئ', u'ئ') text = text.strip() return text
def preprocess(sentences, stopwords, isStopword = False): """ This takes in an array of complete araic sentences, and performs th following operations on all of them: 1.) strips tashkeel 2.) strips harakat 3.) strips lastharaka 4.) strips tatweel 5.) Strips shadda 6.) normalize lam alef ligatures 7.) normalize hamza 8.) tokenize Returns a 2D martix, where each row represents normalized, tokens of each sentence """ #print("SENTENCE INDEX!!!", sentences[0]) output = [] for sentence in sentences: #print("Before Preprocessing:"+ sentence) #print(sentence) text = araby.strip_harakat(sentence) #print("TEXT!!!!", text) text = araby.strip_tashkeel(text) text = araby.strip_lastharaka(text) text = araby.strip_tatweel(text) text = araby.strip_shadda(text) text = araby.normalize_ligature(text) text = araby.normalize_hamza(text) text = clean_str(text) #print("After Preprocessing:"+ text) #print("----") #print(text) try: text = re.match(r'[^\\n\\s\\p{Latin}]+', text).group() tokens = araby.tokenize(text) if not isStopword: tokens = remove_stopwords(stopwords, tokens) tokens = [t for t in tokens if t != '\n'] output.append(tokens) except: pass return output
u"العربية", u"الْعَرَيِيّةُ الفصحى", u"غير مشكول", "Taha", ] word1 = u"" for word in word_list: print(word, '\t', end=" ") if araby.is_vocalized(word): print(' is vocalized', end=" ") if araby.is_vocalizedtext(word): print(' is vocalized text', end=" ") if araby.is_arabicword(word): print(' is valid word', end=" ") else: print("invalid arabic word", end=" ") print(' strip harakat', araby.strip_harakat(word), end=" ") print(' strip tashkeel', araby.strip_tashkeel(word), end=" ") print(' strip tatweel', araby.strip_tatweel(word), end=" ") print(' normalize ligature ', araby.normalize_ligature(word), end=" ") if araby.vocalizedlike(word, word1): print("vocalized_like", end=" ") print() word1 = word if araby.vocalizedlike(u"العربية", u"العرَبية"): print("vocalized_like", end=" ") word = u"الْعَرَيِيّةُ" word_list = [ u"الْعَرَيِيّةُ", u"العربية", u"الْعَرَيِيّةُ الفصحى", u"غير مشكول", "Taha", ] word1 = u"" for word in word_list:
if araby.is_weak(c): print ('weak'), if araby.is_moon(c): print ('moon'), if araby.is_sun(c):print ('sun'), print (araby.order(c)), print (); word=u"الْعَرَيِيّةُ" word_list=[ u"الْعَرَيِيّةُ", u"العربية", u"الْعَرَيِيّةُ الفصحى", u"غير مشكول", "Taha", u"سئل لأنه يؤم الإمام" ] word1=u"" for word in word_list: print (word) if araby.is_vocalized(word): print (' is vocalized') if araby.is_vocalizedtext(word): print (' is vocalized text') if araby.is_arabicword(word): print (' is valid word') else: print ("invalid arabic word") print (' strip harakat', araby.strip_harakat(word)) print (' strip tashkeel', araby.strip_tashkeel(word)) print (' strip tatweel',araby.strip_tatweel(word)) print (' normalize ligature ', araby.normalize_ligature(word)) print (' normalize hamza', araby.normalize_hamza(word)) if araby.vocalizedlike(word, word1): print ("vocalized_like") word1=word; if araby.vocalizedlike(u"العربية",u"العرَبية"): print ("vocalized_like")
u"غير مشكول", "Taha", ] word1=u"" for word in word_list: print word.encode('utf8'),'\t', if araby.is_vocalized(word): print ' is vocalized', ## if araby.isArabicstring(word): print ' iisArabicstring', ## else:print ' invalid arabicstring', if araby.is_vocalizedtext(word): print ' is vocalized text', if araby.is_arabicword(word): print ' is valid word', else: print "invalid arabic word", print ' strip harakat', araby.strip_harakat(word).encode('utf8'), print ' strip tashkeel', araby.strip_tashkeel(word).encode('utf8'), print ' strip tatweel',araby.strip_tatweel(word).encode('utf8'), print ' normalize ligature ', araby.normalize_ligature(word).encode('utf8'), if araby.vocalizedlike(word, word1): print "vocalized_like", print; word1=word; if araby.vocalizedlike(u"العربية",u"العرَبية"): print "vocalized_like", word=u"الْعَرَيِيّةُ" word_list=[ u"الْعَرَيِيّةُ", u"العربية", u"الْعَرَيِيّةُ الفصحى", u"غير مشكول", "Taha", ] word1=u"" for word in word_list: print word.encode('utf8'),'\t',
import numpy as np import pickle as pkl from pyarabic.araby import normalize_hamza, normalize_ligature trn_ids = np.load('data/wiki/ar/tmp/val_ids.npy') it = pkl.load(open('data/wiki/ar/tmp/itos.pkl', 'rb')) t = 'وإن لم يريد أن يفعل بي الفصحى' print(normalize_ligature(t))
def run_diac(gomla, dialect): sos = 'بدايةجملة' if dialect == 'ca' else 'بداية' eos = 'نهايةجملة' if dialect == 'ca' else 'نهاية' token_list_7 = LastNTokens(7, sos) fname = randint(0, 100000) with codecs.open(f'diacritizer/userdata/{dialect}/{fname}.fmt', mode='w', encoding='utf-8') as infile: gomla = strip_tatweel(araby.normalize_ligature(gomla)) # gomla_list = araby.tokenize(gomla.replace('_', '-'), conditions=araby.is_arabicrange, morphs=araby.strip_tashkeel) gomla_list = araby.tokenize(gomla.replace('_', '-'), morphs=araby.strip_tashkeel) # gomla_list = gomla.strip().split() for token in gomla_list: t = ' '.join(token) token_list_7.add_tokens_list(t, 0) infile.write(token_list_7.get_n_tokens() + '\n') else: token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') token_list_7.add_tokens_list(eos, 0) infile.write(token_list_7.get_n_tokens() + '\n') if dialect == 'ca': ca_runner.infer( f"diacritizer/userdata/ca/{fname}.fmt", predictions_file=f"diacritizer/userdata/ca/{fname}.rlt", checkpoint_path=None, log_time=False) elif dialect == 'msa': msa_runner.infer( f"diacritizer/userdata/msa/{fname}.fmt", predictions_file=f"diacritizer/userdata/msa/{fname}.rlt", checkpoint_path=None, log_time=False) elif dialect == 'tun': tn_runner.infer( f"diacritizer/userdata/tun/{fname}.fmt", predictions_file=f"diacritizer/userdata/tun/{fname}.rlt", checkpoint_path=None, log_time=False) elif dialect == 'mor': ma_runner.infer( f"diacritizer/userdata/mor/{fname}.fmt", predictions_file=f"diacritizer/userdata/mor/{fname}.rlt", checkpoint_path=None, log_time=False) with codecs.open(f'diacritizer/userdata/{dialect}/{fname}.rlt', mode='r', encoding='utf-8') as outfile: diacritized_tokens = list() counters = defaultdict(Counter) for i, line in enumerate(outfile): dtokens = line.strip().split(' _ ') # print(len(dtokens), dtokens) for j, _ in enumerate(dtokens): tk = dtokens[j - 1 - i % 7] if tk not in [eos, sos]: counters[j].update([tk]) if sum(counters[j].values()) >= 7: diacritized_tokens.append( counters[j].most_common(1)[0][0].replace(' ', '')) counters[j].clear() else: return ' '.join(diacritized_tokens)
u"غير مشكول", "Taha", ] word1 = u"" for word in word_list: print word.encode('utf8'), '\t', if araby.is_vocalized(word): print ' is vocalized', ## if araby.isArabicstring(word): print ' iisArabicstring', ## else:print ' invalid arabicstring', if araby.is_vocalizedtext(word): print ' is vocalized text', if araby.is_arabicword(word): print ' is valid word', else: print "invalid arabic word", print ' strip harakat', araby.strip_harakat(word).encode('utf8'), print ' strip tashkeel', araby.strip_tashkeel(word).encode('utf8'), print ' strip tatweel', araby.strip_tatweel(word).encode('utf8'), print ' normalize ligature ', araby.normalize_ligature(word).encode( 'utf8'), if araby.vocalizedlike(word, word1): print "vocalized_like", print word1 = word if araby.vocalizedlike(u"العربية", u"العرَبية"): print "vocalized_like", word = u"الْعَرَيِيّةُ" word_list = [ u"الْعَرَيِيّةُ", u"العربية", u"الْعَرَيِيّةُ الفصحى", u"غير مشكول", "Taha", ] word1 = u"" for word in word_list: print word.encode('utf8'), '\t',
def ar_tokenizer(t): return [ wordpunct_tokenize( normalize_ligature(normalize_hamza(strip_tashkeel(k)))) for k in t ]
import sys import pyarabic.araby as araby ayah = sys.argv[1] ayah = araby.strip_tatweel(ayah) ayah = araby.strip_tashkeel(ayah) ayah = araby.normalize_ligature(ayah) ayah = araby.normalize_hamza(ayah) ayah = araby.normalize_alef(ayah) ayah = araby.normalize_teh(ayah) ayah = ayah.replace("ے", "ى") print(ayah) sys.stdout.flush()