def make_alt_verb_forms(parse):
word_forms = []
stem = extract_stem(parse)
if len(stem) < 2:
pass
else:
suffix = extract_suffix(parse)
prefix = extract_prefix(parse)
if suffix:
word_forms.append(prefix + stem)
if prefix:
for suf in vbd_suffixes:
word_forms.append(stem + suf)
for pre in [p for p in vbz_prefixes if p != prefix]:
word_forms.append(pre + stem)
if suffix:
for suf in [s for s in vbz_suff_inflec if s != suffix]:
word_forms.append(pre + stem + suf)
else:
for suf in vbz_suff_inflec:
word_forms.append(pre + stem + suf)
elif suffix and not prefix:
for suf in [s for s in vbd_suffixes if s != suffix]:
word_forms.append(stem + suf)
for pre in vbz_prefixes:
word_forms.append(pre + stem)
for suf in vbz_suff_inflec:
word_forms.append(pre + stem + suf)
return stem, set(word_forms)
def make_alt_unin_verb_forms(parse):
stem = extract_stem(parse)
word_forms = []
for pre in vbz_prefixes:
word_forms.append(pre + stem)
for suf in vbz_suff_inflec:
word_forms.append(pre + stem + suf)
for suf in vbd_suffixes:
word_forms.append(stem + suf)
return stem, set(word_forms)
def make_alt_affixed_pron_forms(parse):
"""P_PRO or C_P_PRO
prepositions = ['ل','ب', 'في', 'علي', 'من']
ind_obj_suffixes = ["ي", "نا", "ن", "كم", "و", "ه", "هو", "ها", "هم"]
P_PRO = oneOf(prepositions)("stem") + oneOf(ind_obj_suffixes)("suffix") + \
FollowedBy(endOfString)
"""
word_forms = []
prep = extract_stem(parse)
suffix = extract_suffix(parse)
if prep:
for ind_pro in [suff for suff in ind_obj_suffixes if suff != suffix]:
if ind_pro:
word_forms.append(prep + ind_pro)
return prep, set(word_forms)
def make_alt_unin_forms(parse):
freq_dict = {}
stem = extract_stem(parse)
# test if it's an uninflected verb
v_stem, verb_forms = make_alt_unin_verb_forms(parse)
ave_verb_freq = compute_ave_freq(verb_forms)
freq_dict['VBD'] = (ave_verb_freq, verb_forms)
# test if it's an uninflected noun
n_stem, noun_forms = make_alt_noun_forms(parse)
ave_noun_freq = compute_ave_freq(noun_forms)
freq_dict['UNIN_N'] = (ave_noun_freq, noun_forms)
chosen_word_type = max(freq_dict, key=freq_dict.get)
freq, word_forms = freq_dict.get(chosen_word_type)
return stem, set(word_forms)
def make_alt_ind_pron_forms(parse):
'''
'PRO' or 'C_PRO'
pronouns = ['انت', 'انتي', 'انا', 'اني', 'هو', 'هي', 'هوّ', 'هيّ'
'احنا', 'انتم', 'انتوما', 'هم', 'هما', 'هوم', 'هوما']
PRO = oneOf(pronouns)("stem")
'''
word_forms = []
stem = extract_stem(parse)
prefix = extract_prefix(parse)
suffix = extract_suffix(parse)
if stem:
for pronoun in [p for p in pronouns if p != stem and p[0] != 'ا']:
if prefix and suffix:
word_forms.append(prefix + pronoun + suffix)
if prefix and not suffix:
word_forms.append(prefix + pronoun)
return stem, set(word_forms)
def choose_best_parse(parse_dict, debug=False):
freq_dict = {}
for word_type, parse in parse_dict.items():
stem = extract_stem(parse)
short_stems = [
'C', 'PART_PRO', 'PRO', 'PREP_PRO', 'PREP', 'C_PART_PRO', 'C_PRO',
'C_P_PRO', 'C_P'
]
if not stem:
continue
if len(stem) < 2 and word_type not in short_stems:
continue
prefix = extract_prefix(parse)
if prefix and extract_prefix(
parse) == 'ال': # automatically return noun if has def art
return parse_dict[word_type], word_type
if 'UNINVBD' in word_type:
func = make_alt_unin_verb_forms
elif 'VB' in word_type:
func = make_alt_verb_forms
elif 'UNIN' in word_type:
func = make_alt_unin_forms
elif 'N' in word_type:
func = make_alt_noun_forms
elif 'PREP_PRO' in word_type:
func = make_alt_affixed_pron_forms
elif 'PRO' or 'C_PRO' in word_type:
func = make_alt_ind_pron_forms
else:
func = None
if func:
stem, word_forms = func(parse)
ave_freq = compute_ave_freq(word_forms)
freq_dict[word_type] = ave_freq
try:
chosen_word_type = max(freq_dict, key=freq_dict.get)
except: # if all are 0
chosen_word_type = 'UNIN'
return parse_dict[chosen_word_type], chosen_word_type
def make_alt_noun_forms(parse):
word_forms = []
fem = False
nisba = False
stem = extract_stem(parse)
if stem and stem[-1] == 'ة':
fem = True
fem_stem = stem[:-1] + 'ت'
if stem and stem[-1] == 'ي':
nisba = True
word_forms.append(stem + 'ة')
if len(stem) < 2:
pass
else:
suffix = extract_suffix(parse)
prefix = extract_prefix(parse)
# change taa marbuta to taa before adding suffixes
for suf in poss_suffixes:
if fem:
word_forms.append(fem_stem + suf)
else:
word_forms.append(stem + suf)
for pre in def_art + def_art_short:
word_forms.append(pre + stem)
if nisba: word_forms.append(pre + stem + 'ة')
if prefix:
try:
word_forms.remove(prefix + stem)
except ValueError:
pass
word_forms.append(stem)
if suffix:
try:
word_forms.remove(stem + suffix)
except ValueError:
pass
word_forms.append(stem)
return stem, set(word_forms)
def choose_best_stem_test(word, debug=False):
parse_dict = stemmer(word)
if debug: print("Parse dict is", parse_dict)
best_parse, word_type = choose_best_parse(parse_dict, debug=debug)
stem = extract_stem(best_parse)
return stem
def test_verb_stem_defined(self):
parse_dict = stemmer(self.vbd)
parse = parse_dict.get('VBD')
self.assertEqual('كتب', extract_stem(parse))
