def __init__(self, source: 'MorphCollection' = None) -> None:
super().__init__()
self.__m_class = MorphClass()
self.__m_gender = MorphGender.UNDEFINED
self.__m_number = MorphNumber.UNDEFINED
self.__m_case = MorphCase()
self.__m_language = MorphLang()
self.__m_voice = MorphVoice.UNDEFINED
self.__m_need_recalc = True
self.__m_items = None
if (source is None):
return
for it in source.items:
mi = None
if (isinstance(it, MorphWordForm)):
wf = MorphWordForm()
wf.copy_from_word_form(Utils.asObjectOrNull(it, MorphWordForm))
mi = (wf)
else:
mi = MorphBaseInfo()
mi.copy_from(it)
if (self.__m_items is None):
self.__m_items = list()
self.__m_items.append(mi)
self.__m_class = MorphClass._new53(source.__m_class.value)
self.__m_gender = source.__m_gender
self.__m_case = MorphCase._new29(source.__m_case.value)
self.__m_number = source.__m_number
self.__m_language = MorphLang._new56(source.__m_language.value)
self.__m_voice = source.__m_voice
self.__m_need_recalc = False
def __process_proper_variants(self, word: str,
res: typing.List['MorphWordForm'],
geo: bool) -> None:
tn = self.m_root_reverce
tn0 = None
nodes_with_vars = None
for i in range(len(word) - 1, -1, -1):
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
ch = ord(word[i])
if (tn.nodes is None):
break
wrapnext34 = RefOutArgWrapper(None)
inoutres35 = Utils.tryGetValue(tn.nodes, ch, wrapnext34)
next0_ = wrapnext34.value
if (not inoutres35):
break
tn = next0_
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
if (tn.reverce_variants is not None):
if (nodes_with_vars is None):
nodes_with_vars = list()
nodes_with_vars.append(tn)
tn0 = tn
else:
i = -1
if (nodes_with_vars is None):
return
for j in range(len(nodes_with_vars) - 1, -1, -1):
tn = nodes_with_vars[j]
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
ok = False
for v in tn.reverce_variants:
if (geo and v.class0_.is_proper_geo):
pass
elif (not geo and v.class0_.is_proper_surname):
pass
else:
continue
r = MorphWordForm(v, word)
if (not MorphWordForm._has_morph_equals(res, r)):
r.undef_coef = v.coef
res.append(r)
ok = True
if (ok):
break
def __deserialize_item(self, stream: Stream) -> 'MorphBaseInfo':
from pullenti.ner.core.internal.SerializerHelper import SerializerHelper
ty = stream.readbyte()
res = (MorphBaseInfo() if ty == 0 else MorphWordForm())
res.class0_ = MorphClass._new53(
SerializerHelper.deserialize_short(stream))
res.case_ = MorphCase._new29(
SerializerHelper.deserialize_short(stream))
res.gender = Utils.valToEnum(
SerializerHelper.deserialize_short(stream), MorphGender)
res.number = Utils.valToEnum(
SerializerHelper.deserialize_short(stream), MorphNumber)
res.language = MorphLang._new56(
SerializerHelper.deserialize_short(stream))
if (ty == 0):
return res
wf = Utils.asObjectOrNull(res, MorphWordForm)
wf.normal_case = SerializerHelper.deserialize_string(stream)
wf.normal_full = SerializerHelper.deserialize_string(stream)
wf.undef_coef = SerializerHelper.deserialize_short(stream)
cou = SerializerHelper.deserialize_int(stream)
i = 0
while i < cou:
if (wf.misc is None):
wf.misc = MorphMiscInfo()
wf.misc.attrs.append(SerializerHelper.deserialize_string(stream))
i += 1
return res
def __process_proper_variants(self, word : str, res : typing.List['MorphWordForm'], geo : bool) -> None:
tn = self.m_root_reverce
nodes_with_vars = None
i = 0
for i in range(len(word) - 1, -1, -1):
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
ch = ord(word[i])
if (tn.nodes is None):
break
if (not ch in tn.nodes):
break
tn = tn.nodes[ch]
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
if (tn.reverce_variants is not None):
if (nodes_with_vars is None):
nodes_with_vars = list()
nodes_with_vars.append(tn)
else: i = -1
if (nodes_with_vars is None):
return
for j in range(len(nodes_with_vars) - 1, -1, -1):
tn = nodes_with_vars[j]
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
ok = False
for vr in tn.reverce_variants:
v = self.get_rule_var(vr.rule_id, vr.variant_id)
if (v is None):
continue
if (geo and v.class0_.is_proper_geo):
pass
elif (not geo and v.class0_.is_proper_surname):
pass
else:
continue
r = MorphWordForm(v, word, self.get_misc_info(v.misc_info_id))
if (not r._has_morph_equals(res)):
r.undef_coef = vr.coef
res.append(r)
ok = True
if (ok):
break
def process_result(self, res : typing.List['MorphWordForm'], word_begin : str, mvs : typing.List['MorphRuleVariant']) -> None:
for mv in mvs:
r = MorphWordForm(mv, None)
if (mv.normal_tail is not None and len(mv.normal_tail) > 0 and mv.normal_tail[0] != '-'):
r.normal_case = (word_begin + mv.normal_tail)
else:
r.normal_case = word_begin
if (mv.full_normal_tail is not None):
if (len(mv.full_normal_tail) > 0 and mv.full_normal_tail[0] != '-'):
r.normal_full = (word_begin + mv.full_normal_tail)
else:
r.normal_full = word_begin
if (not MorphWordForm._has_morph_equals(res, r)):
r.undef_coef = (0)
res.append(r)
def __init__(self, graph_: 'SemGraph') -> None:
self.graph = None
self.morph = MorphWordForm()
self.typ = SemObjectType.UNDEFINED
self.quantity = None
self.concept = None
self.attrs = list()
self.measure = MeasureKind.UNDEFINED
self.not0_ = False
self.tokens = list()
self.links_from = list()
self.links_to = list()
self.tag = None
self.graph = graph_
def __remove_items_morph_case(self, cas: 'MorphCase') -> None:
if (self.__m_items is None):
return
if (len(self.__m_items) == 0):
self.__m_case = ((self.__m_case) & cas)
for i in range(len(self.__m_items) - 1, -1, -1):
if (((self.__m_items[i].case_) & cas).is_undefined):
del self.__m_items[i]
self.__m_need_recalc = True
elif ((((self.__m_items[i].case_) & cas)) !=
self.__m_items[i].case_):
if (isinstance(self.__m_items[i], MorphWordForm)):
wf = MorphWordForm()
wf.copy_from_word_form(
Utils.asObjectOrNull(self.__m_items[i], MorphWordForm))
wf.case_ = (wf.case_) & cas
self.__m_items[i] = (wf)
else:
bi = MorphBaseInfo()
bi.copy_from(self.__m_items[i])
bi.case_ = (bi.case_) & cas
self.__m_items[i] = bi
self.__m_need_recalc = True
self.__m_need_recalc = True
def verb_morph(self) -> 'MorphWordForm':
""" Полное морф.информация (для глагола) """
if (self.__m_verb_morph is not None):
return self.__m_verb_morph
for f in self.morph.items:
if (f.class0_.is_verb and (isinstance(f, MorphWordForm)) and ((f.misc.person) & (MorphPerson.THIRD)) != (MorphPerson.UNDEFINED)):
if (f.normal_case.endswith("СЯ")):
return Utils.asObjectOrNull(f, MorphWordForm)
for f in self.morph.items:
if (f.class0_.is_verb and (isinstance(f, MorphWordForm)) and ((f.misc.person) & (MorphPerson.THIRD)) != (MorphPerson.UNDEFINED)):
return Utils.asObjectOrNull(f, MorphWordForm)
for f in self.morph.items:
if (f.class0_.is_verb and (isinstance(f, MorphWordForm))):
return Utils.asObjectOrNull(f, MorphWordForm)
for f in self.morph.items:
if (f.class0_.is_adjective and (isinstance(f, MorphWordForm))):
return Utils.asObjectOrNull(f, MorphWordForm)
if (self.__m_normal == "НЕТ"):
return MorphWordForm._new605(MorphClass.VERB, MorphMiscInfo())
return None
def get_word_base_info(word: str,
lang: 'MorphLang' = None,
is_case_nominative: bool = False,
in_dict_only: bool = False) -> 'MorphBaseInfo':
""" Получить для словоформы род\число\падеж
Args:
word(str): словоформа
lang(MorphLang): возможный язык
is_case_nominative(bool): исходное слово в именительном падеже (иначе считается падеж любым)
in_dict_only(bool): при true не строить гипотезы для несловарных слов
Returns:
MorphBaseInfo: базовая морфологическая информация
"""
mt = Morphology.__m_inner.run(word, False, lang, None, False)
bi = MorphWordForm()
cla = MorphClass()
if (mt is not None and len(mt) > 0):
for k in range(2):
ok = False
for wf in mt[0].word_forms:
if (k == 0):
if (not wf.is_in_dictionary):
continue
elif (wf.is_in_dictionary):
continue
if (is_case_nominative):
if (not wf.case_.is_nominative
and not wf.case_.is_undefined):
continue
cla.value |= wf.class0_.value
bi.gender = Utils.valToEnum((bi.gender) | (wf.gender),
MorphGender)
bi.case_ = (bi.case_) | wf.case_
bi.number = Utils.valToEnum((bi.number) | (wf.number),
MorphNumber)
if (wf.misc is not None and bi.misc is None):
bi.misc = wf.misc
ok = True
if (ok or in_dict_only):
break
bi.class0_ = cla
return bi
def __try_parse_en(first: 'Token', typ: 'NounPhraseParseAttr',
max_char_pos: int) -> 'NounPhraseToken':
if (first is None):
return None
items = None
has_article = False
has_prop = False
has_misc = False
if (first.previous is not None
and first.previous.morph.class0_.is_preposition
and (first.whitespaces_before_count < 3)):
has_prop = True
t = first
first_pass3048 = True
while True:
if first_pass3048: first_pass3048 = False
else: t = t.next0_
if (not (t is not None)): break
if (max_char_pos > 0 and t.begin_char > max_char_pos):
break
if (not t.chars.is_latin_letter):
break
if (t != first and t.whitespaces_before_count > 2):
if ((((typ) & (NounPhraseParseAttr.MULTILINES))) !=
(NounPhraseParseAttr.NO)):
pass
elif (MiscHelper.is_eng_article(t.previous)):
pass
else:
break
tt = Utils.asObjectOrNull(t, TextToken)
if (t == first and tt is not None):
if (MiscHelper.is_eng_article(tt)):
has_article = True
continue
if (isinstance(t, ReferentToken)):
if ((((typ) & (NounPhraseParseAttr.REFERENTCANBENOUN))) == (
NounPhraseParseAttr.NO)):
break
elif (tt is None):
break
if ((t.is_value("SO", None) and t.next0_ is not None
and t.next0_.is_hiphen) and t.next0_.next0_ is not None):
if (t.next0_.next0_.is_value("CALL", None)):
t = t.next0_.next0_
continue
mc = t.get_morph_class_in_dictionary()
if (mc.is_conjunction or mc.is_preposition):
break
if (mc.is_pronoun or mc.is_personal_pronoun):
if ((((typ) & (NounPhraseParseAttr.PARSEPRONOUNS))) == (
NounPhraseParseAttr.NO)):
break
elif (mc.is_misc):
if (t.is_value("THIS", None) or t.is_value("THAT", None)):
has_misc = True
if ((((typ) & (NounPhraseParseAttr.PARSEPRONOUNS))) == (
NounPhraseParseAttr.NO)):
break
is_adj = False
if (((has_article or has_prop or has_misc)) and items is None):
pass
elif (isinstance(t, ReferentToken)):
pass
else:
if (not mc.is_noun and not mc.is_adjective):
if (mc.is_undefined and has_article):
pass
elif (items is None and mc.is_undefined
and t.chars.is_capital_upper):
pass
elif (mc.is_pronoun):
pass
elif (tt.term.endswith("EAN")):
is_adj = True
elif (MiscHelper.is_eng_adj_suffix(tt.next0_)):
pass
else:
break
if (mc.is_verb):
if (t.next0_ is not None and t.next0_.morph.class0_.is_verb
and (t.whitespaces_after_count < 2)):
pass
elif (t.chars.is_capital_upper
and not MiscHelper.can_be_start_of_sentence(t)):
pass
elif ((t.chars.is_capital_upper and mc.is_noun and
(isinstance(t.next0_, TextToken)))
and t.next0_.chars.is_capital_upper):
pass
elif (isinstance(t, ReferentToken)):
pass
else:
break
if (items is None):
items = list()
it = NounPhraseItem(t, t)
if (mc.is_noun):
it.can_be_noun = True
if (mc.is_adjective or mc.is_pronoun or is_adj):
it.can_be_adj = True
items.append(it)
t = it.end_token
if (len(items) == 1):
if (MiscHelper.is_eng_adj_suffix(t.next0_)):
mc.is_noun = False
mc.is_adjective = True
t = t.next0_.next0_
if (items is None):
return None
noun = items[len(items) - 1]
res = NounPhraseToken(first, noun.end_token)
res.noun = (noun)
res.morph = MorphCollection()
for v in noun.end_token.morph.items:
if (v.class0_.is_verb):
continue
if (v.class0_.is_proper and noun.begin_token.chars.is_all_lower):
continue
if (isinstance(v, MorphWordForm)):
wf = MorphWordForm()
wf.copy_from_word_form(Utils.asObjectOrNull(v, MorphWordForm))
if (has_article and v.number != MorphNumber.SINGULAR):
wf.number = MorphNumber.SINGULAR
res.morph.add_item(wf)
else:
bi = MorphBaseInfo()
bi.copy_from(v)
if (has_article and v.number != MorphNumber.SINGULAR):
bi.number = MorphNumber.SINGULAR
res.morph.add_item(bi)
if (res.morph.items_count == 0 and has_article):
res.morph.add_item(
MorphBaseInfo._new192(MorphClass.NOUN, MorphNumber.SINGULAR))
i = 0
while i < (len(items) - 1):
res.adjectives.append(items[i])
i += 1
return res
def run(self, text: str, only_tokenizing: bool, dlang: 'MorphLang',
progress: EventHandler,
good_text: bool) -> typing.List['MorphToken']:
""" Произвести морфологический анализ текста
Args:
text(str): исходный текст
lang: язык (если null, то попробует определить)
Returns:
typing.List[MorphToken]: последовательность результирующих морфем
"""
if (Utils.isNullOrEmpty(text)):
return None
twr = TextWrapper(text, good_text)
twrch = twr.chars
res = list()
uni_lex = dict()
term0 = None
pure_rus_words = 0
pure_ukr_words = 0
pure_by_words = 0
pure_kz_words = 0
tot_rus_words = 0
tot_ukr_words = 0
tot_by_words = 0
tot_kz_words = 0
i = 0
first_pass2708 = True
while True:
if first_pass2708: first_pass2708 = False
else: i += 1
if (not (i < twr.length)): break
ty = InnerMorphology._getCharTyp(twrch[i])
if (ty == 0):
continue
if (ty > 2):
j = (i + 1)
else:
j = (i + 1)
while j < twr.length:
if (InnerMorphology._getCharTyp(twrch[j]) != ty):
break
j += 1
wstr = text[i:i + j - i]
term = None
if (good_text):
term = wstr
else:
trstr = LanguageHelper.transliteralCorrection(
wstr, term0, False)
term = LanguageHelper.correctWord(trstr)
if (Utils.isNullOrEmpty(term)):
i = (j - 1)
continue
lang = InnerMorphology.__detectLang(twr, i, j - 1, term)
if (lang == MorphLang.UA):
pure_ukr_words += 1
elif (lang == MorphLang.RU):
pure_rus_words += 1
elif (lang == MorphLang.BY):
pure_by_words += 1
elif (lang == MorphLang.KZ):
pure_kz_words += 1
if ((((lang) & MorphLang.RU)) != MorphLang.UNKNOWN):
tot_rus_words += 1
if ((((lang) & MorphLang.UA)) != MorphLang.UNKNOWN):
tot_ukr_words += 1
if ((((lang) & MorphLang.BY)) != MorphLang.UNKNOWN):
tot_by_words += 1
if ((((lang) & MorphLang.KZ)) != MorphLang.UNKNOWN):
tot_kz_words += 1
if (ty == 1):
term0 = term
lemmas = None
if (ty == 1 and not only_tokenizing):
wraplemmas7 = RefOutArgWrapper(None)
inoutres8 = Utils.tryGetValue(uni_lex, term, wraplemmas7)
lemmas = wraplemmas7.value
if (not inoutres8):
lemmas = InnerMorphology.UniLexWrap._new6(lang)
uni_lex[term] = lemmas
tok = MorphToken()
tok.term = term
tok.begin_char = i
if (i == 733860):
pass
tok.end_char = (j - 1)
tok.tag = (lemmas)
res.append(tok)
i = (j - 1)
def_lang = MorphLang(dlang)
if (pure_rus_words > pure_ukr_words and pure_rus_words > pure_by_words
and pure_rus_words > pure_kz_words):
def_lang = MorphLang.RU
elif (tot_rus_words > tot_ukr_words and tot_rus_words > tot_by_words
and tot_rus_words > tot_kz_words):
def_lang = MorphLang.RU
elif (pure_ukr_words > pure_rus_words
and pure_ukr_words > pure_by_words
and pure_ukr_words > pure_kz_words):
def_lang = MorphLang.UA
elif (tot_ukr_words > tot_rus_words and tot_ukr_words > tot_by_words
and tot_ukr_words > tot_kz_words):
def_lang = MorphLang.UA
elif (pure_kz_words > pure_rus_words and pure_kz_words > pure_ukr_words
and pure_kz_words > pure_by_words):
def_lang = MorphLang.KZ
elif (tot_kz_words > tot_rus_words and tot_kz_words > tot_ukr_words
and tot_kz_words > tot_by_words):
def_lang = MorphLang.KZ
elif (pure_by_words > pure_rus_words and pure_by_words > pure_ukr_words
and pure_by_words > pure_kz_words):
def_lang = MorphLang.BY
elif (tot_by_words > tot_rus_words and tot_by_words > tot_ukr_words
and tot_by_words > tot_kz_words):
if (tot_rus_words > 10 and tot_by_words > (tot_rus_words + 20)):
def_lang = MorphLang.BY
elif (tot_rus_words == 0 or tot_by_words >= (tot_rus_words * 2)):
def_lang = MorphLang.BY
if (((def_lang.is_undefined or def_lang.is_ua)) and tot_rus_words > 0):
if (((tot_ukr_words > tot_rus_words
and InnerMorphology.M_ENGINE_UA.language.is_ua))
or ((tot_by_words > tot_rus_words
and InnerMorphology.M_ENGINE_BY.language.is_by))
or ((tot_kz_words > tot_rus_words
and InnerMorphology.M_ENGINE_KZ.language.is_kz))):
cou0 = 0
tot_kz_words = 0
tot_ukr_words = tot_kz_words
tot_by_words = tot_ukr_words
tot_rus_words = tot_by_words
for kp in uni_lex.items():
lang = MorphLang()
wraplang9 = RefOutArgWrapper(lang)
kp[1].word_forms = self.__processOneWord(kp[0], wraplang9)
lang = wraplang9.value
if (kp[1].word_forms is not None):
for wf in kp[1].word_forms:
lang |= wf.language
kp[1].lang = lang
if (lang.is_ru):
tot_rus_words += 1
if (lang.is_ua):
tot_ukr_words += 1
if (lang.is_by):
tot_by_words += 1
if (lang.is_kz):
tot_kz_words += 1
if (lang.is_cyrillic):
cou0 += 1
if (cou0 >= 100):
break
if (tot_rus_words > ((math.floor(tot_by_words / 2)))
and tot_rus_words > ((math.floor(tot_ukr_words / 2)))):
def_lang = MorphLang.RU
elif (tot_ukr_words > ((math.floor(tot_rus_words / 2)))
and tot_ukr_words > ((math.floor(tot_by_words / 2)))):
def_lang = MorphLang.UA
elif (tot_by_words > ((math.floor(tot_rus_words / 2)))
and tot_by_words > ((math.floor(tot_ukr_words / 2)))):
def_lang = MorphLang.BY
elif (def_lang.is_undefined):
def_lang = MorphLang.RU
cou = 0
tot_kz_words = 0
tot_ukr_words = tot_kz_words
tot_by_words = tot_ukr_words
tot_rus_words = tot_by_words
for kp in uni_lex.items():
lang = def_lang
if (lang.is_undefined):
if (tot_rus_words > tot_by_words
and tot_rus_words > tot_ukr_words
and tot_rus_words > tot_kz_words):
lang = MorphLang.RU
elif (tot_ukr_words > tot_rus_words
and tot_ukr_words > tot_by_words
and tot_ukr_words > tot_kz_words):
lang = MorphLang.UA
elif (tot_by_words > tot_rus_words
and tot_by_words > tot_ukr_words
and tot_by_words > tot_kz_words):
lang = MorphLang.BY
elif (tot_kz_words > tot_rus_words
and tot_kz_words > tot_ukr_words
and tot_kz_words > tot_by_words):
lang = MorphLang.KZ
wraplang10 = RefOutArgWrapper(lang)
kp[1].word_forms = self.__processOneWord(kp[0], wraplang10)
lang = wraplang10.value
kp[1].lang = lang
if ((((lang) & MorphLang.RU)) != MorphLang.UNKNOWN):
tot_rus_words += 1
if ((((lang) & MorphLang.UA)) != MorphLang.UNKNOWN):
tot_ukr_words += 1
if ((((lang) & MorphLang.BY)) != MorphLang.UNKNOWN):
tot_by_words += 1
if ((((lang) & MorphLang.KZ)) != MorphLang.UNKNOWN):
tot_kz_words += 1
if (progress is not None):
self.__onProgress(cou, len(uni_lex), progress)
cou += 1
debug_token = None
empty_list = None
for r in res:
uni = Utils.asObjectOrNull(r.tag, InnerMorphology.UniLexWrap)
r.tag = None
if (uni is None or uni.word_forms is None
or len(uni.word_forms) == 0):
if (empty_list is None):
empty_list = list()
r.word_forms = empty_list
if (uni is not None):
r.language = uni.lang
else:
r.word_forms = uni.word_forms
if (r.begin_char == 733860):
debug_token = r
if (not good_text):
i = 0
first_pass2709 = True
while True:
if first_pass2709: first_pass2709 = False
else: i += 1
if (not (i < (len(res) - 2))): break
ui0 = twrch[res[i].begin_char]
ui1 = twrch[res[i + 1].begin_char]
ui2 = twrch[res[i + 2].begin_char]
if (ui1.is_quot):
p = res[i + 1].begin_char
if ((p >= 2 and "БбТт".find(text[p - 1]) >= 0 and
((p + 3) < len(text)))
and "ЕеЯяЁё".find(text[p + 1]) >= 0):
wstr = LanguageHelper.transliteralCorrection(
LanguageHelper.correctWord("{0}Ъ{1}".format(
res[i].getSourceText(text),
res[i + 2].getSourceText(text))), None, False)
li = self.__processOneWord0(wstr)
if (li is not None and len(li) > 0
and li[0].is_in_dictionary):
res[i].end_char = res[i + 2].end_char
res[i].term = wstr
res[i].word_forms = li
del res[i + 1:i + 1 + 2]
elif ((ui1.is_apos and p > 0 and str.isalpha(text[p - 1]))
and ((p + 1) < len(text))
and str.isalpha(text[p + 1])):
if (def_lang == MorphLang.UA
or (((res[i].language) & MorphLang.UA)) !=
MorphLang.UNKNOWN
or (((res[i + 2].language) & MorphLang.UA)) !=
MorphLang.UNKNOWN):
wstr = LanguageHelper.transliteralCorrection(
LanguageHelper.correctWord("{0}{1}".format(
res[i].getSourceText(text),
res[i + 2].getSourceText(text))), None,
False)
li = self.__processOneWord0(wstr)
okk = True
if (okk):
res[i].end_char = res[i + 2].end_char
res[i].term = wstr
if (li is None):
li = list()
res[i].word_forms = li
if (li is not None and len(li) > 0):
res[i].language = li[0].language
del res[i + 1:i + 1 + 2]
elif (((ui1.uni_char == '3' or ui1.uni_char == '4'))
and res[i + 1].length == 1):
src = ("З" if ui1.uni_char == '3' else "Ч")
i0 = i + 1
if ((res[i].end_char + 1) == res[i + 1].begin_char
and ui0.is_cyrillic):
i0 -= 1
src = (res[i0].getSourceText(text) + src)
i1 = i + 1
if ((res[i + 1].end_char + 1) == res[i + 2].begin_char
and ui2.is_cyrillic):
i1 += 1
src += res[i1].getSourceText(text)
if (len(src) > 2):
wstr = LanguageHelper.transliteralCorrection(
LanguageHelper.correctWord(src), None, False)
li = self.__processOneWord0(wstr)
if (li is not None and len(li) > 0
and li[0].is_in_dictionary):
res[i0].end_char = res[i1].end_char
res[i0].term = wstr
res[i0].word_forms = li
del res[i0 + 1:i0 + 1 + i1 - i0]
elif ((ui1.is_hiphen and ui0.is_letter and ui2.is_letter)
and res[i].end_char > res[i].begin_char
and res[i + 2].end_char > res[i + 2].begin_char):
newline = False
sps = 0
j = (res[i + 1].end_char + 1)
while j < res[i + 2].begin_char:
if (text[j] == '\r' or text[j] == '\n'):
newline = True
sps += 1
elif (not Utils.isWhitespace(text[j])):
break
else:
sps += 1
j += 1
full_word = LanguageHelper.correctWord(
res[i].getSourceText(text) +
res[i + 2].getSourceText(text))
if (not newline):
if (full_word in uni_lex or full_word == "ИЗЗА"):
newline = True
elif (text[res[i + 1].begin_char] == (chr(0x00AD))):
newline = True
elif (LanguageHelper.endsWithEx(
res[i].getSourceText(text), "О", "о", None,
None) and len(res[i + 2].word_forms) > 0
and res[i + 2].word_forms[0].is_in_dictionary):
if (text[res[i + 1].begin_char] == '¬'):
li = self.__processOneWord0(full_word)
if (li is not None and len(li) > 0
and li[0].is_in_dictionary):
newline = True
elif ((res[i].end_char + 2) == res[i + 2].begin_char):
if (not str.isupper(text[res[i + 2].begin_char])
and (sps < 2) and len(full_word) > 4):
newline = True
if ((i + 3) < len(res)):
ui3 = twrch[res[i + 3].begin_char]
if (ui3.is_hiphen):
newline = False
elif (((res[i].end_char + 1) == res[i + 1].begin_char
and sps > 0 and (sps < 3))
and len(full_word) > 4):
newline = True
if (newline):
li = self.__processOneWord0(full_word)
if (li is not None and len(li) > 0
and ((li[0].is_in_dictionary
or full_word in uni_lex))):
res[i].end_char = res[i + 2].end_char
res[i].term = full_word
res[i].word_forms = li
del res[i + 1:i + 1 + 2]
else:
pass
elif ((ui1.is_letter and ui0.is_letter and res[i].length > 2)
and res[i + 1].length > 1):
if (ui0.is_upper != ui1.is_upper):
continue
if (not ui0.is_cyrillic or not ui1.is_cyrillic):
continue
newline = False
j = (res[i].end_char + 1)
while j < res[i + 1].begin_char:
if (twrch[j].code == 0xD or twrch[j].code == 0xA):
newline = True
break
j += 1
if (not newline):
continue
full_word = LanguageHelper.correctWord(
res[i].getSourceText(text) +
res[i + 1].getSourceText(text))
if (not full_word in uni_lex):
continue
li = self.__processOneWord0(full_word)
if (li is not None and len(li) > 0
and li[0].is_in_dictionary):
res[i].end_char = res[i + 1].end_char
res[i].term = full_word
res[i].word_forms = li
del res[i + 1]
i = 0
first_pass2710 = True
while True:
if first_pass2710: first_pass2710 = False
else: i += 1
if (not (i < len(res))): break
mt = res[i]
mt.char_info = CharsInfo()
ui0 = twrch[mt.begin_char]
ui00 = UnicodeInfo.ALL_CHARS[ord((res[i].term[0]))]
j = (mt.begin_char + 1)
while j <= mt.end_char:
if (ui0.is_letter):
break
ui0 = twrch[j]
j += 1
if (ui0.is_letter):
res[i].char_info.is_letter = True
if (ui00.is_latin):
res[i].char_info.is_latin_letter = True
elif (ui00.is_cyrillic):
res[i].char_info.is_cyrillic_letter = True
if (res[i].language == MorphLang.UNKNOWN):
if (LanguageHelper.isCyrillic(mt.term)):
res[i].language = (MorphLang.RU if
def_lang.is_undefined else def_lang)
if (good_text):
continue
all_up = True
all_lo = True
j = mt.begin_char
while j <= mt.end_char:
if (twrch[j].is_upper or twrch[j].is_digit):
all_lo = False
else:
all_up = False
j += 1
if (all_up):
mt.char_info.is_all_upper = True
elif (all_lo):
mt.char_info.is_all_lower = True
elif (((ui0.is_upper or twrch[mt.begin_char].is_digit))
and mt.end_char > mt.begin_char):
all_lo = True
j = (mt.begin_char + 1)
while j <= mt.end_char:
if (twrch[j].is_upper or twrch[j].is_digit):
all_lo = False
break
j += 1
if (all_lo):
mt.char_info.is_capital_upper = True
elif (twrch[mt.end_char].is_lower
and (mt.end_char - mt.begin_char) > 1):
all_up = True
j = mt.begin_char
while j < mt.end_char:
if (twrch[j].is_lower):
all_up = False
break
j += 1
if (all_up):
mt.char_info.is_last_lower = True
if (mt.char_info.is_last_lower and mt.length > 2
and mt.char_info.is_cyrillic_letter):
pref = text[mt.begin_char:mt.begin_char + mt.end_char -
mt.begin_char]
ok = False
for wf in mt.word_forms:
if (wf.normal_case == pref or wf.normal_full == pref):
ok = True
break
if (not ok):
mt.word_forms = list(mt.word_forms)
mt.word_forms.insert(
0, MorphWordForm._new11(pref, MorphClass.NOUN, 1))
if (good_text or only_tokenizing):
return res
i = 0
first_pass2711 = True
while True:
if first_pass2711: first_pass2711 = False
else: i += 1
if (not (i < len(res))): break
if (res[i].length == 1 and res[i].char_info.is_latin_letter):
ch = res[i].term[0]
if (ch == 'C' or ch == 'A' or ch == 'P'):
pass
else:
continue
is_rus = False
for ii in range(i - 1, -1, -1):
if ((res[ii].end_char + 1) != res[ii + 1].begin_char):
break
elif (res[ii].char_info.is_letter):
is_rus = res[ii].char_info.is_cyrillic_letter
break
if (not is_rus):
ii = i + 1
while ii < len(res):
if ((res[ii - 1].end_char + 1) != res[ii].begin_char):
break
elif (res[ii].char_info.is_letter):
is_rus = res[ii].char_info.is_cyrillic_letter
break
ii += 1
if (is_rus):
res[i].term = LanguageHelper.transliteralCorrection(
res[i].term, None, True)
res[i].char_info.is_cyrillic_letter = True
res[i].char_info.is_latin_letter = True
for r in res:
if (r.char_info.is_all_upper or r.char_info.is_capital_upper):
if (r.language.is_cyrillic):
ok = False
for wf in r.word_forms:
if (wf.class0_.is_proper_surname):
ok = True
break
if (not ok):
r.word_forms = list(r.word_forms)
InnerMorphology.M_ENGINE_RU.processSurnameVariants(
r.term, r.word_forms)
for r in res:
for mv in r.word_forms:
if (mv.normal_case is None):
mv.normal_case = r.term
i = 0
while i < (len(res) - 2):
if (res[i].char_info.is_latin_letter
and res[i].char_info.is_all_upper and res[i].length == 1):
if (twrch[res[i + 1].begin_char].is_quot
and res[i + 2].char_info.is_latin_letter
and res[i + 2].length > 2):
if ((res[i].end_char + 1) == res[i + 1].begin_char and
(res[i + 1].end_char + 1) == res[i + 2].begin_char):
wstr = "{0}{1}".format(res[i].term, res[i + 2].term)
li = self.__processOneWord0(wstr)
if (li is not None):
res[i].word_forms = li
res[i].end_char = res[i + 2].end_char
res[i].term = wstr
if (res[i + 2].char_info.is_all_lower):
res[i].char_info.is_all_upper = False
res[i].char_info.is_capital_upper = True
elif (not res[i + 2].char_info.is_all_upper):
res[i].char_info.is_all_upper = False
del res[i + 1:i + 1 + 2]
i += 1
i = 0
first_pass2712 = True
while True:
if first_pass2712: first_pass2712 = False
else: i += 1
if (not (i < (len(res) - 1))): break
if (not res[i].char_info.is_letter
and not res[i + 1].char_info.is_letter
and (res[i].end_char + 1) == res[i + 1].begin_char):
if (twrch[res[i].begin_char].is_hiphen
and twrch[res[i + 1].begin_char].is_hiphen):
if (i == 0 or not twrch[res[i - 1].begin_char].is_hiphen):
pass
else:
continue
if ((i + 2) == len(res)
or not twrch[res[i + 2].begin_char].is_hiphen):
pass
else:
continue
res[i].end_char = res[i + 1].end_char
del res[i + 1]
return res
def __try_parse_ru(t: 'Token', can_be_partition: bool,
can_be_adj_partition: bool,
force_parse: bool) -> 'VerbPhraseToken':
res = None
t0 = t
not0_ = None
has_verb = False
verb_be_before = False
prep = None
first_pass3070 = True
while True:
if first_pass3070: first_pass3070 = False
else: t = t.next0_
if (not (t is not None)): break
if (not (isinstance(t, TextToken))):
break
tt = Utils.asObjectOrNull(t, TextToken)
is_participle = False
if (tt.term == "НЕ"):
not0_ = t
continue
ty = 0
norm = None
mc = tt.get_morph_class_in_dictionary()
if (tt.term == "НЕТ"):
if (has_verb):
break
ty = 1
elif (tt.term == "ДОПУСТИМО"):
ty = 3
elif (mc.is_adverb and not mc.is_verb):
ty = 2
elif (tt.is_pure_verb or tt.is_verb_be):
ty = 1
if (has_verb):
if (not tt.morph.contains_attr("инф.", None)):
if (verb_be_before):
pass
else:
break
elif (mc.is_verb):
if (mc.is_preposition or mc.is_misc or mc.is_pronoun):
pass
elif (mc.is_noun):
if (tt.term == "СТАЛИ" or tt.term == "СТЕКЛО"
or tt.term == "БЫЛИ"):
ty = 1
elif (not tt.chars.is_all_lower
and not MiscHelper.can_be_start_of_sentence(tt)):
ty = 1
elif (mc.is_adjective and can_be_partition):
ty = 1
elif (force_parse):
ty = 1
elif (mc.is_proper):
if (tt.chars.is_all_lower):
ty = 1
else:
ty = 1
if (mc.is_adjective):
is_participle = True
if (not tt.morph.case_.is_undefined):
is_participle = True
if (not can_be_partition and is_participle):
break
if (has_verb):
if (tt.morph.contains_attr("инф.", None)):
pass
elif (not is_participle):
pass
else:
break
elif ((mc.is_adjective and tt.morph.contains_attr("к.ф.", None)
and tt.term.endswith("О")) and NounPhraseHelper.try_parse(
tt, NounPhraseParseAttr.NO, 0, None) is None):
ty = 2
elif (mc.is_adjective
and ((can_be_partition or can_be_adj_partition))):
if (tt.morph.contains_attr("к.ф.", None)
and not can_be_adj_partition):
break
norm = tt.get_normal_case_text(MorphClass.ADJECTIVE,
MorphNumber.SINGULAR,
MorphGender.MASCULINE, False)
if (norm.endswith("ЙШИЙ")):
pass
else:
grs = DerivateService.find_derivates(norm, True, None)
if (grs is not None and len(grs) > 0):
hverb = False
hpart = False
for gr in grs:
for w in gr.words:
if (w.class0_.is_adjective
and w.class0_.is_verb):
if (w.spelling == norm):
hpart = True
elif (w.class0_.is_verb):
hverb = True
if (hpart and hverb):
ty = 3
elif (can_be_adj_partition):
ty = 3
if (ty != 3 and not Utils.isNullOrEmpty(grs[0].prefix)
and norm.startswith(grs[0].prefix)):
hverb = False
hpart = False
norm1 = norm[len(grs[0].prefix):]
grs = DerivateService.find_derivates(
norm1, True, None)
if (grs is not None and len(grs) > 0):
for gr in grs:
for w in gr.words:
if (w.class0_.is_adjective
and w.class0_.is_verb):
if (w.spelling == norm1):
hpart = True
elif (w.class0_.is_verb):
hverb = True
if (hpart and hverb):
ty = 3
if (ty == 0 and t == t0 and can_be_partition):
prep = PrepositionHelper.try_parse(t)
if (prep is not None):
t = prep.end_token
continue
if (ty == 0):
break
if (res is None):
res = VerbPhraseToken(t0, t)
res.end_token = t
it = VerbPhraseItemToken._new603(t, t, MorphCollection(t.morph))
if (not0_ is not None):
it.begin_token = not0_
it.not0_ = True
not0_ = (None)
it.is_adverb = ty == 2
if (prep is not None and not t.morph.case_.is_undefined
and len(res.items) == 0):
if (((prep.next_case) & t.morph.case_).is_undefined):
return None
it.morph.remove_items(prep.next_case, False)
res.preposition = prep
if (norm is None):
norm = t.get_normal_case_text(
(MorphClass.ADJECTIVE if ty == 3 else
(MorphClass.ADVERB if ty == 2 else MorphClass.VERB)),
MorphNumber.SINGULAR, MorphGender.MASCULINE, False)
if (ty == 1 and not tt.morph.case_.is_undefined):
mi = MorphWordForm._new604(MorphCase.NOMINATIVE,
MorphNumber.SINGULAR,
MorphGender.MASCULINE)
for mit in tt.morph.items:
if (isinstance(mit, MorphWordForm)):
mi.misc = mit.misc
break
nnn = MorphologyService.get_wordform("КК" + t.term, mi)
if (nnn is not None):
norm = nnn[2:]
it.normal = norm
res.items.append(it)
if (not has_verb and ((ty == 1 or ty == 3))):
res.morph = it.morph
has_verb = True
if (ty == 1 or ty == 3):
if (ty == 1 and tt.is_verb_be):
verb_be_before = True
else:
verb_be_before = False
if (not has_verb):
return None
for i in range(len(res.items) - 1, 0, -1):
if (res.items[i].is_adverb):
del res.items[i]
res.end_token = res.items[i - 1].end_token
else:
break
return res
def process(self, word : str) -> typing.List['MorphWordForm']:
if (Utils.isNullOrEmpty(word)):
return None
res = None
i = 0
if (len(word) > 1):
i = 0
while i < len(word):
ch = word[i]
if (LanguageHelper.is_cyrillic_vowel(ch) or LanguageHelper.is_latin_vowel(ch)):
break
i += 1
if (i >= len(word)):
return res
mvs = [ ]
tn = self.m_root
i = 0
while i <= len(word):
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
if (tn.rule_ids is not None):
word_begin = None
word_end = None
if (i == 0):
word_end = word
elif (i < len(word)):
word_end = word[i:]
else:
word_end = ""
if (res is None):
res = list()
for rid in tn.rule_ids:
r = self.get_rule(rid)
mvs = r.get_vars(word_end)
if (mvs is None):
continue
if (word_begin is None):
if (i == len(word)):
word_begin = word
elif (i > 0):
word_begin = word[0:0+i]
else:
word_begin = ""
self.__process_result(res, word_begin, mvs)
if (tn.nodes is None or i >= len(word)):
break
ch = ord(word[i])
wraptn9 = RefOutArgWrapper(None)
inoutres10 = Utils.tryGetValue(tn.nodes, ch, wraptn9)
tn = wraptn9.value
if (not inoutres10):
break
i += 1
need_test_unknown_vars = True
if (res is not None):
for r in res:
if ((r.class0_.is_pronoun or r.class0_.is_noun or r.class0_.is_adjective) or (r.class0_.is_misc and r.class0_.is_conjunction) or r.class0_.is_preposition):
need_test_unknown_vars = False
elif (r.class0_.is_adverb and r.normal_case is not None):
if (not LanguageHelper.ends_with_ex(r.normal_case, "О", "А", None, None)):
need_test_unknown_vars = False
elif (r.normal_case == "МНОГО"):
need_test_unknown_vars = False
elif (r.class0_.is_verb and len(res) > 1):
ok = False
for rr in res:
if (rr != r and rr.class0_ != r.class0_):
ok = True
break
if (ok and not LanguageHelper.ends_with(word, "ИМ")):
need_test_unknown_vars = False
if (need_test_unknown_vars and LanguageHelper.is_cyrillic_char(word[0])):
gl = 0
sog = 0
j = 0
while j < len(word):
if (LanguageHelper.is_cyrillic_vowel(word[j])):
gl += 1
else:
sog += 1
j += 1
if ((gl < 2) or (sog < 2)):
need_test_unknown_vars = False
if (need_test_unknown_vars and res is not None and len(res) == 1):
if (res[0].class0_.is_verb):
if ("н.вр." in res[0].misc.attrs and "нес.в." in res[0].misc.attrs and not "страд.з." in res[0].misc.attrs):
need_test_unknown_vars = False
elif ("б.вр." in res[0].misc.attrs and "сов.в." in res[0].misc.attrs):
need_test_unknown_vars = False
elif ("инф." in res[0].misc.attrs and "сов.в." in res[0].misc.attrs):
need_test_unknown_vars = False
elif (res[0].normal_case is not None and LanguageHelper.ends_with(res[0].normal_case, "СЯ")):
need_test_unknown_vars = False
if (res[0].class0_.is_undefined and "прдктв." in res[0].misc.attrs):
need_test_unknown_vars = False
if (need_test_unknown_vars):
if (self.m_root_reverce is None):
return res
tn = self.m_root_reverce
tn0 = self.m_root_reverce
for i in range(len(word) - 1, -1, -1):
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
ch = ord(word[i])
if (tn.nodes is None):
break
if (not ch in tn.nodes):
break
tn = tn.nodes[ch]
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
if (tn.reverce_variants is not None):
tn0 = tn
break
else: i = -1
if (tn0 != self.m_root_reverce):
glas = i < 4
while i >= 0:
if (LanguageHelper.is_cyrillic_vowel(word[i]) or LanguageHelper.is_latin_vowel(word[i])):
glas = True
break
i -= 1
if (glas):
for mvref in tn0.reverce_variants:
mv = self.get_rule_var(mvref.rule_id, mvref.variant_id)
if (mv is None):
continue
if (((not mv.class0_.is_verb and not mv.class0_.is_adjective and not mv.class0_.is_noun) and not mv.class0_.is_proper_surname and not mv.class0_.is_proper_geo) and not mv.class0_.is_proper_secname):
continue
ok = False
for rr in res:
if (rr.is_in_dictionary):
if (rr.class0_ == mv.class0_ or rr.class0_.is_noun):
ok = True
break
if (not mv.class0_.is_adjective and rr.class0_.is_verb):
ok = True
break
if (ok):
continue
if (len(mv.tail) > 0 and not LanguageHelper.ends_with(word, mv.tail)):
continue
r = MorphWordForm(mv, word, self.get_misc_info(mv.misc_info_id))
if (not r._has_morph_equals(res)):
r.undef_coef = mvref.coef
if (res is None):
res = list()
res.append(r)
if (word == "ПРИ" and res is not None):
for i in range(len(res) - 1, -1, -1):
if (res[i].class0_.is_proper_geo):
del res[i]
else: i = -1
if (res is None or len(res) == 0):
return None
self.__sort(res, word)
for v in res:
if (v.normal_case is None):
v.normal_case = word
if (v.class0_.is_verb):
if (v.normal_full is None and LanguageHelper.ends_with(v.normal_case, "ТЬСЯ")):
v.normal_full = v.normal_case[0:0+len(v.normal_case) - 2]
v.language = self.language
if (v.class0_.is_preposition):
v.normal_case = LanguageHelper.normalize_preposition(v.normal_case)
mc = MorphClass()
for i in range(len(res) - 1, -1, -1):
if (not res[i].is_in_dictionary and res[i].class0_.is_adjective and len(res) > 1):
if ("к.ф." in res[i].misc.attrs or "неизм." in res[i].misc.attrs):
del res[i]
continue
if (res[i].is_in_dictionary):
mc.value |= res[i].class0_.value
else: i = -1
if (mc == MorphClass.VERB and len(res) > 1):
for r in res:
if (r.undef_coef > (100) and r.class0_ == MorphClass.ADJECTIVE):
r.undef_coef = (0)
if (len(res) == 0):
return None
return res
def getAllWordforms(self, word : str) -> typing.List['MorphWordForm']:
res = list()
tn = self.m_root
i = 0
while i <= len(word):
if (tn.lazy_pos > 0):
self.__loadTreeNode(tn)
if (tn.rules is not None):
word_begin = ""
word_end = ""
if (i > 0):
word_begin = word[0:0+i]
else:
word_end = word
if (i < len(word)):
word_end = word[i:]
else:
word_begin = word
for r in tn.rules:
if (word_end in r.variants):
for vl in r.variants_list:
for v in vl:
wf = MorphWordForm(v, None)
if (not MorphWordForm._hasMorphEquals(res, wf)):
wf.normal_case = (word_begin + v.tail)
wf.undef_coef = (0)
res.append(wf)
if (tn.nodes is None or i >= len(word)):
break
ch = ord(word[i])
wraptn20 = RefOutArgWrapper(None)
inoutres21 = Utils.tryGetValue(tn.nodes, ch, wraptn20)
tn = wraptn20.value
if (not inoutres21):
break
i += 1
i = 0
first_pass2713 = True
while True:
if first_pass2713: first_pass2713 = False
else: i += 1
if (not (i < len(res))): break
wf = res[i]
if (wf.containsAttr("инф.", None)):
continue
j = i + 1
first_pass2714 = True
while True:
if first_pass2714: first_pass2714 = False
else: j += 1
if (not (j < len(res))): break
wf1 = res[j]
if (wf1.containsAttr("инф.", None)):
continue
if ((wf.class0_ == wf1.class0_ and wf.gender == wf1.gender and wf.number == wf1.number) and wf.normal_case == wf1.normal_case):
wf.case_ = (wf.case_) | wf1.case_
del res[j]
j -= 1
i = 0
first_pass2715 = True
while True:
if first_pass2715: first_pass2715 = False
else: i += 1
if (not (i < len(res))): break
wf = res[i]
if (wf.containsAttr("инф.", None)):
continue
j = i + 1
first_pass2716 = True
while True:
if first_pass2716: first_pass2716 = False
else: j += 1
if (not (j < len(res))): break
wf1 = res[j]
if (wf1.containsAttr("инф.", None)):
continue
if ((wf.class0_ == wf1.class0_ and wf.case_ == wf1.case_ and wf.number == wf1.number) and wf.normal_case == wf1.normal_case):
wf.gender = Utils.valToEnum((wf.gender) | (wf1.gender), MorphGender)
del res[j]
j -= 1
return res
def get_all_wordforms(self, word : str) -> typing.List['MorphWordForm']:
res = list()
i = 0
tn = self.m_root
i = 0
while i <= len(word):
if (tn.lazy_pos > 0):
self.__load_tree_node(tn)
if (tn.rule_ids is not None):
word_begin = ""
word_end = ""
if (i > 0):
word_begin = word[0:0+i]
else:
word_end = word
if (i < len(word)):
word_end = word[i:]
else:
word_begin = word
for rid in tn.rule_ids:
r = self.get_rule(rid)
if (r.contains_var(word_end)):
for vl in r.morph_vars:
for v in vl:
wf = MorphWordForm(v, None, self.get_misc_info(v.misc_info_id))
if (not wf._has_morph_equals(res)):
wf.normal_case = (word_begin + v.tail)
wf.undef_coef = (0)
res.append(wf)
if (tn.nodes is None or i >= len(word)):
break
ch = ord(word[i])
wraptn11 = RefOutArgWrapper(None)
inoutres12 = Utils.tryGetValue(tn.nodes, ch, wraptn11)
tn = wraptn11.value
if (not inoutres12):
break
i += 1
i = 0
first_pass2983 = True
while True:
if first_pass2983: first_pass2983 = False
else: i += 1
if (not (i < len(res))): break
wf = res[i]
if (wf.contains_attr("инф.", None)):
continue
cas = wf.case_
j = i + 1
first_pass2984 = True
while True:
if first_pass2984: first_pass2984 = False
else: j += 1
if (not (j < len(res))): break
wf1 = res[j]
if (wf1.contains_attr("инф.", None)):
continue
if ((wf.class0_ == wf1.class0_ and wf.gender == wf1.gender and wf.number == wf1.number) and wf.normal_case == wf1.normal_case):
cas |= wf1.case_
del res[j]
j -= 1
if (cas != wf.case_):
res[i].case_ = cas
i = 0
first_pass2985 = True
while True:
if first_pass2985: first_pass2985 = False
else: i += 1
if (not (i < len(res))): break
wf = res[i]
if (wf.contains_attr("инф.", None)):
continue
j = i + 1
first_pass2986 = True
while True:
if first_pass2986: first_pass2986 = False
else: j += 1
if (not (j < len(res))): break
wf1 = res[j]
if (wf1.contains_attr("инф.", None)):
continue
if ((wf.class0_ == wf1.class0_ and wf.case_ == wf1.case_ and wf.number == wf1.number) and wf.normal_case == wf1.normal_case):
wf.gender = Utils.valToEnum((wf.gender) | (wf1.gender), MorphGender)
del res[j]
j -= 1
return res
def process(self, word : str) -> typing.List['MorphWordForm']:
""" Обработка одного слова
Args:
word(str): слово должно быть в верхнем регистре
"""
if (Utils.isNullOrEmpty(word)):
return None
res = None
if (len(word) > 1):
i = 0
while i < len(word):
ch = word[i]
if (LanguageHelper.isCyrillicVowel(ch) or LanguageHelper.isLatinVowel(ch)):
break
i += 1
if (i >= len(word)):
return res
mvs = [ ]
tn = self.m_root
i = 0
while i <= len(word):
if (tn.lazy_pos > 0):
self.__loadTreeNode(tn)
if (tn.rules is not None):
word_begin = None
word_end = None
if (i == 0):
word_end = word
elif (i < len(word)):
word_end = word[i:]
else:
word_end = ""
if (res is None):
res = list()
for r in tn.rules:
wrapmvs14 = RefOutArgWrapper(None)
inoutres15 = Utils.tryGetValue(r.variants, word_end, wrapmvs14)
mvs = wrapmvs14.value
if (inoutres15):
if (word_begin is None):
if (i == len(word)):
word_begin = word
elif (i > 0):
word_begin = word[0:0+i]
else:
word_begin = ""
r.processResult(res, word_begin, mvs)
if (tn.nodes is None or i >= len(word)):
break
ch = ord(word[i])
wraptn16 = RefOutArgWrapper(None)
inoutres17 = Utils.tryGetValue(tn.nodes, ch, wraptn16)
tn = wraptn16.value
if (not inoutres17):
break
i += 1
need_test_unknown_vars = True
if (res is not None):
for r in res:
if ((r.class0_.is_pronoun or r.class0_.is_noun or r.class0_.is_adjective) or (r.class0_.is_misc and r.class0_.is_conjunction) or r.class0_.is_preposition):
need_test_unknown_vars = False
elif (r.class0_.is_adverb and r.normal_case is not None):
if (not LanguageHelper.endsWithEx(r.normal_case, "О", "А", None, None)):
need_test_unknown_vars = False
elif (r.normal_case == "МНОГО"):
need_test_unknown_vars = False
elif (r.class0_.is_verb and len(res) > 1):
ok = False
for rr in res:
if (rr != r and rr.class0_ != r.class0_):
ok = True
break
if (ok and not LanguageHelper.endsWith(word, "ИМ")):
need_test_unknown_vars = False
if (need_test_unknown_vars and LanguageHelper.isCyrillicChar(word[0])):
gl = 0
sog = 0
j = 0
while j < len(word):
if (LanguageHelper.isCyrillicVowel(word[j])):
gl += 1
else:
sog += 1
j += 1
if ((gl < 2) or (sog < 2)):
need_test_unknown_vars = False
if (need_test_unknown_vars and res is not None and len(res) == 1):
if (res[0].class0_.is_verb):
if ("н.вр." in res[0].misc.attrs and "нес.в." in res[0].misc.attrs and not "страд.з." in res[0].misc.attrs):
need_test_unknown_vars = False
elif ("б.вр." in res[0].misc.attrs and "сов.в." in res[0].misc.attrs):
need_test_unknown_vars = False
elif (res[0].normal_case is not None and LanguageHelper.endsWith(res[0].normal_case, "СЯ")):
need_test_unknown_vars = False
if (res[0].class0_.is_undefined and "прдктв." in res[0].misc.attrs):
need_test_unknown_vars = False
if (need_test_unknown_vars):
if (self.m_root_reverce is None):
return res
tn = self.m_root_reverce
tn0 = None
for i in range(len(word) - 1, -1, -1):
if (tn.lazy_pos > 0):
self.__loadTreeNode(tn)
ch = ord(word[i])
if (tn.nodes is None):
break
wrapnext18 = RefOutArgWrapper(None)
inoutres19 = Utils.tryGetValue(tn.nodes, ch, wrapnext18)
next0_ = wrapnext18.value
if (not inoutres19):
break
tn = next0_
if (tn.lazy_pos > 0):
self.__loadTreeNode(tn)
if (tn.reverce_variants is not None):
tn0 = tn
break
else: i = -1
if (tn0 is not None):
glas = i < 4
while i >= 0:
if (LanguageHelper.isCyrillicVowel(word[i]) or LanguageHelper.isLatinVowel(word[i])):
glas = True
break
i -= 1
if (glas):
for mv in tn0.reverce_variants:
if (((not mv.class0_.is_verb and not mv.class0_.is_adjective and not mv.class0_.is_noun) and not mv.class0_.is_proper_surname and not mv.class0_.is_proper_geo) and not mv.class0_.is_proper_secname):
continue
ok = False
for rr in res:
if (rr.is_in_dictionary):
if (rr.class0_ == mv.class0_ or rr.class0_.is_noun):
ok = True
break
if (not mv.class0_.is_adjective and rr.class0_.is_verb):
ok = True
break
if (ok):
continue
if (len(mv.tail) > 0 and not LanguageHelper.endsWith(word, mv.tail)):
continue
r = MorphWordForm(mv, word)
if (not MorphWordForm._hasMorphEquals(res, r)):
r.undef_coef = mv.coef
if (res is None):
res = list()
res.append(r)
if (word == "ПРИ" and res is not None):
for i in range(len(res) - 1, -1, -1):
if (res[i].class0_.is_proper_geo):
del res[i]
else: i = -1
if (res is None or len(res) == 0):
return None
MorphEngine.__sort(res, word)
for v in res:
if (v.normal_case is None):
v.normal_case = word
if (v.class0_.is_verb):
if (v.normal_full is None and LanguageHelper.endsWith(v.normal_case, "ТЬСЯ")):
v.normal_full = v.normal_case[0:0+len(v.normal_case) - 2]
v.language = self.language
if (v.class0_.is_preposition):
v.normal_case = LanguageHelper.normalizePreposition(v.normal_case)
mc = MorphClass()
for i in range(len(res) - 1, -1, -1):
if (not res[i].is_in_dictionary and res[i].class0_.is_adjective and len(res) > 1):
if ("к.ф." in res[i].misc.attrs or "неизм." in res[i].misc.attrs):
del res[i]
continue
if (res[i].is_in_dictionary):
mc.value |= res[i].class0_.value
else: i = -1
if (mc == MorphClass.VERB and len(res) > 1):
for r in res:
if (r.undef_coef > (100) and r.class0_ == MorphClass.ADJECTIVE):
r.undef_coef = (0)
if (len(res) == 0):
return None
return res
def __create_result(self, blk : 'SemBlock') -> None:
if (self.best_var is not None):
for s in self.best_var.segs:
if (s is not None):
s.correct_morph()
self.best_var.create_alt_links()
all_items = list()
for it in self.items:
if (it.res_graph is None):
continue
if (it.result is None):
if (isinstance(it.source, NounPhraseToken)):
npt = Utils.asObjectOrNull(it.source, NounPhraseToken)
if (it.plural == 1 and ((it.source.morph.number) & (MorphNumber.PLURAL)) != (MorphNumber.UNDEFINED)):
it.source.morph.remove_items(MorphNumber.PLURAL, False)
it.result = CreateHelper.create_noun_group(it.res_graph, npt)
if (npt.multi_nouns and it.result.quantity is None):
it.result_list = list()
it.result_list.append(it.result)
if (len(npt.adjectives) > 0 and ((npt.adjectives[0].begin_token.morph.number) & (MorphNumber.SINGULAR)) == (MorphNumber.SINGULAR)):
it.result.morph.number = MorphNumber.SINGULAR
if (it.result.morph.normal_full is not None):
it.result.morph.normal_case = it.result.morph.normal_full
i = 1
while i < len(npt.adjectives):
so = SemObject._new2933(it.res_graph, it.result.typ)
so.tokens.append(npt.noun)
wf = MorphWordForm()
wf.copy_from_word_form(it.result.morph)
so.morph = wf
for a in it.result.attrs:
so.attrs.append(a)
so.concept = it.result.concept
so.not0_ = it.result.not0_
asem = CreateHelper.create_npt_adj(it.res_graph, npt, npt.adjectives[i])
if (asem is not None):
it.res_graph.add_link(SemLinkType.DETAIL, so, asem, "какой", False, None)
it.result_list.append(so)
it.res_graph.objects.append(so)
i += 1
elif (isinstance(it.source, VerbPhraseToken)):
it.result = CreateHelper.create_verb_group(it.res_graph, Utils.asObjectOrNull(it.source, VerbPhraseToken))
it.result_verb_last = (Utils.asObjectOrNull(it.source.last_verb.tag, SemObject))
elif (isinstance(it.source, NumbersWithUnitToken)):
it.result = CreateHelper.create_number(it.res_graph, Utils.asObjectOrNull(it.source, NumbersWithUnitToken))
if (it.result is not None and it.quant is not None):
it.result.quantity = it.quant
if (it.result is not None and it.attrs is not None):
for a in it.attrs:
it.result.attrs.append(a.attr)
it.result.tokens.append(a.token)
if (it.result is not None):
if (it.result.graph != it.res_graph):
pass
all_items.append(it)
if (self.best_var is not None):
for s in self.best_var.segs:
if (s is not None):
self.__create_lists(s)
if (self.best_var is not None):
for s in self.best_var.segs:
if (s is not None):
self.__create_links(s)
i = 0
first_pass3460 = True
while True:
if first_pass3460: first_pass3460 = False
else: i += 1
if (not (i < len(self.items))): break
it = self.items[i]
if (it.typ != SentItemType.ADVERB or it.res_graph is None):
continue
adv = Utils.asObjectOrNull(it.source, AdverbToken)
if (adv.typ != SemAttributeType.UNDEFINED):
continue
before = None
after = None
for ii in range(i - 1, -1, -1):
it0 = self.items[ii]
if (it0.typ == SentItemType.VERB):
before = it0
break
elif (it0.typ == SentItemType.ADVERB or it0.typ == SentItemType.NOUN):
pass
else:
break
if (before is None):
for ii in range(i - 1, -1, -1):
it0 = self.items[ii]
if (it0.typ == SentItemType.VERB or it0.typ == SentItemType.NOUN):
before = it0
break
elif (it0.typ == SentItemType.ADVERB):
pass
else:
break
comma_after = False
ii = i + 1
while ii < len(self.items):
it0 = self.items[ii]
if (it0.typ == SentItemType.VERB or it0.typ == SentItemType.NOUN):
after = it0
break
elif (it0.typ == SentItemType.ADVERB):
pass
elif (it0.can_be_comma_end):
if (before is not None and before.typ == SentItemType.VERB):
break
if (((ii + 1) < len(self.items)) and ((self.items[ii + 1].typ == SentItemType.ADVERB or self.items[ii + 1].typ == SentItemType.VERB))):
pass
else:
comma_after = True
else:
break
ii += 1
if (before is not None and after is not None):
if (comma_after):
after = (None)
elif (before.typ == SentItemType.NOUN and after.typ == SentItemType.VERB):
before = (None)
elif (before.typ == SentItemType.VERB and after.typ == SentItemType.NOUN):
after = (None)
it.result = CreateHelper.create_adverb(it.res_graph, adv)
if (it.attrs is not None):
for a in it.attrs:
it.result.attrs.append(a.attr)
it.result.tokens.append(a.token)
if (after is not None or before is not None):
it.res_graph.add_link(SemLinkType.DETAIL, (before.result if after is None else after.result), it.result, "как", False, None)
preds = list()
agent = None
for it in self.items:
if (it.result is not None and it.typ == SentItemType.VERB and (isinstance(it.source, VerbPhraseToken))):
if (agent is not None):
has_pac = False
for li in it.res_graph.links:
if (li.typ == SemLinkType.PACIENT and li.source == it.result):
has_pac = True
break
if (not has_pac):
ni0 = NGItem._new2926(agent)
gli0 = NGLink._new2939(ni0, Utils.asObjectOrNull(it.source, VerbPhraseToken), NGLinkType.PACIENT)
if (agent.result_list is not None):
gli0.from_is_plural = True
gli0.calc_coef(False)
if (gli0.coef > 0 and gli0.plural == 1):
for ii in agent.result_list:
it.res_graph.add_link(SemLinkType.PACIENT, it.result, ii, None, False, None)
self.coef += (1)
else:
gli0.calc_coef(True)
if (gli0.coef > 0):
it.res_graph.add_link(SemLinkType.PACIENT, it.result, agent.result, None, False, None)
self.coef += (1)
ali = None
for li in it.res_graph.links:
if (li.typ == SemLinkType.AGENT and li.source == it.result):
ali = li
break
if (ali is not None):
agent = self.__find_item_by_res(ali.target)
continue
if (agent is None):
continue
ni = NGItem._new2926(agent)
gli = NGLink._new2939(ni, Utils.asObjectOrNull(it.source, VerbPhraseToken), NGLinkType.AGENT)
if (agent.result_list is not None):
gli.from_is_plural = True
gli.calc_coef(False)
if (gli.coef > 0 and gli.plural == 1):
for ii in agent.result_list:
it.res_graph.add_link(SemLinkType.AGENT, it.result, ii, None, False, None)
self.coef += (1)
else:
gli.calc_coef(True)
if (gli.coef > 0):
it.res_graph.add_link(SemLinkType.AGENT, it.result, agent.result, None, False, None)
self.coef += (1)
agent = (None)
i = 0
first_pass3461 = True
while True:
if first_pass3461: first_pass3461 = False
else: i += 1
if (not (i < len(self.items))): break
it = self.items[i]
if (it.result is not None and it.typ == SentItemType.DEEPART):
pass
else:
continue
link = None
for j in range(i - 1, -1, -1):
itt = self.items[j]
if (itt.typ != SentItemType.NOUN):
continue
if (not ((itt.source.morph.case_.is_nominative))):
continue
ispacad = False
for li in itt.res_graph.links:
if (((li.typ == SemLinkType.AGENT or li.typ == SemLinkType.PACIENT)) and li.target == itt.result):
ispacad = True
if (not ispacad):
continue
if (link is None):
link = itt.res_graph.add_link(SemLinkType.AGENT, it.result, itt.result, None, False, None)
elif (link.alt_link is None):
link.alt_link = itt.res_graph.add_link(SemLinkType.AGENT, it.result, itt.result, None, False, None)
link.alt_link.alt_link = link
break
if (link is None):
j = i + 1
first_pass3462 = True
while True:
if first_pass3462: first_pass3462 = False
else: j += 1
if (not (j < len(self.items))): break
itt = self.items[j]
if (itt.typ != SentItemType.NOUN):
continue
if (not ((itt.source.morph.case_.is_nominative))):
continue
ispacad = False
for li in itt.res_graph.links:
if (((li.typ == SemLinkType.AGENT or li.typ == SemLinkType.PACIENT)) and li.target == itt.result):
ispacad = True
if (not ispacad):
continue
if (link is None):
link = itt.res_graph.add_link(SemLinkType.AGENT, it.result, itt.result, None, False, None)
elif (link.alt_link is None):
link.alt_link = itt.res_graph.add_link(SemLinkType.AGENT, it.result, itt.result, None, False, None)
link.alt_link.alt_link = link
break
if (link is not None):
self.coef += 1
for fr in self.res_block.fragments:
if (fr.can_be_error_structure):
self.coef /= (2)
if (len(self.res_block.fragments) > 0 and len(self.res_block.fragments[0].graph.objects) > 0):
it = self.res_block.fragments[0].graph.objects[0]
if (self.last_char is not None and self.last_char.is_char('?')):
if (it.morph.normal_full == "КАКОЙ" or it.morph.normal_full == "СКОЛЬКО"):
it.typ = SemObjectType.QUESTION