def _lang2dict_alias2server(cls, lang):
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
h = merge_dicts([{alias: server}
for server in cls.list_all()
for alias in cls.server_langs2aliases(server, langs_recognizable)],
vwrite=vwrite_no_duplicate_key)
return h
def lang2pattern(cls, lang):
j_me = cls.j_yaml()
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
me_list = [
me for lang in langs_recognizable for me in j_me.get(lang, [])
]
rstr = RegexTool.rstr_iter2or(map(re.escape, me_list))
pattern = re.compile(rstr, re.I)
return pattern
def _lang2dict_alias2codename(cls, lang):
langs = HenriqueLocale.lang2langs_recognizable(lang)
h = cls.dict_lang2codename2texts()
# h_codename2aliases = cls.dict_lang2codename2texts().get(lang)
h_alias2codename = merge_dicts([{
cls.text2norm(alias): codename
} for lang in langs for codename, aliases in h.get(lang).items()
for alias in aliases],
vwrite=vwrite_no_duplicate_key)
return h_alias2codename
def lang2matcher(cls, lang):
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
h_codename2aliases = merge_dicts([{Port.port2codename(port): Port.port_langs2aliases(port, langs_recognizable)}
for port in Port.list_all()],
vwrite=vwrite_no_duplicate_key)
config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm,
GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2pattern_port_tradegood,
}
matcher = GazetteerMatcher(h_codename2aliases, config)
return matcher
def lang2matcher(cls, lang):
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
def server2h_codename2aliases(server):
aliases = Server.server_langs2aliases(server, langs_recognizable)
return {Server.server2codename(server): aliases}
h_codename2aliases = merge_dicts(map(server2h_codename2aliases,
Server.list_all()),
vwrite=vwrite_no_duplicate_key)
assert_is_not_none(h_codename2aliases)
config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm}
matcher = GazetteerMatcher(h_codename2aliases, config)
return matcher
def text2element_list(cls, text_in, lang):
logger = HenriqueLogger.func_level2logger(cls.text2element_list,
logging.DEBUG)
langs = HenriqueLocale.lang2langs_recognizable(lang)
logger.debug({"langs": langs})
match_list_number = list(cls.pattern_number().finditer(text_in))
span_list_number = lmap(lambda m: m.span(), match_list_number)
matcher = TimedeltaEntityUnit.langs2matcher(langs)
span_value_list_timedelta_unit = list(
matcher.text2span_value_iter(text_in))
spans_list = [
span_list_number,
lmap(ig(0), span_value_list_timedelta_unit),
]
gap2is_valid = partial(StringTool.str_span2match_blank_or_nullstr,
text_in)
indextuple_list = ContextfreeTool.spans_list2reducible_indextuple_list(
spans_list, gap2is_valid)
def indextuple2element(indextuple):
i, j = indextuple
quantity = int(match_list_number[i].group())
unit = span_value_list_timedelta_unit[j][1]
span = (
span_list_number[i][0],
span_value_list_timedelta_unit[j][0][1],
)
element = {
cls.Field.QUANTITY: quantity,
cls.Field.UNIT: unit,
cls.Field.SPAN: span,
}
return element
element_list = lmap(indextuple2element, indextuple_list)
return element_list
def lang2matcher(cls, lang):
langs = HenriqueLocale.lang2langs_recognizable(lang)
h_lang2codename2aliases = cls.dict_lang2codename2aliases()
def codename2texts(codename):
for lang in langs:
aliases = JsonTool.down(h_lang2codename2aliases, [lang, codename])
if not aliases:
continue
yield from aliases
h_codename2texts = {codename: list(codename2texts(codename))
for codename in cls.codenames()}
config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm}
matcher = GazetteerMatcher(h_codename2texts, config)
return matcher
def lang2matcher(cls, lang):
tgt_list = Tradegoodtype.list_all()
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
def tgt2aliases(tgt):
for _lang in langs_recognizable:
yield from Tradegoodtype.tradegoodtype_lang2aliases(tgt, _lang)
h_value2aliases = merge_dicts([{
Tradegoodtype.tradegoodtype2codename(tgt):
list(tgt2aliases(tgt))
} for tgt in tgt_list],
vwrite=vwrite_no_duplicate_key)
config = {
GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm,
# GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2rstr_word_with_cardinal_suffix,
}
matcher = GazetteerMatcher(h_value2aliases, config)
return matcher
def lang2matcher(cls, lang):
tg_list = Tradegood.list_all()
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
def tg2aliases(tg):
for _lang in langs_recognizable:
yield from Tradegood.tradegood_lang2aliases(tg, _lang)
h_value2aliases = merge_dicts(
[{
Tradegood.tradegood2codename(tg): list(tg2aliases(tg))
} for tg in tg_list],
vwrite=vwrite_no_duplicate_key)
config = {
GazetteerMatcher.Config.Key.NORMALIZER:
cls.text2norm,
GazetteerMatcher.Config.Key.TEXTS2PATTERN:
HenriqueEntity.texts2pattern_port_tradegood,
}
matcher = GazetteerMatcher(h_value2aliases, config)
return matcher
def text2entity_list(cls, text_in, config=None):
locale = HenriqueEntity.Config.config2locale(
config) or HenriqueLocale.DEFAULT
lang = LocaleTool.locale2lang(locale)
langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang)
if "ko" not in langs_recognizable:
return []
match_list = list(cls.pattern_ko().finditer(text_in))
def match2entity_list(match):
span = match.span()
assert_in(SpanTool.span2len(span), (2, 3))
entity_list = []
s, e = span
span_nutmeg = (s, s + 1)
entity_nutmeg = {
FoxylibEntity.Field.SPAN:
span_nutmeg,
FoxylibEntity.Field.TEXT:
StringTool.str_span2substr(text_in, span_nutmeg),
FoxylibEntity.Field.VALUE:
"Nutmeg",
FoxylibEntity.Field.TYPE:
TradegoodEntity.entity_type(),
}
entity_list.append(entity_nutmeg)
span_mace = (s + 1, s + 2)
entity_mace = {
FoxylibEntity.Field.SPAN:
span_mace,
FoxylibEntity.Field.TEXT:
StringTool.str_span2substr(text_in, span_mace),
FoxylibEntity.Field.VALUE:
"Mace",
FoxylibEntity.Field.TYPE:
TradegoodEntity.entity_type(),
}
entity_list.append(entity_mace)
if SpanTool.span2len(span) == 3:
span_clove = (s + 2, s + 3)
entity_cloves = {
FoxylibEntity.Field.SPAN:
span_clove,
FoxylibEntity.Field.TEXT:
StringTool.str_span2substr(text_in, span_clove),
FoxylibEntity.Field.VALUE:
"Cloves",
FoxylibEntity.Field.TYPE:
TradegoodEntity.entity_type(),
}
entity_list.append(entity_cloves)
return entity_list
entity_list = [
entity for m in match_list for entity in match2entity_list(m)
]
return entity_list