def _lang2dict_alias2server(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) h = merge_dicts([{alias: server} for server in cls.list_all() for alias in cls.server_langs2aliases(server, langs_recognizable)], vwrite=vwrite_no_duplicate_key) return h
def datetime_nanban2str_out(dt_nanban): filepath = os.path.join(FILE_DIR, "tmplt.{}.part.txt".format(lang)) has_dt_nanban = (dt_nanban is not None) if not has_dt_nanban: data = { "server": Server.server_lang2name(server, lang), "has_dt_nanban": has_dt_nanban, } str_out = HenriqueJinja2.textfile2text(filepath, data) return str_out td_nanban = dt_nanban - utc_now tzdb = HenriqueLocale.lang2tzdb(lang) str_timedelta_nanban = NanbanTimedelta.timedelta_lang2text( td_nanban, lang) logger.debug({ "server_codename": server_codename, "server": server, "dt_nanban": dt_nanban, }) data = { "server": Server.server_lang2name(server, lang), "dt_nanban": NanbanTimedelta.datetime2text(dt_nanban, tzdb), "dt_now": NanbanTimedelta.datetime2text(utc_now, tzdb), "timedelta_nanban": str_timedelta_nanban, "has_dt_nanban": has_dt_nanban, } str_out = HenriqueJinja2.textfile2text(filepath, data) return str_out
def lang2pattern(cls, lang): j_me = cls.j_yaml() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) me_list = [ me for lang in langs_recognizable for me in j_me.get(lang, []) ] rstr = RegexTool.rstr_iter2or(map(re.escape, me_list)) pattern = re.compile(rstr, re.I) return pattern
class ServerEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def server2h_codename2aliases(server): aliases = Server.server_langs2aliases(server, langs_recognizable) return {Server.server2codename(server): aliases} h_codename2aliases = merge_dicts(map(server2h_codename2aliases, Server.list_all()), vwrite=vwrite_no_duplicate_key) assert_is_not_none(h_codename2aliases) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
def _lang2dict_alias2codename(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h = cls.dict_lang2codename2texts() # h_codename2aliases = cls.dict_lang2codename2texts().get(lang) h_alias2codename = merge_dicts([{ cls.text2norm(alias): codename } for lang in langs for codename, aliases in h.get(lang).items() for alias in aliases], vwrite=vwrite_no_duplicate_key) return h_alias2codename
def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) h_codename2aliases = merge_dicts([{Port.port2codename(port): Port.port_langs2aliases(port, langs_recognizable)} for port in Port.list_all()], vwrite=vwrite_no_duplicate_key) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2pattern_port_tradegood, } matcher = GazetteerMatcher(h_codename2aliases, config) return matcher
def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def server2h_codename2aliases(server): aliases = Server.server_langs2aliases(server, langs_recognizable) return {Server.server2codename(server): aliases} h_codename2aliases = merge_dicts(map(server2h_codename2aliases, Server.list_all()), vwrite=vwrite_no_duplicate_key) assert_is_not_none(h_codename2aliases) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2aliases, config) return matcher
class PortEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) h_codename2aliases = merge_dicts([{Port.port2codename(port): Port.port_langs2aliases(port, langs_recognizable)} for port in Port.list_all()], vwrite=vwrite_no_duplicate_key) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2pattern_port_tradegood, } matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod # @CacheTool.cache2hashable(cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), # f_pair=CacheTool.JSON.func_pair(), ) @CacheTool.cache2hashable(cache=cached(cache=TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL), ), f_pair=CacheTool.JSON.func_pair(),) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale(config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang(HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
def text2element_list(cls, text_in, lang): logger = HenriqueLogger.func_level2logger(cls.text2element_list, logging.DEBUG) langs = HenriqueLocale.lang2langs_recognizable(lang) logger.debug({"langs": langs}) match_list_number = list(cls.pattern_number().finditer(text_in)) span_list_number = lmap(lambda m: m.span(), match_list_number) matcher = TimedeltaEntityUnit.langs2matcher(langs) span_value_list_timedelta_unit = list( matcher.text2span_value_iter(text_in)) spans_list = [ span_list_number, lmap(ig(0), span_value_list_timedelta_unit), ] gap2is_valid = partial(StringTool.str_span2match_blank_or_nullstr, text_in) indextuple_list = ContextfreeTool.spans_list2reducible_indextuple_list( spans_list, gap2is_valid) def indextuple2element(indextuple): i, j = indextuple quantity = int(match_list_number[i].group()) unit = span_value_list_timedelta_unit[j][1] span = ( span_list_number[i][0], span_value_list_timedelta_unit[j][0][1], ) element = { cls.Field.QUANTITY: quantity, cls.Field.UNIT: unit, cls.Field.SPAN: span, } return element element_list = lmap(indextuple2element, indextuple_list) return element_list
def test_03(self): logger = HenriqueLogger.func_level2logger(self.test_02, logging.DEBUG) Chatroom.chatrooms2upsert([ChatroomKakaotalk.chatroom()]) sender_name = "iris" channel_user_codename = ChannelUserKakaotalk.sender_name2codename( sender_name) ChannelUser.channel_users2upsert( [ChannelUserKakaotalk.sender_name2channel_user(sender_name)]) now_seoul = datetime.now( tz=pytz.timezone(HenriqueLocale.lang2tzdb("ko"))) dt_target = now_seoul - timedelta(seconds=3 * 60) text = "?남만 {}".format(dt_target.strftime("%I:%M %p").lstrip("0")) logger.debug({ "text": text, "now_seoul": now_seoul, }) packet = { KhalaPacket.Field.TEXT: text, KhalaPacket.Field.CHATROOM: KakaotalkUWOChatroom.codename(), KhalaPacket.Field.CHANNEL_USER: channel_user_codename, KhalaPacket.Field.SENDER_NAME: sender_name, } response = NanbanSkill.packet2response(packet) # pprint(text) # pprint(response) response_lines = response.splitlines() span = ( len("다음 남만 시각: "), len("다음 남만 시각: 3:58:00 PM (KST) "), ) hyp = SpanTool.list_span2sublist(response_lines[2], span).strip() dt_nanban = dt_target + NanbanTimedelta.period() ref = dt_nanban.strftime("%I:%M:00 %p (KST)").lstrip("0") self.assertEqual( hyp, ref, )
def lang2matcher(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h_lang2codename2aliases = cls.dict_lang2codename2aliases() def codename2texts(codename): for lang in langs: aliases = JsonTool.down(h_lang2codename2aliases, [lang, codename]) if not aliases: continue yield from aliases h_codename2texts = {codename: list(codename2texts(codename)) for codename in cls.codenames()} config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2texts, config) return matcher
def lang2matcher(cls, lang): tgt_list = Tradegoodtype.list_all() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def tgt2aliases(tgt): for _lang in langs_recognizable: yield from Tradegoodtype.tradegoodtype_lang2aliases(tgt, _lang) h_value2aliases = merge_dicts([{ Tradegoodtype.tradegoodtype2codename(tgt): list(tgt2aliases(tgt)) } for tgt in tgt_list], vwrite=vwrite_no_duplicate_key) config = { GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, # GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2rstr_word_with_cardinal_suffix, } matcher = GazetteerMatcher(h_value2aliases, config) return matcher
class Me: @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def j_yaml(cls): filepath = os.path.join(FILE_DIR, "me.yaml") j_yaml = YAMLTool.filepath2j(filepath, Loader=yaml.SafeLoader) return j_yaml @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2pattern(cls, lang): j_me = cls.j_yaml() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) me_list = [ me for lang in langs_recognizable for me in j_me.get(lang, []) ] rstr = RegexTool.rstr_iter2or(map(re.escape, me_list)) pattern = re.compile(rstr, re.I) return pattern
def lang2matcher(cls, lang): tg_list = Tradegood.list_all() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def tg2aliases(tg): for _lang in langs_recognizable: yield from Tradegood.tradegood_lang2aliases(tg, _lang) h_value2aliases = merge_dicts( [{ Tradegood.tradegood2codename(tg): list(tg2aliases(tg)) } for tg in tg_list], vwrite=vwrite_no_duplicate_key) config = { GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2pattern_port_tradegood, } matcher = GazetteerMatcher(h_value2aliases, config) return matcher
def _dict_lang2matcher(cls, ): return { lang: cls.lang2matcher(lang) for lang in HenriqueLocale.langs() }
class TradegoodtypeEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @WARMER.add(cond=not HenriqueEnv.is_skip_warmup()) def _dict_lang2matcher(cls, ): return { lang: cls.lang2matcher(lang) for lang in HenriqueLocale.langs() } @classmethod @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): tgt_list = Tradegoodtype.list_all() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def tgt2aliases(tgt): for _lang in langs_recognizable: yield from Tradegoodtype.tradegoodtype_lang2aliases(tgt, _lang) h_value2aliases = merge_dicts([{ Tradegoodtype.tradegoodtype2codename(tgt): list(tgt2aliases(tgt)) } for tgt in tgt_list], vwrite=vwrite_no_duplicate_key) config = { GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, # GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2rstr_word_with_cardinal_suffix, } matcher = GazetteerMatcher(h_value2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=cached( TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL), ), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class SkillEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def entity2skill_codename(cls, entity): return FoxylibEntity.entity2value(entity) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod def dict_lang2codename2aliases(cls): from henrique.main.document.skill.googlesheets.skill_googlesheets import SkillGooglesheets return SkillGooglesheets.dict_lang2codename2aliases() @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def codenames(cls): codename_sets = [set(h_codename2aliases.keys()) for lang, h_codename2aliases in cls.dict_lang2codename2aliases().items()] return iter2singleton(codename_sets) @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h_lang2codename2aliases = cls.dict_lang2codename2aliases() def codename2texts(codename): for lang in langs: aliases = JsonTool.down(h_lang2codename2aliases, [lang, codename]) if not aliases: continue yield from aliases h_codename2texts = {codename: list(codename2texts(codename)) for codename in cls.codenames()} config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2texts, config) return matcher @classmethod @CacheTool.cache2hashable(cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): lang = LocaleTool.locale2lang(HenriqueEntity.Config.config2locale(config)) span_value_list = list(cls.lang2matcher(lang).text2span_value_iter(text_in)) entity_list = [{FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
def packet2response(cls, packet): logger = HenriqueLogger.func_level2logger(cls.packet2response, logging.DEBUG) logger.debug({"packet": packet}) server_codename = HenriquePacket.packet2server(packet) chatroom = Chatroom.codename2chatroom( KhalaPacket.packet2chatroom(packet)) locale = Chatroom.chatroom2locale(chatroom) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) tz = pytz.timezone(HenriqueLocale.lang2tzdb(lang)) dt_now = datetime.now(tz) text_in = KhalaPacket.packet2text(packet) config = {HenriqueEntity.Config.Field.LOCALE: locale} # entity_list = RelativeTimedeltaEntity.text2entity_list(text_in, config=config) entity_list = HenriqueEntity.text_extractors2entity_list( text_in, cls.config2extractors(config), ) logger.debug({ "len(entity_list)": len(entity_list), "entity_list": entity_list, }) if not entity_list: return cls.server_lang2lookup(server_codename, lang) if len(entity_list) != 1: return # Invalid request entity = l_singleton2obj(entity_list) if FoxylibEntity.entity2type( entity) == RelativeTimedeltaEntity.entity_type(): reldelta = RelativeTimedeltaEntity.entity2relativedelta(entity) dt_in = cls.relativedelta2nanban_datetime( server_codename, reldelta, ) # raise Exception({"dt_in":dt_in, "reldelta":reldelta}) if dt_in is None: msg_error = NanbanSkillError.codename_lang2text( NanbanSkillError.Codename.NO_PREV_NANBAN_ERROR, lang) raise HenriqueCommandError(msg_error) logger.debug({ "reldelta": reldelta, }) elif FoxylibEntity.entity2type(entity) == TimeEntity.entity_type(): time_in = TimeEntity.value2datetime_time( FoxylibEntity.entity2value(entity)) dt_in = PytzTool.localize(datetime.combine(dt_now.date(), time_in), tz) logger.debug({ "time_in": time_in, "dt_in": dt_in, }) else: raise RuntimeError({ "Invalid entity type: {}".format( FoxylibEntity.entity2type(entity)) }) dt_nearest = DatetimeTool.datetime2nearest(dt_in, dt_now, NanbanTimedelta.period(), Nearest.COMING) logger.debug({ "text_in": text_in, "dt_now": dt_now, "dt_in": dt_in, "dt_nearest": dt_nearest, }) cls.nanban_datetime2upsert_mongo(packet, dt_nearest) return cls.server_lang2lookup(server_codename, lang)
class TrendEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def dict_lang2codename2texts(cls): from henrique.main.document.price.trend.googlesheets.trend_googlesheets import TrendGooglesheets h = TrendGooglesheets.dict_lang2codename2texts() return h @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def _lang2dict_alias2codename(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h = cls.dict_lang2codename2texts() # h_codename2aliases = cls.dict_lang2codename2texts().get(lang) h_alias2codename = merge_dicts([{ cls.text2norm(alias): codename } for lang in langs for codename, aliases in h.get(lang).items() for alias in aliases], vwrite=vwrite_no_duplicate_key) return h_alias2codename @classmethod def lang_alias2codename(cls, lang, alias): h_alias2codename = cls._lang2dict_alias2codename(lang) return h_alias2codename.get(alias) @classmethod def lang2rstr(cls, lang): aliases = cls._lang2dict_alias2codename(lang).keys() return RegexTool.rstr_iter2or( map(lambda x: re.escape(cls.text2norm(x)), aliases)) @classmethod def lang2pattern(cls, lang): from henrique.main.document.price.rate.rate_entity import RateEntity logger = HenriqueLogger.func_level2logger(cls.lang2pattern, logging.DEBUG) left_bounds = [ RateEntity.rstr_last_char(), r"\s", ] right_bounds = RegexTool.right_wordbounds() rstr = RegexTool.rstr2bounded(cls.lang2rstr(lang), left_bounds, right_bounds) logger.debug({"left_bounds": left_bounds, "rstr": rstr}) return re.compile(rstr, re.I) @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) pattern = cls.lang2pattern(lang) m_list = list(pattern.finditer(text_in)) def match2entity(match): span = match.span() text = StringTool.str_span2substr(text_in, span) codename = cls.lang_alias2codename(lang, text) entity = { FoxylibEntity.Field.VALUE: codename, FoxylibEntity.Field.TEXT: text, FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TYPE: cls.entity_type(), } return entity entity_list = lmap(match2entity, m_list) return entity_list
class RateEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod def rstr(cls): rstr = format_str( r"{}\s*{}?", RegexTool.name_rstr2named( "cardinal", "\d+", ), RegexTool.name_rstr2named( "Metricprefix", Metricprefix.rstr(), ), ) return rstr @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def rstr_last_char(cls): rstr_suffix_list = [r"\d", Metricprefix.rstr()] return RegexTool.rstr_iter2or(rstr_suffix_list) @classmethod def match2value(cls, m): v = int(m.group("cardinal")) multiple = Metricprefix.text2multiple(m.group("Metricprefix")) or 1 return v * multiple @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2pattern(cls, lang): from henrique.main.document.price.trend.trend_entity import TrendEntity logger = HenriqueLogger.func_level2logger(cls.lang2pattern, logging.DEBUG) # rstr_suffix = format_str("{}?", # RegexTool.rstr2wrapped(TrendEntity.lang2rstr(lang)), # ) ### may be concatenated with port/tradegood name # rstr_prefixed = RegexTool.rstr2rstr_words_prefixed(cls.rstr()) # raise Exception({"rstr_suffix":rstr_suffix}) rstr_trend = TrendEntity.lang2rstr(lang) # bound_right_list_raw = RegexTool.right_wordbounds() right_bounds = lchain( RegexTool.bounds2prefixed(RegexTool.right_wordbounds(), rstr_trend), RegexTool.right_wordbounds(), ) rstr_rightbound = RegexTool.rstr2right_bounded(cls.rstr(), right_bounds) logger.debug({#"rstr_trend":rstr_trend, #"right_bounds":right_bounds, "rstr_rightbound":rstr_rightbound, }) # rstr_suffixed = RegexTool.rstr2rstr_words_suffixed(cls.rstr(), rstr_suffix=rstr_suffix) # raise Exception({"rstr_trend": rstr_trend, "rstr_suffixed": rstr_suffixed}) # return re.compile(RegexTool.rstr2wordbounded(cls.rstr())) return re.compile(rstr_rightbound, re.I) @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) pattern = cls.lang2pattern(lang) m_list = list(pattern.finditer(text_in)) def match2entity(m): span = m.span() entity = { FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: cls.match2value(m), FoxylibEntity.Field.TYPE: cls.entity_type(), } return entity entity_list = lmap(match2entity, m_list) return entity_list
class CultureEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) # @classmethod # @WARMER.add(cond=not HenriqueEnv.is_skip_warmup()) # def _dict_lang2matcher(cls,): # return {lang: cls.lang2matcher(lang) for lang in HenriqueLocale.langs()} @classmethod @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def culture2h_codename2aliases(culture): aliases = Culture.culture_langs2aliases(culture, langs_recognizable) return {Culture.culture2codename(culture): aliases} h_codename2aliases = merge_dicts(map(culture2h_codename2aliases, Culture.list_all()), vwrite=vwrite_no_duplicate_key) assert_is_not_none(h_codename2aliases) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=cached(cache=TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL)), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) if "ko" not in langs_recognizable: return [] match_list = list(cls.pattern_ko().finditer(text_in)) def match2entity_list(match): span = match.span() assert_in(SpanTool.span2len(span), (2, 3)) entity_list = [] s, e = span span_nutmeg = (s, s + 1) entity_nutmeg = { FoxylibEntity.Field.SPAN: span_nutmeg, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span_nutmeg), FoxylibEntity.Field.VALUE: "Nutmeg", FoxylibEntity.Field.TYPE: TradegoodEntity.entity_type(), } entity_list.append(entity_nutmeg) span_mace = (s + 1, s + 2) entity_mace = { FoxylibEntity.Field.SPAN: span_mace, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span_mace), FoxylibEntity.Field.VALUE: "Mace", FoxylibEntity.Field.TYPE: TradegoodEntity.entity_type(), } entity_list.append(entity_mace) if SpanTool.span2len(span) == 3: span_clove = (s + 2, s + 3) entity_cloves = { FoxylibEntity.Field.SPAN: span_clove, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span_clove), FoxylibEntity.Field.VALUE: "Cloves", FoxylibEntity.Field.TYPE: TradegoodEntity.entity_type(), } entity_list.append(entity_cloves) return entity_list entity_list = [ entity for m in match_list for entity in match2entity_list(m) ] return entity_list