class ServerEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def server2h_codename2aliases(server): aliases = Server.server_langs2aliases(server, langs_recognizable) return {Server.server2codename(server): aliases} h_codename2aliases = merge_dicts(map(server2h_codename2aliases, Server.list_all()), vwrite=vwrite_no_duplicate_key) assert_is_not_none(h_codename2aliases) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class PortEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) h_codename2aliases = merge_dicts([{Port.port2codename(port): Port.port_langs2aliases(port, langs_recognizable)} for port in Port.list_all()], vwrite=vwrite_no_duplicate_key) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2pattern_port_tradegood, } matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod # @CacheTool.cache2hashable(cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), # f_pair=CacheTool.JSON.func_pair(), ) @CacheTool.cache2hashable(cache=cached(cache=TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL), ), f_pair=CacheTool.JSON.func_pair(),) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale(config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang(HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class Me: @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def j_yaml(cls): filepath = os.path.join(FILE_DIR, "me.yaml") j_yaml = YAMLTool.filepath2j(filepath, Loader=yaml.SafeLoader) return j_yaml @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2pattern(cls, lang): j_me = cls.j_yaml() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) me_list = [ me for lang in langs_recognizable for me in j_me.get(lang, []) ] rstr = RegexTool.rstr_iter2or(map(re.escape, me_list)) pattern = re.compile(rstr, re.I) return pattern
class TradegoodtypeEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @WARMER.add(cond=not HenriqueEnv.is_skip_warmup()) def _dict_lang2matcher(cls, ): return { lang: cls.lang2matcher(lang) for lang in HenriqueLocale.langs() } @classmethod @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): tgt_list = Tradegoodtype.list_all() langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def tgt2aliases(tgt): for _lang in langs_recognizable: yield from Tradegoodtype.tradegoodtype_lang2aliases(tgt, _lang) h_value2aliases = merge_dicts([{ Tradegoodtype.tradegoodtype2codename(tgt): list(tgt2aliases(tgt)) } for tgt in tgt_list], vwrite=vwrite_no_duplicate_key) config = { GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm, # GazetteerMatcher.Config.Key.TEXTS2PATTERN: HenriqueEntity.texts2rstr_word_with_cardinal_suffix, } matcher = GazetteerMatcher(h_value2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=cached( TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL), ), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class SkillEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def entity2skill_codename(cls, entity): return FoxylibEntity.entity2value(entity) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod def dict_lang2codename2aliases(cls): from henrique.main.document.skill.googlesheets.skill_googlesheets import SkillGooglesheets return SkillGooglesheets.dict_lang2codename2aliases() @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def codenames(cls): codename_sets = [set(h_codename2aliases.keys()) for lang, h_codename2aliases in cls.dict_lang2codename2aliases().items()] return iter2singleton(codename_sets) @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h_lang2codename2aliases = cls.dict_lang2codename2aliases() def codename2texts(codename): for lang in langs: aliases = JsonTool.down(h_lang2codename2aliases, [lang, codename]) if not aliases: continue yield from aliases h_codename2texts = {codename: list(codename2texts(codename)) for codename in cls.codenames()} config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2texts, config) return matcher @classmethod @CacheTool.cache2hashable(cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): lang = LocaleTool.locale2lang(HenriqueEntity.Config.config2locale(config)) span_value_list = list(cls.lang2matcher(lang).text2span_value_iter(text_in)) entity_list = [{FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class RateEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod def rstr(cls): rstr = format_str( r"{}\s*{}?", RegexTool.name_rstr2named( "cardinal", "\d+", ), RegexTool.name_rstr2named( "Metricprefix", Metricprefix.rstr(), ), ) return rstr @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def rstr_last_char(cls): rstr_suffix_list = [r"\d", Metricprefix.rstr()] return RegexTool.rstr_iter2or(rstr_suffix_list) @classmethod def match2value(cls, m): v = int(m.group("cardinal")) multiple = Metricprefix.text2multiple(m.group("Metricprefix")) or 1 return v * multiple @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2pattern(cls, lang): from henrique.main.document.price.trend.trend_entity import TrendEntity logger = HenriqueLogger.func_level2logger(cls.lang2pattern, logging.DEBUG) # rstr_suffix = format_str("{}?", # RegexTool.rstr2wrapped(TrendEntity.lang2rstr(lang)), # ) ### may be concatenated with port/tradegood name # rstr_prefixed = RegexTool.rstr2rstr_words_prefixed(cls.rstr()) # raise Exception({"rstr_suffix":rstr_suffix}) rstr_trend = TrendEntity.lang2rstr(lang) # bound_right_list_raw = RegexTool.right_wordbounds() right_bounds = lchain( RegexTool.bounds2prefixed(RegexTool.right_wordbounds(), rstr_trend), RegexTool.right_wordbounds(), ) rstr_rightbound = RegexTool.rstr2right_bounded(cls.rstr(), right_bounds) logger.debug({#"rstr_trend":rstr_trend, #"right_bounds":right_bounds, "rstr_rightbound":rstr_rightbound, }) # rstr_suffixed = RegexTool.rstr2rstr_words_suffixed(cls.rstr(), rstr_suffix=rstr_suffix) # raise Exception({"rstr_trend": rstr_trend, "rstr_suffixed": rstr_suffixed}) # return re.compile(RegexTool.rstr2wordbounded(cls.rstr())) return re.compile(rstr_rightbound, re.I) @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) pattern = cls.lang2pattern(lang) m_list = list(pattern.finditer(text_in)) def match2entity(m): span = m.span() entity = { FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: cls.match2value(m), FoxylibEntity.Field.TYPE: cls.entity_type(), } return entity entity_list = lmap(match2entity, m_list) return entity_list
class CultureEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) # @classmethod # @WARMER.add(cond=not HenriqueEnv.is_skip_warmup()) # def _dict_lang2matcher(cls,): # return {lang: cls.lang2matcher(lang) for lang in HenriqueLocale.langs()} @classmethod @cached(cache=TTLCache(maxsize=HenriqueLocale.lang_count(), ttl=HenriqueEntity.Cache.DEFAULT_TTL)) # @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=HenriqueLocale.lang_count())) def lang2matcher(cls, lang): langs_recognizable = HenriqueLocale.lang2langs_recognizable(lang) def culture2h_codename2aliases(culture): aliases = Culture.culture_langs2aliases(culture, langs_recognizable) return {Culture.culture2codename(culture): aliases} h_codename2aliases = merge_dicts(map(culture2h_codename2aliases, Culture.list_all()), vwrite=vwrite_no_duplicate_key) assert_is_not_none(h_codename2aliases) config = {GazetteerMatcher.Config.Key.NORMALIZER: cls.text2norm} matcher = GazetteerMatcher(h_codename2aliases, config) return matcher @classmethod @CacheTool.cache2hashable( cache=cached(cache=TTLCache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE, ttl=HenriqueEntity.Cache.DEFAULT_TTL)), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) or LocaleTool.locale2lang( HenriqueLocale.DEFAULT) matcher = cls.lang2matcher(lang) span_value_list = list(matcher.text2span_value_iter(text_in)) entity_list = [{ FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TEXT: StringTool.str_span2substr(text_in, span), FoxylibEntity.Field.VALUE: value, FoxylibEntity.Field.TYPE: cls.entity_type(), } for span, value in span_value_list] return entity_list
class TrendEntity: @classmethod def entity_type(cls): return ClassTool.class2fullpath(cls) @classmethod def text2norm(cls, text): return str2lower(text) @classmethod @FunctionTool.wrapper2wraps_applied(lru_cache(maxsize=2)) def dict_lang2codename2texts(cls): from henrique.main.document.price.trend.googlesheets.trend_googlesheets import TrendGooglesheets h = TrendGooglesheets.dict_lang2codename2texts() return h @classmethod @FunctionTool.wrapper2wraps_applied( lru_cache(maxsize=HenriqueLocale.lang_count())) def _lang2dict_alias2codename(cls, lang): langs = HenriqueLocale.lang2langs_recognizable(lang) h = cls.dict_lang2codename2texts() # h_codename2aliases = cls.dict_lang2codename2texts().get(lang) h_alias2codename = merge_dicts([{ cls.text2norm(alias): codename } for lang in langs for codename, aliases in h.get(lang).items() for alias in aliases], vwrite=vwrite_no_duplicate_key) return h_alias2codename @classmethod def lang_alias2codename(cls, lang, alias): h_alias2codename = cls._lang2dict_alias2codename(lang) return h_alias2codename.get(alias) @classmethod def lang2rstr(cls, lang): aliases = cls._lang2dict_alias2codename(lang).keys() return RegexTool.rstr_iter2or( map(lambda x: re.escape(cls.text2norm(x)), aliases)) @classmethod def lang2pattern(cls, lang): from henrique.main.document.price.rate.rate_entity import RateEntity logger = HenriqueLogger.func_level2logger(cls.lang2pattern, logging.DEBUG) left_bounds = [ RateEntity.rstr_last_char(), r"\s", ] right_bounds = RegexTool.right_wordbounds() rstr = RegexTool.rstr2bounded(cls.lang2rstr(lang), left_bounds, right_bounds) logger.debug({"left_bounds": left_bounds, "rstr": rstr}) return re.compile(rstr, re.I) @classmethod @CacheTool.cache2hashable( cache=lru_cache(maxsize=HenriqueEntity.Cache.DEFAULT_SIZE), f_pair=CacheTool.JSON.func_pair(), ) def text2entity_list(cls, text_in, config=None): locale = HenriqueEntity.Config.config2locale( config) or HenriqueLocale.DEFAULT lang = LocaleTool.locale2lang(locale) pattern = cls.lang2pattern(lang) m_list = list(pattern.finditer(text_in)) def match2entity(match): span = match.span() text = StringTool.str_span2substr(text_in, span) codename = cls.lang_alias2codename(lang, text) entity = { FoxylibEntity.Field.VALUE: codename, FoxylibEntity.Field.TEXT: text, FoxylibEntity.Field.SPAN: span, FoxylibEntity.Field.TYPE: cls.entity_type(), } return entity entity_list = lmap(match2entity, m_list) return entity_list