def app(cls): HenriqueLogger.attach_stderr2loggers(logging.DEBUG) logger = HenriqueLogger.func_level2logger(cls.app, logging.DEBUG) logger.debug({"START": "START"}) logging.getLogger('googleapicliet.discovery_cache').setLevel( logging.ERROR) # quiet googleapiclient # WTForm WTFormsTool.json_init() # warmup from henrique.main.singleton.warmer.henrique_warmer import HenriqueWarmer HenriqueWarmer.warmup_all() application = connexion.FlaskApp(__name__, ) # application.add_api('swagger.yaml', resolver=RestyResolver("ariana.main")) application.add_api('swagger.yaml') app = application.app app.static_folder = HenriqueFront.dirpath_static() app.config.update(HenriqueFlaskConfig.config()) logger.debug({"app.secret_key": app.secret_key}) # from henrique.singleton.auth0.henrique_auth0 import HenriqueAuth0 # HenriqueAuth0.app2auth0(app) cls._load_urls2app(app) # raise Exception() logger.debug({"END": "END"}) return app
def main(): HenriqueLogger.attach_stderr2loggers(logging.DEBUG) logger = HenriqueLogger.func_level2logger(main, logging.DEBUG) j_list = list(Porttradegoodstate2Markettrend.postgres2j_iter()) logger.debug({"# j_list": len(j_list)}) Porttradegoodstate2Markettrend.j_iter2mongodb(j_list)
def main(): HenriqueLogger.attach_stderr2loggers(logging.DEBUG) logger = HenriqueLogger.func_level2logger(main, logging.DEBUG) j_list = list(Tradegood2MongoDB.postgres2j_iter()) logger.debug({"# j_list": len(j_list)}) Tradegood2MongoDB.j_iter2mongodb(j_list)
def main(): from henrique.main.singleton.logger.henrique_logger import HenriqueLogger # KhalaLogger.attach_stderr2loggers(logging.DEBUG) DiscordLogger.attach_stderr2loggers(logging.DEBUG) HenriqueLogger.attach_stderr2loggers(logging.DEBUG) start_discord()
def postgres2j_iter(cls): logger = HenriqueLogger.func_level2logger(cls.postgres2j_iter, logging.DEBUG) with HenriquePostgres.cursor() as cursor: sql = SQL("SELECT * from {}").format( Identifier(TradegoodTable.NAME)) cursor.execute(sql) for t in PostgresTool.fetch_iter(cursor): j = t[TradegoodTable.index_json()] # logger.debug({"j":j}) h_lang2names = {} for lang, name in j["name"].items(): h_lang2names[lang] = lchain(h_lang2names.get(lang, []), [name]) for lang, nickname_list in j.get("nicknames", {}).items(): h_lang2names[lang] = lchain(h_lang2names.get(lang, []), nickname_list) j[TradegoodDoc.F.NAMES] = { lang: luniq(name_list) for lang, name_list in h_lang2names.items() } for k in ["name", "nicknames"]: j.pop(k, None) # logger.debug({'j["names"]':j["names"]}) j[TradegoodDoc.F.KEY] = j["names"]["en"][0] yield j
def _langs2matcher(cls, langs): logger = HenriqueLogger.func_level2logger(cls._langs2matcher, logging.DEBUG) gazetteer = cls.langs2gazetteer(langs) def texts2pattern(texts): rstr_raw = RegexTool.rstr_iter2or(map(re.escape, texts)) left_bounds = lchain( RegexTool.bounds2suffixed(RegexTool.left_wordbounds(), "\d"), RegexTool.left_wordbounds(), ) right_bounds = RegexTool.right_wordbounds() rstr = RegexTool.rstr2bounded(rstr_raw, left_bounds, right_bounds) logger.debug({ "rstr": rstr, "rstr_raw": rstr_raw, }) return re.compile(rstr, re.I) config = { GazetteerMatcher.Config.Key.TEXTS2PATTERN: texts2pattern, GazetteerMatcher.Config.Key.NORMALIZER: cls.normalize, } matcher = GazetteerMatcher(gazetteer, config=config) return matcher
def env2compile(cls, env): logger = HenriqueLogger.func_level2logger(cls.env2compile, logging.DEBUG) logger.debug({"env": env}) uwsgi_mode = HenriqueUwsgi.env2mode(env) nginx_mode = HenriqueNginx.env2mode(env) uid = HenriqueNginx.Mode.mode2user(nginx_mode) gid = HenriqueNginx.Mode.mode2group(nginx_mode) data = { "env": env, "uid": uid, "gid": gid, "uwsgi_mode": uwsgi_mode, } filepath = os.path.join(FILE_DIR, "conf", "henrique.supervisord.{}.conf".format(env)) utf8 = Jinja2Renderer.template2text(cls.template(), data=data) FileTool.utf82file(utf8, filepath) logger.debug({ "filepath": filepath, "data": data, })
def price_lang2text(cls, price, lang): logger = HenriqueLogger.func_level2logger(cls.price_lang2text, logging.DEBUG) if price is None: return cls.lang2text_idk(lang) rate = MarketpriceDoc.price2rate(price) trend = MarketpriceDoc.price2trend(price) # raise Exception({"price":price}) channel_user_codename = MarketpriceDoc.price2channel_user(price) channel_user = l_singleton2obj( ChannelUser.codenames2channel_users([channel_user_codename])) logger.debug({"price": price, "channel_user": channel_user}) # raise Exception({"price":price}) created_at = MarketpriceDoc.price2created_at( price) or MarketpriceDoc.created_at_backoff() td = datetime.now(tz=pytz.utc) - created_at str_timedelta = HenriqueDatetime.timedelta_lang2str(td, lang) arrow = Trend.trend2arrow(trend) text_out_base = format_str("{}{} @ {}", str(rate), arrow, str_timedelta) if td >= HenriqueDatetime.Constant.TIMEDELTA_OUTDATED: return text_out_base user_alias = ChannelUser.channel_user2alias(channel_user) str_user_alias = "[by {}]".format(user_alias) text_out = " ".join([text_out_base, str_user_alias]) return text_out
def port_tradegood_lists2blocks(cls, port_tradegood_list, price_dict, lang, groupby_parameter_type): logger = HenriqueLogger.func_level2logger( cls.port_tradegood_lists2blocks, logging.DEBUG) logger.debug({"port_tradegood_list": port_tradegood_list}) if groupby_parameter_type == PriceSkillParameter.Type.PORTLIKE: from henrique.main.skill.price.by_port.price_by_port import PriceByPort blocks = [ PriceByPort.port2text(port_codename, lmap(ig(1), l), price_dict, lang) for port_codename, l in gb_tree_global(port_tradegood_list, [ig(0)]) ] return blocks if groupby_parameter_type == PriceSkillParameter.Type.TRADEGOOD: from henrique.main.skill.price.by_tradegood.price_by_tradegood import PriceByTradegood blocks = [ PriceByTradegood.tradegood2text(tg_codename, lmap(ig(0), l), price_dict, lang) for tg_codename, l in gb_tree_global(port_tradegood_list, [ig(1)]) ] return blocks raise Exception(groupby_parameter_type)
def marketprice_doc_iter(cls): logger = HenriqueLogger.func_level2logger(cls.marketprice_doc_iter, logging.DEBUG) # h_id2culture = CultureTable.dict_id2codename() # logger.debug({"h_id2culture":h_id2culture}) with HenriquePostgres.cursor() as cursor: sql = SQL("SELECT * FROM {} ORDER BY id ASC").format( Identifier(cls.NAME), ) cursor.execute(sql) for t in PostgresTool.fetch_iter(cursor): port_codename = cls.tuple2port_name_en(t) tradegood_codename = cls.tuple2tradegood_name_en(t) rate = cls.tuple2rate(t) trend = Trend.int2trend(cls.tuple2trend(t)) data = cls.tuple2j(t) server_codename = JsonTool.down(data, ["server"]) doc = { MarketpriceDoc.Field.PORT: port_codename, MarketpriceDoc.Field.TRADEGOOD: tradegood_codename, MarketpriceDoc.Field.RATE: rate, MarketpriceDoc.Field.TREND: trend, MarketpriceDoc.Field.SERVER: server_codename, # MarketpriceDoc.Field.CHATROOM_USER: channel_user, } yield doc
def pair_list2mongodb(cls, pair_list): logger = HenriqueLogger.func_level2logger(cls.pair_list2mongodb, logging.DEBUG) logger.debug({"len(pair_list)": len(pair_list)}) n = len(pair_list) write_concern = WriteConcern(w=3, wtimeout=n) def upsert_channel_user(): channel_user_list_raw = lfilter(is_not_none, map(ig(1), pair_list)) logger.debug({"len(channel_user_list_raw)": len(channel_user_list_raw)}) # raise Exception({"channel_user_list_raw[0]": channel_user_list_raw[0]}) channel_user_list = luniq(channel_user_list_raw, idfun=ChannelUser.channel_user2codename) collection = ChannelUserCollection.collection().with_options(write_concern=write_concern) def doc2pair(channel_user): cond = {ChannelUser.Field.CODENAME: ChannelUser.channel_user2codename(channel_user)} return cond, channel_user j_pair_list = lmap(doc2pair, channel_user_list) MongoDBTool.j_pair_list2upsert(collection, j_pair_list) def upsert_marketprice(): marketprice_list = lmap(ig(0), pair_list) collection = MarketpriceCollection.collection().with_options(write_concern=write_concern) j_pair_list = [(x,x) for x in marketprice_list] MongoDBTool.j_pair_list2upsert(collection, j_pair_list) upsert_channel_user() upsert_marketprice()
def lang2pattern(cls, lang): from henrique.main.document.price.trend.trend_entity import TrendEntity logger = HenriqueLogger.func_level2logger(cls.lang2pattern, logging.DEBUG) # rstr_suffix = format_str("{}?", # RegexTool.rstr2wrapped(TrendEntity.lang2rstr(lang)), # ) ### may be concatenated with port/tradegood name # rstr_prefixed = RegexTool.rstr2rstr_words_prefixed(cls.rstr()) # raise Exception({"rstr_suffix":rstr_suffix}) rstr_trend = TrendEntity.lang2rstr(lang) # bound_right_list_raw = RegexTool.right_wordbounds() right_bounds = lchain( RegexTool.bounds2prefixed(RegexTool.right_wordbounds(), rstr_trend), RegexTool.right_wordbounds(), ) rstr_rightbound = RegexTool.rstr2right_bounded(cls.rstr(), right_bounds) logger.debug({#"rstr_trend":rstr_trend, #"right_bounds":right_bounds, "rstr_rightbound":rstr_rightbound, }) # rstr_suffixed = RegexTool.rstr2rstr_words_suffixed(cls.rstr(), rstr_suffix=rstr_suffix) # raise Exception({"rstr_trend": rstr_trend, "rstr_suffixed": rstr_suffixed}) # return re.compile(RegexTool.rstr2wordbounded(cls.rstr())) return re.compile(rstr_rightbound, re.I)
def nanban_datetime2upsert_mongo( cls, packet, datetime_nanban, ): logger = HenriqueLogger.func_level2logger( cls.nanban_datetime2upsert_mongo, logging.DEBUG) server_codename = HenriquePacket.packet2server(packet) text_in = KhalaPacket.packet2text(packet) # server = Server.codename2server(server_codename) dt_utc = DatetimeTool.astimezone(datetime_nanban, pytz.utc) # raise Exception({"datetime_in":datetime_in,"dt_utc":dt_utc}) nanban = { ServerNanban.Field.DATETIME: dt_utc, ServerNanban.Field.COMMAND_IN: text_in, } doc_this = { ServerDoc.Field.CODENAME: server_codename, ServerDoc.Field.NANBAN: nanban, } logger.debug({ "datetime_nanban": datetime_nanban, "doc_this": doc_this, }) ServerDoc.docs2upsert([doc_this])
def culture_list_all(cls): logger = HenriqueLogger.func_level2logger(cls.culture_list_all, logging.DEBUG) h_codename2aliases_en = NamesenSheet.dict_codename2aliases() h_codename2aliases_ko = NameskoSheet.dict_codename2aliases() h_codename2prefers = PrefersSheet.dict_codename2prefers() codename_list = luniq( chain( h_codename2aliases_en.keys(), h_codename2aliases_ko.keys(), h_codename2prefers.keys(), )) def codename2culture(codename): aliases = DictTool.filter( lambda k, v: v, { "en": h_codename2aliases_en.get(codename), "ko": h_codename2aliases_ko.get(codename), }) culture = { Culture.Field.CODENAME: codename, Culture.Field.ALIASES: aliases, Culture.Field.PREFERS: h_codename2prefers.get(codename) or [], } return DictTool.filter(lambda k, v: v, culture) list_all = lmap(codename2culture, codename_list) # logger.debug({"list_all":list_all}) return list_all
def get(cls): logger = HenriqueLogger.func_level2logger(cls.get, logging.DEBUG) sender_name = request.args.get(cls.Data.Field.SENDER_NAME) text_in = request.args.get(cls.Data.Field.TEXT) newline = request.args.get(cls.Data.Field.NEWLINE) logger.debug({"sender_name":sender_name, "text_in":text_in, "newline":newline}) if not HenriqueCommand.text2is_query(text_in): return None chatroom = ChatroomKakaotalk.chatroom() Chatroom.chatrooms2upsert([chatroom]) channel_user = ChannelUserKakaotalk.sender_name2channel_user(sender_name) ChannelUser.channel_users2upsert([channel_user]) packet = {KhalaPacket.Field.TEXT: text_in, KhalaPacket.Field.CHATROOM: Chatroom.chatroom2codename(chatroom), KhalaPacket.Field.CHANNEL_USER: ChannelUser.channel_user2codename(channel_user), KhalaPacket.Field.SENDER_NAME: sender_name, } logger.debug({"packet": packet,}) text_response = HenriquePacket.packet2response(packet) if not text_response: return None if cls.packet2skip_response(packet): # run packet but do not respond return None text_out = newline.join(text_response.splitlines()) if newline else text_response return text_out, 200
def entity2response_block(cls, entity, packet, ): logger = HenriqueLogger.func_level2logger(cls.packet2response, logging.DEBUG) chatroom = Chatroom.codename2chatroom(KhalaPacket.packet2chatroom(packet)) if Chatroom.chatroom2codename(chatroom) != ChatroomKakaotalk.codename(): return locale = Chatroom.chatroom2locale(chatroom) lang = LocaleTool.locale2lang(locale) v = FoxylibEntity.entity2value(entity) codename = ChatroomuserEntity.value_packet2codename(v, packet) logger.debug({"codename": codename, "entity": entity, "v": v, }) chatroomuser = Chatroomuser.codename2chatroomuser(codename) comments = Chatroomuser.chatroomuser2comments(chatroomuser) comment = choice(comments) if comments else None filepath = os.path.join(FILE_DIR, "tmplt.{}.part.txt".format(lang)) data = {"name": codename, "comment": comment, "str_aliases": ", ".join(Chatroomuser.chatroomuser2aliases(chatroomuser)), } text_out = str2strip(HenriqueJinja2.textfile2text(filepath, data)) return text_out
def test_01(self): logger = HenriqueLogger.func_level2logger(self.test_01, logging.DEBUG) with HenriquePostgres.cursor() as cursor: cursor.execute("""SELECT * from unchartedwatersonline_port""") l = cursor.fetchall() logger.debug({"l": l})
def test_client(cls): logger = HenriqueLogger.func_level2logger(cls.test_client, logging.DEBUG) logger.debug({"START": "START"}) c = cls.app().test_client() logger.debug({"END": "END"}) return c
def env2ip(cls, env): logger = HenriqueLogger.func_level2logger(cls.env2ip, logging.DEBUG) j_server = cls.j_server() # logger.debug({"j_server":j_server}) ip = JsonTool.down(j_server, [cls.Constant.ROOT, env, "ip"]) #logger.debug({"[cls.Constant.ROOT,env,ip]":[cls.Constant.ROOT,env,"ip"]}) return ip
def key2nullboolean(cls, key): logger = HenriqueLogger.func_level2logger(cls.key2nullboolean, logging.DEBUG) v = cls.key2value(key) nb = BooleanTool.parse2nullboolean(v) # logger.debug({"key":key, "v":v, "nb":nb}) return nb
def element2relativedelta(cls, element): logger = HenriqueLogger.func_level2logger(cls.element2relativedelta, logging.DEBUG) unit = cls.element2unit(element) quantity = cls.element2quantity(element) kwargs = {TimedeltaEntityUnit.unit2plural(unit): quantity} # logger.debug({"kwargs":kwargs}) return relativedelta(**kwargs)
def entity2relativedelta(cls, entity): logger = HenriqueLogger.func_level2logger(cls.entity2relativedelta, logging.DEBUG) element_list = FoxylibEntity.entity2value(entity) relativedelta_list = lmap(TimedeltaElement.element2relativedelta, element_list) logger.debug({"relativedelta_list": relativedelta_list}) return sum(relativedelta_list, relativedelta(days=0))
def test_06(self): logger = HenriqueLogger.func_level2logger(self.test_03, logging.DEBUG) config = {HenriqueEntity.Config.Field.LOCALE: "en-US"} hyp = TrendEntity.text2entity_list("95 하", config=config) ref = [] # pprint({"hyp":hyp}) self.assertEqual(hyp, ref)
def postgres2tuple_iter(cls): logger = HenriqueLogger.func_level2logger(cls.postgres2tuple_iter, logging.DEBUG) with HenriquePostgres.cursor() as cursor: sql = SQL("SELECT * from {}").format(Identifier(PortTradegoodStateTable.NAME)) cursor.execute(sql) for t_list_chunk in ChunkTool.chunk_size2chunks(PostgresTool.fetch_iter(cursor), 100000): yield from t_list_chunk
def _load_urls2app(cls, app): logger = HenriqueLogger.func_level2logger(cls._load_urls2app, logging.DEBUG) from henrique.front.path.view import IndexView FlaskTool.add_url2app(app, IndexView.url(), IndexView.get, methods=[ "GET", ])
def xoxb_token(cls): logger = HenriqueLogger.func_level2logger(cls.xoxb_token, logging.DEBUG) key = "SLACK_HENRIQUE_BOT_USER_OAUTH_ACCESS_TOKEN" token = HenriqueEnv.key2value(key) logger.debug({ "key": key, "bool(token)": bool(token), "token is None": token is None, }) return token
def tuple2pair(cls, t): logger = HenriqueLogger.func_level2logger(cls.tuple2pair, logging.DEBUG) def chatuser_uuid2channel_user(chatuser_uuid, alias=None): if not chatuser_uuid: return None row = ChatuserTable.uuid2row(chatuser_uuid) if not row: logger.debug({"chatuser_uuid":chatuser_uuid}) return None channel_user = ChatuserTable.row2channel_user(row, alias=alias) return channel_user j_postgres = t[PortTradegoodStateTable.index_json()] sender_name = j_postgres.get("sender_name") created_at = DatetimeTool.fromisoformat(j_postgres["created_at"]) port_name_en = PortTradegoodStateTable.tuple2port_name_en(t) port = cls.name_en2port(port_name_en) tradegood_name_en = PortTradegoodStateTable.tuple2tradegood_name_en(t) tradegood = cls.name_en2tradegood(tradegood_name_en) rate = PortTradegoodStateTable.tuple2rate(t) trend = PortTradegoodStateTable.tuple2trend(t) chatuser_uuid = PortTradegoodStateTable.tuple2chatuser_uuid(t) channel_user = chatuser_uuid2channel_user(chatuser_uuid, alias=sender_name) channel_user_code = ChannelUser.channel_user2codename(channel_user) if channel_user else None server_alias = str2lower(jdown(j_postgres, ["server", "name"])) # logger.debug({"server_alias":server_alias}) server = Server.alias_lang2server(server_alias, "ko") assert_true(server,server_alias) marketprice = {MarketpriceDoc.Field.CREATED_AT: created_at, MarketpriceDoc.Field.PORT: Port.port2codename(port), MarketpriceDoc.Field.TRADEGOOD: Tradegood.tradegood2codename(tradegood), MarketpriceDoc.Field.RATE: rate, MarketpriceDoc.Field.TREND: trend, MarketpriceDoc.Field.CHANNEL_USER: channel_user_code, MarketpriceDoc.Field.SERVER: Server.server2codename(server), } return marketprice, channel_user
def warmup_all(cls): logger = HenriqueLogger.func_level2logger(cls.warmup_all, logging.DEBUG) logger.debug("START") # not really necessary cuz warmer will run once loaded in each file, but still to avoid unnecessary warnings warmer_list = list(cls.warmer_iter()) # warmer_list = [IterTool.first(cls.warmer_iter())] logger.debug({"warmer_list": warmer_list}) for x in warmer_list: logger.debug({"warmer": x}) x.warmup() logger.debug("END")
def test_06(self): logger = HenriqueLogger.func_level2logger(self.test_06, logging.DEBUG) Chatroom.chatrooms2upsert([ChatroomKakaotalk.chatroom()]) ServerDoc.codenames2delete([Server.Codename.MARIS]) sender_name = "iris" channel_user_codename = ChannelUserKakaotalk.sender_name2codename( sender_name) ChannelUser.channel_users2upsert( [ChannelUserKakaotalk.sender_name2channel_user(sender_name)]) tz = pytz.timezone("Asia/Seoul") now_tz = DatetimeTool.datetime2truncate_seconds(datetime.now(tz)) # hour = (now_tz + timedelta(seconds=3600)).hour packet1 = { KhalaPacket.Field.TEXT: "?남만 {}".format(now_tz.strftime("%I:%M %p")), KhalaPacket.Field.CHATROOM: KakaotalkUWOChatroom.codename(), KhalaPacket.Field.CHANNEL_USER: channel_user_codename, KhalaPacket.Field.SENDER_NAME: sender_name, } NanbanSkill.packet2response(packet1) packet2 = { KhalaPacket.Field.TEXT: "?남만 +2분 1초", KhalaPacket.Field.CHATROOM: KakaotalkUWOChatroom.codename(), KhalaPacket.Field.CHANNEL_USER: channel_user_codename, KhalaPacket.Field.SENDER_NAME: sender_name, } NanbanSkill.packet2response(packet2) doc = ServerDoc.codename2doc(Server.Codename.MARIS) dt_nanban = DatetimeTool.astimezone( ServerNanban.nanban2datetime(ServerDoc.doc2nanban(doc)), tz) logger.debug({ "now_tz": now_tz, "dt_nanban": dt_nanban, 'now_tz.strftime("%I:%M %p")': now_tz.strftime("%I:%M %p"), }) ref = (now_tz + timedelta(seconds=2 * 60 + 1)).timetz() self.assertEquals(dt_nanban.timetz(), ref)
def test_05(self): logger = HenriqueLogger.func_level2logger(self.test_03, logging.DEBUG) config = {HenriqueEntity.Config.Field.LOCALE: "en-US"} hyp = TrendEntity.text2entity_list("95 down", config=config) ref = [{ 'span': (3, 7), 'text': 'down', 'type': TrendEntity.entity_type(), 'value': 'down' }] # pprint({"hyp":hyp}) self.assertEqual(hyp, ref)