def app(cls):
HenriqueLogger.attach_stderr2loggers(logging.DEBUG)
logger = HenriqueLogger.func_level2logger(cls.app, logging.DEBUG)
logger.debug({"START": "START"})
logging.getLogger('googleapicliet.discovery_cache').setLevel(
logging.ERROR) # quiet googleapiclient
# WTForm
WTFormsTool.json_init()
# warmup
from henrique.main.singleton.warmer.henrique_warmer import HenriqueWarmer
HenriqueWarmer.warmup_all()
application = connexion.FlaskApp(__name__, )
# application.add_api('swagger.yaml', resolver=RestyResolver("ariana.main"))
application.add_api('swagger.yaml')
app = application.app
app.static_folder = HenriqueFront.dirpath_static()
app.config.update(HenriqueFlaskConfig.config())
logger.debug({"app.secret_key": app.secret_key})
# from henrique.singleton.auth0.henrique_auth0 import HenriqueAuth0
# HenriqueAuth0.app2auth0(app)
cls._load_urls2app(app)
# raise Exception()
logger.debug({"END": "END"})
return app
def main():
HenriqueLogger.attach_stderr2loggers(logging.DEBUG)
logger = HenriqueLogger.func_level2logger(main, logging.DEBUG)
j_list = list(Porttradegoodstate2Markettrend.postgres2j_iter())
logger.debug({"# j_list": len(j_list)})
Porttradegoodstate2Markettrend.j_iter2mongodb(j_list)
def main():
HenriqueLogger.attach_stderr2loggers(logging.DEBUG)
logger = HenriqueLogger.func_level2logger(main, logging.DEBUG)
j_list = list(Tradegood2MongoDB.postgres2j_iter())
logger.debug({"# j_list": len(j_list)})
Tradegood2MongoDB.j_iter2mongodb(j_list)
def main():
from henrique.main.singleton.logger.henrique_logger import HenriqueLogger
# KhalaLogger.attach_stderr2loggers(logging.DEBUG)
DiscordLogger.attach_stderr2loggers(logging.DEBUG)
HenriqueLogger.attach_stderr2loggers(logging.DEBUG)
start_discord()
def postgres2j_iter(cls):
logger = HenriqueLogger.func_level2logger(cls.postgres2j_iter,
logging.DEBUG)
with HenriquePostgres.cursor() as cursor:
sql = SQL("SELECT * from {}").format(
Identifier(TradegoodTable.NAME))
cursor.execute(sql)
for t in PostgresTool.fetch_iter(cursor):
j = t[TradegoodTable.index_json()]
# logger.debug({"j":j})
h_lang2names = {}
for lang, name in j["name"].items():
h_lang2names[lang] = lchain(h_lang2names.get(lang, []),
[name])
for lang, nickname_list in j.get("nicknames", {}).items():
h_lang2names[lang] = lchain(h_lang2names.get(lang, []),
nickname_list)
j[TradegoodDoc.F.NAMES] = {
lang: luniq(name_list)
for lang, name_list in h_lang2names.items()
}
for k in ["name", "nicknames"]:
j.pop(k, None)
# logger.debug({'j["names"]':j["names"]})
j[TradegoodDoc.F.KEY] = j["names"]["en"][0]
yield j
def _langs2matcher(cls, langs):
logger = HenriqueLogger.func_level2logger(cls._langs2matcher,
logging.DEBUG)
gazetteer = cls.langs2gazetteer(langs)
def texts2pattern(texts):
rstr_raw = RegexTool.rstr_iter2or(map(re.escape, texts))
left_bounds = lchain(
RegexTool.bounds2suffixed(RegexTool.left_wordbounds(), "\d"),
RegexTool.left_wordbounds(),
)
right_bounds = RegexTool.right_wordbounds()
rstr = RegexTool.rstr2bounded(rstr_raw, left_bounds, right_bounds)
logger.debug({
"rstr": rstr,
"rstr_raw": rstr_raw,
})
return re.compile(rstr, re.I)
config = {
GazetteerMatcher.Config.Key.TEXTS2PATTERN: texts2pattern,
GazetteerMatcher.Config.Key.NORMALIZER: cls.normalize,
}
matcher = GazetteerMatcher(gazetteer, config=config)
return matcher
def env2compile(cls, env):
logger = HenriqueLogger.func_level2logger(cls.env2compile,
logging.DEBUG)
logger.debug({"env": env})
uwsgi_mode = HenriqueUwsgi.env2mode(env)
nginx_mode = HenriqueNginx.env2mode(env)
uid = HenriqueNginx.Mode.mode2user(nginx_mode)
gid = HenriqueNginx.Mode.mode2group(nginx_mode)
data = {
"env": env,
"uid": uid,
"gid": gid,
"uwsgi_mode": uwsgi_mode,
}
filepath = os.path.join(FILE_DIR, "conf",
"henrique.supervisord.{}.conf".format(env))
utf8 = Jinja2Renderer.template2text(cls.template(), data=data)
FileTool.utf82file(utf8, filepath)
logger.debug({
"filepath": filepath,
"data": data,
})
def price_lang2text(cls, price, lang):
logger = HenriqueLogger.func_level2logger(cls.price_lang2text,
logging.DEBUG)
if price is None:
return cls.lang2text_idk(lang)
rate = MarketpriceDoc.price2rate(price)
trend = MarketpriceDoc.price2trend(price)
# raise Exception({"price":price})
channel_user_codename = MarketpriceDoc.price2channel_user(price)
channel_user = l_singleton2obj(
ChannelUser.codenames2channel_users([channel_user_codename]))
logger.debug({"price": price, "channel_user": channel_user})
# raise Exception({"price":price})
created_at = MarketpriceDoc.price2created_at(
price) or MarketpriceDoc.created_at_backoff()
td = datetime.now(tz=pytz.utc) - created_at
str_timedelta = HenriqueDatetime.timedelta_lang2str(td, lang)
arrow = Trend.trend2arrow(trend)
text_out_base = format_str("{}{} @ {}", str(rate), arrow,
str_timedelta)
if td >= HenriqueDatetime.Constant.TIMEDELTA_OUTDATED:
return text_out_base
user_alias = ChannelUser.channel_user2alias(channel_user)
str_user_alias = "[by {}]".format(user_alias)
text_out = " ".join([text_out_base, str_user_alias])
return text_out
def port_tradegood_lists2blocks(cls, port_tradegood_list, price_dict, lang,
groupby_parameter_type):
logger = HenriqueLogger.func_level2logger(
cls.port_tradegood_lists2blocks, logging.DEBUG)
logger.debug({"port_tradegood_list": port_tradegood_list})
if groupby_parameter_type == PriceSkillParameter.Type.PORTLIKE:
from henrique.main.skill.price.by_port.price_by_port import PriceByPort
blocks = [
PriceByPort.port2text(port_codename, lmap(ig(1), l),
price_dict, lang) for port_codename, l in
gb_tree_global(port_tradegood_list, [ig(0)])
]
return blocks
if groupby_parameter_type == PriceSkillParameter.Type.TRADEGOOD:
from henrique.main.skill.price.by_tradegood.price_by_tradegood import PriceByTradegood
blocks = [
PriceByTradegood.tradegood2text(tg_codename, lmap(ig(0), l),
price_dict, lang) for
tg_codename, l in gb_tree_global(port_tradegood_list, [ig(1)])
]
return blocks
raise Exception(groupby_parameter_type)
def marketprice_doc_iter(cls):
logger = HenriqueLogger.func_level2logger(cls.marketprice_doc_iter,
logging.DEBUG)
# h_id2culture = CultureTable.dict_id2codename()
# logger.debug({"h_id2culture":h_id2culture})
with HenriquePostgres.cursor() as cursor:
sql = SQL("SELECT * FROM {} ORDER BY id ASC").format(
Identifier(cls.NAME), )
cursor.execute(sql)
for t in PostgresTool.fetch_iter(cursor):
port_codename = cls.tuple2port_name_en(t)
tradegood_codename = cls.tuple2tradegood_name_en(t)
rate = cls.tuple2rate(t)
trend = Trend.int2trend(cls.tuple2trend(t))
data = cls.tuple2j(t)
server_codename = JsonTool.down(data, ["server"])
doc = {
MarketpriceDoc.Field.PORT: port_codename,
MarketpriceDoc.Field.TRADEGOOD: tradegood_codename,
MarketpriceDoc.Field.RATE: rate,
MarketpriceDoc.Field.TREND: trend,
MarketpriceDoc.Field.SERVER: server_codename,
# MarketpriceDoc.Field.CHATROOM_USER: channel_user,
}
yield doc
def pair_list2mongodb(cls, pair_list):
logger = HenriqueLogger.func_level2logger(cls.pair_list2mongodb, logging.DEBUG)
logger.debug({"len(pair_list)": len(pair_list)})
n = len(pair_list)
write_concern = WriteConcern(w=3, wtimeout=n)
def upsert_channel_user():
channel_user_list_raw = lfilter(is_not_none, map(ig(1), pair_list))
logger.debug({"len(channel_user_list_raw)": len(channel_user_list_raw)})
# raise Exception({"channel_user_list_raw[0]": channel_user_list_raw[0]})
channel_user_list = luniq(channel_user_list_raw, idfun=ChannelUser.channel_user2codename)
collection = ChannelUserCollection.collection().with_options(write_concern=write_concern)
def doc2pair(channel_user):
cond = {ChannelUser.Field.CODENAME: ChannelUser.channel_user2codename(channel_user)}
return cond, channel_user
j_pair_list = lmap(doc2pair, channel_user_list)
MongoDBTool.j_pair_list2upsert(collection, j_pair_list)
def upsert_marketprice():
marketprice_list = lmap(ig(0), pair_list)
collection = MarketpriceCollection.collection().with_options(write_concern=write_concern)
j_pair_list = [(x,x) for x in marketprice_list]
MongoDBTool.j_pair_list2upsert(collection, j_pair_list)
upsert_channel_user()
upsert_marketprice()
def lang2pattern(cls, lang):
from henrique.main.document.price.trend.trend_entity import TrendEntity
logger = HenriqueLogger.func_level2logger(cls.lang2pattern,
logging.DEBUG)
# rstr_suffix = format_str("{}?",
# RegexTool.rstr2wrapped(TrendEntity.lang2rstr(lang)),
# )
### may be concatenated with port/tradegood name
# rstr_prefixed = RegexTool.rstr2rstr_words_prefixed(cls.rstr())
# raise Exception({"rstr_suffix":rstr_suffix})
rstr_trend = TrendEntity.lang2rstr(lang)
# bound_right_list_raw = RegexTool.right_wordbounds()
right_bounds = lchain(
RegexTool.bounds2prefixed(RegexTool.right_wordbounds(),
rstr_trend),
RegexTool.right_wordbounds(),
)
rstr_rightbound = RegexTool.rstr2right_bounded(cls.rstr(),
right_bounds)
logger.debug({#"rstr_trend":rstr_trend,
#"right_bounds":right_bounds,
"rstr_rightbound":rstr_rightbound,
})
# rstr_suffixed = RegexTool.rstr2rstr_words_suffixed(cls.rstr(), rstr_suffix=rstr_suffix)
# raise Exception({"rstr_trend": rstr_trend, "rstr_suffixed": rstr_suffixed})
# return re.compile(RegexTool.rstr2wordbounded(cls.rstr()))
return re.compile(rstr_rightbound, re.I)
def nanban_datetime2upsert_mongo(
cls,
packet,
datetime_nanban,
):
logger = HenriqueLogger.func_level2logger(
cls.nanban_datetime2upsert_mongo, logging.DEBUG)
server_codename = HenriquePacket.packet2server(packet)
text_in = KhalaPacket.packet2text(packet)
# server = Server.codename2server(server_codename)
dt_utc = DatetimeTool.astimezone(datetime_nanban, pytz.utc)
# raise Exception({"datetime_in":datetime_in,"dt_utc":dt_utc})
nanban = {
ServerNanban.Field.DATETIME: dt_utc,
ServerNanban.Field.COMMAND_IN: text_in,
}
doc_this = {
ServerDoc.Field.CODENAME: server_codename,
ServerDoc.Field.NANBAN: nanban,
}
logger.debug({
"datetime_nanban": datetime_nanban,
"doc_this": doc_this,
})
ServerDoc.docs2upsert([doc_this])
def culture_list_all(cls):
logger = HenriqueLogger.func_level2logger(cls.culture_list_all,
logging.DEBUG)
h_codename2aliases_en = NamesenSheet.dict_codename2aliases()
h_codename2aliases_ko = NameskoSheet.dict_codename2aliases()
h_codename2prefers = PrefersSheet.dict_codename2prefers()
codename_list = luniq(
chain(
h_codename2aliases_en.keys(),
h_codename2aliases_ko.keys(),
h_codename2prefers.keys(),
))
def codename2culture(codename):
aliases = DictTool.filter(
lambda k, v: v, {
"en": h_codename2aliases_en.get(codename),
"ko": h_codename2aliases_ko.get(codename),
})
culture = {
Culture.Field.CODENAME: codename,
Culture.Field.ALIASES: aliases,
Culture.Field.PREFERS: h_codename2prefers.get(codename) or [],
}
return DictTool.filter(lambda k, v: v, culture)
list_all = lmap(codename2culture, codename_list)
# logger.debug({"list_all":list_all})
return list_all
def get(cls):
logger = HenriqueLogger.func_level2logger(cls.get, logging.DEBUG)
sender_name = request.args.get(cls.Data.Field.SENDER_NAME)
text_in = request.args.get(cls.Data.Field.TEXT)
newline = request.args.get(cls.Data.Field.NEWLINE)
logger.debug({"sender_name":sender_name, "text_in":text_in, "newline":newline})
if not HenriqueCommand.text2is_query(text_in):
return None
chatroom = ChatroomKakaotalk.chatroom()
Chatroom.chatrooms2upsert([chatroom])
channel_user = ChannelUserKakaotalk.sender_name2channel_user(sender_name)
ChannelUser.channel_users2upsert([channel_user])
packet = {KhalaPacket.Field.TEXT: text_in,
KhalaPacket.Field.CHATROOM: Chatroom.chatroom2codename(chatroom),
KhalaPacket.Field.CHANNEL_USER: ChannelUser.channel_user2codename(channel_user),
KhalaPacket.Field.SENDER_NAME: sender_name,
}
logger.debug({"packet": packet,})
text_response = HenriquePacket.packet2response(packet)
if not text_response:
return None
if cls.packet2skip_response(packet): # run packet but do not respond
return None
text_out = newline.join(text_response.splitlines()) if newline else text_response
return text_out, 200
def entity2response_block(cls, entity, packet, ):
logger = HenriqueLogger.func_level2logger(cls.packet2response, logging.DEBUG)
chatroom = Chatroom.codename2chatroom(KhalaPacket.packet2chatroom(packet))
if Chatroom.chatroom2codename(chatroom) != ChatroomKakaotalk.codename():
return
locale = Chatroom.chatroom2locale(chatroom)
lang = LocaleTool.locale2lang(locale)
v = FoxylibEntity.entity2value(entity)
codename = ChatroomuserEntity.value_packet2codename(v, packet)
logger.debug({"codename": codename,
"entity": entity,
"v": v,
})
chatroomuser = Chatroomuser.codename2chatroomuser(codename)
comments = Chatroomuser.chatroomuser2comments(chatroomuser)
comment = choice(comments) if comments else None
filepath = os.path.join(FILE_DIR, "tmplt.{}.part.txt".format(lang))
data = {"name": codename,
"comment": comment,
"str_aliases": ", ".join(Chatroomuser.chatroomuser2aliases(chatroomuser)),
}
text_out = str2strip(HenriqueJinja2.textfile2text(filepath, data))
return text_out
def test_01(self):
logger = HenriqueLogger.func_level2logger(self.test_01, logging.DEBUG)
with HenriquePostgres.cursor() as cursor:
cursor.execute("""SELECT * from unchartedwatersonline_port""")
l = cursor.fetchall()
logger.debug({"l": l})
def test_client(cls):
logger = HenriqueLogger.func_level2logger(cls.test_client,
logging.DEBUG)
logger.debug({"START": "START"})
c = cls.app().test_client()
logger.debug({"END": "END"})
return c
def env2ip(cls, env):
logger = HenriqueLogger.func_level2logger(cls.env2ip, logging.DEBUG)
j_server = cls.j_server()
# logger.debug({"j_server":j_server})
ip = JsonTool.down(j_server, [cls.Constant.ROOT, env, "ip"])
#logger.debug({"[cls.Constant.ROOT,env,ip]":[cls.Constant.ROOT,env,"ip"]})
return ip
def key2nullboolean(cls, key):
logger = HenriqueLogger.func_level2logger(cls.key2nullboolean,
logging.DEBUG)
v = cls.key2value(key)
nb = BooleanTool.parse2nullboolean(v)
# logger.debug({"key":key, "v":v, "nb":nb})
return nb
def element2relativedelta(cls, element):
logger = HenriqueLogger.func_level2logger(cls.element2relativedelta,
logging.DEBUG)
unit = cls.element2unit(element)
quantity = cls.element2quantity(element)
kwargs = {TimedeltaEntityUnit.unit2plural(unit): quantity}
# logger.debug({"kwargs":kwargs})
return relativedelta(**kwargs)
def entity2relativedelta(cls, entity):
logger = HenriqueLogger.func_level2logger(cls.entity2relativedelta,
logging.DEBUG)
element_list = FoxylibEntity.entity2value(entity)
relativedelta_list = lmap(TimedeltaElement.element2relativedelta,
element_list)
logger.debug({"relativedelta_list": relativedelta_list})
return sum(relativedelta_list, relativedelta(days=0))
def test_06(self):
logger = HenriqueLogger.func_level2logger(self.test_03, logging.DEBUG)
config = {HenriqueEntity.Config.Field.LOCALE: "en-US"}
hyp = TrendEntity.text2entity_list("95 하", config=config)
ref = []
# pprint({"hyp":hyp})
self.assertEqual(hyp, ref)
def postgres2tuple_iter(cls):
logger = HenriqueLogger.func_level2logger(cls.postgres2tuple_iter, logging.DEBUG)
with HenriquePostgres.cursor() as cursor:
sql = SQL("SELECT * from {}").format(Identifier(PortTradegoodStateTable.NAME))
cursor.execute(sql)
for t_list_chunk in ChunkTool.chunk_size2chunks(PostgresTool.fetch_iter(cursor), 100000):
yield from t_list_chunk
def _load_urls2app(cls, app):
logger = HenriqueLogger.func_level2logger(cls._load_urls2app,
logging.DEBUG)
from henrique.front.path.view import IndexView
FlaskTool.add_url2app(app,
IndexView.url(),
IndexView.get,
methods=[
"GET",
])
def xoxb_token(cls):
logger = HenriqueLogger.func_level2logger(cls.xoxb_token,
logging.DEBUG)
key = "SLACK_HENRIQUE_BOT_USER_OAUTH_ACCESS_TOKEN"
token = HenriqueEnv.key2value(key)
logger.debug({
"key": key,
"bool(token)": bool(token),
"token is None": token is None,
})
return token
def tuple2pair(cls, t):
logger = HenriqueLogger.func_level2logger(cls.tuple2pair, logging.DEBUG)
def chatuser_uuid2channel_user(chatuser_uuid, alias=None):
if not chatuser_uuid:
return None
row = ChatuserTable.uuid2row(chatuser_uuid)
if not row:
logger.debug({"chatuser_uuid":chatuser_uuid})
return None
channel_user = ChatuserTable.row2channel_user(row, alias=alias)
return channel_user
j_postgres = t[PortTradegoodStateTable.index_json()]
sender_name = j_postgres.get("sender_name")
created_at = DatetimeTool.fromisoformat(j_postgres["created_at"])
port_name_en = PortTradegoodStateTable.tuple2port_name_en(t)
port = cls.name_en2port(port_name_en)
tradegood_name_en = PortTradegoodStateTable.tuple2tradegood_name_en(t)
tradegood = cls.name_en2tradegood(tradegood_name_en)
rate = PortTradegoodStateTable.tuple2rate(t)
trend = PortTradegoodStateTable.tuple2trend(t)
chatuser_uuid = PortTradegoodStateTable.tuple2chatuser_uuid(t)
channel_user = chatuser_uuid2channel_user(chatuser_uuid, alias=sender_name)
channel_user_code = ChannelUser.channel_user2codename(channel_user) if channel_user else None
server_alias = str2lower(jdown(j_postgres, ["server", "name"]))
# logger.debug({"server_alias":server_alias})
server = Server.alias_lang2server(server_alias, "ko")
assert_true(server,server_alias)
marketprice = {MarketpriceDoc.Field.CREATED_AT: created_at,
MarketpriceDoc.Field.PORT: Port.port2codename(port),
MarketpriceDoc.Field.TRADEGOOD: Tradegood.tradegood2codename(tradegood),
MarketpriceDoc.Field.RATE: rate,
MarketpriceDoc.Field.TREND: trend,
MarketpriceDoc.Field.CHANNEL_USER: channel_user_code,
MarketpriceDoc.Field.SERVER: Server.server2codename(server),
}
return marketprice, channel_user
def warmup_all(cls):
logger = HenriqueLogger.func_level2logger(cls.warmup_all,
logging.DEBUG)
logger.debug("START")
# not really necessary cuz warmer will run once loaded in each file, but still to avoid unnecessary warnings
warmer_list = list(cls.warmer_iter())
# warmer_list = [IterTool.first(cls.warmer_iter())]
logger.debug({"warmer_list": warmer_list})
for x in warmer_list:
logger.debug({"warmer": x})
x.warmup()
logger.debug("END")
def test_06(self):
logger = HenriqueLogger.func_level2logger(self.test_06, logging.DEBUG)
Chatroom.chatrooms2upsert([ChatroomKakaotalk.chatroom()])
ServerDoc.codenames2delete([Server.Codename.MARIS])
sender_name = "iris"
channel_user_codename = ChannelUserKakaotalk.sender_name2codename(
sender_name)
ChannelUser.channel_users2upsert(
[ChannelUserKakaotalk.sender_name2channel_user(sender_name)])
tz = pytz.timezone("Asia/Seoul")
now_tz = DatetimeTool.datetime2truncate_seconds(datetime.now(tz))
# hour = (now_tz + timedelta(seconds=3600)).hour
packet1 = {
KhalaPacket.Field.TEXT:
"?남만 {}".format(now_tz.strftime("%I:%M %p")),
KhalaPacket.Field.CHATROOM: KakaotalkUWOChatroom.codename(),
KhalaPacket.Field.CHANNEL_USER: channel_user_codename,
KhalaPacket.Field.SENDER_NAME: sender_name,
}
NanbanSkill.packet2response(packet1)
packet2 = {
KhalaPacket.Field.TEXT: "?남만 +2분 1초",
KhalaPacket.Field.CHATROOM: KakaotalkUWOChatroom.codename(),
KhalaPacket.Field.CHANNEL_USER: channel_user_codename,
KhalaPacket.Field.SENDER_NAME: sender_name,
}
NanbanSkill.packet2response(packet2)
doc = ServerDoc.codename2doc(Server.Codename.MARIS)
dt_nanban = DatetimeTool.astimezone(
ServerNanban.nanban2datetime(ServerDoc.doc2nanban(doc)), tz)
logger.debug({
"now_tz":
now_tz,
"dt_nanban":
dt_nanban,
'now_tz.strftime("%I:%M %p")':
now_tz.strftime("%I:%M %p"),
})
ref = (now_tz + timedelta(seconds=2 * 60 + 1)).timetz()
self.assertEquals(dt_nanban.timetz(), ref)
def test_05(self):
logger = HenriqueLogger.func_level2logger(self.test_03, logging.DEBUG)
config = {HenriqueEntity.Config.Field.LOCALE: "en-US"}
hyp = TrendEntity.text2entity_list("95 down", config=config)
ref = [{
'span': (3, 7),
'text': 'down',
'type': TrendEntity.entity_type(),
'value': 'down'
}]
# pprint({"hyp":hyp})
self.assertEqual(hyp, ref)
