async def events(event: EventCallback):
if event.type == 'url_verification':
# AppにRequest URLを登録した際に初回だけ送信されるURLの検証
# ref: https://api.slack.com/events/url_verification
return JSONResponse({'challenge': event.challenge})
try:
team_conf = TeamConf.get(event.team_id)
except TeamConf.DoesNotExist:
return Response(status_code=HTTPStatus.BAD_REQUEST)
client = WebClient(team_conf.access_token)
if event.event:
if event.event.type == 'reaction_added':
# 投稿にemojiでリアクションがあったイベントを処理する
# ref: https://api.slack.com/events/reaction_added
if event.event.reaction in team_conf.emoji_set:
# リアクションのemojiが設定されている場合
if event.event.item:
item = event.event.item
url = client.chat_getPermalink(
channel=item.channel,
message_ts=item.ts).get('permalink')
blocks = [
SectionBlock(text=MarkdownTextObject(text=f'<{url}>')),
ActionsBlock(elements=[
ButtonElement(text='読んだ',
action_id='mark_as_read',
value='mark_as_read')
])
]
client.chat_postMessage(text=url,
channel=event.event.user,
unfurl_links=True,
blocks=blocks)
return Response()
class CountStamp:
startdate = None
startdatetime = None
enddatetime = None
stamp_counter = {}
user_counter = {}
recv_counter = {}
client = None
bot = None
message = ""
chat_list = []
# インストラクタ
def __init__(self):
print("CountStamp Start")
self.client = WebClient(const.USER_TOKEN)
self.bot = WebClient(const.BOT_TOKEN)
#Start of time
self.startdate = datetime.date.today() - datetime.timedelta(
days=const.DAYS_AGO)
self.startdatetime = datetime.datetime.combine(self.startdate,
datetime.time())
self.enddatetime = self.startdatetime + datetime.timedelta(
days=const.DAYS_TERM)
print("term:{}-{}".format(self.startdatetime, self.enddatetime))
# チャンネルリストを取得する
def setChannelList(self):
self.channel_list = self.client.conversations_list(
exclude_archived=True, limit=const.CHANNEL_LIMIT)
print("channels:{}".format(len(self.channel_list["channels"])))
# スタンプをカウントする
def cntStamp(self):
channel_cnt = 0
for channel in self.channel_list["channels"]:
channel_cnt = channel_cnt + 1
channel_id = channel["id"]
#apiの呼び出し回数の制限（1分間に50回まで）を回避する
time.sleep(1)
history = self.client.conversations_history(
channel=channel_id,
oldest=self.startdatetime.timestamp(),
latest=self.enddatetime.timestamp())
#履歴内のスタンプをカウントする
self.cntReactions(history=history, channel=channel)
# 履歴内のスタンプをカウントする
def cntReactions(self, history, channel):
if (len(history['messages']) > 0):
print("channel_name:{} messages:{}".format(
channel['name'], len(history['messages'])))
for message in history["messages"]:
try:
if (message.get("reactions")):
reactions_cnt = 0
for reaction in message["reactions"]:
# ユーザー別のスタンプ数のカウント
self.cntUsers(users=reaction["users"])
# スタンプ別のスタンプ数のカウント
key = reaction["name"]
if (self.stamp_counter.get(key)):
self.stamp_counter[key] = self.stamp_counter[
key] + reaction["count"]
else:
self.stamp_counter[key] = reaction["count"]
# スタンプを受け取ったユーザー別のスタンプ数のカウント
if (message.get("user")):
key = message["user"]
if (self.recv_counter.get(key)):
self.recv_counter[key] = self.recv_counter[
key] + reaction["count"]
else:
self.recv_counter[key] = reaction["count"]
# スレッドについたスタンプ数のカウント
reactions_cnt = reactions_cnt + reaction["count"]
# スレッド別のスタンプ数
self.chat_list.append(
[channel['id'], message["ts"], reactions_cnt])
except KeyError as e:
print("KeyError:")
print(e.args)
#スタンプしたユーザーをカウント
def cntUsers(self, users):
for user_id in users:
if (self.user_counter.get(user_id)):
self.user_counter[user_id] = self.user_counter[user_id] + 1
else:
self.user_counter[user_id] = 1
# カウントをポストする
def setMessage(self):
sorted_stamp = sorted(self.stamp_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_user = sorted(self.user_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_recv = sorted(self.recv_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_chat = sorted(self.chat_list, key=lambda x: x[2], reverse=True)
w_list = ['月', '火', '水', '木', '金', '土', '日']
self.message = "{}({})のスタンプランキングTOP{}を発表します。\n".format(
self.startdate.strftime('%Y年%m月%d日'),
w_list[self.startdate.weekday()], const.RANK_LIMIT)
self.message = self.message + "\n\n:+1:このスタンプが良く使われました:+1:\n"
self.setRankingMessage(sorted_stamp, False)
self.message = self.message + "\n\n:tera_感謝_赤:このユーザーがたくさんスタンプしました:tera_感謝_赤:\n"
self.setRankingMessage(sorted_user, True)
self.message = self.message + "\n\n:gift:このユーザーがたくさんスタンプを受け取りました:gift:\n"
self.setRankingMessage(sorted_recv, True)
self.message = self.message + "\n\n:trophy:スタンプを集めたメッセージはこちら:trophy:\n"
self.setChatRankingMessage(sorted_chat)
total_stamp = sum(self.stamp_counter.values())
self.message = self.message + "\n\nすべてのスタンプを合計すると {} でした！".format(
total_stamp)
i = 1
while i <= int(total_stamp / const.CLAP_LOOP):
self.message = self.message + ":clap:"
i = i + 1
today_weekday = datetime.date.today().weekday()
# 土日はメッセージを変える。
if (today_weekday != 5 and today_weekday != 6):
self.message = self.message + "\nそれでは今日もはりきってスタンプしましょう！"
else:
self.message = self.message + "\n休日対応おつかれさまです。"
def postMessage(self):
self.bot.chat_postMessage(channel=const.CHANNEL_NAME,
text=self.message)
#ランキング処理
def setRankingMessage(self, rank_list, user_flag):
rank = 1
i = 0
while i < len(rank_list):
if (user_flag):
self.message = self.message + '\n{}位 {} {}'.format(
rank, self.getUsername(rank_list[i][0]), rank_list[i][1])
else:
self.message = self.message + '\n{}位 :{}: {}'.format(
rank, rank_list[i][0], rank_list[i][1])
#同列順位の処理
j = 1
while (i + j) < len(rank_list):
if (rank_list[i][1] == rank_list[i + j][1]):
if (user_flag):
self.message = self.message + ' {} {}'.format(
self.getUsername(rank_list[i + j][0]),
rank_list[i + j][1])
else:
self.message = self.message + ' :{}: {}'.format(
rank_list[i + j][0], rank_list[i + j][1])
j = j + 1
else:
break
i = i + j
rank = rank + j
if (rank > const.RANK_LIMIT):
break
self.message = self.message + '\n'
#チャットのランク処理
def setChatRankingMessage(self, sorted_chat):
rank = 1
for chat in sorted_chat:
link = self.bot.chat_getPermalink(channel=chat[0],
message_ts=chat[1])
self.message = self.message + link["permalink"] + '\n'
rank = rank + 1
if (rank > const.CHAT_RANK_LIMIT):
break
#ユーザーの表示名を取得する
def getUsername(self, user_id):
user_info = self.bot.users_info(user=user_id)
return user_info['user']['profile']['real_name']
# デストラクタ
def __del__(self):
print("CountStamp End")
class CSDigest:
def __init__(self, config_path) -> None:
print("loading data")
# handling files
with open(config_path) as cfg:
config = yaml.load(cfg, Loader=yaml.FullLoader)
with open(config["token_path"]) as tk:
self.token = tk.readline()
with open(config["template_path"]) as tp:
self.temp = BeautifulSoup(tp, "html.parser")
self.cache_im = os.path.join("cache", "images")
os.makedirs(self.cache_im, exist_ok=True)
self.ts_old = dateparser.parse(config["time_span"]).timestamp()
# initialize objects
self.foodnet = models.load_model("./foodnet/model")
self.datagen = ImageDataGenerator(rescale=1.0 / 255)
self.client = WebClient(token=self.token)
self.channels = pd.DataFrame(
self.client.conversations_list(limit=1000)["channels"]
).set_index("name")
self.users = pd.DataFrame(
self.client.users_list(limit=1000)["members"]
).set_index("id")
self.users["display_name"] = self.users["profile"].apply(
self.extract_profile, key="display_name"
)
print("fetching messages")
# get messages
ms_general = self.get_msg("general", same_user=False)
ms_home = self.get_msg("homesanity", ts_thres=0)
ms_quote = self.get_msg("quotablequotes", ts_thres=120, same_user=False)
print("building newsletter")
# handle carousel
if len(ms_general) > 0:
ms_general["class"] = ms_general.apply(self.classify_msg, axis="columns")
ms_tada = ms_general[ms_general["class"] == "tada"]
if len(ms_tada) > 0:
ms_tada["permalink"] = ms_tada.apply(self.get_permalink, axis="columns")
self.build_carousel(ms_tada)
# handle food
ms_files = pd.concat([ms_general, ms_home])
if len(ms_files) > 0:
ms_files["file_path"] = ms_files.apply(self.download_images, axis="columns")
ms_files = ms_files[ms_files["file_path"].astype(bool)]
ms_files["food_prob"] = ms_files["file_path"].apply(self.classify_food)
ms_files["food_path"] = ms_files.apply(self.filter_food, axis="columns")
ms_food = ms_files[ms_files["food_path"].notnull()]
ms_food["permalink"] = ms_food.apply(self.get_permalink, axis="columns")
ms_food["aspect"] = ms_food["food_path"].apply(self.get_img_aspect)
ms_food = ms_food.sort_values("aspect", ascending=True)
self.build_portfolio(ms_food)
# handle quotes
if len(ms_quote) > 0:
ms_quote = ms_quote[~ms_quote["files"].astype(bool)]
ms_quote["permalink"] = ms_quote.apply(self.get_permalink, axis="columns")
self.build_quotes(ms_quote)
def get_msg(self, channel, ts_thres=5, same_user=True):
ms = pd.DataFrame(
self.client.conversations_history(
channel=self.channels.loc[channel]["id"],
oldest=self.ts_old,
limit=1000,
)["messages"]
)
if len(ms) > 0:
try:
ms = ms[ms["subtype"].isnull()]
except KeyError:
pass
if len(ms) > 0:
ms = (
self.cluster_msg(ms, ts_thres=ts_thres, same_user=same_user)
.groupby("component")
.apply(self.merge_msg)
.reset_index()
.apply(self.translate_msg_user, axis="columns")
)
ms["text"] = ms["text"].apply(
emojize, use_aliases=True, variant="emoji_type"
)
ms["channel"] = self.channels.loc[channel]["id"]
return ms
def classify_msg(self, msg_df):
if msg_df["reactions"]:
tada = list(filter(lambda r: r["name"] == "tada", msg_df["reactions"]))
if tada and tada[0]["count"] > 5:
return "tada"
def cluster_msg(self, msg_df, ts_thres, same_user):
ts_dist = pdist(msg_df["ts"].values.astype(float).reshape((-1, 1)))
txt_dist = pdist(CountVectorizer().fit_transform(msg_df["text"]).toarray())
adj = squareform(ts_dist) < ts_thres * 60
if same_user:
user_dist = pdist(
msg_df["user"].values.reshape((-1, 1)),
metric=lambda u, v: 0 if u == v else 1,
)
adj = adj * (squareform(user_dist) < 1)
n_comp, lab = connected_components(adj, directed=False)
msg_df["component"] = lab
return msg_df
def merge_msg(self, msg_df, multiple_users="first"):
msg_df = msg_df.sort_values("ts")
if multiple_users == "forbid":
user = msg_df["user"].unique()
assert len(user) == 1
user = user.item()
elif multiple_users == "first":
user = msg_df.iloc[0]["user"]
msg_df = msg_df[msg_df["user"] == user]
else:
raise ValueError("multiple_users=={} not understood".format(multiple_users))
try:
reactions = msg_df["reactions"].dropna().values
reactions = sum(reactions, [])
except KeyError:
reactions = []
try:
files = msg_df["files"].dropna().values
files = sum(files, [])
except KeyError:
files = []
try:
attch = msg_df["attachments"].dropna().values
attch = sum(attch, [])
except KeyError:
attch = []
return pd.Series(
{
"user": user,
"text": "\n".join(msg_df["text"].values),
"ts": msg_df.iloc[0].loc["ts"],
"reactions": reactions,
"files": files,
"attachments": attch,
}
)
def translate_msg_user(self, msg_row, substitute=["display_name", "name"]):
try:
msg_row["user"] = self.translate_user(msg_row["user"], substitute)
msg_row["text"] = re.sub(
r"\<\@(.*?)\>",
lambda u: self.translate_user(u.group(1), substitute),
msg_row["text"],
)
except TypeError:
pass
return msg_row
def translate_user(self, uid, substitute):
for sub in substitute:
if sub == "real_name":
prefix = ""
else:
prefix = "@"
user = self.users.loc[uid, sub]
if type(user) == str and bool(user):
return prefix + user
def extract_profile(self, prof, key):
try:
return prof[key]
except KeyError:
return np.nan
def get_permalink(self, msg_row):
return self.client.chat_getPermalink(
channel=msg_row["channel"], message_ts=str(msg_row["ts"])
)["permalink"]
def download_images(self, msg_row):
fpaths = []
for fdict in msg_row["files"]:
try:
mimietype = fdict["mimetype"]
except KeyError:
continue
if mimietype.startswith("image"):
fpath = os.path.join(
self.cache_im,
".".join([fdict["id"], fdict["filetype"]]),
)
resp = requests.get(
fdict["url_private_download"],
headers={"Authorization": "Bearer {}".format(self.token)},
)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
for fdict in msg_row["attachments"]:
try:
url = fdict["image_url"]
except KeyError:
continue
fpath = os.path.join(self.cache_im, url.split("/")[-1].split("?")[0])
resp = requests.get(url)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
return fpaths
def build_carousel(self, msg_df):
indicator = self.temp.find("ol", {"id": "carousel-inds"})
ind_temp = indicator.find("li", {"id": "carousel-ind-template"}).extract()
sld_wrapper = self.temp.find("div", {"id": "carousel-slides"})
tada_temp = self.temp.find("div", {"id": "carousel-slide-template"}).extract()
for (imsg, msg), icss in zip(
msg_df.reset_index(drop=True).iterrows(), itt.cycle(np.arange(3) + 1)
):
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
cur_tada = copy.copy(tada_temp)
cur_tada.find("h3", {"id": "carousel-slide-message"}).string = (
msg["text"] if len(msg["text"]) <= 320 else msg["text"][:320] + "..."
)
cur_tada.find(True, {"id": "carousel-slide-author"}).string = " ".join(
[
msg["user"],
datetime.fromtimestamp(float(msg["ts"])).strftime("%b %d"),
]
)
cur_tada.find("a")["href"] = msg["permalink"]
if re.search("birthday", msg["text"].lower()):
cur_tada["class"] = [
"carousel-birthday" if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
else:
cur_tada["class"] = [
"carousel-tada-{}".format(icss) if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
if not imsg == 0:
del cur_ind["class"]
cur_tada["class"] = list(
filter(lambda c: c != "active", cur_tada["class"])
)
indicator.append(cur_ind)
sld_wrapper.append(cur_tada)
def write_html(self):
with open("csdigest.html", "w", encoding="utf-8") as outf:
outf.write(str(self.temp))
def classify_food(self, img_path):
try:
imgs = [img_to_array(load_img(imp).resize((512, 512))) for imp in img_path]
except:
return np.atleast_1d(np.ones(len(img_path))).tolist()
predict = self.foodnet.predict(self.datagen.flow(np.stack(imgs)))
return np.atleast_1d(predict.squeeze()).tolist()
def filter_food(self, msg_row, thres=0.1):
minval, minidx = np.min(msg_row["food_prob"]), np.argmin(msg_row["food_prob"])
if minval < thres:
return msg_row["file_path"][minidx]
else:
return np.nan
def get_img_aspect(self, path):
img = load_img(path)
return img.size[0] / img.size[1]
def build_portfolio(self, msg_df):
porto = self.temp.find("div", {"id": "portfolio-container"})
port_temp = self.temp.find("div", {"id": "portfolio-template"}).extract()
del port_temp["id"]
for imsg, msg in msg_df.iterrows():
cur_temp = copy.copy(port_temp)
cur_temp.img["src"] = msg["food_path"]
cur_temp.find("a", {"id": "port-zoom-link"})["href"] = msg["food_path"]
cur_temp.find("a", {"id": "port-msg-link"})["href"] = msg["permalink"]
txt = msg["text"]
if len(txt) > 150:
txt = txt[:150] + "..."
cur_temp.find(True, {"id": "port-item-text"}).string = txt
# cur_temp.find(True, {"id": "port-item-text"}).string = str(
# np.min(msg["food_prob"])
# )
porto.append(cur_temp)
def build_quotes(self, msg_df):
quotes = self.temp.find("div", {"id": "quote-block"})
quote_temp = quotes.find("div", {"id": "quote-template"}).extract()
indicators = self.temp.find("ol", {"id": "quote-indicator-wrap"})
ind_temp = indicators.find("li", {"id": "quote-indicator"}).extract()
for imsg, msg in msg_df.reset_index(drop=True).iterrows():
cur_quote = copy.copy(quote_temp)
cur_quote.find("a", {"id": "quote-link"})["href"] = msg["permalink"]
cur_quote.find("p", {"id": "quote-content"}).string = (
msg["text"] if len(msg["text"]) <= 400 else msg["text"][:400] + "..."
)
cur_quote.find("span", {"id": "quote-name"}).string = msg["user"]
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
if imsg > 0:
cur_quote["class"] = list(
filter(lambda c: c != "active", cur_quote["class"])
)
cur_ind["class"] = list(
filter(lambda c: c != "active", cur_ind["class"])
)
quotes.append(cur_quote)
indicators.append(cur_ind)
class Slack:
def __init__(self, token, channel_name=None, channel_id=None):
self.client = WebClient(token)
self.channel_id = channel_id
channels = self.client.conversations_list(
types='public_channel,private_channel')
if channel_name:
for channel in channels['channels']:
if channel['name'] == channel_name:
self.channel_id = channel['id']
break
if not self.channel_id:
self.channel_id = self.client.conversations_create(
name=channel_name.lower(), is_private=True)['channel']['id']
admins = [
u['id'] for u in self.client.users_list()['members']
if u.get('is_admin') or u.get('is_owner')
]
self.client.conversations_invite(channel=self.channel_id,
users=admins)
def send_snippet(self,
title,
initial_comment,
code,
code_type='python',
thread_ts=None):
return self.client.files_upload(
channels=self.channel_id,
title=title,
initial_comment=initial_comment.replace('<br>', ''),
content=code,
filetype=code_type,
thread_ts=thread_ts)['ts']
def send_exception_snippet(self,
domain,
event,
code_type='python',
thread_ts=None):
message = traceback.format_exc() + '\n\n\n' + dumps(event, indent=2)
subject = 'Error occurred in ' + domain
self.send_snippet(subject,
subject,
message,
code_type=code_type,
thread_ts=thread_ts)
def send_raw_message(self, blocks, thread_ts=None):
return self.client.chat_postMessage(channel=self.channel_id,
blocks=blocks,
thread_ts=thread_ts)['ts']
def update_raw_message(self, ts, blocks):
self.client.chat_update(channel=self.channel_id, blocks=blocks, ts=ts)
def get_perm_link(self, ts):
return self.client.chat_getPermalink(channel=self.channel_id,
message_ts=ts)['permalink']
def send_message(self, message, attachment=None, thread_ts=None):
blocks = [{
'type': 'section',
'text': {
'type': 'mrkdwn',
'text': message.replace('<br>', '')
}
}, {
'type': 'divider'
}]
if attachment:
blocks[0]['accessory'] = {
'type': 'button',
'text': {
'type': 'plain_text',
'text': attachment['text'],
'emoji': True
},
'url': attachment['value']
}
return self.send_raw_message(blocks, thread_ts)
class SlackMessaging:
"""A class used to parse in-game messages and send them to Slack."""
def __init__(
self,
slack_bot_token: str,
slack_channel_id: str,
) -> None:
"""Class constructor."""
super().__init__()
self.slack_channel_id = slack_channel_id
self.slack_client = WebClient(token=slack_bot_token)
self.logger = logging.getLogger(__name__)
def send_slack_message(
self, text: str, sender: Dict, parent: str = None
) -> SlackResponse:
"""Send Slack message to the configured channel."""
try:
icon = sender["icons"]["league"]
if sender["icons"]["race"]:
icon = sender["icons"]["race"]
elif sender["icons"]["player"]:
icon = sender["icons"]["player"]
# Call the chat.postMessage method using the WebClient
result = self.slack_client.chat_postMessage(
channel=self.slack_channel_id,
text=text,
username=sender["name"],
icon_url=icon,
thread_ts=parent,
)
self.logger.debug(result)
if parent:
time.sleep(1)
child_pl = self.slack_client.chat_getPermalink(
channel=self.slack_channel_id, message_ts=result["ts"]
)
text = f"<{child_pl['permalink']}|New message in thread>"
channel_result = self.slack_client.chat_postMessage(
channel=self.slack_channel_id,
text=text,
username=sender["name"],
icon_url=icon,
)
self.logger.debug(channel_result)
return result
except SlackApiError as err:
self.logger.error("Error posting message: %s", err)
raise
def send_new_messages_to_slack(
self, messages: List, planets_game_id: str, mbox_path: Path = None
) -> None:
"""Fetch messages from a game and send new ones to Slack."""
if not mbox_path:
mbox_path = Path(f"messages-{planets_game_id}.mbox")
mbox = Mbox(mbox_path)
message_ids = mbox.get_message_ids()
for msg in messages:
slack_thread_id = None
# Check that we haven't sent the message earlier, and filter out
# some spammy messages generated by the system.
if msg["msgid"] not in message_ids:
if not (
msg["sourcename"] == msg["gamename"]
and len(msg["replies"]) == 0
and re.search(
"joined the game in slot|earned the award", msg["message"]
)
):
# Let's not overwhelm Slack API
time.sleep(2)
# Construct the Slack message and send it
slack_resp = self.send_slack_message(
self.construct_slack_message(msg),
msg["sender"],
)
if slack_resp:
self.logger.info(
"Message %s sent to Slack successfully.", msg["msgid"]
)
slack_thread_id = slack_resp["ts"]
if mbox.save_email_message(msg, slack_thread_id):
message_ids[msg["msgid"]] = slack_thread_id
else:
self.logger.debug("Message %s already sent.", msg["msgid"])
slack_thread_id = message_ids[msg["msgid"]]
for reply in sorted(msg["replies"], key=lambda x: x["dateadded"]):
if reply["msgid"] not in message_ids:
# Let's not overwhelm Slack API
time.sleep(2)
# Construct the Slack message and send it
slack_reply_resp = self.send_slack_message(
self.construct_slack_message(reply),
reply["sender"],
slack_thread_id,
)
if slack_reply_resp:
self.logger.info(
"Reply %s (parent %s) sent to Slack successfully.",
reply["msgid"],
msg["msgid"],
)
if mbox.save_email_message(reply, slack_reply_resp["ts"]):
message_ids[reply["msgid"]] = slack_thread_id
else:
self.logger.debug(
"Reply %s (parent %s) already sent.",
reply["msgid"],
msg["msgid"],
)
@staticmethod
def construct_slack_message(message: Dict) -> str:
"""Construct a Slack message from an in-game message."""
turn_str = f'*Turn {message["turn"]}*'
to_str = f'*To:* {", ".join(message["recipients"].keys())}'
date_str = f'*Date*: {message["dateadded"]}'
body_str = message["message"].replace("<br/>", "\n")
return f"{turn_str}\n{to_str}\n{date_str}\n\n{body_str}"
