def client_conversations_history(slack_token="",
channel="",
retrieve_messages_from=0
):
# WebClient insantiates a client that can call API methods
# When using Bolt, you can use either `app.client` or the `client` passed to listeners.
client = WebClient(token=slack_token)
# Store conversation history
conversation_history = []
# ID of the channel you want to send the message to
channel_id = channel
try:
# Call the conversations.history method using the WebClient
# conversations.history returns the first 100 messages by default
# These results are paginated, see: https://api.slack.com/methods/conversations.history$pagination
result = client.conversations_history(channel=channel_id, oldest=retrieve_messages_from)
# Print results
logging.info("{} messages found in {}".format(len(conversation_history), id))
return result
except SlackApiError as e:
logging.error("Error creating conversation: {}".format(e))
pass
class SlackChannel:
"""Abstraction of slack channel API
"""
def __init__(self, *, token: str, channel: str):
self._client = WebClient(token=token)
self._channel = channel
self._msgs = None
def _load_conversation(self, *, latest: float) -> Iterable:
cursor = None
while True:
try:
response = self._client.conversations_history(
channel=self._channel,
latest=latest,
cursor=cursor,
)
if response.status_code != 200:
_log.error("%r", response)
return
if not response.data.get("ok", False):
_log.error("%s",
response.data.get("error", "unknown failure"))
return
for msg in response.data.get("messages", []):
yield msg
if not response.data.get("has_more", False):
return
cursor = response.data.get("response_metadata",
{}).get("next_cursor")
except SlackApiError:
_log.exception("Failed to retrieve conversation messages")
return
def messages(self, *, upto: str) -> Iterable:
"""Yield all messages in the channel upto given time
"""
yield from self._load_conversation(latest=parse(upto).timestamp())
def delete_message(self, *, message_id: str) -> bool:
"""Delete specific message from the chat
"""
try:
response = self._client.chat_delete(channel=self._channel,
ts=message_id)
return response.status_code == 200
except SlackApiError:
_log.exception("Failed to delete message")
return False
def get_messages(config: ScraperConfig) -> List[Dict[str, Any]]:
slack_client = WebClient(token=config.slack_token)
messages = []
cursor = None
while True:
response = slack_client.conversations_history(
channel=config.slack_channel_id,
oldest=config.start_datetime.timestamp(),
cursor=cursor)
messages += response.data["messages"]
if not response.data["has_more"]:
break
cursor = response.data["response_metadata"]["next_cursor"]
return messages
def command_backup(ack, body, command, logger, client):
ack()
logger.info(command)
result = client.users_list()
users = save_users(result['members'])
wClient = WebClient(token=os.environ.get("SLACK_BOT_TOKEN"))
response = wClient.conversations_history(channel=command['channel_id'])
file1 = open("CondensedMessages.txt", 'w+')
for message in response["messages"]:
file1.write(users[message["user"]]['name'] + " " + message["text"] +
'\n')
# file1.write(users[message["user"]] + " " + message["text"] + '\n')
file1.close()
file2 = open("completeText.txt", 'w+')
file2.write(str(response["messages"]))
file2.close()
# The name of the file you're going to upload
file_name = "./backup.zip"
# ID of channel that you want to upload file to
channel_id = command['channel_id']
zipObj = ZipFile('backup.zip', 'w')
zipObj.write('CondensedMessages.txt')
zipObj.write('completeText.txt')
zipObj.close()
try:
# Call the files.upload method using the WebClient
# Uploading files requires the `files:write` scope
result = client.files_upload(
channels=channel_id,
initial_comment=
"Here's a back up of the channel's conversations:smile:",
file=file_name,
)
# Log the result
logger.info(result)
except SlackApiError as e:
logger.error("Error uploading file: {}".format(e))
def get_messages(message):
global today_epoch
client = WebClient(token=os.environ.get("SLACK_BOT_TOKEN"))
channel_id = message["channel"]
try:
result = client.conversations_history(channel=channel_id,
inclusive=True,
oldest=today_epoch)
conversation_history = result["messages"]
text = ""
list_form = []
word_count = 0
for s_ind in range(len(conversation_history) - 1, -1, -1):
s = conversation_history[s_ind]
try:
# Filter out the slackbot messages
if s['type'] == 'message' and 'bot_id' not in s and s[
'client_msg_id'] != message['client_msg_id']:
text += s['text'] + " "
word_count += s['text'].count(" ") + 1
list_form.append(s['text'])
print(s['text'])
# Try to only look at the 150 most recent words for the naive multiple context handing feature
if word_count > word_count_thresh:
break
except:
continue
text = text[:-1]
# By default, assume topic is changing when a question is asked.
today_epoch = datetime.datetime.now().timestamp()
return text, list_form
except SlackApiError as e:
print("Error: {}".format(e))
return None
def get_fitbod_data():
print('get 4')
if os.environ.get('HEROKU'):
token = os.environ.get('fitbod_token_luke')
channel_id = os.environ.get('workout_channel_luke')
else:
token = yaml.safe_load(open('config.yaml'))['fitbod_token_luke']
channel_id = yaml.safe_load(open('config.yaml'))['workout_channel_luke']
print('get 5')
client = WebClient(token)
try:
# Call the conversations.history method using the WebClient
# conversations.history returns the first 100 messages by default
# These results are paginated, see: https://api.slack.com/methods/conversations.history$pagination
result = client.conversations_history(channel=channel_id)
conversation_history = result["messages"]
print('get 6')
# Print results
#print("{} messages found in {}".format(len(conversation_history), id))
newest_workout_url = conversation_history[0]['files'][0]['url_private']
raw_workout_data = requests.get(newest_workout_url, headers={'Authorization': 'Bearer %s' % token})
workout_data_text = io.StringIO(raw_workout_data.text)
x = pd.read_csv(workout_data_text)
final = x.sort_values('Date')
return final
except SlackApiError as e:
logger.error("Error creating conversation: {}".format(e))
def _fetch_messages(
client: WebClient,
*,
channel: str,
to_date: datetime.datetime,
from_date: datetime.datetime,
) -> Iterable[Message]:
latest = to_date.timestamp()
for _ in range(MAX_ITERATIONS):
conversations = client.conversations_history(channel=channel,
latest=latest,
limit=LIMIT)
assert isinstance(conversations.data, dict)
messages = [
Message(**message) for message in conversations.data["messages"]
]
if not messages:
return
for message in messages:
if message.ts.timestamp() < from_date.timestamp():
return
yield message
latest = messages[-1].ts.timestamp()
def get_channel_to_message(channel):
# APIを叩く
client = WebClient(token=os.environ['SLACK_BOT_TOKEN'])
response = client.conversations_history(channel=channel)
messages = []
# レスポンスをチェック
for msg in response['messages']:
# メッセージでなければ次へ
if msg['type'] != 'message':
continue
# リプライがついて無ければそのまま登録
if not ('reply_count' in msg.keys()):
messages.append(msg)
continue
# リプライがついていればリプライを取得する
if 1 < msg['reply_count']:
messages = messages + get_message_to_reply(channel, msg['ts'])
else:
messages.append(msg)
return messages
def retrieve_message(channel, message_id):
"""
Retrieve a single message from Slack
:param channel: The channel the message was posted to
:param message_id: The timestamp of the message
:return: The message details
"""
if not settings.SLACK_TOKEN:
return {'ok': False, 'error': 'config_error'}
client = WebClient(token=settings.SLACK_TOKEN)
try:
response = client.conversations_history(channel=channel,
latest=message_id,
inclusive=True,
limit=1)
assert response['ok'] is True
return response
except SlackApiError as e:
assert e.response['ok'] is False
return e.response
class Slack(threading.Thread):
def __init__(self, interval=3, config=None, auto_start=True, **commands):
"""
Initializes Slack bot, including auto-updating widget if in notebook
and using multiprocessing.
Args:
interval (int): Update interval for widget (must be over 1s).
config (Optional[dict]): Config dict
If not given, uses qc.config['user']['slack']
The config dict must contain the following keys:
- 'bot_name': Name of the bot
- 'bot_token': Token from bot (obtained from slack website)
- 'names': Usernames to periodically check for IM messages
auto_start (bool): Defaults to True.
"""
if config is not None:
self.config = config
else:
self.config = qc_config.user.slack
self.slack = WebClient(token=self.config['token'])
self.users = self.get_users(self.config['names'])
self.get_im_ids(self.users)
self.commands = {
'plot': self.upload_latest_plot,
'msmt': self.print_measurement_information,
'measurement': self.print_measurement_information,
'notify': self.add_task,
'help': self.help_message,
'task': self.add_task,
**commands
}
self.task_commands = {'finished': self.check_msmt_finished}
self.interval = interval
self.tasks = []
# Flag that exits loop when set to True (called via self.exit())
self._exit = False
# Flag that enables actions to be performed in the event loop
# Enabled via self.start(), disabled via self.stop()
self._is_active = False
# Call Thread init
super().__init__()
if auto_start:
self.start()
def start(self):
self._is_active = True
try:
# Start thread, can only be called once
super().start()
except RuntimeError:
# Thread already started, ignoring
pass
def run(self):
"""
Thread event loop that periodically checks for updates.
Can be stopped via :meth:`stop` , after which the Thread is stopped.
Returns:
None.
"""
while not self._exit:
# Continue event loop
if self._is_active:
# check for updates
self.update()
sleep(self.interval)
def stop(self):
"""
Stop checking for updates. Can be started again via :meth:`start`.
Returns:
None.
"""
self._is_active = False
def exit(self):
"""
Exit event loop, stop Thread.
Returns:
None
"""
self._stop = True
def user_from_id(self, user_id):
"""
Retrieve user from user id.
Args:
user_id: Id from which to retrieve user information.
Returns:
dict: User information.
"""
return self.slack.users_info(user=user_id)['user']
def get_users(self, usernames):
"""
Extracts user information for users.
Args:
usernames: Slack usernames of users.
Returns:
dict: {username: user}
"""
users = {}
response = self.slack.users_list()
for member in response['members']:
if member['name'] in usernames:
users[member['name']] = member
if len(users) != len(usernames):
remaining_names = [name for name in usernames if name not in users]
raise RuntimeError(f'Could not find names {remaining_names}')
return users
def get_im_ids(self, users):
"""
Adds IM ids of users to users dict.
Also adds `last_ts` to the latest IM message
Args:
users (dict): {username: user}
Returns:
None.
"""
response = self.slack.conversations_list(types='im')
user_ids = {username: user['id'] for username, user in users.items()}
im_ids = {chan['user']: chan['id'] for chan in response['channels']}
for username, user_id in user_ids.items():
if user_id in im_ids.keys():
users[username]['im_id'] = im_ids[user_id]
# update last ts
messages = self.get_im_messages(username=username, limit=1)
if messages:
users[username]['last_ts'] = float(messages[0]['ts'])
else:
users[username]['last_ts'] = None
def get_im_messages(self, username, **kwargs):
"""
Retrieves IM messages from username.
Args:
username: Name of user.
**kwargs: Additional kwargs for retrieving IM messages.
Returns:
List of IM messages.
"""
# provide backward compatibility with 'count' keyword. It still works,
# but is undocumented. 'count' likely does the same as 'limit', but
# 'limit' takes precedence
if 'limit' not in kwargs.keys():
kwargs['limit'] = kwargs.pop('count', None)
channel = self.users[username].get('im_id', None)
if channel is None:
return []
else:
response = self.slack.conversations_history(channel=channel,
**kwargs)
return response['messages']
def get_new_im_messages(self):
"""
Retrieves new IM messages for each user in self.users.
Updates user['last_ts'] to ts of newest message.
Returns:
im_messages (Dict): {username: [messages list]} newer than last_ts.
"""
im_messages = {}
for username, user in self.users.items():
last_ts = user.get('last_ts', None)
new_messages = self.get_im_messages(username=username,
oldest=last_ts)
# Kwarg 'oldest' sometimes also returns message with ts==last_ts
new_messages = [
m for m in new_messages if float(m['ts']) != last_ts
]
im_messages[username] = new_messages
if new_messages:
self.users[username]['last_ts'] = float(new_messages[0]['ts'])
return im_messages
def update(self):
"""
Performs tasks, and checks for new messages.
Periodically called from widget update.
Returns:
None.
"""
new_tasks = []
for task in self.tasks:
task_finished = task()
if not task_finished:
new_tasks.append(task)
self.tasks = new_tasks
new_messages = {}
try:
new_messages = self.get_new_im_messages()
except (ReadTimeout, HTTPError, ConnectTimeout, ReadTimeoutError) as e:
# catch any timeouts caused by network delays
warnings.warn('error retrieving slack messages',
SlackTimeoutWarning)
logging.info(e)
self.handle_messages(new_messages)
def help_message(self):
"""Return simple help message"""
cc = ", ".join("`" + str(k) + "`" for k in self.commands.keys())
return "\nAvailable commands: %s" % cc
def handle_messages(self, messages):
"""
Performs commands depending on messages.
This includes adding tasks to be performed during each update.
"""
for user, user_messages in messages.items():
for message in user_messages:
if message.get('user', None) != self.users[user]['id']:
# Filter out bot messages
continue
channel = self.users[user]['im_id']
# Extract command (first word) and possible args
command, args, kwargs = convert_command(message['text'])
if command in self.commands:
msg = f'Executing {command}'
if args:
msg += f' {args}'
if kwargs:
msg += f' {kwargs}'
self.slack.chat_postMessage(text=msg, channel=channel)
func = self.commands[command]
try:
if isinstance(func, _BaseParameter):
results = func(*args, **kwargs)
else:
# Only add channel and Slack if they are explicit
# kwargs
func_sig = inspect.signature(func)
if 'channel' in func_sig.parameters:
kwargs['channel'] = channel
if 'slack' in func_sig.parameters:
kwargs['slack'] = self
results = func(*args, **kwargs)
if results is not None:
self.slack.chat_postMessage(
text=f'Results: {results}', channel=channel)
except Exception:
self.slack.chat_postMessage(
text=f'Error: {traceback.format_exc()}',
channel=channel)
else:
self.slack.chat_postMessage(
text='Command {} not understood. Try `help`'.format(
command),
channel=channel)
def add_task(self, command, *args, channel, **kwargs):
"""
Add a task to self.tasks, which will be executed during each update
Args:
command: Task command.
*args: Additional args for command.
channel: Slack channel (can also be IM channel).
**kwargs: Additional kwargs for particular.
Returns:
None.
"""
if command in self.task_commands:
self.slack.chat_postMessage(text=f'Added task "{command}"',
channel=channel)
func = self.task_commands[command]
self.tasks.append(partial(func, *args, channel=channel, **kwargs))
else:
self.slack.chat_postMessage(
text=f'Task command {command} not understood', channel=channel)
def upload_latest_plot(self, channel, **kwargs):
"""
Uploads latest plot (if any) to slack channel.
The latest plot is retrieved from
:class:`qcodes.plots.base.BasePlot`, which is updated
every time a new qcodes plot is instantiated.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
# Create temporary filename
temp_filename = tempfile.mktemp(suffix='.jpg')
# Retrieve latest plot
latest_plot = BasePlot.latest_plot
if latest_plot is not None:
# Saves latest plot to filename
latest_plot.save(filename=temp_filename)
# Upload plot to slack
self.slack.files_upload(file=temp_filename, channels=channel)
os.remove(temp_filename)
else:
self.slack.chat_postMessage(text='No latest plot', channel=channel)
def print_measurement_information(self, channel, **kwargs):
"""
Prints information about the current measurement.
Information printed is percentage complete, and dataset representation.
Dataset is retrieved from DataSet.latest_dataset, which updates itself
every time a new dataset is created
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
dataset = active_data_set()
if dataset is not None:
self.slack.chat_postMessage(
text='Measurement is {:.0f}% complete'.format(
100 * dataset.fraction_complete()),
channel=channel)
self.slack.chat_postMessage(text=repr(dataset), channel=channel)
else:
self.slack.chat_postMessage(text='No latest dataset found',
channel=channel)
def check_msmt_finished(self, channel, **kwargs):
"""
Checks if the latest measurement is completed.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
bool: True if measurement is finished, False otherwise.
"""
if active_loop() is None:
self.slack.chat_postMessage(text='Measurement complete',
channel=channel)
return True
else:
return False
class CountStamp:
startdate = None
startdatetime = None
enddatetime = None
stamp_counter = {}
user_counter = {}
recv_counter = {}
client = None
bot = None
message = ""
chat_list = []
# インストラクタ
def __init__(self):
print("CountStamp Start")
self.client = WebClient(const.USER_TOKEN)
self.bot = WebClient(const.BOT_TOKEN)
#Start of time
self.startdate = datetime.date.today() - datetime.timedelta(
days=const.DAYS_AGO)
self.startdatetime = datetime.datetime.combine(self.startdate,
datetime.time())
self.enddatetime = self.startdatetime + datetime.timedelta(
days=const.DAYS_TERM)
print("term:{}-{}".format(self.startdatetime, self.enddatetime))
# チャンネルリストを取得する
def setChannelList(self):
self.channel_list = self.client.conversations_list(
exclude_archived=True, limit=const.CHANNEL_LIMIT)
print("channels:{}".format(len(self.channel_list["channels"])))
# スタンプをカウントする
def cntStamp(self):
channel_cnt = 0
for channel in self.channel_list["channels"]:
channel_cnt = channel_cnt + 1
channel_id = channel["id"]
#apiの呼び出し回数の制限（1分間に50回まで）を回避する
time.sleep(1)
history = self.client.conversations_history(
channel=channel_id,
oldest=self.startdatetime.timestamp(),
latest=self.enddatetime.timestamp())
#履歴内のスタンプをカウントする
self.cntReactions(history=history, channel=channel)
# 履歴内のスタンプをカウントする
def cntReactions(self, history, channel):
if (len(history['messages']) > 0):
print("channel_name:{} messages:{}".format(
channel['name'], len(history['messages'])))
for message in history["messages"]:
try:
if (message.get("reactions")):
reactions_cnt = 0
for reaction in message["reactions"]:
# ユーザー別のスタンプ数のカウント
self.cntUsers(users=reaction["users"])
# スタンプ別のスタンプ数のカウント
key = reaction["name"]
if (self.stamp_counter.get(key)):
self.stamp_counter[key] = self.stamp_counter[
key] + reaction["count"]
else:
self.stamp_counter[key] = reaction["count"]
# スタンプを受け取ったユーザー別のスタンプ数のカウント
if (message.get("user")):
key = message["user"]
if (self.recv_counter.get(key)):
self.recv_counter[key] = self.recv_counter[
key] + reaction["count"]
else:
self.recv_counter[key] = reaction["count"]
# スレッドについたスタンプ数のカウント
reactions_cnt = reactions_cnt + reaction["count"]
# スレッド別のスタンプ数
self.chat_list.append(
[channel['id'], message["ts"], reactions_cnt])
except KeyError as e:
print("KeyError:")
print(e.args)
#スタンプしたユーザーをカウント
def cntUsers(self, users):
for user_id in users:
if (self.user_counter.get(user_id)):
self.user_counter[user_id] = self.user_counter[user_id] + 1
else:
self.user_counter[user_id] = 1
# カウントをポストする
def setMessage(self):
sorted_stamp = sorted(self.stamp_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_user = sorted(self.user_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_recv = sorted(self.recv_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_chat = sorted(self.chat_list, key=lambda x: x[2], reverse=True)
w_list = ['月', '火', '水', '木', '金', '土', '日']
self.message = "{}({})のスタンプランキングTOP{}を発表します。\n".format(
self.startdate.strftime('%Y年%m月%d日'),
w_list[self.startdate.weekday()], const.RANK_LIMIT)
self.message = self.message + "\n\n:+1:このスタンプが良く使われました:+1:\n"
self.setRankingMessage(sorted_stamp, False)
self.message = self.message + "\n\n:tera_感謝_赤:このユーザーがたくさんスタンプしました:tera_感謝_赤:\n"
self.setRankingMessage(sorted_user, True)
self.message = self.message + "\n\n:gift:このユーザーがたくさんスタンプを受け取りました:gift:\n"
self.setRankingMessage(sorted_recv, True)
self.message = self.message + "\n\n:trophy:スタンプを集めたメッセージはこちら:trophy:\n"
self.setChatRankingMessage(sorted_chat)
total_stamp = sum(self.stamp_counter.values())
self.message = self.message + "\n\nすべてのスタンプを合計すると {} でした！".format(
total_stamp)
i = 1
while i <= int(total_stamp / const.CLAP_LOOP):
self.message = self.message + ":clap:"
i = i + 1
today_weekday = datetime.date.today().weekday()
# 土日はメッセージを変える。
if (today_weekday != 5 and today_weekday != 6):
self.message = self.message + "\nそれでは今日もはりきってスタンプしましょう！"
else:
self.message = self.message + "\n休日対応おつかれさまです。"
def postMessage(self):
self.bot.chat_postMessage(channel=const.CHANNEL_NAME,
text=self.message)
#ランキング処理
def setRankingMessage(self, rank_list, user_flag):
rank = 1
i = 0
while i < len(rank_list):
if (user_flag):
self.message = self.message + '\n{}位 {} {}'.format(
rank, self.getUsername(rank_list[i][0]), rank_list[i][1])
else:
self.message = self.message + '\n{}位 :{}: {}'.format(
rank, rank_list[i][0], rank_list[i][1])
#同列順位の処理
j = 1
while (i + j) < len(rank_list):
if (rank_list[i][1] == rank_list[i + j][1]):
if (user_flag):
self.message = self.message + ' {} {}'.format(
self.getUsername(rank_list[i + j][0]),
rank_list[i + j][1])
else:
self.message = self.message + ' :{}: {}'.format(
rank_list[i + j][0], rank_list[i + j][1])
j = j + 1
else:
break
i = i + j
rank = rank + j
if (rank > const.RANK_LIMIT):
break
self.message = self.message + '\n'
#チャットのランク処理
def setChatRankingMessage(self, sorted_chat):
rank = 1
for chat in sorted_chat:
link = self.bot.chat_getPermalink(channel=chat[0],
message_ts=chat[1])
self.message = self.message + link["permalink"] + '\n'
rank = rank + 1
if (rank > const.CHAT_RANK_LIMIT):
break
#ユーザーの表示名を取得する
def getUsername(self, user_id):
user_info = self.bot.users_info(user=user_id)
return user_info['user']['profile']['real_name']
# デストラクタ
def __del__(self):
print("CountStamp End")
class CSDigest:
def __init__(self, config_path) -> None:
print("loading data")
# handling files
with open(config_path) as cfg:
config = yaml.load(cfg, Loader=yaml.FullLoader)
with open(config["token_path"]) as tk:
self.token = tk.readline()
with open(config["template_path"]) as tp:
self.temp = BeautifulSoup(tp, "html.parser")
self.cache_im = os.path.join("cache", "images")
os.makedirs(self.cache_im, exist_ok=True)
self.ts_old = dateparser.parse(config["time_span"]).timestamp()
# initialize objects
self.foodnet = models.load_model("./foodnet/model")
self.datagen = ImageDataGenerator(rescale=1.0 / 255)
self.client = WebClient(token=self.token)
self.channels = pd.DataFrame(
self.client.conversations_list(limit=1000)["channels"]
).set_index("name")
self.users = pd.DataFrame(
self.client.users_list(limit=1000)["members"]
).set_index("id")
self.users["display_name"] = self.users["profile"].apply(
self.extract_profile, key="display_name"
)
print("fetching messages")
# get messages
ms_general = self.get_msg("general", same_user=False)
ms_home = self.get_msg("homesanity", ts_thres=0)
ms_quote = self.get_msg("quotablequotes", ts_thres=120, same_user=False)
print("building newsletter")
# handle carousel
if len(ms_general) > 0:
ms_general["class"] = ms_general.apply(self.classify_msg, axis="columns")
ms_tada = ms_general[ms_general["class"] == "tada"]
if len(ms_tada) > 0:
ms_tada["permalink"] = ms_tada.apply(self.get_permalink, axis="columns")
self.build_carousel(ms_tada)
# handle food
ms_files = pd.concat([ms_general, ms_home])
if len(ms_files) > 0:
ms_files["file_path"] = ms_files.apply(self.download_images, axis="columns")
ms_files = ms_files[ms_files["file_path"].astype(bool)]
ms_files["food_prob"] = ms_files["file_path"].apply(self.classify_food)
ms_files["food_path"] = ms_files.apply(self.filter_food, axis="columns")
ms_food = ms_files[ms_files["food_path"].notnull()]
ms_food["permalink"] = ms_food.apply(self.get_permalink, axis="columns")
ms_food["aspect"] = ms_food["food_path"].apply(self.get_img_aspect)
ms_food = ms_food.sort_values("aspect", ascending=True)
self.build_portfolio(ms_food)
# handle quotes
if len(ms_quote) > 0:
ms_quote = ms_quote[~ms_quote["files"].astype(bool)]
ms_quote["permalink"] = ms_quote.apply(self.get_permalink, axis="columns")
self.build_quotes(ms_quote)
def get_msg(self, channel, ts_thres=5, same_user=True):
ms = pd.DataFrame(
self.client.conversations_history(
channel=self.channels.loc[channel]["id"],
oldest=self.ts_old,
limit=1000,
)["messages"]
)
if len(ms) > 0:
try:
ms = ms[ms["subtype"].isnull()]
except KeyError:
pass
if len(ms) > 0:
ms = (
self.cluster_msg(ms, ts_thres=ts_thres, same_user=same_user)
.groupby("component")
.apply(self.merge_msg)
.reset_index()
.apply(self.translate_msg_user, axis="columns")
)
ms["text"] = ms["text"].apply(
emojize, use_aliases=True, variant="emoji_type"
)
ms["channel"] = self.channels.loc[channel]["id"]
return ms
def classify_msg(self, msg_df):
if msg_df["reactions"]:
tada = list(filter(lambda r: r["name"] == "tada", msg_df["reactions"]))
if tada and tada[0]["count"] > 5:
return "tada"
def cluster_msg(self, msg_df, ts_thres, same_user):
ts_dist = pdist(msg_df["ts"].values.astype(float).reshape((-1, 1)))
txt_dist = pdist(CountVectorizer().fit_transform(msg_df["text"]).toarray())
adj = squareform(ts_dist) < ts_thres * 60
if same_user:
user_dist = pdist(
msg_df["user"].values.reshape((-1, 1)),
metric=lambda u, v: 0 if u == v else 1,
)
adj = adj * (squareform(user_dist) < 1)
n_comp, lab = connected_components(adj, directed=False)
msg_df["component"] = lab
return msg_df
def merge_msg(self, msg_df, multiple_users="first"):
msg_df = msg_df.sort_values("ts")
if multiple_users == "forbid":
user = msg_df["user"].unique()
assert len(user) == 1
user = user.item()
elif multiple_users == "first":
user = msg_df.iloc[0]["user"]
msg_df = msg_df[msg_df["user"] == user]
else:
raise ValueError("multiple_users=={} not understood".format(multiple_users))
try:
reactions = msg_df["reactions"].dropna().values
reactions = sum(reactions, [])
except KeyError:
reactions = []
try:
files = msg_df["files"].dropna().values
files = sum(files, [])
except KeyError:
files = []
try:
attch = msg_df["attachments"].dropna().values
attch = sum(attch, [])
except KeyError:
attch = []
return pd.Series(
{
"user": user,
"text": "\n".join(msg_df["text"].values),
"ts": msg_df.iloc[0].loc["ts"],
"reactions": reactions,
"files": files,
"attachments": attch,
}
)
def translate_msg_user(self, msg_row, substitute=["display_name", "name"]):
try:
msg_row["user"] = self.translate_user(msg_row["user"], substitute)
msg_row["text"] = re.sub(
r"\<\@(.*?)\>",
lambda u: self.translate_user(u.group(1), substitute),
msg_row["text"],
)
except TypeError:
pass
return msg_row
def translate_user(self, uid, substitute):
for sub in substitute:
if sub == "real_name":
prefix = ""
else:
prefix = "@"
user = self.users.loc[uid, sub]
if type(user) == str and bool(user):
return prefix + user
def extract_profile(self, prof, key):
try:
return prof[key]
except KeyError:
return np.nan
def get_permalink(self, msg_row):
return self.client.chat_getPermalink(
channel=msg_row["channel"], message_ts=str(msg_row["ts"])
)["permalink"]
def download_images(self, msg_row):
fpaths = []
for fdict in msg_row["files"]:
try:
mimietype = fdict["mimetype"]
except KeyError:
continue
if mimietype.startswith("image"):
fpath = os.path.join(
self.cache_im,
".".join([fdict["id"], fdict["filetype"]]),
)
resp = requests.get(
fdict["url_private_download"],
headers={"Authorization": "Bearer {}".format(self.token)},
)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
for fdict in msg_row["attachments"]:
try:
url = fdict["image_url"]
except KeyError:
continue
fpath = os.path.join(self.cache_im, url.split("/")[-1].split("?")[0])
resp = requests.get(url)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
return fpaths
def build_carousel(self, msg_df):
indicator = self.temp.find("ol", {"id": "carousel-inds"})
ind_temp = indicator.find("li", {"id": "carousel-ind-template"}).extract()
sld_wrapper = self.temp.find("div", {"id": "carousel-slides"})
tada_temp = self.temp.find("div", {"id": "carousel-slide-template"}).extract()
for (imsg, msg), icss in zip(
msg_df.reset_index(drop=True).iterrows(), itt.cycle(np.arange(3) + 1)
):
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
cur_tada = copy.copy(tada_temp)
cur_tada.find("h3", {"id": "carousel-slide-message"}).string = (
msg["text"] if len(msg["text"]) <= 320 else msg["text"][:320] + "..."
)
cur_tada.find(True, {"id": "carousel-slide-author"}).string = " ".join(
[
msg["user"],
datetime.fromtimestamp(float(msg["ts"])).strftime("%b %d"),
]
)
cur_tada.find("a")["href"] = msg["permalink"]
if re.search("birthday", msg["text"].lower()):
cur_tada["class"] = [
"carousel-birthday" if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
else:
cur_tada["class"] = [
"carousel-tada-{}".format(icss) if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
if not imsg == 0:
del cur_ind["class"]
cur_tada["class"] = list(
filter(lambda c: c != "active", cur_tada["class"])
)
indicator.append(cur_ind)
sld_wrapper.append(cur_tada)
def write_html(self):
with open("csdigest.html", "w", encoding="utf-8") as outf:
outf.write(str(self.temp))
def classify_food(self, img_path):
try:
imgs = [img_to_array(load_img(imp).resize((512, 512))) for imp in img_path]
except:
return np.atleast_1d(np.ones(len(img_path))).tolist()
predict = self.foodnet.predict(self.datagen.flow(np.stack(imgs)))
return np.atleast_1d(predict.squeeze()).tolist()
def filter_food(self, msg_row, thres=0.1):
minval, minidx = np.min(msg_row["food_prob"]), np.argmin(msg_row["food_prob"])
if minval < thres:
return msg_row["file_path"][minidx]
else:
return np.nan
def get_img_aspect(self, path):
img = load_img(path)
return img.size[0] / img.size[1]
def build_portfolio(self, msg_df):
porto = self.temp.find("div", {"id": "portfolio-container"})
port_temp = self.temp.find("div", {"id": "portfolio-template"}).extract()
del port_temp["id"]
for imsg, msg in msg_df.iterrows():
cur_temp = copy.copy(port_temp)
cur_temp.img["src"] = msg["food_path"]
cur_temp.find("a", {"id": "port-zoom-link"})["href"] = msg["food_path"]
cur_temp.find("a", {"id": "port-msg-link"})["href"] = msg["permalink"]
txt = msg["text"]
if len(txt) > 150:
txt = txt[:150] + "..."
cur_temp.find(True, {"id": "port-item-text"}).string = txt
# cur_temp.find(True, {"id": "port-item-text"}).string = str(
# np.min(msg["food_prob"])
# )
porto.append(cur_temp)
def build_quotes(self, msg_df):
quotes = self.temp.find("div", {"id": "quote-block"})
quote_temp = quotes.find("div", {"id": "quote-template"}).extract()
indicators = self.temp.find("ol", {"id": "quote-indicator-wrap"})
ind_temp = indicators.find("li", {"id": "quote-indicator"}).extract()
for imsg, msg in msg_df.reset_index(drop=True).iterrows():
cur_quote = copy.copy(quote_temp)
cur_quote.find("a", {"id": "quote-link"})["href"] = msg["permalink"]
cur_quote.find("p", {"id": "quote-content"}).string = (
msg["text"] if len(msg["text"]) <= 400 else msg["text"][:400] + "..."
)
cur_quote.find("span", {"id": "quote-name"}).string = msg["user"]
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
if imsg > 0:
cur_quote["class"] = list(
filter(lambda c: c != "active", cur_quote["class"])
)
cur_ind["class"] = list(
filter(lambda c: c != "active", cur_ind["class"])
)
quotes.append(cur_quote)
indicators.append(cur_ind)
with mydb.cursor() as cursor:
sql = "SELECT channel_id, ao FROM aos WHERE backblast = 1 and archived = 0"
cursor.execute(sql)
channels = cursor.fetchall()
channels_df = pd.DataFrame(channels, columns={'channel_id', 'ao'})
finally:
print('Finding all PAX that attended recent workouts - stand by.')
# Get all channel conversation
messages_df = pd.DataFrame([]) # creates an empty dataframe to append to
for id in channels_df['channel_id']:
data = ''
while True:
try:
#print("Checking channel " + id) # <-- Use this if debugging any slack channels throwing errors
response = slack.conversations_history(channel=id, cursor=data)
response_metadata = response.get('response_metadata', {})
next_cursor = response_metadata.get('next_cursor')
messages = response.data['messages']
temp_df = pd.json_normalize(messages)
temp_df = temp_df[['user', 'type', 'text', 'ts']]
temp_df = temp_df.rename(columns={'user' : 'user_id', 'type' : 'message_type', 'ts' : 'timestamp'})
temp_df["channel_id"] = id
messages_df = messages_df.append(temp_df, ignore_index=True)
except:
print("Error: Unable to access Slack channel:", id, "in region:",db)
logging.warning("Error: Unable to access Slack channel %s in region %s", id, db)
if next_cursor:
# Keep going from next offset.
#print('Next Page Cursor:', next_cursor)
data = next_cursor
