def client_conversations_history(slack_token="", channel="", retrieve_messages_from=0 ): # WebClient insantiates a client that can call API methods # When using Bolt, you can use either `app.client` or the `client` passed to listeners. client = WebClient(token=slack_token) # Store conversation history conversation_history = [] # ID of the channel you want to send the message to channel_id = channel try: # Call the conversations.history method using the WebClient # conversations.history returns the first 100 messages by default # These results are paginated, see: https://api.slack.com/methods/conversations.history$pagination result = client.conversations_history(channel=channel_id, oldest=retrieve_messages_from) # Print results logging.info("{} messages found in {}".format(len(conversation_history), id)) return result except SlackApiError as e: logging.error("Error creating conversation: {}".format(e)) pass
class SlackChannel: """Abstraction of slack channel API """ def __init__(self, *, token: str, channel: str): self._client = WebClient(token=token) self._channel = channel self._msgs = None def _load_conversation(self, *, latest: float) -> Iterable: cursor = None while True: try: response = self._client.conversations_history( channel=self._channel, latest=latest, cursor=cursor, ) if response.status_code != 200: _log.error("%r", response) return if not response.data.get("ok", False): _log.error("%s", response.data.get("error", "unknown failure")) return for msg in response.data.get("messages", []): yield msg if not response.data.get("has_more", False): return cursor = response.data.get("response_metadata", {}).get("next_cursor") except SlackApiError: _log.exception("Failed to retrieve conversation messages") return def messages(self, *, upto: str) -> Iterable: """Yield all messages in the channel upto given time """ yield from self._load_conversation(latest=parse(upto).timestamp()) def delete_message(self, *, message_id: str) -> bool: """Delete specific message from the chat """ try: response = self._client.chat_delete(channel=self._channel, ts=message_id) return response.status_code == 200 except SlackApiError: _log.exception("Failed to delete message") return False
def get_messages(config: ScraperConfig) -> List[Dict[str, Any]]: slack_client = WebClient(token=config.slack_token) messages = [] cursor = None while True: response = slack_client.conversations_history( channel=config.slack_channel_id, oldest=config.start_datetime.timestamp(), cursor=cursor) messages += response.data["messages"] if not response.data["has_more"]: break cursor = response.data["response_metadata"]["next_cursor"] return messages
def command_backup(ack, body, command, logger, client): ack() logger.info(command) result = client.users_list() users = save_users(result['members']) wClient = WebClient(token=os.environ.get("SLACK_BOT_TOKEN")) response = wClient.conversations_history(channel=command['channel_id']) file1 = open("CondensedMessages.txt", 'w+') for message in response["messages"]: file1.write(users[message["user"]]['name'] + " " + message["text"] + '\n') # file1.write(users[message["user"]] + " " + message["text"] + '\n') file1.close() file2 = open("completeText.txt", 'w+') file2.write(str(response["messages"])) file2.close() # The name of the file you're going to upload file_name = "./backup.zip" # ID of channel that you want to upload file to channel_id = command['channel_id'] zipObj = ZipFile('backup.zip', 'w') zipObj.write('CondensedMessages.txt') zipObj.write('completeText.txt') zipObj.close() try: # Call the files.upload method using the WebClient # Uploading files requires the `files:write` scope result = client.files_upload( channels=channel_id, initial_comment= "Here's a back up of the channel's conversations:smile:", file=file_name, ) # Log the result logger.info(result) except SlackApiError as e: logger.error("Error uploading file: {}".format(e))
def get_messages(message): global today_epoch client = WebClient(token=os.environ.get("SLACK_BOT_TOKEN")) channel_id = message["channel"] try: result = client.conversations_history(channel=channel_id, inclusive=True, oldest=today_epoch) conversation_history = result["messages"] text = "" list_form = [] word_count = 0 for s_ind in range(len(conversation_history) - 1, -1, -1): s = conversation_history[s_ind] try: # Filter out the slackbot messages if s['type'] == 'message' and 'bot_id' not in s and s[ 'client_msg_id'] != message['client_msg_id']: text += s['text'] + " " word_count += s['text'].count(" ") + 1 list_form.append(s['text']) print(s['text']) # Try to only look at the 150 most recent words for the naive multiple context handing feature if word_count > word_count_thresh: break except: continue text = text[:-1] # By default, assume topic is changing when a question is asked. today_epoch = datetime.datetime.now().timestamp() return text, list_form except SlackApiError as e: print("Error: {}".format(e)) return None
def get_fitbod_data(): print('get 4') if os.environ.get('HEROKU'): token = os.environ.get('fitbod_token_luke') channel_id = os.environ.get('workout_channel_luke') else: token = yaml.safe_load(open('config.yaml'))['fitbod_token_luke'] channel_id = yaml.safe_load(open('config.yaml'))['workout_channel_luke'] print('get 5') client = WebClient(token) try: # Call the conversations.history method using the WebClient # conversations.history returns the first 100 messages by default # These results are paginated, see: https://api.slack.com/methods/conversations.history$pagination result = client.conversations_history(channel=channel_id) conversation_history = result["messages"] print('get 6') # Print results #print("{} messages found in {}".format(len(conversation_history), id)) newest_workout_url = conversation_history[0]['files'][0]['url_private'] raw_workout_data = requests.get(newest_workout_url, headers={'Authorization': 'Bearer %s' % token}) workout_data_text = io.StringIO(raw_workout_data.text) x = pd.read_csv(workout_data_text) final = x.sort_values('Date') return final except SlackApiError as e: logger.error("Error creating conversation: {}".format(e))
def _fetch_messages( client: WebClient, *, channel: str, to_date: datetime.datetime, from_date: datetime.datetime, ) -> Iterable[Message]: latest = to_date.timestamp() for _ in range(MAX_ITERATIONS): conversations = client.conversations_history(channel=channel, latest=latest, limit=LIMIT) assert isinstance(conversations.data, dict) messages = [ Message(**message) for message in conversations.data["messages"] ] if not messages: return for message in messages: if message.ts.timestamp() < from_date.timestamp(): return yield message latest = messages[-1].ts.timestamp()
def get_channel_to_message(channel): # APIを叩く client = WebClient(token=os.environ['SLACK_BOT_TOKEN']) response = client.conversations_history(channel=channel) messages = [] # レスポンスをチェック for msg in response['messages']: # メッセージでなければ次へ if msg['type'] != 'message': continue # リプライがついて無ければそのまま登録 if not ('reply_count' in msg.keys()): messages.append(msg) continue # リプライがついていればリプライを取得する if 1 < msg['reply_count']: messages = messages + get_message_to_reply(channel, msg['ts']) else: messages.append(msg) return messages
def retrieve_message(channel, message_id): """ Retrieve a single message from Slack :param channel: The channel the message was posted to :param message_id: The timestamp of the message :return: The message details """ if not settings.SLACK_TOKEN: return {'ok': False, 'error': 'config_error'} client = WebClient(token=settings.SLACK_TOKEN) try: response = client.conversations_history(channel=channel, latest=message_id, inclusive=True, limit=1) assert response['ok'] is True return response except SlackApiError as e: assert e.response['ok'] is False return e.response
class Slack(threading.Thread): def __init__(self, interval=3, config=None, auto_start=True, **commands): """ Initializes Slack bot, including auto-updating widget if in notebook and using multiprocessing. Args: interval (int): Update interval for widget (must be over 1s). config (Optional[dict]): Config dict If not given, uses qc.config['user']['slack'] The config dict must contain the following keys: - 'bot_name': Name of the bot - 'bot_token': Token from bot (obtained from slack website) - 'names': Usernames to periodically check for IM messages auto_start (bool): Defaults to True. """ if config is not None: self.config = config else: self.config = qc_config.user.slack self.slack = WebClient(token=self.config['token']) self.users = self.get_users(self.config['names']) self.get_im_ids(self.users) self.commands = { 'plot': self.upload_latest_plot, 'msmt': self.print_measurement_information, 'measurement': self.print_measurement_information, 'notify': self.add_task, 'help': self.help_message, 'task': self.add_task, **commands } self.task_commands = {'finished': self.check_msmt_finished} self.interval = interval self.tasks = [] # Flag that exits loop when set to True (called via self.exit()) self._exit = False # Flag that enables actions to be performed in the event loop # Enabled via self.start(), disabled via self.stop() self._is_active = False # Call Thread init super().__init__() if auto_start: self.start() def start(self): self._is_active = True try: # Start thread, can only be called once super().start() except RuntimeError: # Thread already started, ignoring pass def run(self): """ Thread event loop that periodically checks for updates. Can be stopped via :meth:`stop` , after which the Thread is stopped. Returns: None. """ while not self._exit: # Continue event loop if self._is_active: # check for updates self.update() sleep(self.interval) def stop(self): """ Stop checking for updates. Can be started again via :meth:`start`. Returns: None. """ self._is_active = False def exit(self): """ Exit event loop, stop Thread. Returns: None """ self._stop = True def user_from_id(self, user_id): """ Retrieve user from user id. Args: user_id: Id from which to retrieve user information. Returns: dict: User information. """ return self.slack.users_info(user=user_id)['user'] def get_users(self, usernames): """ Extracts user information for users. Args: usernames: Slack usernames of users. Returns: dict: {username: user} """ users = {} response = self.slack.users_list() for member in response['members']: if member['name'] in usernames: users[member['name']] = member if len(users) != len(usernames): remaining_names = [name for name in usernames if name not in users] raise RuntimeError(f'Could not find names {remaining_names}') return users def get_im_ids(self, users): """ Adds IM ids of users to users dict. Also adds `last_ts` to the latest IM message Args: users (dict): {username: user} Returns: None. """ response = self.slack.conversations_list(types='im') user_ids = {username: user['id'] for username, user in users.items()} im_ids = {chan['user']: chan['id'] for chan in response['channels']} for username, user_id in user_ids.items(): if user_id in im_ids.keys(): users[username]['im_id'] = im_ids[user_id] # update last ts messages = self.get_im_messages(username=username, limit=1) if messages: users[username]['last_ts'] = float(messages[0]['ts']) else: users[username]['last_ts'] = None def get_im_messages(self, username, **kwargs): """ Retrieves IM messages from username. Args: username: Name of user. **kwargs: Additional kwargs for retrieving IM messages. Returns: List of IM messages. """ # provide backward compatibility with 'count' keyword. It still works, # but is undocumented. 'count' likely does the same as 'limit', but # 'limit' takes precedence if 'limit' not in kwargs.keys(): kwargs['limit'] = kwargs.pop('count', None) channel = self.users[username].get('im_id', None) if channel is None: return [] else: response = self.slack.conversations_history(channel=channel, **kwargs) return response['messages'] def get_new_im_messages(self): """ Retrieves new IM messages for each user in self.users. Updates user['last_ts'] to ts of newest message. Returns: im_messages (Dict): {username: [messages list]} newer than last_ts. """ im_messages = {} for username, user in self.users.items(): last_ts = user.get('last_ts', None) new_messages = self.get_im_messages(username=username, oldest=last_ts) # Kwarg 'oldest' sometimes also returns message with ts==last_ts new_messages = [ m for m in new_messages if float(m['ts']) != last_ts ] im_messages[username] = new_messages if new_messages: self.users[username]['last_ts'] = float(new_messages[0]['ts']) return im_messages def update(self): """ Performs tasks, and checks for new messages. Periodically called from widget update. Returns: None. """ new_tasks = [] for task in self.tasks: task_finished = task() if not task_finished: new_tasks.append(task) self.tasks = new_tasks new_messages = {} try: new_messages = self.get_new_im_messages() except (ReadTimeout, HTTPError, ConnectTimeout, ReadTimeoutError) as e: # catch any timeouts caused by network delays warnings.warn('error retrieving slack messages', SlackTimeoutWarning) logging.info(e) self.handle_messages(new_messages) def help_message(self): """Return simple help message""" cc = ", ".join("`" + str(k) + "`" for k in self.commands.keys()) return "\nAvailable commands: %s" % cc def handle_messages(self, messages): """ Performs commands depending on messages. This includes adding tasks to be performed during each update. """ for user, user_messages in messages.items(): for message in user_messages: if message.get('user', None) != self.users[user]['id']: # Filter out bot messages continue channel = self.users[user]['im_id'] # Extract command (first word) and possible args command, args, kwargs = convert_command(message['text']) if command in self.commands: msg = f'Executing {command}' if args: msg += f' {args}' if kwargs: msg += f' {kwargs}' self.slack.chat_postMessage(text=msg, channel=channel) func = self.commands[command] try: if isinstance(func, _BaseParameter): results = func(*args, **kwargs) else: # Only add channel and Slack if they are explicit # kwargs func_sig = inspect.signature(func) if 'channel' in func_sig.parameters: kwargs['channel'] = channel if 'slack' in func_sig.parameters: kwargs['slack'] = self results = func(*args, **kwargs) if results is not None: self.slack.chat_postMessage( text=f'Results: {results}', channel=channel) except Exception: self.slack.chat_postMessage( text=f'Error: {traceback.format_exc()}', channel=channel) else: self.slack.chat_postMessage( text='Command {} not understood. Try `help`'.format( command), channel=channel) def add_task(self, command, *args, channel, **kwargs): """ Add a task to self.tasks, which will be executed during each update Args: command: Task command. *args: Additional args for command. channel: Slack channel (can also be IM channel). **kwargs: Additional kwargs for particular. Returns: None. """ if command in self.task_commands: self.slack.chat_postMessage(text=f'Added task "{command}"', channel=channel) func = self.task_commands[command] self.tasks.append(partial(func, *args, channel=channel, **kwargs)) else: self.slack.chat_postMessage( text=f'Task command {command} not understood', channel=channel) def upload_latest_plot(self, channel, **kwargs): """ Uploads latest plot (if any) to slack channel. The latest plot is retrieved from :class:`qcodes.plots.base.BasePlot`, which is updated every time a new qcodes plot is instantiated. Args: channel: Slack channel (can also be IM channel). **kwargs: Not used. Returns: None. """ # Create temporary filename temp_filename = tempfile.mktemp(suffix='.jpg') # Retrieve latest plot latest_plot = BasePlot.latest_plot if latest_plot is not None: # Saves latest plot to filename latest_plot.save(filename=temp_filename) # Upload plot to slack self.slack.files_upload(file=temp_filename, channels=channel) os.remove(temp_filename) else: self.slack.chat_postMessage(text='No latest plot', channel=channel) def print_measurement_information(self, channel, **kwargs): """ Prints information about the current measurement. Information printed is percentage complete, and dataset representation. Dataset is retrieved from DataSet.latest_dataset, which updates itself every time a new dataset is created Args: channel: Slack channel (can also be IM channel). **kwargs: Not used. Returns: None. """ dataset = active_data_set() if dataset is not None: self.slack.chat_postMessage( text='Measurement is {:.0f}% complete'.format( 100 * dataset.fraction_complete()), channel=channel) self.slack.chat_postMessage(text=repr(dataset), channel=channel) else: self.slack.chat_postMessage(text='No latest dataset found', channel=channel) def check_msmt_finished(self, channel, **kwargs): """ Checks if the latest measurement is completed. Args: channel: Slack channel (can also be IM channel). **kwargs: Not used. Returns: bool: True if measurement is finished, False otherwise. """ if active_loop() is None: self.slack.chat_postMessage(text='Measurement complete', channel=channel) return True else: return False
class CountStamp: startdate = None startdatetime = None enddatetime = None stamp_counter = {} user_counter = {} recv_counter = {} client = None bot = None message = "" chat_list = [] # インストラクタ def __init__(self): print("CountStamp Start") self.client = WebClient(const.USER_TOKEN) self.bot = WebClient(const.BOT_TOKEN) #Start of time self.startdate = datetime.date.today() - datetime.timedelta( days=const.DAYS_AGO) self.startdatetime = datetime.datetime.combine(self.startdate, datetime.time()) self.enddatetime = self.startdatetime + datetime.timedelta( days=const.DAYS_TERM) print("term:{}-{}".format(self.startdatetime, self.enddatetime)) # チャンネルリストを取得する def setChannelList(self): self.channel_list = self.client.conversations_list( exclude_archived=True, limit=const.CHANNEL_LIMIT) print("channels:{}".format(len(self.channel_list["channels"]))) # スタンプをカウントする def cntStamp(self): channel_cnt = 0 for channel in self.channel_list["channels"]: channel_cnt = channel_cnt + 1 channel_id = channel["id"] #apiの呼び出し回数の制限(1分間に50回まで)を回避する time.sleep(1) history = self.client.conversations_history( channel=channel_id, oldest=self.startdatetime.timestamp(), latest=self.enddatetime.timestamp()) #履歴内のスタンプをカウントする self.cntReactions(history=history, channel=channel) # 履歴内のスタンプをカウントする def cntReactions(self, history, channel): if (len(history['messages']) > 0): print("channel_name:{} messages:{}".format( channel['name'], len(history['messages']))) for message in history["messages"]: try: if (message.get("reactions")): reactions_cnt = 0 for reaction in message["reactions"]: # ユーザー別のスタンプ数のカウント self.cntUsers(users=reaction["users"]) # スタンプ別のスタンプ数のカウント key = reaction["name"] if (self.stamp_counter.get(key)): self.stamp_counter[key] = self.stamp_counter[ key] + reaction["count"] else: self.stamp_counter[key] = reaction["count"] # スタンプを受け取ったユーザー別のスタンプ数のカウント if (message.get("user")): key = message["user"] if (self.recv_counter.get(key)): self.recv_counter[key] = self.recv_counter[ key] + reaction["count"] else: self.recv_counter[key] = reaction["count"] # スレッドについたスタンプ数のカウント reactions_cnt = reactions_cnt + reaction["count"] # スレッド別のスタンプ数 self.chat_list.append( [channel['id'], message["ts"], reactions_cnt]) except KeyError as e: print("KeyError:") print(e.args) #スタンプしたユーザーをカウント def cntUsers(self, users): for user_id in users: if (self.user_counter.get(user_id)): self.user_counter[user_id] = self.user_counter[user_id] + 1 else: self.user_counter[user_id] = 1 # カウントをポストする def setMessage(self): sorted_stamp = sorted(self.stamp_counter.items(), key=lambda x: x[1], reverse=True) sorted_user = sorted(self.user_counter.items(), key=lambda x: x[1], reverse=True) sorted_recv = sorted(self.recv_counter.items(), key=lambda x: x[1], reverse=True) sorted_chat = sorted(self.chat_list, key=lambda x: x[2], reverse=True) w_list = ['月', '火', '水', '木', '金', '土', '日'] self.message = "{}({})のスタンプランキングTOP{}を発表します。\n".format( self.startdate.strftime('%Y年%m月%d日'), w_list[self.startdate.weekday()], const.RANK_LIMIT) self.message = self.message + "\n\n:+1:このスタンプが良く使われました:+1:\n" self.setRankingMessage(sorted_stamp, False) self.message = self.message + "\n\n:tera_感謝_赤:このユーザーがたくさんスタンプしました:tera_感謝_赤:\n" self.setRankingMessage(sorted_user, True) self.message = self.message + "\n\n:gift:このユーザーがたくさんスタンプを受け取りました:gift:\n" self.setRankingMessage(sorted_recv, True) self.message = self.message + "\n\n:trophy:スタンプを集めたメッセージはこちら:trophy:\n" self.setChatRankingMessage(sorted_chat) total_stamp = sum(self.stamp_counter.values()) self.message = self.message + "\n\nすべてのスタンプを合計すると {} でした!".format( total_stamp) i = 1 while i <= int(total_stamp / const.CLAP_LOOP): self.message = self.message + ":clap:" i = i + 1 today_weekday = datetime.date.today().weekday() # 土日はメッセージを変える。 if (today_weekday != 5 and today_weekday != 6): self.message = self.message + "\nそれでは今日もはりきってスタンプしましょう!" else: self.message = self.message + "\n休日対応おつかれさまです。" def postMessage(self): self.bot.chat_postMessage(channel=const.CHANNEL_NAME, text=self.message) #ランキング処理 def setRankingMessage(self, rank_list, user_flag): rank = 1 i = 0 while i < len(rank_list): if (user_flag): self.message = self.message + '\n{}位 {} {}'.format( rank, self.getUsername(rank_list[i][0]), rank_list[i][1]) else: self.message = self.message + '\n{}位 :{}: {}'.format( rank, rank_list[i][0], rank_list[i][1]) #同列順位の処理 j = 1 while (i + j) < len(rank_list): if (rank_list[i][1] == rank_list[i + j][1]): if (user_flag): self.message = self.message + ' {} {}'.format( self.getUsername(rank_list[i + j][0]), rank_list[i + j][1]) else: self.message = self.message + ' :{}: {}'.format( rank_list[i + j][0], rank_list[i + j][1]) j = j + 1 else: break i = i + j rank = rank + j if (rank > const.RANK_LIMIT): break self.message = self.message + '\n' #チャットのランク処理 def setChatRankingMessage(self, sorted_chat): rank = 1 for chat in sorted_chat: link = self.bot.chat_getPermalink(channel=chat[0], message_ts=chat[1]) self.message = self.message + link["permalink"] + '\n' rank = rank + 1 if (rank > const.CHAT_RANK_LIMIT): break #ユーザーの表示名を取得する def getUsername(self, user_id): user_info = self.bot.users_info(user=user_id) return user_info['user']['profile']['real_name'] # デストラクタ def __del__(self): print("CountStamp End")
class CSDigest: def __init__(self, config_path) -> None: print("loading data") # handling files with open(config_path) as cfg: config = yaml.load(cfg, Loader=yaml.FullLoader) with open(config["token_path"]) as tk: self.token = tk.readline() with open(config["template_path"]) as tp: self.temp = BeautifulSoup(tp, "html.parser") self.cache_im = os.path.join("cache", "images") os.makedirs(self.cache_im, exist_ok=True) self.ts_old = dateparser.parse(config["time_span"]).timestamp() # initialize objects self.foodnet = models.load_model("./foodnet/model") self.datagen = ImageDataGenerator(rescale=1.0 / 255) self.client = WebClient(token=self.token) self.channels = pd.DataFrame( self.client.conversations_list(limit=1000)["channels"] ).set_index("name") self.users = pd.DataFrame( self.client.users_list(limit=1000)["members"] ).set_index("id") self.users["display_name"] = self.users["profile"].apply( self.extract_profile, key="display_name" ) print("fetching messages") # get messages ms_general = self.get_msg("general", same_user=False) ms_home = self.get_msg("homesanity", ts_thres=0) ms_quote = self.get_msg("quotablequotes", ts_thres=120, same_user=False) print("building newsletter") # handle carousel if len(ms_general) > 0: ms_general["class"] = ms_general.apply(self.classify_msg, axis="columns") ms_tada = ms_general[ms_general["class"] == "tada"] if len(ms_tada) > 0: ms_tada["permalink"] = ms_tada.apply(self.get_permalink, axis="columns") self.build_carousel(ms_tada) # handle food ms_files = pd.concat([ms_general, ms_home]) if len(ms_files) > 0: ms_files["file_path"] = ms_files.apply(self.download_images, axis="columns") ms_files = ms_files[ms_files["file_path"].astype(bool)] ms_files["food_prob"] = ms_files["file_path"].apply(self.classify_food) ms_files["food_path"] = ms_files.apply(self.filter_food, axis="columns") ms_food = ms_files[ms_files["food_path"].notnull()] ms_food["permalink"] = ms_food.apply(self.get_permalink, axis="columns") ms_food["aspect"] = ms_food["food_path"].apply(self.get_img_aspect) ms_food = ms_food.sort_values("aspect", ascending=True) self.build_portfolio(ms_food) # handle quotes if len(ms_quote) > 0: ms_quote = ms_quote[~ms_quote["files"].astype(bool)] ms_quote["permalink"] = ms_quote.apply(self.get_permalink, axis="columns") self.build_quotes(ms_quote) def get_msg(self, channel, ts_thres=5, same_user=True): ms = pd.DataFrame( self.client.conversations_history( channel=self.channels.loc[channel]["id"], oldest=self.ts_old, limit=1000, )["messages"] ) if len(ms) > 0: try: ms = ms[ms["subtype"].isnull()] except KeyError: pass if len(ms) > 0: ms = ( self.cluster_msg(ms, ts_thres=ts_thres, same_user=same_user) .groupby("component") .apply(self.merge_msg) .reset_index() .apply(self.translate_msg_user, axis="columns") ) ms["text"] = ms["text"].apply( emojize, use_aliases=True, variant="emoji_type" ) ms["channel"] = self.channels.loc[channel]["id"] return ms def classify_msg(self, msg_df): if msg_df["reactions"]: tada = list(filter(lambda r: r["name"] == "tada", msg_df["reactions"])) if tada and tada[0]["count"] > 5: return "tada" def cluster_msg(self, msg_df, ts_thres, same_user): ts_dist = pdist(msg_df["ts"].values.astype(float).reshape((-1, 1))) txt_dist = pdist(CountVectorizer().fit_transform(msg_df["text"]).toarray()) adj = squareform(ts_dist) < ts_thres * 60 if same_user: user_dist = pdist( msg_df["user"].values.reshape((-1, 1)), metric=lambda u, v: 0 if u == v else 1, ) adj = adj * (squareform(user_dist) < 1) n_comp, lab = connected_components(adj, directed=False) msg_df["component"] = lab return msg_df def merge_msg(self, msg_df, multiple_users="first"): msg_df = msg_df.sort_values("ts") if multiple_users == "forbid": user = msg_df["user"].unique() assert len(user) == 1 user = user.item() elif multiple_users == "first": user = msg_df.iloc[0]["user"] msg_df = msg_df[msg_df["user"] == user] else: raise ValueError("multiple_users=={} not understood".format(multiple_users)) try: reactions = msg_df["reactions"].dropna().values reactions = sum(reactions, []) except KeyError: reactions = [] try: files = msg_df["files"].dropna().values files = sum(files, []) except KeyError: files = [] try: attch = msg_df["attachments"].dropna().values attch = sum(attch, []) except KeyError: attch = [] return pd.Series( { "user": user, "text": "\n".join(msg_df["text"].values), "ts": msg_df.iloc[0].loc["ts"], "reactions": reactions, "files": files, "attachments": attch, } ) def translate_msg_user(self, msg_row, substitute=["display_name", "name"]): try: msg_row["user"] = self.translate_user(msg_row["user"], substitute) msg_row["text"] = re.sub( r"\<\@(.*?)\>", lambda u: self.translate_user(u.group(1), substitute), msg_row["text"], ) except TypeError: pass return msg_row def translate_user(self, uid, substitute): for sub in substitute: if sub == "real_name": prefix = "" else: prefix = "@" user = self.users.loc[uid, sub] if type(user) == str and bool(user): return prefix + user def extract_profile(self, prof, key): try: return prof[key] except KeyError: return np.nan def get_permalink(self, msg_row): return self.client.chat_getPermalink( channel=msg_row["channel"], message_ts=str(msg_row["ts"]) )["permalink"] def download_images(self, msg_row): fpaths = [] for fdict in msg_row["files"]: try: mimietype = fdict["mimetype"] except KeyError: continue if mimietype.startswith("image"): fpath = os.path.join( self.cache_im, ".".join([fdict["id"], fdict["filetype"]]), ) resp = requests.get( fdict["url_private_download"], headers={"Authorization": "Bearer {}".format(self.token)}, ) open(fpath, "wb").write(resp.content) fpaths.append(fpath) for fdict in msg_row["attachments"]: try: url = fdict["image_url"] except KeyError: continue fpath = os.path.join(self.cache_im, url.split("/")[-1].split("?")[0]) resp = requests.get(url) open(fpath, "wb").write(resp.content) fpaths.append(fpath) return fpaths def build_carousel(self, msg_df): indicator = self.temp.find("ol", {"id": "carousel-inds"}) ind_temp = indicator.find("li", {"id": "carousel-ind-template"}).extract() sld_wrapper = self.temp.find("div", {"id": "carousel-slides"}) tada_temp = self.temp.find("div", {"id": "carousel-slide-template"}).extract() for (imsg, msg), icss in zip( msg_df.reset_index(drop=True).iterrows(), itt.cycle(np.arange(3) + 1) ): cur_ind = copy.copy(ind_temp) cur_ind["data-slide-to"] = str(imsg) cur_tada = copy.copy(tada_temp) cur_tada.find("h3", {"id": "carousel-slide-message"}).string = ( msg["text"] if len(msg["text"]) <= 320 else msg["text"][:320] + "..." ) cur_tada.find(True, {"id": "carousel-slide-author"}).string = " ".join( [ msg["user"], datetime.fromtimestamp(float(msg["ts"])).strftime("%b %d"), ] ) cur_tada.find("a")["href"] = msg["permalink"] if re.search("birthday", msg["text"].lower()): cur_tada["class"] = [ "carousel-birthday" if c == "carousel-tada-1" else c for c in cur_tada["class"] ] else: cur_tada["class"] = [ "carousel-tada-{}".format(icss) if c == "carousel-tada-1" else c for c in cur_tada["class"] ] if not imsg == 0: del cur_ind["class"] cur_tada["class"] = list( filter(lambda c: c != "active", cur_tada["class"]) ) indicator.append(cur_ind) sld_wrapper.append(cur_tada) def write_html(self): with open("csdigest.html", "w", encoding="utf-8") as outf: outf.write(str(self.temp)) def classify_food(self, img_path): try: imgs = [img_to_array(load_img(imp).resize((512, 512))) for imp in img_path] except: return np.atleast_1d(np.ones(len(img_path))).tolist() predict = self.foodnet.predict(self.datagen.flow(np.stack(imgs))) return np.atleast_1d(predict.squeeze()).tolist() def filter_food(self, msg_row, thres=0.1): minval, minidx = np.min(msg_row["food_prob"]), np.argmin(msg_row["food_prob"]) if minval < thres: return msg_row["file_path"][minidx] else: return np.nan def get_img_aspect(self, path): img = load_img(path) return img.size[0] / img.size[1] def build_portfolio(self, msg_df): porto = self.temp.find("div", {"id": "portfolio-container"}) port_temp = self.temp.find("div", {"id": "portfolio-template"}).extract() del port_temp["id"] for imsg, msg in msg_df.iterrows(): cur_temp = copy.copy(port_temp) cur_temp.img["src"] = msg["food_path"] cur_temp.find("a", {"id": "port-zoom-link"})["href"] = msg["food_path"] cur_temp.find("a", {"id": "port-msg-link"})["href"] = msg["permalink"] txt = msg["text"] if len(txt) > 150: txt = txt[:150] + "..." cur_temp.find(True, {"id": "port-item-text"}).string = txt # cur_temp.find(True, {"id": "port-item-text"}).string = str( # np.min(msg["food_prob"]) # ) porto.append(cur_temp) def build_quotes(self, msg_df): quotes = self.temp.find("div", {"id": "quote-block"}) quote_temp = quotes.find("div", {"id": "quote-template"}).extract() indicators = self.temp.find("ol", {"id": "quote-indicator-wrap"}) ind_temp = indicators.find("li", {"id": "quote-indicator"}).extract() for imsg, msg in msg_df.reset_index(drop=True).iterrows(): cur_quote = copy.copy(quote_temp) cur_quote.find("a", {"id": "quote-link"})["href"] = msg["permalink"] cur_quote.find("p", {"id": "quote-content"}).string = ( msg["text"] if len(msg["text"]) <= 400 else msg["text"][:400] + "..." ) cur_quote.find("span", {"id": "quote-name"}).string = msg["user"] cur_ind = copy.copy(ind_temp) cur_ind["data-slide-to"] = str(imsg) if imsg > 0: cur_quote["class"] = list( filter(lambda c: c != "active", cur_quote["class"]) ) cur_ind["class"] = list( filter(lambda c: c != "active", cur_ind["class"]) ) quotes.append(cur_quote) indicators.append(cur_ind)
with mydb.cursor() as cursor: sql = "SELECT channel_id, ao FROM aos WHERE backblast = 1 and archived = 0" cursor.execute(sql) channels = cursor.fetchall() channels_df = pd.DataFrame(channels, columns={'channel_id', 'ao'}) finally: print('Finding all PAX that attended recent workouts - stand by.') # Get all channel conversation messages_df = pd.DataFrame([]) # creates an empty dataframe to append to for id in channels_df['channel_id']: data = '' while True: try: #print("Checking channel " + id) # <-- Use this if debugging any slack channels throwing errors response = slack.conversations_history(channel=id, cursor=data) response_metadata = response.get('response_metadata', {}) next_cursor = response_metadata.get('next_cursor') messages = response.data['messages'] temp_df = pd.json_normalize(messages) temp_df = temp_df[['user', 'type', 'text', 'ts']] temp_df = temp_df.rename(columns={'user' : 'user_id', 'type' : 'message_type', 'ts' : 'timestamp'}) temp_df["channel_id"] = id messages_df = messages_df.append(temp_df, ignore_index=True) except: print("Error: Unable to access Slack channel:", id, "in region:",db) logging.warning("Error: Unable to access Slack channel %s in region %s", id, db) if next_cursor: # Keep going from next offset. #print('Next Page Cursor:', next_cursor) data = next_cursor