def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
try:
client = WebClient(token=config["token"])
client.conversations_list()
return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except SlackApiError as e:
logger.error(f"Got an error: {e.args[0]}")
return AirbyteConnectionStatus(status=Status.FAILED, message=str(e.args[0]))
def load_channels(archived=False):
"""
Get a list of all the public channels in Slack
:param archived: Boolean - Include archived channels
:returns: Response object (Dictionary)
"""
if not settings.SLACK_TOKEN:
return {'ok': False, 'error': 'config_error'}
client = WebClient(token=settings.SLACK_TOKEN)
try:
response = client.conversations_list(exclude_archived=not archived)
assert response['ok'] is True
channels = []
for channel in response['channels']:
channels.append((channel['id'], channel['name']))
return {'ok': True, 'channels': channels}
except SlackApiError as e:
assert e.response['ok'] is False
return e.response
def get_slack_conversations():
try:
slack_token = config["SLACK"]["oauth_token"]
client = WebClient(token=slack_token)
response = client.conversations_list()
conversations = response["channels"]
return conversations
except Exception as error:
logger.error(str(error))
def get_slack_channel_id(channel_name):
wc = WebClient(slack_token)
channels = wc.conversations_list(exclude_archived=True,
types="public_channel")
for c in channels["channels"]:
if c["name"] == channel_name:
channel_id = c["id"]
return channel_id
def deviceStartComm(self, device):
self.logger.debug(f"{device.name}: Starting Device")
client = WebClient(token=device.pluginProps['bot_token'])
auth_info = client.auth_test()
self.slack_accounts[auth_info['team_id']] = device.id
self.logger.info(f"{device.name}: Connected to Slack Workspace '{auth_info['team']}'")
self.channels[device.id] = [
(channel["id"], channel["name"])
for channel in client.conversations_list()['channels']
]
self.logger.info("{}: Channel List Updated".format(device.name))
self.logger.debug("{}: Channels: {}".format(device.name, self.channels[device.id]))
class SlackBot:
def __init__(self, token):
self.client = WebClient(token=token)
self.channels_dict = dict()
try:
for response in self.client.conversations_list():
self.channels_dict.update({
channel["name"]: channel["id"]
for channel in response["channels"]
})
except SlackApiError as e:
print(f"Error: {e}")
def check_channel(self, channel: str):
return channel in self.channels_dict
def post_in_channel(self, channel_name: str, message: str,
articles: [Article]):
if articles:
marked_articles = '>- ' + '\n>- '.join(
[f'<{art.url}|{art.header}>' for art in articles])
try:
self.client.conversations_join(
channel=self.channels_dict[channel_name])
self.client.chat_postMessage(
channel=self.channels_dict[channel_name],
text=f"{message}",
blocks=[{
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"{message}"
}
}, {
"type": "section",
"text": {
"type": "mrkdwn",
"text": f"{marked_articles}"
}
}])
except SlackApiError as e:
print(f"Error: {e}")
def return_joining_channels(say, logger, context):
'''tell channel names where the bot joins
`@BOTNAME where`
'''
client = WebClient(token=slack_bot_token)
# if you want only channels, ch['is_channel'] will be good filter
channels = []
for ch in client.conversations_list()['channels']:
members = client.conversations_members(channel=ch['id'])['members']
if bot_user_id in members:
channels.append(ch['name'])
if len(channels) > 0:
say(msgs.imin().format('\n'.join(channels)))
else:
say(msgs.noplace())
# Instanciate WebClient
client = WebClient(token=os.environ.get("SLACK_BOT_TOKEN"))
# Instanciate WebClient
app = Flask(__name__)
handler = SlackRequestHandler(bolt_app)
# Slack
# multi channel translation
list_channel_basename = os.environ.get("MULTI_CHANNEL").split(",")
channels = []
# retrieve channel object from Slack API
try:
# Call the conversations.list method using the built-in WebClient
for paginated_response in client.conversations_list(limit=10):
channels += paginated_response['channels']
except SlackApiError as e:
logging.error("Error fetching conversations: {}".format(e))
#logger.error("Error fetching conversations: {}".format(e))
name_dict, id_dict = {}, {}
for i in channels:
name_dict[i['name']] = i['id']
id_dict[i['id']] = i['name']
# not good way :(
del i
# Logging
if DEBUG == "True":
class TestWebClient(unittest.TestCase):
def setUp(self):
setup_mock_web_api_server(self)
self.client = WebClient(
token="xoxb-api_test",
base_url="http://localhost:8888",
)
def tearDown(self):
cleanup_mock_web_api_server(self)
pattern_for_language = re.compile("python/(\\S+)", re.IGNORECASE)
pattern_for_package_identifier = re.compile("slackclient/(\\S+)")
def test_subsequent_requests_with_a_session_succeeds(self):
resp = self.client.api_test()
assert resp["ok"]
resp = self.client.api_test()
assert resp["ok"]
def test_api_calls_include_user_agent(self):
self.client.token = "xoxb-api_test"
resp = self.client.api_test()
self.assertEqual(200, resp.status_code)
def test_builtin_api_methods_send_json(self):
self.client.token = "xoxb-api_test"
resp = self.client.api_test(msg="bye")
self.assertEqual(200, resp.status_code)
self.assertEqual("bye", resp["args"]["msg"])
def test_requests_can_be_paginated(self):
self.client.token = "xoxb-users_list_pagination"
users = []
for page in self.client.users_list(limit=2):
users = users + page["members"]
self.assertTrue(len(users) == 4)
def test_response_can_be_paginated_multiple_times(self):
self.client.token = "xoxb-conversations_list_pagination"
# This test suite verifies the changes in #521 work as expected
response = self.client.conversations_list(limit=1)
ids = []
for page in response:
ids.append(page["channels"][0]["id"])
self.assertEqual(ids, ["C1", "C2", "C3"])
# The second iteration starting with page 2
# (page1 is already cached in `response`)
self.client.token = "xoxb-conversations_list_pagination2"
ids = []
for page in response:
ids.append(page["channels"][0]["id"])
self.assertEqual(ids, ["C1", "C2", "C3"])
def test_request_pagination_stops_when_next_cursor_is_missing(self):
self.client.token = "xoxb-users_list_pagination_1"
users = []
for page in self.client.users_list(limit=2):
users = users + page["members"]
self.assertTrue(len(users) == 2)
def test_json_can_only_be_sent_with_post_requests(self):
with self.assertRaises(err.SlackRequestError):
self.client.api_call("fake.method", http_verb="GET", json={})
def test_slack_api_error_is_raised_on_unsuccessful_responses(self):
self.client.token = "xoxb-api_test_false"
with self.assertRaises(err.SlackApiError):
self.client.api_test()
self.client.token = "xoxb-500"
with self.assertRaises(err.SlackApiError):
self.client.api_test()
def test_slack_api_rate_limiting_exception_returns_retry_after(self):
self.client.token = "xoxb-rate_limited"
try:
self.client.api_test()
except err.SlackApiError as slack_api_error:
self.assertFalse(slack_api_error.response["ok"])
self.assertEqual(429, slack_api_error.response.status_code)
self.assertEqual(30, int(slack_api_error.response.headers["Retry-After"]))
def test_the_api_call_files_argument_creates_the_expected_data(self):
self.client.token = "xoxb-users_setPhoto"
resp = self.client.users_setPhoto(
image="tests/slack_sdk_fixture/slack_logo.png"
)
self.assertEqual(200, resp.status_code)
def test_issue_560_bool_in_params_sync(self):
self.client.token = "xoxb-conversations_list"
self.client.conversations_list(exclude_archived=1) # ok
self.client.conversations_list(exclude_archived="true") # ok
self.client.conversations_list(exclude_archived=True) # ok
def test_issue_690_oauth_v2_access(self):
self.client.token = ""
resp = self.client.oauth_v2_access(
client_id="111.222", client_secret="secret", code="codeeeeeeeeee"
)
self.assertIsNone(resp["error"])
with self.assertRaises(err.SlackApiError):
self.client.oauth_v2_access(
client_id="999.999", client_secret="secret", code="codeeeeeeeeee"
)
def test_issue_690_oauth_access(self):
self.client.token = ""
resp = self.client.oauth_access(
client_id="111.222", client_secret="secret", code="codeeeeeeeeee"
)
self.assertIsNone(resp["error"])
with self.assertRaises(err.SlackApiError):
self.client.oauth_access(
client_id="999.999", client_secret="secret", code="codeeeeeeeeee"
)
def test_issue_705_no_param_request_pagination(self):
self.client.token = "xoxb-users_list_pagination"
users = []
for page in self.client.users_list():
users = users + page["members"]
self.assertTrue(len(users) == 4)
def test_token_param(self):
client = WebClient(base_url="http://localhost:8888")
with self.assertRaises(err.SlackApiError):
client.users_list()
resp = client.users_list(token="xoxb-users_list_pagination")
self.assertIsNone(resp["error"])
with self.assertRaises(err.SlackApiError):
client.users_list()
def test_timeout_issue_712(self):
client = WebClient(base_url="http://localhost:8888", timeout=1)
with self.assertRaises(socket.timeout):
client.users_list(token="xoxb-timeout")
def test_html_response_body_issue_718(self):
client = WebClient(base_url="http://localhost:8888")
try:
client.users_list(token="xoxb-html_response")
self.fail("SlackApiError expected here")
except err.SlackApiError as e:
self.assertTrue(
str(e).startswith(
"Received a response in a non-JSON format: <!DOCTYPE HTML PUBLIC"
),
e,
)
def test_user_agent_customization_issue_769(self):
client = WebClient(
base_url="http://localhost:8888",
token="xoxb-user-agent this_is test",
user_agent_prefix="this_is",
user_agent_suffix="test",
)
resp = client.api_test()
self.assertTrue(resp["ok"])
def test_default_team_id(self):
client = WebClient(base_url="http://localhost:8888", team_id="T_DEFAULT")
resp = client.users_list(token="xoxb-users_list_pagination")
self.assertIsNone(resp["error"])
class Slack(threading.Thread):
def __init__(self, interval=3, config=None, auto_start=True, **commands):
"""
Initializes Slack bot, including auto-updating widget if in notebook
and using multiprocessing.
Args:
interval (int): Update interval for widget (must be over 1s).
config (Optional[dict]): Config dict
If not given, uses qc.config['user']['slack']
The config dict must contain the following keys:
- 'bot_name': Name of the bot
- 'bot_token': Token from bot (obtained from slack website)
- 'names': Usernames to periodically check for IM messages
auto_start (bool): Defaults to True.
"""
if config is not None:
self.config = config
else:
self.config = qc_config.user.slack
self.slack = WebClient(token=self.config['token'])
self.users = self.get_users(self.config['names'])
self.get_im_ids(self.users)
self.commands = {
'plot': self.upload_latest_plot,
'msmt': self.print_measurement_information,
'measurement': self.print_measurement_information,
'notify': self.add_task,
'help': self.help_message,
'task': self.add_task,
**commands
}
self.task_commands = {'finished': self.check_msmt_finished}
self.interval = interval
self.tasks = []
# Flag that exits loop when set to True (called via self.exit())
self._exit = False
# Flag that enables actions to be performed in the event loop
# Enabled via self.start(), disabled via self.stop()
self._is_active = False
# Call Thread init
super().__init__()
if auto_start:
self.start()
def start(self):
self._is_active = True
try:
# Start thread, can only be called once
super().start()
except RuntimeError:
# Thread already started, ignoring
pass
def run(self):
"""
Thread event loop that periodically checks for updates.
Can be stopped via :meth:`stop` , after which the Thread is stopped.
Returns:
None.
"""
while not self._exit:
# Continue event loop
if self._is_active:
# check for updates
self.update()
sleep(self.interval)
def stop(self):
"""
Stop checking for updates. Can be started again via :meth:`start`.
Returns:
None.
"""
self._is_active = False
def exit(self):
"""
Exit event loop, stop Thread.
Returns:
None
"""
self._stop = True
def user_from_id(self, user_id):
"""
Retrieve user from user id.
Args:
user_id: Id from which to retrieve user information.
Returns:
dict: User information.
"""
return self.slack.users_info(user=user_id)['user']
def get_users(self, usernames):
"""
Extracts user information for users.
Args:
usernames: Slack usernames of users.
Returns:
dict: {username: user}
"""
users = {}
response = self.slack.users_list()
for member in response['members']:
if member['name'] in usernames:
users[member['name']] = member
if len(users) != len(usernames):
remaining_names = [name for name in usernames if name not in users]
raise RuntimeError(f'Could not find names {remaining_names}')
return users
def get_im_ids(self, users):
"""
Adds IM ids of users to users dict.
Also adds `last_ts` to the latest IM message
Args:
users (dict): {username: user}
Returns:
None.
"""
response = self.slack.conversations_list(types='im')
user_ids = {username: user['id'] for username, user in users.items()}
im_ids = {chan['user']: chan['id'] for chan in response['channels']}
for username, user_id in user_ids.items():
if user_id in im_ids.keys():
users[username]['im_id'] = im_ids[user_id]
# update last ts
messages = self.get_im_messages(username=username, limit=1)
if messages:
users[username]['last_ts'] = float(messages[0]['ts'])
else:
users[username]['last_ts'] = None
def get_im_messages(self, username, **kwargs):
"""
Retrieves IM messages from username.
Args:
username: Name of user.
**kwargs: Additional kwargs for retrieving IM messages.
Returns:
List of IM messages.
"""
# provide backward compatibility with 'count' keyword. It still works,
# but is undocumented. 'count' likely does the same as 'limit', but
# 'limit' takes precedence
if 'limit' not in kwargs.keys():
kwargs['limit'] = kwargs.pop('count', None)
channel = self.users[username].get('im_id', None)
if channel is None:
return []
else:
response = self.slack.conversations_history(channel=channel,
**kwargs)
return response['messages']
def get_new_im_messages(self):
"""
Retrieves new IM messages for each user in self.users.
Updates user['last_ts'] to ts of newest message.
Returns:
im_messages (Dict): {username: [messages list]} newer than last_ts.
"""
im_messages = {}
for username, user in self.users.items():
last_ts = user.get('last_ts', None)
new_messages = self.get_im_messages(username=username,
oldest=last_ts)
# Kwarg 'oldest' sometimes also returns message with ts==last_ts
new_messages = [
m for m in new_messages if float(m['ts']) != last_ts
]
im_messages[username] = new_messages
if new_messages:
self.users[username]['last_ts'] = float(new_messages[0]['ts'])
return im_messages
def update(self):
"""
Performs tasks, and checks for new messages.
Periodically called from widget update.
Returns:
None.
"""
new_tasks = []
for task in self.tasks:
task_finished = task()
if not task_finished:
new_tasks.append(task)
self.tasks = new_tasks
new_messages = {}
try:
new_messages = self.get_new_im_messages()
except (ReadTimeout, HTTPError, ConnectTimeout, ReadTimeoutError) as e:
# catch any timeouts caused by network delays
warnings.warn('error retrieving slack messages',
SlackTimeoutWarning)
logging.info(e)
self.handle_messages(new_messages)
def help_message(self):
"""Return simple help message"""
cc = ", ".join("`" + str(k) + "`" for k in self.commands.keys())
return "\nAvailable commands: %s" % cc
def handle_messages(self, messages):
"""
Performs commands depending on messages.
This includes adding tasks to be performed during each update.
"""
for user, user_messages in messages.items():
for message in user_messages:
if message.get('user', None) != self.users[user]['id']:
# Filter out bot messages
continue
channel = self.users[user]['im_id']
# Extract command (first word) and possible args
command, args, kwargs = convert_command(message['text'])
if command in self.commands:
msg = f'Executing {command}'
if args:
msg += f' {args}'
if kwargs:
msg += f' {kwargs}'
self.slack.chat_postMessage(text=msg, channel=channel)
func = self.commands[command]
try:
if isinstance(func, _BaseParameter):
results = func(*args, **kwargs)
else:
# Only add channel and Slack if they are explicit
# kwargs
func_sig = inspect.signature(func)
if 'channel' in func_sig.parameters:
kwargs['channel'] = channel
if 'slack' in func_sig.parameters:
kwargs['slack'] = self
results = func(*args, **kwargs)
if results is not None:
self.slack.chat_postMessage(
text=f'Results: {results}', channel=channel)
except Exception:
self.slack.chat_postMessage(
text=f'Error: {traceback.format_exc()}',
channel=channel)
else:
self.slack.chat_postMessage(
text='Command {} not understood. Try `help`'.format(
command),
channel=channel)
def add_task(self, command, *args, channel, **kwargs):
"""
Add a task to self.tasks, which will be executed during each update
Args:
command: Task command.
*args: Additional args for command.
channel: Slack channel (can also be IM channel).
**kwargs: Additional kwargs for particular.
Returns:
None.
"""
if command in self.task_commands:
self.slack.chat_postMessage(text=f'Added task "{command}"',
channel=channel)
func = self.task_commands[command]
self.tasks.append(partial(func, *args, channel=channel, **kwargs))
else:
self.slack.chat_postMessage(
text=f'Task command {command} not understood', channel=channel)
def upload_latest_plot(self, channel, **kwargs):
"""
Uploads latest plot (if any) to slack channel.
The latest plot is retrieved from
:class:`qcodes.plots.base.BasePlot`, which is updated
every time a new qcodes plot is instantiated.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
# Create temporary filename
temp_filename = tempfile.mktemp(suffix='.jpg')
# Retrieve latest plot
latest_plot = BasePlot.latest_plot
if latest_plot is not None:
# Saves latest plot to filename
latest_plot.save(filename=temp_filename)
# Upload plot to slack
self.slack.files_upload(file=temp_filename, channels=channel)
os.remove(temp_filename)
else:
self.slack.chat_postMessage(text='No latest plot', channel=channel)
def print_measurement_information(self, channel, **kwargs):
"""
Prints information about the current measurement.
Information printed is percentage complete, and dataset representation.
Dataset is retrieved from DataSet.latest_dataset, which updates itself
every time a new dataset is created
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
dataset = active_data_set()
if dataset is not None:
self.slack.chat_postMessage(
text='Measurement is {:.0f}% complete'.format(
100 * dataset.fraction_complete()),
channel=channel)
self.slack.chat_postMessage(text=repr(dataset), channel=channel)
else:
self.slack.chat_postMessage(text='No latest dataset found',
channel=channel)
def check_msmt_finished(self, channel, **kwargs):
"""
Checks if the latest measurement is completed.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
bool: True if measurement is finished, False otherwise.
"""
if active_loop() is None:
self.slack.chat_postMessage(text='Measurement complete',
channel=channel)
return True
else:
return False
class CountStamp:
startdate = None
startdatetime = None
enddatetime = None
stamp_counter = {}
user_counter = {}
recv_counter = {}
client = None
bot = None
message = ""
chat_list = []
# インストラクタ
def __init__(self):
print("CountStamp Start")
self.client = WebClient(const.USER_TOKEN)
self.bot = WebClient(const.BOT_TOKEN)
#Start of time
self.startdate = datetime.date.today() - datetime.timedelta(
days=const.DAYS_AGO)
self.startdatetime = datetime.datetime.combine(self.startdate,
datetime.time())
self.enddatetime = self.startdatetime + datetime.timedelta(
days=const.DAYS_TERM)
print("term:{}-{}".format(self.startdatetime, self.enddatetime))
# チャンネルリストを取得する
def setChannelList(self):
self.channel_list = self.client.conversations_list(
exclude_archived=True, limit=const.CHANNEL_LIMIT)
print("channels:{}".format(len(self.channel_list["channels"])))
# スタンプをカウントする
def cntStamp(self):
channel_cnt = 0
for channel in self.channel_list["channels"]:
channel_cnt = channel_cnt + 1
channel_id = channel["id"]
#apiの呼び出し回数の制限（1分間に50回まで）を回避する
time.sleep(1)
history = self.client.conversations_history(
channel=channel_id,
oldest=self.startdatetime.timestamp(),
latest=self.enddatetime.timestamp())
#履歴内のスタンプをカウントする
self.cntReactions(history=history, channel=channel)
# 履歴内のスタンプをカウントする
def cntReactions(self, history, channel):
if (len(history['messages']) > 0):
print("channel_name:{} messages:{}".format(
channel['name'], len(history['messages'])))
for message in history["messages"]:
try:
if (message.get("reactions")):
reactions_cnt = 0
for reaction in message["reactions"]:
# ユーザー別のスタンプ数のカウント
self.cntUsers(users=reaction["users"])
# スタンプ別のスタンプ数のカウント
key = reaction["name"]
if (self.stamp_counter.get(key)):
self.stamp_counter[key] = self.stamp_counter[
key] + reaction["count"]
else:
self.stamp_counter[key] = reaction["count"]
# スタンプを受け取ったユーザー別のスタンプ数のカウント
if (message.get("user")):
key = message["user"]
if (self.recv_counter.get(key)):
self.recv_counter[key] = self.recv_counter[
key] + reaction["count"]
else:
self.recv_counter[key] = reaction["count"]
# スレッドについたスタンプ数のカウント
reactions_cnt = reactions_cnt + reaction["count"]
# スレッド別のスタンプ数
self.chat_list.append(
[channel['id'], message["ts"], reactions_cnt])
except KeyError as e:
print("KeyError:")
print(e.args)
#スタンプしたユーザーをカウント
def cntUsers(self, users):
for user_id in users:
if (self.user_counter.get(user_id)):
self.user_counter[user_id] = self.user_counter[user_id] + 1
else:
self.user_counter[user_id] = 1
# カウントをポストする
def setMessage(self):
sorted_stamp = sorted(self.stamp_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_user = sorted(self.user_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_recv = sorted(self.recv_counter.items(),
key=lambda x: x[1],
reverse=True)
sorted_chat = sorted(self.chat_list, key=lambda x: x[2], reverse=True)
w_list = ['月', '火', '水', '木', '金', '土', '日']
self.message = "{}({})のスタンプランキングTOP{}を発表します。\n".format(
self.startdate.strftime('%Y年%m月%d日'),
w_list[self.startdate.weekday()], const.RANK_LIMIT)
self.message = self.message + "\n\n:+1:このスタンプが良く使われました:+1:\n"
self.setRankingMessage(sorted_stamp, False)
self.message = self.message + "\n\n:tera_感謝_赤:このユーザーがたくさんスタンプしました:tera_感謝_赤:\n"
self.setRankingMessage(sorted_user, True)
self.message = self.message + "\n\n:gift:このユーザーがたくさんスタンプを受け取りました:gift:\n"
self.setRankingMessage(sorted_recv, True)
self.message = self.message + "\n\n:trophy:スタンプを集めたメッセージはこちら:trophy:\n"
self.setChatRankingMessage(sorted_chat)
total_stamp = sum(self.stamp_counter.values())
self.message = self.message + "\n\nすべてのスタンプを合計すると {} でした！".format(
total_stamp)
i = 1
while i <= int(total_stamp / const.CLAP_LOOP):
self.message = self.message + ":clap:"
i = i + 1
today_weekday = datetime.date.today().weekday()
# 土日はメッセージを変える。
if (today_weekday != 5 and today_weekday != 6):
self.message = self.message + "\nそれでは今日もはりきってスタンプしましょう！"
else:
self.message = self.message + "\n休日対応おつかれさまです。"
def postMessage(self):
self.bot.chat_postMessage(channel=const.CHANNEL_NAME,
text=self.message)
#ランキング処理
def setRankingMessage(self, rank_list, user_flag):
rank = 1
i = 0
while i < len(rank_list):
if (user_flag):
self.message = self.message + '\n{}位 {} {}'.format(
rank, self.getUsername(rank_list[i][0]), rank_list[i][1])
else:
self.message = self.message + '\n{}位 :{}: {}'.format(
rank, rank_list[i][0], rank_list[i][1])
#同列順位の処理
j = 1
while (i + j) < len(rank_list):
if (rank_list[i][1] == rank_list[i + j][1]):
if (user_flag):
self.message = self.message + ' {} {}'.format(
self.getUsername(rank_list[i + j][0]),
rank_list[i + j][1])
else:
self.message = self.message + ' :{}: {}'.format(
rank_list[i + j][0], rank_list[i + j][1])
j = j + 1
else:
break
i = i + j
rank = rank + j
if (rank > const.RANK_LIMIT):
break
self.message = self.message + '\n'
#チャットのランク処理
def setChatRankingMessage(self, sorted_chat):
rank = 1
for chat in sorted_chat:
link = self.bot.chat_getPermalink(channel=chat[0],
message_ts=chat[1])
self.message = self.message + link["permalink"] + '\n'
rank = rank + 1
if (rank > const.CHAT_RANK_LIMIT):
break
#ユーザーの表示名を取得する
def getUsername(self, user_id):
user_info = self.bot.users_info(user=user_id)
return user_info['user']['profile']['real_name']
# デストラクタ
def __del__(self):
print("CountStamp End")
class CSDigest:
def __init__(self, config_path) -> None:
print("loading data")
# handling files
with open(config_path) as cfg:
config = yaml.load(cfg, Loader=yaml.FullLoader)
with open(config["token_path"]) as tk:
self.token = tk.readline()
with open(config["template_path"]) as tp:
self.temp = BeautifulSoup(tp, "html.parser")
self.cache_im = os.path.join("cache", "images")
os.makedirs(self.cache_im, exist_ok=True)
self.ts_old = dateparser.parse(config["time_span"]).timestamp()
# initialize objects
self.foodnet = models.load_model("./foodnet/model")
self.datagen = ImageDataGenerator(rescale=1.0 / 255)
self.client = WebClient(token=self.token)
self.channels = pd.DataFrame(
self.client.conversations_list(limit=1000)["channels"]
).set_index("name")
self.users = pd.DataFrame(
self.client.users_list(limit=1000)["members"]
).set_index("id")
self.users["display_name"] = self.users["profile"].apply(
self.extract_profile, key="display_name"
)
print("fetching messages")
# get messages
ms_general = self.get_msg("general", same_user=False)
ms_home = self.get_msg("homesanity", ts_thres=0)
ms_quote = self.get_msg("quotablequotes", ts_thres=120, same_user=False)
print("building newsletter")
# handle carousel
if len(ms_general) > 0:
ms_general["class"] = ms_general.apply(self.classify_msg, axis="columns")
ms_tada = ms_general[ms_general["class"] == "tada"]
if len(ms_tada) > 0:
ms_tada["permalink"] = ms_tada.apply(self.get_permalink, axis="columns")
self.build_carousel(ms_tada)
# handle food
ms_files = pd.concat([ms_general, ms_home])
if len(ms_files) > 0:
ms_files["file_path"] = ms_files.apply(self.download_images, axis="columns")
ms_files = ms_files[ms_files["file_path"].astype(bool)]
ms_files["food_prob"] = ms_files["file_path"].apply(self.classify_food)
ms_files["food_path"] = ms_files.apply(self.filter_food, axis="columns")
ms_food = ms_files[ms_files["food_path"].notnull()]
ms_food["permalink"] = ms_food.apply(self.get_permalink, axis="columns")
ms_food["aspect"] = ms_food["food_path"].apply(self.get_img_aspect)
ms_food = ms_food.sort_values("aspect", ascending=True)
self.build_portfolio(ms_food)
# handle quotes
if len(ms_quote) > 0:
ms_quote = ms_quote[~ms_quote["files"].astype(bool)]
ms_quote["permalink"] = ms_quote.apply(self.get_permalink, axis="columns")
self.build_quotes(ms_quote)
def get_msg(self, channel, ts_thres=5, same_user=True):
ms = pd.DataFrame(
self.client.conversations_history(
channel=self.channels.loc[channel]["id"],
oldest=self.ts_old,
limit=1000,
)["messages"]
)
if len(ms) > 0:
try:
ms = ms[ms["subtype"].isnull()]
except KeyError:
pass
if len(ms) > 0:
ms = (
self.cluster_msg(ms, ts_thres=ts_thres, same_user=same_user)
.groupby("component")
.apply(self.merge_msg)
.reset_index()
.apply(self.translate_msg_user, axis="columns")
)
ms["text"] = ms["text"].apply(
emojize, use_aliases=True, variant="emoji_type"
)
ms["channel"] = self.channels.loc[channel]["id"]
return ms
def classify_msg(self, msg_df):
if msg_df["reactions"]:
tada = list(filter(lambda r: r["name"] == "tada", msg_df["reactions"]))
if tada and tada[0]["count"] > 5:
return "tada"
def cluster_msg(self, msg_df, ts_thres, same_user):
ts_dist = pdist(msg_df["ts"].values.astype(float).reshape((-1, 1)))
txt_dist = pdist(CountVectorizer().fit_transform(msg_df["text"]).toarray())
adj = squareform(ts_dist) < ts_thres * 60
if same_user:
user_dist = pdist(
msg_df["user"].values.reshape((-1, 1)),
metric=lambda u, v: 0 if u == v else 1,
)
adj = adj * (squareform(user_dist) < 1)
n_comp, lab = connected_components(adj, directed=False)
msg_df["component"] = lab
return msg_df
def merge_msg(self, msg_df, multiple_users="first"):
msg_df = msg_df.sort_values("ts")
if multiple_users == "forbid":
user = msg_df["user"].unique()
assert len(user) == 1
user = user.item()
elif multiple_users == "first":
user = msg_df.iloc[0]["user"]
msg_df = msg_df[msg_df["user"] == user]
else:
raise ValueError("multiple_users=={} not understood".format(multiple_users))
try:
reactions = msg_df["reactions"].dropna().values
reactions = sum(reactions, [])
except KeyError:
reactions = []
try:
files = msg_df["files"].dropna().values
files = sum(files, [])
except KeyError:
files = []
try:
attch = msg_df["attachments"].dropna().values
attch = sum(attch, [])
except KeyError:
attch = []
return pd.Series(
{
"user": user,
"text": "\n".join(msg_df["text"].values),
"ts": msg_df.iloc[0].loc["ts"],
"reactions": reactions,
"files": files,
"attachments": attch,
}
)
def translate_msg_user(self, msg_row, substitute=["display_name", "name"]):
try:
msg_row["user"] = self.translate_user(msg_row["user"], substitute)
msg_row["text"] = re.sub(
r"\<\@(.*?)\>",
lambda u: self.translate_user(u.group(1), substitute),
msg_row["text"],
)
except TypeError:
pass
return msg_row
def translate_user(self, uid, substitute):
for sub in substitute:
if sub == "real_name":
prefix = ""
else:
prefix = "@"
user = self.users.loc[uid, sub]
if type(user) == str and bool(user):
return prefix + user
def extract_profile(self, prof, key):
try:
return prof[key]
except KeyError:
return np.nan
def get_permalink(self, msg_row):
return self.client.chat_getPermalink(
channel=msg_row["channel"], message_ts=str(msg_row["ts"])
)["permalink"]
def download_images(self, msg_row):
fpaths = []
for fdict in msg_row["files"]:
try:
mimietype = fdict["mimetype"]
except KeyError:
continue
if mimietype.startswith("image"):
fpath = os.path.join(
self.cache_im,
".".join([fdict["id"], fdict["filetype"]]),
)
resp = requests.get(
fdict["url_private_download"],
headers={"Authorization": "Bearer {}".format(self.token)},
)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
for fdict in msg_row["attachments"]:
try:
url = fdict["image_url"]
except KeyError:
continue
fpath = os.path.join(self.cache_im, url.split("/")[-1].split("?")[0])
resp = requests.get(url)
open(fpath, "wb").write(resp.content)
fpaths.append(fpath)
return fpaths
def build_carousel(self, msg_df):
indicator = self.temp.find("ol", {"id": "carousel-inds"})
ind_temp = indicator.find("li", {"id": "carousel-ind-template"}).extract()
sld_wrapper = self.temp.find("div", {"id": "carousel-slides"})
tada_temp = self.temp.find("div", {"id": "carousel-slide-template"}).extract()
for (imsg, msg), icss in zip(
msg_df.reset_index(drop=True).iterrows(), itt.cycle(np.arange(3) + 1)
):
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
cur_tada = copy.copy(tada_temp)
cur_tada.find("h3", {"id": "carousel-slide-message"}).string = (
msg["text"] if len(msg["text"]) <= 320 else msg["text"][:320] + "..."
)
cur_tada.find(True, {"id": "carousel-slide-author"}).string = " ".join(
[
msg["user"],
datetime.fromtimestamp(float(msg["ts"])).strftime("%b %d"),
]
)
cur_tada.find("a")["href"] = msg["permalink"]
if re.search("birthday", msg["text"].lower()):
cur_tada["class"] = [
"carousel-birthday" if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
else:
cur_tada["class"] = [
"carousel-tada-{}".format(icss) if c == "carousel-tada-1" else c
for c in cur_tada["class"]
]
if not imsg == 0:
del cur_ind["class"]
cur_tada["class"] = list(
filter(lambda c: c != "active", cur_tada["class"])
)
indicator.append(cur_ind)
sld_wrapper.append(cur_tada)
def write_html(self):
with open("csdigest.html", "w", encoding="utf-8") as outf:
outf.write(str(self.temp))
def classify_food(self, img_path):
try:
imgs = [img_to_array(load_img(imp).resize((512, 512))) for imp in img_path]
except:
return np.atleast_1d(np.ones(len(img_path))).tolist()
predict = self.foodnet.predict(self.datagen.flow(np.stack(imgs)))
return np.atleast_1d(predict.squeeze()).tolist()
def filter_food(self, msg_row, thres=0.1):
minval, minidx = np.min(msg_row["food_prob"]), np.argmin(msg_row["food_prob"])
if minval < thres:
return msg_row["file_path"][minidx]
else:
return np.nan
def get_img_aspect(self, path):
img = load_img(path)
return img.size[0] / img.size[1]
def build_portfolio(self, msg_df):
porto = self.temp.find("div", {"id": "portfolio-container"})
port_temp = self.temp.find("div", {"id": "portfolio-template"}).extract()
del port_temp["id"]
for imsg, msg in msg_df.iterrows():
cur_temp = copy.copy(port_temp)
cur_temp.img["src"] = msg["food_path"]
cur_temp.find("a", {"id": "port-zoom-link"})["href"] = msg["food_path"]
cur_temp.find("a", {"id": "port-msg-link"})["href"] = msg["permalink"]
txt = msg["text"]
if len(txt) > 150:
txt = txt[:150] + "..."
cur_temp.find(True, {"id": "port-item-text"}).string = txt
# cur_temp.find(True, {"id": "port-item-text"}).string = str(
# np.min(msg["food_prob"])
# )
porto.append(cur_temp)
def build_quotes(self, msg_df):
quotes = self.temp.find("div", {"id": "quote-block"})
quote_temp = quotes.find("div", {"id": "quote-template"}).extract()
indicators = self.temp.find("ol", {"id": "quote-indicator-wrap"})
ind_temp = indicators.find("li", {"id": "quote-indicator"}).extract()
for imsg, msg in msg_df.reset_index(drop=True).iterrows():
cur_quote = copy.copy(quote_temp)
cur_quote.find("a", {"id": "quote-link"})["href"] = msg["permalink"]
cur_quote.find("p", {"id": "quote-content"}).string = (
msg["text"] if len(msg["text"]) <= 400 else msg["text"][:400] + "..."
)
cur_quote.find("span", {"id": "quote-name"}).string = msg["user"]
cur_ind = copy.copy(ind_temp)
cur_ind["data-slide-to"] = str(imsg)
if imsg > 0:
cur_quote["class"] = list(
filter(lambda c: c != "active", cur_quote["class"])
)
cur_ind["class"] = list(
filter(lambda c: c != "active", cur_ind["class"])
)
quotes.append(cur_quote)
indicators.append(cur_ind)
class SlackMonitor(Monitor):
"""
Create a monitoring service that alerts on Task failures / completion in a Slack channel
"""
def __init__(self,
slack_api_token,
channel,
message_prefix=None,
filters=None):
# type: (str, str, Optional[str], Optional[List[Callable[[Task], bool]]]) -> ()
"""
Create a Slack Monitoring object.
It will alert on any Task/Experiment that failed or completed
:param slack_api_token: Slack bot API Token. Token should start with "xoxb-"
:param channel: Name of the channel to post alerts to
:param message_prefix: optional message prefix to add before any message posted
For example: message_prefix="Hey <!here>,"
:param filters: An optional collection of callables that will be passed a Task
object and return True/False if it should be filtered away
"""
super(SlackMonitor, self).__init__()
self.channel = "{}".format(channel[1:] if channel[0] ==
"#" else channel)
self.slack_client = WebClient(token=slack_api_token)
self.min_num_iterations = 0
self.filters = filters or list()
self.status_alerts = [
"failed",
]
self.include_manual_experiments = False
self.include_archived = False
self.verbose = False
self._channel_id = None
self._message_prefix = "{} ".format(
message_prefix) if message_prefix else ""
self.check_credentials()
def check_credentials(self):
# type: () -> ()
"""
Check we have the correct credentials for the slack channel
"""
self.slack_client.api_test()
# Find channel ID
channels = []
cursor = None
while True:
response = self.slack_client.conversations_list(cursor=cursor)
channels.extend(response.data["channels"])
cursor = response.data["response_metadata"].get("next_cursor")
if not cursor:
break
channel_id = [
channel_info.get("id") for channel_info in channels
if channel_info.get("name") == self.channel
]
if not channel_id:
raise ValueError(
"Error: Could not locate channel name '{}'".format(
self.channel))
# test bot permission (join channel)
self._channel_id = channel_id[0]
self.slack_client.conversations_join(channel=self._channel_id)
def post_message(self, message, retries=1, wait_period=10.0):
# type: (str, int, float) -> ()
"""
Post message on our slack channel
:param message: Message to be sent (markdown style)
:param retries: Number of retries before giving up
:param wait_period: wait between retries in seconds
"""
for i in range(retries):
if i != 0:
sleep(wait_period)
try:
self.slack_client.chat_postMessage(
channel=self._channel_id,
blocks=[
dict(type="section",
text={
"type": "mrkdwn",
"text": message
})
],
)
return
except SlackApiError as e:
print(
'While trying to send message: "\n{}\n"\nGot an error: {}'.
format(message, e.response["error"]))
def get_query_parameters(self):
# type: () -> dict
"""
Return the query parameters for the monitoring.
:return dict: Example dictionary: {'status': ['failed'], 'order_by': ['-last_update']}
"""
filter_tags = list() if self.include_archived else ["-archived"]
if not self.include_manual_experiments:
filter_tags.append("-development")
return dict(status=self.status_alerts,
order_by=["-last_update"],
system_tags=filter_tags)
def process_task(self, task):
"""
# type: (Task) -> ()
Called on every Task that we monitor.
This is where we send the Slack alert
:return: None
"""
# skipping failed tasks with low number of iterations
if self.min_num_iterations and task.get_last_iteration(
) < self.min_num_iterations:
print("Skipping {} experiment id={}, number of iterations {} < {}".
format(task.status, task.id, task.get_last_iteration(),
self.min_num_iterations))
return
if any(f(task) for f in self.filters):
if self.verbose:
print("Experiment id={} {} did not pass all filters".format(
task.id, task.status))
return
print('Experiment id={} {}, raising alert on channel "{}"'.format(
task.id, task.status, self.channel))
console_output = task.get_reported_console_output(number_of_reports=3)
message = "{}Experiment ID <{}|{}> *{}*\nProject: *{}* - Name: *{}*\n" "```\n{}\n```".format(
self._message_prefix,
task.get_output_log_web_page(),
task.id,
task.status,
task.get_project_name(),
task.name,
("\n".join(console_output))[-2048:],
)
self.post_message(message, retries=5)
class ChannelManager(object):
def __init__(self, token=Path('SLACK_OAUTH_TOKEN').read_text()):
self.token = token
self.client = WebClient(token=self.token)
self.channels = [ ]
# Sets the internal channels field
def list(self):
resp = self.client.conversations_list(exclude_archived=True, types="public_channel,private_channel", limit=500)
assert resp.data['ok']
self.channels = []
for channel in resp.data['channels']:
self.channels.append({
'ID': channel['id'],
'Channel Name': channel['name'],
'Purpose': channel['purpose']['value']
})
def exists(self, name):
for channel in self.channels:
if channel['Channel Name'] == name:
return True
return False
def get_id(self, name):
for channel in self.channels:
if channel['Channel Name'] == name:
return channel['ID']
return False
def get_purpose(self, name):
for channel in self.channels:
if channel['Channel Name'] == name:
return channel['Purpose']
return False
def create(self, name, is_private):
print(f'Creating channel {name} ...')
resp = self.client.conversations_create(name=name, is_private=is_private)
return resp
def set_purpose(self, name, purpose):
chan_id = self.get_id(name)
# TODO(arjun): Hack. The name that comes back has HTML entities for
# special characters. God help us all.
if self.get_purpose(name) != "":
return
print(f'Setting purpose for {name} ...')
self.client.conversations_setPurpose(channel=chan_id, purpose=purpose)
def post(self, name, message):
channel_id = self.get_id(name)
if not channel_id:
return False
try:
resp = self.client.chat_postMessage(channel=channel_id, text=message)
if resp.data['ok']:
return True
else:
print(f"Response was {resp.data}")
return False
except SlackApiError as e:
if e.response["error"] != "not_in_channel":
raise e
self.client.conversations_join(channel=channel_id)
self.client.chat_postMessage(channel=channel_id, text=message)
return True
class Slack:
def __init__(self, token, channel_name=None, channel_id=None):
self.client = WebClient(token)
self.channel_id = channel_id
channels = self.client.conversations_list(
types='public_channel,private_channel')
if channel_name:
for channel in channels['channels']:
if channel['name'] == channel_name:
self.channel_id = channel['id']
break
if not self.channel_id:
self.channel_id = self.client.conversations_create(
name=channel_name.lower(), is_private=True)['channel']['id']
admins = [
u['id'] for u in self.client.users_list()['members']
if u.get('is_admin') or u.get('is_owner')
]
self.client.conversations_invite(channel=self.channel_id,
users=admins)
def send_snippet(self,
title,
initial_comment,
code,
code_type='python',
thread_ts=None):
return self.client.files_upload(
channels=self.channel_id,
title=title,
initial_comment=initial_comment.replace('<br>', ''),
content=code,
filetype=code_type,
thread_ts=thread_ts)['ts']
def send_exception_snippet(self,
domain,
event,
code_type='python',
thread_ts=None):
message = traceback.format_exc() + '\n\n\n' + dumps(event, indent=2)
subject = 'Error occurred in ' + domain
self.send_snippet(subject,
subject,
message,
code_type=code_type,
thread_ts=thread_ts)
def send_raw_message(self, blocks, thread_ts=None):
return self.client.chat_postMessage(channel=self.channel_id,
blocks=blocks,
thread_ts=thread_ts)['ts']
def update_raw_message(self, ts, blocks):
self.client.chat_update(channel=self.channel_id, blocks=blocks, ts=ts)
def get_perm_link(self, ts):
return self.client.chat_getPermalink(channel=self.channel_id,
message_ts=ts)['permalink']
def send_message(self, message, attachment=None, thread_ts=None):
blocks = [{
'type': 'section',
'text': {
'type': 'mrkdwn',
'text': message.replace('<br>', '')
}
}, {
'type': 'divider'
}]
if attachment:
blocks[0]['accessory'] = {
'type': 'button',
'text': {
'type': 'plain_text',
'text': attachment['text'],
'emoji': True
},
'url': attachment['value']
}
return self.send_raw_message(blocks, thread_ts)
# Set Slack tokens
key = sys.argv[2]
slack = WebClient(token=key)
#Define AWS Database connection criteria
mydb = pymysql.connect(host=host,
port=port,
user=user,
password=password,
db=db,
charset='utf8mb4',
cursorclass=pymysql.cursors.DictCursor)
# Get channel list
channels_response = slack.conversations_list(limit=999)
channels = channels_response.data['channels']
channels_df = pd.json_normalize(channels)
channels_df = channels_df[['id', 'name', 'created', 'is_archived']]
channels_df = channels_df.rename(
columns={
'id': 'channel_id',
'name': 'ao',
'created': 'channel_created',
'is_archived': 'archived'
})
# Now connect to the AWS database and insert some rows!
print('Updating Slack channel list / AOs for region...')
try:
with mydb.cursor() as cursor:
