def send_slack_report(env: str, suite: str):
attachments = _generate_slack_attachments(env, suite)
webhook_url = os.getenv('SLACK_TEST_REPORT_WEBHOOK_URL')
slack_bot_token = os.getenv('SLACK_BOT_TOKEN')
assert webhook_url, print(
"SLACK_TEST_REPORT_WEBHOOK_URL env variable needs to be set")
assert slack_bot_token, print(
"SLACK_BOT_TOKEN env variable needs to be set")
response = requests.post(url=webhook_url,
data=json.dumps({"attachments": attachments}),
headers={'Content-Type': 'application/json'})
assert response.status_code == 200, print(
f"Response wasn't 200, it was {response}")
print("Sent UI test statistics to #build")
if _tests_failed():
client = WebClient(token=slack_bot_token)
shutil.make_archive('UI-test-report', 'zip', PATH)
try:
client.files_upload(
channels='#build',
file='UI-test-report.zip',
title='test-report.zip',
)
except SlackApiError as e:
print(f'Error uploading test report: {e}')
os.remove('UI-test-report.zip')
print('Sent UI test report to #build')
def upload(attachment, filename, title=None, message=None, channels=None):
"""
Upload a new file to Slack
:param attachment: File path to the file
:param filename: Filename with file extension (i.e. example.pdf)
:param title: Title of the file to display in Slack
:param message: The message text introducing the file in the specified ``channels``
:param channels: Comma-separated list of channel names or ids where the file should be posted (i.e. C1234567890)
:returns: Response object (Dictionary)
"""
if not settings.SLACK_TOKEN:
return {'ok': False, 'error': 'config_error'}
client = WebClient(token=settings.SLACK_TOKEN)
client.timeout = 600
try:
if channels:
response = client.files_upload(channels=channels,
file=attachment,
filename=filename,
initial_comment=message,
title=title)
else:
response = client.files_upload(file=attachment,
filename=filename,
title=title)
assert response['ok'] is True
return {'ok': True, 'file': response['file']}
except SlackApiError as e:
assert e.response['ok'] is False
return e.response
def post_image_to_slack(
slack_oath_token = '',
file_name = '',
channels = '#tmp-slack-webhook-testing-playground',
comment = 'Here\'s my test file :smile:'
) :
# Import WebClient from Python SDK (github.com/slackapi/python-slack-sdk)
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
# Request user password if not provided already
if slack_oath_token == '' :
slack_oath_token = getpass.getpass('Slack OAuth token:')
# WebClient insantiates a client that can call API methods
# When using Bolt, you can use either `app.client` or the `client` passed to listeners.
client = WebClient(token=slack_oath_token)
# The name of the file you're going to upload
try:
# Call the files.upload method using the WebClient
# Uploading files requires the `files:write` scope
client.files_upload(
channels=channels,
initial_comment=comment,
file=file_name
)
except SlackApiError as e:
print("Error uploading file: {}".format(e))
def send_slack_message(text, files):
import os
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
client = WebClient(token=os.environ['SLACK_TOKEN'])
if text != None:
try:
client.chat_postMessage(channel='#data-collection-notifications',
text=text)
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
if files != None:
for one_file in files:
if one_file == '':
continue
try:
client.files_upload(channels='#data-collection-notifications',
file=os.path.expanduser(one_file))
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
def upload_slack_file(filename,
caption="Here's my file :smile:",
channel='#dev-logging-etl'):
bot_token = os.getenv('SLACKBOT_TOKEN')
client = WebClient(bot_token)
client.files_upload(
channels=channel,
initial_comment=caption,
file=filename,
)
return
def _handle_task_failure(self, task_to_track, log_process):
[process.kill() for process in log_process]
[process.wait() for process in log_process]
subprocess.call(["stty", "echo"])
# Implement your own alert methods here
subprocess.Popen([
'mv', f'{self.config.log_label}/{task_to_track}.bag',
f'{self.config.log_label}/{task_to_track}-failed.bag'
]).communicate()
subprocess.Popen([
'mv', f'{self.config.log_label}/{task_to_track}.mp4',
f'{self.config.log_label}/{task_to_track}-failed.mp4'
]).communicate()
try:
# Example slack integration
os.environ['SLACK_BOT_TOKEN']
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
client = WebClient(token=os.environ['SLACK_BOT_TOKEN'])
try:
# Zip bag up
subprocess.Popen([
'zip', '-r',
f'{self.config.log_label}/{task_to_track}-failed.zip',
f'{self.config.log_label}/{task_to_track}-failed.bag'
]).communicate()
filepath = f'{self.config.log_label}/{task_to_track}-failed.zip'
response = client.files_upload(
channels=os.environ['SLACK_BOT_CHANNEL'], file=filepath)
assert response["file"] # the uploaded file
filepath = f'{self.config.log_label}/{task_to_track}-failed.mp4'
response = client.files_upload(
channels=os.environ['SLACK_BOT_CHANNEL'], file=filepath)
assert response["file"] # the uploaded file
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
# str like 'invalid_auth', 'channel_not_found'
assert e.response["error"]
print(f"Got an error: {e.response['error']}")
except KeyError:
pass
shutdown(1)
class Slack(Config):
token = None
client = None
def __init__(self):
super().__init__()
self.set_token()
def set_token(self):
if os.path.exists(".env"):
load_dotenv(verbose=True)
self.token = os.environ["SLACK_TOKEN"]
self.client = WebClient(token=self.token)
def post_image(self, file):
channel_name = self.SLACK_CHANNEL_NAME
try:
response = self.client.files_upload(
channels=channel_name,
file=file,
)
if not response["ok"]:
raise SlackApiError("슬랙 전송 실패")
return True
except SlackApiError:
return False
class SlackApp:
def __init__(self, token, channel):
self.token = token
self.client = WebClient(token=token)
self.channel = channel
self.queue = []
def send(self, sendable: Sendable, ts=None):
if isinstance(sendable, Message):
response = self.client.chat_postMessage(channel=self.channel,
text=sendable.text,
thread_ts=ts)
elif isinstance(sendable, File):
response = self.client.files_upload(
channels=self.channel,
initial_comment=sendable.comment,
file=sendable.path,
thread_ts=ts,
)
else:
raise RuntimeError(
f'Not supported sendable type: {type(sendable)}')
assert response.status_code == 200, f'Failed to send a message to {self.channel}\n{response}'
return response.data.get('ts')
def img_to_slack(imagefile):
token_file = os.path.join(
os.path.dirname(get_ipython_module_path('BMM.functions')),
'image_uploader_token')
try:
with open(token_file, "r") as f:
token = f.read().replace('\n', '')
except:
post_to_slack(f'failed to post image: {imagefile}')
return ()
client = WebClient(token=token)
#client = WebClient(token=os.environ['SLACK_API_TOKEN'])
try:
response = client.files_upload(channels='#beamtime', file=imagefile)
assert response["file"] # the uploaded file
except SlackApiError as e:
post_to_slack('failed to post image: {imagefile}')
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
except Exception as em:
print("EXCEPTION: " + str(em))
report(f'failed to post image: {imagefile}', level='bold', slack=True)
def sendMessage(self, pluginAction, slackDevice, callerWaitingForResult):
msgText = self.prepareTextValue(pluginAction.props['msgBody'])
channel = pluginAction.props['channel']
client = WebClient(token=slackDevice.pluginProps['bot_token'])
client.chat_postMessage(channel=channel, text=msgText)
attach = pluginAction.props.get("attachments", "")
if len(attach) > 0:
files = indigo.activePlugin.substitute(attach)
fileList = files.split(",")
for file in fileList:
path = os.path.expanduser(file)
name = os.path.basename(path)
client.files_upload(channels=channel, file=path, title=name)
def upload_files_to_slack(files):
slack_token = os.environ["SLACK_BOT_TOKEN"]
client = WebClient(token=slack_token)
for f in files:
basename = os.path.basename(f)
resp = client.files_upload(
channels=os.environ["SLACK_POSTING_CHANNEL"],
file=f,
title=f"*{basename}* from '{socket.gethostname()}'",
)
print(f"Status code for uploading of {basename} is {resp.status_code}")
def send_user_message_with_img(image_file_name):
client = WebClient(token=SLACK_BOT_TOKEN)
try:
response = client.files_upload(channels=f"@{SLACK_USER_ID}",
title="Motion Capture",
initial_comment="Motion Detected.",
file=image_file_name)
except SlackApiError as e:
print(f"Error uploading file: {e}")
def upload_file(filepath: str, initial_comment=None):
api_token = os.getenv('SLACK_BOT_TOKEN')
if api_token is None:
print('Cannot find slack api token.')
return
client = WebClient(token=api_token)
response = client.files_upload(channels=notify_channel,
file=filepath,
initial_comment=initial_comment)
return response
def send_to_slack(fig):
client = WebClient(os.environ["SLACK_BOT_TOKEN"])
image_path = "image/test03.png"
fig.write_image(image_path)
upload_text_file = client.files_upload(
channels='#github-actions',
title="graph.png",
file=image_path,
initial_comment="graph"
)
def send_slack_file(channel, filename, content):
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
client = WebClient(token='{token}')
try:
return client.files_upload(
content=content,
filename=filename,
channels=channel
)
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
print(f"Got an error: {e.response['error']}")
def send_report(report_path):
client = WebClient(token=config["slack"]["slack_bot_token"])
log(f"Sending: {report_path}")
try:
response = client.files_upload(
channels=config["slack"]["slack_channel_id"],
file=report_path,
initial_comment="A new Kowalski performance report is ready!",
)
assert response["file"] # the uploaded file
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response["error"] # str like 'invalid_auth', 'channel_not_found'
log(f"Got an error: {e.response['error']}")
class PylabnetSlackBot():
""" Initializes Pylabnet Slack Bot"""
def __init__(self):
self.client = WebClient(token=SLACKBOT_ACCESS_TOKEN)
self.subscribed_channels = []
def subscribe_channel(self, channel):
""" Add Slack channel to list of subscribed channels."""
channel_list = channel if isinstance(channel, list) else [channel]
for channel in channel_list:
self.subscribed_channels.append(channel)
def post_to_channel(self, channel, message):
"""Post message to channel"""
try:
response = self.client.chat_postMessage(channel=channel,
text=message)
assert response["message"]["text"] == message
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
def broadcast_to_channels(self, message):
""" Post message to all subscribed channels"""
for channel in self.subscribed_channels:
self.post_to_channel(channel, message)
def upload_file(self, channel, filepath):
"""Upload file to channel"""
try:
response = self.client.files_upload(channels=channel,
file=filepath)
assert response["file"] # the uploaded file
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
def upload_to_channels(self, filepath):
"""Upload file to all subscribed channels"""
for channel in self.subscribed_channels:
self.upload_file(channel, filepath)
import os
import time
from slack_sdk import WebClient
from slack_sdk.errors import SlackApiError
client = WebClient(token='token')
while True:
try:
filepath = "./TemperatureData.csv"
response = client.files_upload(channels='#updates', file=filepath)
assert response["file"] # the uploaded file
except SlackApiError as e:
# You will get a SlackApiError if "ok" is False
assert e.response["ok"] is False
assert e.response[
"error"] # str like 'invalid_auth', 'channel_not_found'
print(f"Got an error: {e.response['error']}")
time.sleep(3600)
"Posts": "orange"
})
plt.title("Monthly Leaderboard - " + thismonthnamelong + ", " + yearnum)
plt.xlabel("")
plt.ylabel("# Posts for " + thismonthname + ", 2021")
plt.savefig('./plots/' + db + '/PAX_Leaderboard_' + region +
thismonthname + yearnum + '.jpg',
bbox_inches='tight') # save the figure to a file
print('Monthly Leaderboard Graph created for region', region,
'Sending to Slack now... hang tight!')
#slack.chat.post_message(firstf, 'Hey ' + region + "! Check out the current posting leaderboards for " + thismonthnamelong + ", " + yearnum + " as well as for Year to Date (includes all beatdowns, rucks, Qsource, etc.). Here are the top 20 posters! T-CLAPS to these HIMs. The month isn't over yet, SYITG and get on the board!")
slack.files_upload(
channels=firstf,
initial_comment='Hey ' + region +
"! Check out the current posting leaderboards for " +
thismonthnamelong + ", " + yearnum +
" as well as for Year to Date (includes all beatdowns, rucks, Qsource, etc.). Here are the top 20 posters! T-CLAPS to these HIMs.",
file='./plots/' + db + '/PAX_Leaderboard_' + region + thismonthname +
yearnum + '.jpg',
)
total_graphs = total_graphs + 1
print('Total graphs made:', total_graphs)
try:
with mydb.cursor() as cursor:
sql = "select PAX, count(distinct AO) as UniqueAOs, count(Date) as Posts\
from attendance_view \
WHERE YEAR(Date) = %s \
group by PAX \
order by count(Date) desc\
limit 20"
plt.legend('')
plt.ioff()
#ax = bd_tmp_df.plot.bar(x='Q', color={"ao": "orange"})
#plt.title("Q Counts - " + ao + " " + thismonthnamelong + ", " + yearnum)
#plt.xlabel("")
#plt.ylabel("# Q Counts for " + thismonthname + ", " + yearnum)
plt.savefig('./plots/' + db + '/Q_Counts_' + ao + "_" +
thismonthname + yearnum + '.jpg',
bbox_inches='tight') # save the figure to a file
print('Q Graph created for AO', ao,
'Sending to Slack now... hang tight!')
#ao2 = 'U0187M4NWG4' # Use this for testing to send all charts to a specific user
#slack.chat.post_message(ao, 'Hey ' + ao + '! Here is a look at who Qd last month. Is your name on this list? Remember Core Principle #4 - F3 is peer led on a rotating fashion. Exercise your leadership muscles. Sign up to Q!')
slack.files_upload(
channels=ao,
initial_comment='Hey ' + ao +
'! Here is a look at who has been stepping up to Q at this AO. Is your name on this list? Remember Core Principle #4 - F3 is peer led on a rotating fashion. Exercise your leadership muscles. Sign up to Q!',
file='./plots/' + db + '/Q_Counts_' + ao + "_" +
thismonthname + yearnum + '.jpg')
total_graphs = total_graphs + 1
except:
print('An Error Occurred in Sending')
finally:
print('Message Sent')
print('Total AO graphs made:', total_graphs)
try:
total_graphs = 0
month = []
day = []
year = []
with mydb.cursor() as cursor:
class Slack(threading.Thread):
def __init__(self, interval=3, config=None, auto_start=True, **commands):
"""
Initializes Slack bot, including auto-updating widget if in notebook
and using multiprocessing.
Args:
interval (int): Update interval for widget (must be over 1s).
config (Optional[dict]): Config dict
If not given, uses qc.config['user']['slack']
The config dict must contain the following keys:
- 'bot_name': Name of the bot
- 'bot_token': Token from bot (obtained from slack website)
- 'names': Usernames to periodically check for IM messages
auto_start (bool): Defaults to True.
"""
if config is not None:
self.config = config
else:
self.config = qc_config.user.slack
self.slack = WebClient(token=self.config['token'])
self.users = self.get_users(self.config['names'])
self.get_im_ids(self.users)
self.commands = {
'plot': self.upload_latest_plot,
'msmt': self.print_measurement_information,
'measurement': self.print_measurement_information,
'notify': self.add_task,
'help': self.help_message,
'task': self.add_task,
**commands
}
self.task_commands = {'finished': self.check_msmt_finished}
self.interval = interval
self.tasks = []
# Flag that exits loop when set to True (called via self.exit())
self._exit = False
# Flag that enables actions to be performed in the event loop
# Enabled via self.start(), disabled via self.stop()
self._is_active = False
# Call Thread init
super().__init__()
if auto_start:
self.start()
def start(self):
self._is_active = True
try:
# Start thread, can only be called once
super().start()
except RuntimeError:
# Thread already started, ignoring
pass
def run(self):
"""
Thread event loop that periodically checks for updates.
Can be stopped via :meth:`stop` , after which the Thread is stopped.
Returns:
None.
"""
while not self._exit:
# Continue event loop
if self._is_active:
# check for updates
self.update()
sleep(self.interval)
def stop(self):
"""
Stop checking for updates. Can be started again via :meth:`start`.
Returns:
None.
"""
self._is_active = False
def exit(self):
"""
Exit event loop, stop Thread.
Returns:
None
"""
self._stop = True
def user_from_id(self, user_id):
"""
Retrieve user from user id.
Args:
user_id: Id from which to retrieve user information.
Returns:
dict: User information.
"""
return self.slack.users_info(user=user_id)['user']
def get_users(self, usernames):
"""
Extracts user information for users.
Args:
usernames: Slack usernames of users.
Returns:
dict: {username: user}
"""
users = {}
response = self.slack.users_list()
for member in response['members']:
if member['name'] in usernames:
users[member['name']] = member
if len(users) != len(usernames):
remaining_names = [name for name in usernames if name not in users]
raise RuntimeError(f'Could not find names {remaining_names}')
return users
def get_im_ids(self, users):
"""
Adds IM ids of users to users dict.
Also adds `last_ts` to the latest IM message
Args:
users (dict): {username: user}
Returns:
None.
"""
response = self.slack.conversations_list(types='im')
user_ids = {username: user['id'] for username, user in users.items()}
im_ids = {chan['user']: chan['id'] for chan in response['channels']}
for username, user_id in user_ids.items():
if user_id in im_ids.keys():
users[username]['im_id'] = im_ids[user_id]
# update last ts
messages = self.get_im_messages(username=username, limit=1)
if messages:
users[username]['last_ts'] = float(messages[0]['ts'])
else:
users[username]['last_ts'] = None
def get_im_messages(self, username, **kwargs):
"""
Retrieves IM messages from username.
Args:
username: Name of user.
**kwargs: Additional kwargs for retrieving IM messages.
Returns:
List of IM messages.
"""
# provide backward compatibility with 'count' keyword. It still works,
# but is undocumented. 'count' likely does the same as 'limit', but
# 'limit' takes precedence
if 'limit' not in kwargs.keys():
kwargs['limit'] = kwargs.pop('count', None)
channel = self.users[username].get('im_id', None)
if channel is None:
return []
else:
response = self.slack.conversations_history(channel=channel,
**kwargs)
return response['messages']
def get_new_im_messages(self):
"""
Retrieves new IM messages for each user in self.users.
Updates user['last_ts'] to ts of newest message.
Returns:
im_messages (Dict): {username: [messages list]} newer than last_ts.
"""
im_messages = {}
for username, user in self.users.items():
last_ts = user.get('last_ts', None)
new_messages = self.get_im_messages(username=username,
oldest=last_ts)
# Kwarg 'oldest' sometimes also returns message with ts==last_ts
new_messages = [
m for m in new_messages if float(m['ts']) != last_ts
]
im_messages[username] = new_messages
if new_messages:
self.users[username]['last_ts'] = float(new_messages[0]['ts'])
return im_messages
def update(self):
"""
Performs tasks, and checks for new messages.
Periodically called from widget update.
Returns:
None.
"""
new_tasks = []
for task in self.tasks:
task_finished = task()
if not task_finished:
new_tasks.append(task)
self.tasks = new_tasks
new_messages = {}
try:
new_messages = self.get_new_im_messages()
except (ReadTimeout, HTTPError, ConnectTimeout, ReadTimeoutError) as e:
# catch any timeouts caused by network delays
warnings.warn('error retrieving slack messages',
SlackTimeoutWarning)
logging.info(e)
self.handle_messages(new_messages)
def help_message(self):
"""Return simple help message"""
cc = ", ".join("`" + str(k) + "`" for k in self.commands.keys())
return "\nAvailable commands: %s" % cc
def handle_messages(self, messages):
"""
Performs commands depending on messages.
This includes adding tasks to be performed during each update.
"""
for user, user_messages in messages.items():
for message in user_messages:
if message.get('user', None) != self.users[user]['id']:
# Filter out bot messages
continue
channel = self.users[user]['im_id']
# Extract command (first word) and possible args
command, args, kwargs = convert_command(message['text'])
if command in self.commands:
msg = f'Executing {command}'
if args:
msg += f' {args}'
if kwargs:
msg += f' {kwargs}'
self.slack.chat_postMessage(text=msg, channel=channel)
func = self.commands[command]
try:
if isinstance(func, _BaseParameter):
results = func(*args, **kwargs)
else:
# Only add channel and Slack if they are explicit
# kwargs
func_sig = inspect.signature(func)
if 'channel' in func_sig.parameters:
kwargs['channel'] = channel
if 'slack' in func_sig.parameters:
kwargs['slack'] = self
results = func(*args, **kwargs)
if results is not None:
self.slack.chat_postMessage(
text=f'Results: {results}', channel=channel)
except Exception:
self.slack.chat_postMessage(
text=f'Error: {traceback.format_exc()}',
channel=channel)
else:
self.slack.chat_postMessage(
text='Command {} not understood. Try `help`'.format(
command),
channel=channel)
def add_task(self, command, *args, channel, **kwargs):
"""
Add a task to self.tasks, which will be executed during each update
Args:
command: Task command.
*args: Additional args for command.
channel: Slack channel (can also be IM channel).
**kwargs: Additional kwargs for particular.
Returns:
None.
"""
if command in self.task_commands:
self.slack.chat_postMessage(text=f'Added task "{command}"',
channel=channel)
func = self.task_commands[command]
self.tasks.append(partial(func, *args, channel=channel, **kwargs))
else:
self.slack.chat_postMessage(
text=f'Task command {command} not understood', channel=channel)
def upload_latest_plot(self, channel, **kwargs):
"""
Uploads latest plot (if any) to slack channel.
The latest plot is retrieved from
:class:`qcodes.plots.base.BasePlot`, which is updated
every time a new qcodes plot is instantiated.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
# Create temporary filename
temp_filename = tempfile.mktemp(suffix='.jpg')
# Retrieve latest plot
latest_plot = BasePlot.latest_plot
if latest_plot is not None:
# Saves latest plot to filename
latest_plot.save(filename=temp_filename)
# Upload plot to slack
self.slack.files_upload(file=temp_filename, channels=channel)
os.remove(temp_filename)
else:
self.slack.chat_postMessage(text='No latest plot', channel=channel)
def print_measurement_information(self, channel, **kwargs):
"""
Prints information about the current measurement.
Information printed is percentage complete, and dataset representation.
Dataset is retrieved from DataSet.latest_dataset, which updates itself
every time a new dataset is created
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
None.
"""
dataset = active_data_set()
if dataset is not None:
self.slack.chat_postMessage(
text='Measurement is {:.0f}% complete'.format(
100 * dataset.fraction_complete()),
channel=channel)
self.slack.chat_postMessage(text=repr(dataset), channel=channel)
else:
self.slack.chat_postMessage(text='No latest dataset found',
channel=channel)
def check_msmt_finished(self, channel, **kwargs):
"""
Checks if the latest measurement is completed.
Args:
channel: Slack channel (can also be IM channel).
**kwargs: Not used.
Returns:
bool: True if measurement is finished, False otherwise.
"""
if active_loop() is None:
self.slack.chat_postMessage(text='Measurement complete',
channel=channel)
return True
else:
return False
try:
bd_tmp_df.groupby(['Q', 'Year']).size().unstack().sort_values(
['Q'], ascending=True).plot(kind='bar')
plt.title('Number of Qs by individual at ' + ao +
' year to date ' + yearnum)
#plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), frameon=False)
plt.legend('')
plt.ioff()
#ax = bd_tmp_df.plot.bar(x='Q', color={"ao": "orange"})
#plt.title("Q Counts - " + ao + " " + thismonthnamelong + ", " + yearnum)
#plt.xlabel("")
#plt.ylabel("# Q Counts for " + thismonthname + ", " + yearnum)
plt.savefig('./plots/' + db + '/Q_Counts_YTD_' + ao + "_" +
yearnum + '.jpg',
bbox_inches='tight') # save the figure to a file
print('Q Graph created for AO', ao,
'Sending to Slack now... hang tight!')
#ao2 = 'U01FFCMHUG2' # Use this for testing to send all charts to a specific user
#slack.chat.post_message(ao, 'Hey ' + ao + '! Here is a look at who Qd last month. Is your name on this list? Remember Core Principle #4 - F3 is peer led on a rotating fashion. Exercise your leadership muscles. Sign up to Q!')
slack.files_upload(
channels=ao,
initial_comment='Hey ' + ao +
'! Who is crushing it this year so far as Q at this AO? Check below.',
file='./plots/' + db + '/Q_Counts_YTD_' + ao + "_" +
yearnum + '.jpg')
total_graphs = total_graphs + 1
except:
print('An Error Occurred in Sending')
finally:
print('Message Sent')
print('Total AO graphs made:', total_graphs)
class Slack:
def __init__(self, token, channel_name=None, channel_id=None):
self.client = WebClient(token)
self.channel_id = channel_id
channels = self.client.conversations_list(
types='public_channel,private_channel')
if channel_name:
for channel in channels['channels']:
if channel['name'] == channel_name:
self.channel_id = channel['id']
break
if not self.channel_id:
self.channel_id = self.client.conversations_create(
name=channel_name.lower(), is_private=True)['channel']['id']
admins = [
u['id'] for u in self.client.users_list()['members']
if u.get('is_admin') or u.get('is_owner')
]
self.client.conversations_invite(channel=self.channel_id,
users=admins)
def send_snippet(self,
title,
initial_comment,
code,
code_type='python',
thread_ts=None):
return self.client.files_upload(
channels=self.channel_id,
title=title,
initial_comment=initial_comment.replace('<br>', ''),
content=code,
filetype=code_type,
thread_ts=thread_ts)['ts']
def send_exception_snippet(self,
domain,
event,
code_type='python',
thread_ts=None):
message = traceback.format_exc() + '\n\n\n' + dumps(event, indent=2)
subject = 'Error occurred in ' + domain
self.send_snippet(subject,
subject,
message,
code_type=code_type,
thread_ts=thread_ts)
def send_raw_message(self, blocks, thread_ts=None):
return self.client.chat_postMessage(channel=self.channel_id,
blocks=blocks,
thread_ts=thread_ts)['ts']
def update_raw_message(self, ts, blocks):
self.client.chat_update(channel=self.channel_id, blocks=blocks, ts=ts)
def get_perm_link(self, ts):
return self.client.chat_getPermalink(channel=self.channel_id,
message_ts=ts)['permalink']
def send_message(self, message, attachment=None, thread_ts=None):
blocks = [{
'type': 'section',
'text': {
'type': 'mrkdwn',
'text': message.replace('<br>', '')
}
}, {
'type': 'divider'
}]
if attachment:
blocks[0]['accessory'] = {
'type': 'button',
'text': {
'type': 'plain_text',
'text': attachment['text'],
'emoji': True
},
'url': attachment['value']
}
return self.send_raw_message(blocks, thread_ts)
result = []
with open(os.path.join(os.path.dirname(__file__), "iplist.txt"), "r") as f:
lines = f.read().splitlines()
for line in lines:
item = fetch_scan_result(line)
# respect shodan API rate limit
time.sleep(1)
if not item:
continue
result.extend(item)
sorted_result = sort_result(result)
for item in sorted_result:
report += item
report += "========================================\n"
report += "Have a good day!"
try:
# response = client.chat_postMessage(channel=SLACK_CHANNEL, text=report)
response = client.files_upload(channels=SLACK_CHANNEL, content=report, title="Shodan_Notifier")
except SlackApiError as e:
assert e.response["ok"] is False
assert e.response["error"]
print(f"Got an error: {e.response['error']}")
cursor.execute(sql, val)
bd_tmp = cursor.fetchall()
bd_tmp_df = pd.DataFrame(bd_tmp)
bd_tmp_df['Month'] = bd_tmp_df['Month'].replace(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12], [
'January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December'
])
bd_df_styled = bd_tmp_df.style.background_gradient(
cmap=cm,
subset=['TotalPosts',
'TotalUniquePax']).set_caption("This region is ON FIRE!")
dfi.export(bd_df_styled,
'./plots/' + db + '/AO_SummaryTable' + thismonthname +
yearnum + '.jpg') # save the figure to a file
#fig = ff.create_table(bd_df_styled)
#fig.write_image('./plots/' + db + '/AO_SummaryTable' + thismonthname + yearnum + '.jpg')
print(
'AO summary table created for all AOs. Sending to Slack now... hang tight!'
)
#slack.chat_postMessage(channel=firstf, text="Hey " + region + " - it's the First of the Month! (queue Bone, Thungz and Harmony tunes here)! Here is a detailed summary of AO posting stats for the region last month.")
slack.files_upload(
channels=firstf,
initial_comment="Hey " + region +
" - it's that time of the Month again. Here is a detailed summary of AO posting stats for the region last month.",
file='./plots/' + db + '/AO_SummaryTable' + thismonthname +
yearnum + '.jpg')
total_graphs = total_graphs + 1
finally:
print('Total graphs made:', total_graphs)
mydb.close()
if not posts_df.empty:
ax = posts_df.plot.bar(x='PAX', color={"Posts": "orange"})
plt.title("Monthly Leaderboard - " + thismonthnamelong + ", " +
yearnum)
plt.xlabel("")
plt.ylabel("# Posts for " + thismonthname + ", 2021")
plt.savefig('./plots/' + db + '/PAX_Leaderboard_' + ao +
thismonthname + yearnum + '.jpg',
bbox_inches='tight') # save the figure to a file
print('Monthly Leaderboard Graph created for AO', ao,
'Sending to Slack now... hang tight!')
#slack.chat.post_message(ao, 'Hey ' + ao + "! Here are the posting leaderboards for " + thismonthnamelong + ", " + yearnum + " as well as for Year to Date (includes all beatdowns, rucks, Qsource, etc.) with the top 20 posters! T-CLAPS to these HIMs.")
slack.files_upload(
channels=ao,
initial_comment='Hey ' + ao +
"! Here are the posting leaderboards for " + thismonthnamelong +
", " + yearnum +
" as well as for Year to Date (includes all beatdowns, rucks, Qsource, etc.) with the top 20 posters! T-CLAPS to these HIMs.",
file='./plots/' + db + '/PAX_Leaderboard_' + ao + thismonthname +
yearnum + '.jpg')
total_graphs = total_graphs + 1
print('Total graphs made:', total_graphs)
try:
with mydb.cursor() as cursor:
sql = "select PAX, count(Date) as Posts\
from attendance_view \
WHERE YEAR(Date) = %s \
AND ao = %s \
group by PAX \
order by count(Date) desc\
limit 20"
# Query AWS by for beatdown history
try:
with mydb.cursor() as cursor:
sql = "SELECT DISTINCT AO, MONTHNAME(Date) as Month, PAX FROM attendance_view WHERE YEAR(Date) = %s"
val = yearnum
cursor.execute(sql, val)
bd_tmp = cursor.fetchall()
bd_tmp_df = pd.DataFrame(bd_tmp, columns={'AO', 'Month', 'PAX'})
bd_tmp_df.groupby(['AO', 'Month']).size().unstack().plot(kind='bar')
plt.title('Number of unique PAX attending each AO by month in ' +
yearnum)
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5), frameon=False)
plt.ioff()
plt.savefig('./plots/' + db + '/PAX_Counts_By_AO_' + thismonthname +
yearnum + '.jpg',
bbox_inches='tight') # save the figure to a file
print(
'Unique PAX graph created for unique PAX across all AOs. Sending to Slack now... hang tight!'
)
#slack.chat.post_message(firstf, "Hello " + region + "! Here is a quick look at how many UNIQUE PAX attended beatdowns by AO by Month for " + yearnum + "!")
slack.files_upload(
channels=firstf,
initial_comment="Hello " + region +
"! Here is a quick look at how many UNIQUE PAX attended beatdowns by AO by Month for "
+ yearnum + "!",
file='./plots/' + db + '/PAX_Counts_By_AO_' + thismonthname +
yearnum + '.jpg')
total_graphs = total_graphs + 1
finally:
print('Total graphs made:', total_graphs)
mydb.close()
class PyLogger(logging.Filter, metaclass=Singleton):
def __init__(self,
log_name='py',
td_log=False,
slack_notify=False,
useFileHandler=True):
"""
init function
:param log_name: log 파일에 접미사로 붙을 이름
:param useFileHandler : 로그파일을 기록합니다.
:param td_log : fluentd 로그를 활성화 합니다. (PyConfig 설정되는 config파일에 [LOG] 태그의 IP,PORT,TAG 값이 설정되어야 합니다.
:param slack_notify: slack 메세징 기능을 활성화 합니다. (PyConfig 설정되는 config파일에 [SLACK] 태그의 TAG,CHANNELS 값이 설정되어야 합니다.
:param useFileHandler : 로그파일을 파일로 기록합니다.
"""
# slack notification info
self.slack_client = WebClient(
py_config['SLACK']['TOKEN']) if slack_notify else None
# fluentd-log parameters
self.td_log = td_log
self.td_ip = py_config['FLUENTD']['IP']
self.td_port = py_config['FLUENTD']['PORT']
self.td_tag = py_config['FLUENTD']['TAG']
# 로그 저장 경로
self.log_name = log_name
self.__logger = logging.getLogger(self.log_name)
self.__logger.setLevel(logging.INFO)
# 포매터를 만든다
self.formatter = logging.Formatter(
'%(levelname)s\t%(process)d\t%(asctime)s\t%(user_ip)s\t%(user_agent)s\t%(svr_protocol)s\t%(req_method)s\t%(full_path)s\t%(message)s',
datefmt='%y-%m-%d %H:%M:%S')
# input image path 설정
super().__init__()
self.__logger.addFilter(self)
self.set_request_info()
# fileHandler set
if useFileHandler:
self.logs_info = {"format": ".log"}
self.log_dir = py_config['LOG']['PATH'] + '/' + log_name
self._set_fileHandler('info')
self._set_fileHandler('error')
def info(self, message, **kwargs):
"""
info log를 남깁니다.
Logger 초기화시 td_log 가 True이면
kwargs를 추가하여 td-log를 전송합니다.
:param message: 메세지
:param kwargs: log에 추가할 키워드 파라미터
:return:
"""
if self.td_log:
data = {
'language': self.accept_language,
'deviceId': self.deviceId,
'appType': self.appType,
'userIp': self.userIp,
'processId': os.getpid(),
'userNo': self.userNo,
'user-agent': self.user_agent,
'type': 'REQ' if message.split('\t')[0] == 'input' else 'RES',
'hostname': self.host,
'uri': self.path,
'path_var': self.full_path,
'method': self.req_method,
'event_time': datetime.now().strftime('%Y%m%d%H%M%S.%f')[:-3],
'payload': json.loads(message.split('\t')[1]),
}
for k, v in kwargs.items():
data[k] = v
requests.post('http://{}:{}/{}'.format(self.td_ip, self.td_port,
self.td_tag),
json=data)
else:
for k, v in kwargs.items():
self.__logger.info('{}\t{}'.format(k, v))
self.__logger.info(message)
def warning(self, message, **kwargs):
"""
warning log를 남깁니다.
:param message: 메세지
:param kwargs: log에 추가할 키워드 파라미터
:return:
"""
for k, v in kwargs.items():
self.__logger.error('{}\t{}'.format(k, v))
self.__logger.warning(message)
def error(self, message, **kwargs):
"""
error log를 남깁니다.
:param message: 메세지
:param kwargs: log에 추가할 키워드 파라미터
:return:
"""
for k, v in kwargs.items():
self.__logger.error('{}\t{}'.format(k, v))
self.__logger.error(message)
from io import BytesIO, StringIO
if self.slack_client is not None:
channels = [
v for v in py_config['SLACK']['CHANNELS'].split(',') if v
]
for channel in channels:
if str(message).startswith('input'):
file = message.split('\t')[-1].encode()
initial_comment = '[ 에러 발생 : {} ]'.format(
self.log_name.upper())
title = " ".join([
datetime.now().strftime('%y-%m-%d %H:%M:%S'),
self.userIp, self.user_agent, self.req_method,
self.full_path
])
self.slack_client.files_upload(
channels=channel,
file=file,
title=title,
initial_comment=initial_comment,
filename='{}.json'.format(message.split('\t')[0]))
elif str(message).startswith('output'):
file = message.split('\t')[-1].encode()
self.slack_client.files_upload(channels=channel,
file=file,
filename='{}.json'.format(
message.split('\t')[0]))
else:
# add api name, ip info
self.slack_client.chat_postMessage(channel=channel,
text=message)
def set_request_info(self,
userIp=None,
user_agent=None,
svr_protocol=None,
req_method=None,
path=None,
full_path=None,
host=None,
language=None,
deviceId=None,
appType=None,
userNo=None):
"""
input으로 들어온 데이터를 로그에 기록합니다.
:param user_ip: user ip
:param user_agent: user agent
:param svr_protocol: server protocol
:param req_method: GET or POST
:param full_path: url full_path
:return:
"""
self.userIp = userIp
self.user_agent = user_agent
self.svr_protocol = svr_protocol
self.req_method = req_method
self.path = path
self.full_path = full_path
self.host = host
self.accept_language = language
self.deviceId = deviceId
self.appType = appType
self.userNo = userNo
def filter(self, record):
"""
오버로딩 메소드
로그에 input 이미지 경로를 나타내는 필터
Autor : heewinkim
:param record: input data
:return:
"""
record.user_ip = self.userIp
record.user_agent = self.user_agent
record.svr_protocol = self.svr_protocol
record.req_method = self.req_method
record.full_path = self.full_path
return True
def _set_fileHandler(self, log_type) -> None:
"""
한개 로그 모듈 설정
:param log_type: (str) 'info' or 'error'
"""
log_save_dir = "{}/{}_log/".format(self.log_dir, log_type)
log_save_name = "{}_{}".format(self.log_name, log_type)
self.logs_info[log_type] = log_save_name
self.logs_info[log_type + "_dir"] = log_save_dir
# 로그 저장 경로 존재 확인 및 생성
try:
if not os.path.exists(log_save_dir):
os.makedirs(log_save_dir)
print("The log dir was created: ", log_save_dir)
time.sleep(0.3)
except FileExistsError as e:
print("log dir exist, ignore create command..")
time.sleep(0.1)
# 스트림과 파일로 로그를 출력하는 핸들러를 각각 만든다.
fileHandler = logging.FileHandler(log_save_dir + log_save_name +
self.logs_info["format"])
# 각 핸들러에 포매터를 지정한다.
fileHandler.setFormatter(self.formatter)
# 로그 레벨 설정
if log_type == "info":
fileHandler.setLevel(logging.INFO)
elif log_type == "error":
fileHandler.setLevel(logging.ERROR)
# 로거 인스턴스에 스트림 핸들러와 파일핸들러를 붙인다.
self.__logger.addHandler(fileHandler)
bbox_to_anchor=(1, 0.5),
frameon=False)
plt.ioff()
plt.savefig('./plots/' + db + '/' + user_id_tmp + "_" +
thismonthname + yearnum + '.jpg',
bbox_inches='tight') #save the figure to a file
total_graphs = total_graphs + 1
#manual_graphs = [240,241,242,244,245,246,247,249,250]
if total_graphs > 0: # This is a count of total users processed, in case of error during processing. Set the total_graphs > to whatever # comes next in the log file row count.
print(total_graphs, 'PAX posting graph created for user',
pax, 'Sending to Slack now... hang tight!')
#slack.chat.post_message(user_id_tmp, 'Hey ' + pax + "! Here is your monthly posting summary for " + yearnum + ". \nPush yourself, get those bars higher every month! SYITG!")
slack.files_upload(
channels=user_id_tmp,
initial_comment='Hey ' + pax +
"! Here is your monthly posting summary for " +
yearnum +
". \nPush yourself, get those bars higher every month! SYITG!",
file='./plots/' + db + '/' + user_id_tmp + "_" +
thismonthname + yearnum + '.jpg')
attendance_tmp_df.hist()
os.system("echo " + user_id_tmp + " >>" + "./logs/" + db +
"/PAXcharter.log")
else:
print(pax + 'skipped')
except:
print("An exception occurred")
finally:
plt.close(
'all'
) #Note - this was added after the December 2020 processing, make sure this works
print('Total graphs made:', total_graphs)
def plugin(srv, item):
srv.logging.debug("*** MODULE=%s: service=%s, target=%s", __file__,
item.service, item.target)
# check for service level token
token = item.config.get('token')
# get the target tokens
addrs = list(item.addrs)
# check for target level tokens (which have preference)
try:
if len(addrs) == 4:
token, channel, username, icon = addrs
else:
channel, username, icon = addrs
except Exception as e:
srv.logging.error("Incorrect target configuration for target=%s: %s",
item.target, e)
return False
# if no token then fail
if token is None:
srv.logging.error("No token found for slack")
return False
# if the incoming payload has been transformed, use that,
# else the original payload
text = item.message
# check if the message has been decoded from a JSON payload
if 'message' in item.data:
text = item.data['message']
else:
text = item.message
# check if there is an image contained in a JSON payload
# (support either an image URL or base64 encoded image)
try:
image = None
if 'imageurl' in item.data:
imageurl = item.data['imageurl']
srv.logging.debug("Image url detected - %s" % imageurl)
#Image payload has auth parms, so use the correct method for authenticating.
if 'auth' in item.data:
authtype = item.data['auth']
authuser = item.data['user']
authpass = item.data['password']
if authtype == 'digest':
image = requests.get(imageurl,
stream=True,
auth=HTTPDigestAuth(
authuser, authpass)).raw
else:
image = requests.get(imageurl,
stream=True,
auth=HTTPBasicAuth(
authuser, authpass)).raw
else:
image = requests.get(imageurl, stream=True).raw
elif 'imagebase64' in item.data:
imagebase64 = item.data['imagebase64']
srv.logging.debug("Image (base64 encoded) detected")
image = base64.b64decode(str(imagebase64))
except Exception as e:
srv.logging.warning("Cannot download image: %s", e)
try:
slack = WebClient(token=token)
if image is None:
slack.chat_postMessage(channel=channel,
text=text,
username=username,
icon_emoji=icon,
unfurl_links=True)
else:
srv.logging.debug("Channel id: %s" % channel)
slack.files_upload(file=image, title=text, channels=channel)
srv.logging.debug("image posted")
except SlackApiError as e:
assert e.response["ok"] is False
assert e.response["error"]
srv.logging.warning("Cannot post to slack %s: %s" %
(channel, e.response['error']))
return False
except Exception as e:
srv.logging.warning("Cannot post to slack %s: %s" % (channel, e))
return False
return True
