def handle(alert,
type='sms',
recipient_phone=None,
sender_phone=None,
message=None):
if not os.environ.get('TWILIO_API_SID'):
log.info(f"No TWILIO_API_SID in env, skipping handler.")
return None
twilio_sid = os.environ["TWILIO_API_SID"]
twilio_token = vault.decrypt_if_encrypted(os.environ['TWILIO_API_TOKEN'])
# check if phone is not empty if yes notification will be delivered to twilio
if recipient_phone is None:
log.error(f'Cannot identify assignee phone number')
return None
if message is None:
log.error(f'SMS Message is empty')
return None
log.debug(
f'Twilio message for recipient with phone number {recipient_phone}',
message)
client = Client(twilio_sid, twilio_token)
response = client.messages.create(body=message,
from_=sender_phone,
to=recipient_phone)
return response
def message_template(vars):
payload = None
# if we have Slack user data, send it to template
if 'user' in vars:
params = {
'alert': vars['alert'],
'properties': vars['properties'],
'user': vars['user']
}
else:
params = {'alert': vars['alert'], 'properties': vars['properties']}
try:
# retrieve Slack message structure from javascript UDF
rows = db.connect_and_fetchall("select " + vars['template'] +
"(parse_json('" + json.dumps(params) +
"'))")
row = rows[1]
if len(row) > 0:
log.debug(f"Template {vars['template']}", ''.join(row[0]))
payload = json.loads(''.join(row[0]))
else:
log.error(f"Error loading javascript template {vars['template']}")
raise Exception("Error loading javascript template " +
{vars['template']})
except Exception as e:
log.error(f"Error loading javascript template", e)
raise
return payload
def GET(resource, key=None, limit=100, offset=0):
if key is None:
key = resource
log.debug(f'GET {resource} limit={limit} offset={offset}')
response = requests.get(
url=f'https://cloud.tenable.com/{resource}',
params={
'limit': limit,
'offset': offset
},
headers={"X-ApiKeys": f"accessKey={token}; secretKey={secret}"},
)
if response.status_code != 200:
log.info(
f'response status {response.status_code}: {response.text}')
return
result = response.json()
elements = result.get(key)
if elements is None:
log.error(f'no {key} in :', result)
return
yield from elements
pages = result.get('pagination', {})
total = pages.get('total', 0)
limit = pages.get('limit', 0)
offset = pages.get('offset', 0)
if total > limit + offset:
yield from GET(resource, key, limit, offset + limit)
def get_data(organization_id: int,
key: str,
secret: str,
params: dict = {}) -> dict:
url = f"https://management.api.umbrella.com/v1/organizations/{organization_id}/roamingcomputers"
headers: dict = {
"Content-Type": "application/json",
"Accept": "application/json"
}
try:
req = requests.get(
url,
params=params,
headers=headers,
auth=requests.auth.HTTPBasicAuth(key, secret),
)
req.raise_for_status()
except requests.HTTPError as http_err:
log.error(f"Error GET: url={url}")
log.error(f"HTTP error occurred: {http_err}")
raise
try:
log.debug(req.status_code)
json = req.json()
except Exception as json_error:
log.error(f"JSON error occurred: {json_error}")
log.debug(f"requests response {req}")
raise
return json
def get_agent_data():
scanners = list(GET('scanners'))
log.debug(f'got {len(scanners)} scanners')
for s in scanners:
sid = s['id']
agents = list(GET(f'scanners/{sid}/agents', 'agents', 5000))
log.debug(f'scanner {sid} has {len(agents)} agents')
yield from agents
def handle(alert, procedure=None, parameters=None):
log.debug(f"Procedure name {procedure}")
log.debug(f"Procedure parameters {parameters}")
if procedure is not None:
# call Snowflake stored procedure
try:
result = call_procedure(procedure, parameters)
return result
except Exception:
return None
else:
return None
def ingest(table_name, options):
landing_table = f'data.{table_name}'
token = options['token']
asset_entity_id = options['asset_entity_id']
general_url = (
f"https://api.assetpanda.com:443//v2/entities/{asset_entity_id}/objects"
)
fields_url = f"https://api.assetpanda.com:443//v2/entities/{asset_entity_id}"
params = {"offset": 0, "limit": PAGE_SIZE}
total_object_count = 0
insert_time = datetime.utcnow()
while params['offset'] <= total_object_count:
log.debug("total_object_count: ", total_object_count)
assets = get_data(token=token, url=general_url, params=params)
list_object, total_object_count = get_list_objects_and_total_from_get_object(
assets)
dict_fields = get_data(token, fields_url, params=params)
list_field = dict_fields["fields"]
# Stripping down the metadata to remove unnecessary fields. We only really care about the following:
# {"field_140": "MAC_Address", "field_135" :"IP"}
clear_fields: dict = reduce(reduce_fields, list_field, {})
# replace every key "field_NO" by the value of the clear_field["field_NO"]
list_object_without_field_id = replace_device_key(
list_object, clear_fields)
db.insert(
landing_table,
values=[(entry, entry.get('id', None), insert_time)
for entry in list_object_without_field_id],
select=db.derive_insert_select(LANDING_TABLE_COLUMNS),
columns=db.derive_insert_columns(LANDING_TABLE_COLUMNS),
)
log.info(
f'Inserted {len(list_object_without_field_id)} rows ({landing_table}).'
)
yield len(list_object_without_field_id)
# increment the offset to get new entries each iteration in the while loop
params["offset"] += PAGE_SIZE
def get_data(token: str, url: str, params: dict = {}) -> dict:
headers: dict = {"Authorization": f"Bearer {token}"}
try:
log.debug(f"Preparing GET: url={url} with params={params}")
req = requests.get(url, params=params, headers=headers)
req.raise_for_status()
except HTTPError as http_err:
log.error(f"Error GET: url={url}")
log.error(f"HTTP error occurred: {http_err}")
raise http_err
log.debug(req.status_code)
return req.json()
def get_data(url: str, token: str, params: dict = {}) -> dict:
headers: dict = {
"Content-Type": "application/json",
"Accept": "application/json",
"X-Cisco-Meraki-API-Key": f"{token}",
}
try:
log.debug(f"Preparing GET: url={url} with params={params}")
req = requests.get(url, params=params, headers=headers)
req.raise_for_status()
except HTTPError as http_err:
log.error(f"Error GET: url={url}")
log.error(f"HTTP error occurred: {http_err}")
raise
log.debug(req.status_code)
return req.json()
def get_data(url: str, cms_auth: str, api_key: str, params: dict = {}) -> dict:
headers: dict = {
'Content-Type': 'application/json',
'aw-tenant-code': api_key,
'Accept': 'application/json',
'Authorization': cms_auth,
}
try:
log.debug(f"Preparing GET: url={url} with params={params}")
req = requests.get(url, params=params, headers=headers)
req.raise_for_status()
except HTTPError as http_err:
log.error(f"Error GET: url={url}")
log.error(f"HTTP error occurred: {http_err}")
raise
return req.json()
def handle(alert,
type='msteams',
webhook=None,
title=None,
color=None,
message=None):
""" Handler for the MS Teams integration utilizing the pymsteams library """
if not webhook and not os.environ.get('MSTEAMS_WEBHOOK'):
# log.info(f"No Webhook is provided nor there is a MSTEAMS_WEBHOOK in env, skipping handler.")
return None
webhook = webhook or vault.decrypt_if_encrypted(
os.environ['MSTEAMS_WEBHOOK'])
if message is None:
log.error('Message is empty')
return None
# You must create the connectorcard object with the Microsoft Webhook URL
m = connectorcard(webhook)
if title:
m.title(f'SnowAlert: {title}')
else:
m.title('SnowAlert')
if color:
# setting a hex color for the message
m.color(color)
# Add text to the message.
if message:
m.text(message)
log.debug('Microsoft Teams message for via webhook', message)
# send the message.
m.send()
if m.last_http_status.status_code != 300:
log.error(f"MS Teams handler error", m.last_http_status.text)
return None
return m.last_http_status
def ingest_agents(table_name, options):
last_export_time = next(
db.fetch(
f'SELECT MAX(export_at) as time FROM data.{table_name}'))['TIME']
timestamp = datetime.now(timezone.utc)
if (last_export_time is None
or (timestamp - last_export_time).total_seconds() > 86400):
all_agents = sorted(get_agent_data(),
key=lambda a: a.get('last_connect', 0))
unique_agents = {a['uuid']: a for a in all_agents}.values()
rows = [{'raw': ua, 'export_at': timestamp} for ua in unique_agents]
log.debug(f'inserting {len(unique_agents)} unique (by uuid) agents')
db.insert(f'data.{table_name}', rows)
return len(rows)
else:
log.info('Not time to import Tenable Agents')
return 0
def get_token_basic(client_id: str, client_secret: str) -> str:
headers: dict = {
"Content-Type": "application/x-www-form-urlencoded;charset=UTF-8"
}
try:
log.debug(f"Preparing POST: url={CROWDSTRIKE_AUTH_TOKEN_URL}")
req = requests.post(
CROWDSTRIKE_AUTH_TOKEN_URL,
headers=headers,
auth=requests.auth.HTTPBasicAuth(client_id, client_secret),
)
req.raise_for_status()
except requests.HTTPError as http_err:
log.error(f"Error GET: url={CROWDSTRIKE_AUTH_TOKEN_URL}")
log.error(f"HTTP error occurred: {http_err}")
raise http_err
try:
credential = req.json()
except Exception as json_error:
log.debug(f"JSON error occurred: {json_error}")
log.debug(f"requests response {req}")
raise (json_error)
try:
access_token = credential["access_token"]
except BaseException:
log.error("error auth request token")
raise AttributeError("error auth request token")
return access_token
def ingest_vulns(table_name):
last_export_time = next(
db.fetch(
f'SELECT MAX(export_at) AS time FROM data.{table_name}'))['TIME']
now = datetime.now(timezone.utc)
if (last_export_time is None
or (now - last_export_time) > timedelta(days=1)):
log.debug('TIO export vulns')
# insert empty row...
db.insert(f'data.{table_name}', [{'export_at': now}])
# ...because this line takes awhile
vulns = TIO.exports.vulns()
rows = [{'raw': v, 'export_at': now} for v in vulns]
db.insert(f'data.{table_name}', rows)
return len(rows)
else:
log.info('Not time to import Tenable vulnerabilities yet')
return 0
def call_procedure(procedure, parameters):
payload = None
try:
# call stored procedure
if parameters is not None and len(parameters) > 0:
params = "("
for i in range(len(parameters)):
params = params + "%s"
if i < len(parameters) - 1:
params = params + ","
params = params + ")"
else:
params = "()"
sql = "call " + procedure + params
log.debug(f"Procedure call sql {sql}")
connection = db.connect()
cur = connection.cursor()
cur.execute(sql, tuple(parameters))
rows = cur.fetchall()
if len(rows) > 0:
row = rows[0]
if len(row) > 0:
log.debug(f"Stored procedure {procedure} response",
''.join(row[0]))
payload = ''.join(row[0])
cur.close()
except Exception as e:
log.error(f"Error executing stored procedure", e)
raise
return payload
def handle(
alert,
type='sns',
topic=None,
target=None,
recipient_phone=None,
subject=None,
message_structure=None,
message=None,
):
# check if phone is nit empty if yes notification will be delivered to twilio
if recipient_phone is None and topic is None and target is None:
log.error(f'Cannot identify recipient')
return None
if message is None:
log.error(f'SNS Message is empty')
return None
log.debug(f'SNS message ', message)
client = boto3.client('sns', region_name=REGION)
params = {}
if message_structure is not None:
params['MessageStructure'] = message_structure
if message_structure == 'json':
message = json.dumps(message)
if topic is not None:
params['TopicArn'] = topic
if target is not None:
params['TargetArn'] = target
if recipient_phone is not None:
params['PhoneNumber'] = recipient_phone
if subject is not None:
params['Subject'] = subject
log.debug(f"SNS message", message)
params['Message'] = message
# Try to send the message.
try:
# Provide the contents of the message.
response = client.publish(**params)
# Display an error if something goes wrong.
except ClientError as e:
log.error(f'Failed to send message {e}')
return None
else:
log.debug("SNS message sent!")
return response
def get_data(token: str, url: str, params: dict = {}) -> dict:
headers: dict = {"Authorization": f"Bearer {token}"}
try:
log.debug(f"Preparing GET: url={url} with params={params}")
req = requests.get(url, params=params, headers=headers)
req.raise_for_status()
except requests.HTTPError as http_err:
log.error(f"Error GET: url={url}")
log.error(f"Error GET: url={url}")
log.error(f"HTTP error occurred: {http_err}")
raise http_err
try:
json = req.json()
except Exception as json_error:
log.debug(f"JSON error occurred: {json_error}")
log.debug(f"requests response {req}")
json = {}
return json
def message_template(vars):
payload = None
# remove handlers data, it might contain JSON incompatible strucutres
vars['alert'].pop('HANDLERS')
# if we have Slack user data, send it to template
if 'user' in vars:
params = {
'alert': vars['alert'],
'properties': vars['properties'],
'user': vars['user'],
}
else:
params = {'alert': vars['alert'], 'properties': vars['properties']}
log.debug(f"Javascript template parameters", params)
try:
# retrieve Slack message structure from javascript UDF
rows = db.connect_and_fetchall(
"select " + vars['template'] + "(parse_json(%s))",
params=[json.dumps(params)],
)
row = rows[1]
if len(row) > 0:
log.debug(f"Template {vars['template']}", ''.join(row[0]))
payload = json.loads(''.join(row[0]))
else:
log.error(f"Error loading javascript template {vars['template']}")
raise Exception(
f"Error loading javascript template {vars['template']}")
except Exception as e:
log.error(f"Error loading javascript template", e)
raise
log.debug(f"Template payload", payload)
return payload
def handle(alert,
recipient_email=None,
channel=None,
template=None,
message=None):
if 'SLACK_API_TOKEN' not in os.environ:
log.info(f"No SLACK_API_TOKEN in env, skipping handler.")
return None
slack_token = vault.decrypt_if_encrypted(os.environ['SLACK_API_TOKEN'])
sc = SlackClient(slack_token)
# otherwise we will retrieve email from assignee and use it to identify Slack user
# Slack user id will be assigned as a channel
title = alert['TITLE']
if recipient_email is not None:
result = sc.api_call("users.lookupByEmail", email=recipient_email)
# log.info(f'Slack user info for {email}', result)
if result['ok'] is True and 'error' not in result:
user = result['user']
userid = user['id']
else:
log.error(
f'Cannot identify Slack user for email {recipient_email}')
return None
# check if channel exists, if yes notification will be delivered to the channel
if channel is not None:
log.info(f'Creating new SLACK message for {title} in channel', channel)
else:
if recipient_email is not None:
channel = userid
log.info(
f'Creating new SLACK message for {title} for user {recipient_email}'
)
else:
log.error(f'Cannot identify assignee email')
return None
blocks = None
attachments = None
text = title
if template is not None:
properties = {'channel': channel, 'message': message}
# create Slack message structure in Snowflake javascript UDF
try:
payload = message_template(locals())
except Exception:
return None
if payload is not None:
if 'blocks' in payload:
blocks = json.dumps(payload['blocks'])
if 'attachments' in payload:
attachments = json.dumps(payload['attachments'])
if 'text' in payload:
text = payload['text']
else:
log.error(f'Payload is empty for template {template}')
return None
else:
# does not have template, will send just simple message
if message is not None:
text = message
response = sc.api_call("chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments)
log.debug(f'Slack response', response)
if response['ok'] is False:
log.error(f"Slack handler error", response['error'])
return None
if 'message' in response:
del response['message']
return response
def handle(
alert,
recipient_email=None,
channel=None,
template=None,
message=None,
file_content=None,
file_type=None,
file_name=None,
blocks=None,
attachments=None,
api_token=API_TOKEN,
slack_api_token=None,
):
slack_token_ct = slack_api_token or api_token
slack_token = vault.decrypt_if_encrypted(slack_token_ct)
sc = SlackClient(slack_token)
# otherwise we will retrieve email from assignee and use it to identify Slack user
# Slack user id will be assigned as a channel
title = alert['TITLE']
if recipient_email is not None:
result = sc.api_call("users.lookupByEmail", email=recipient_email)
# log.info(f'Slack user info for {email}', result)
if result['ok'] is True and 'error' not in result:
user = result['user']
userid = user['id']
else:
log.error(
f'Cannot identify Slack user for email {recipient_email}')
return None
# check if channel exists, if yes notification will be delivered to the channel
if channel is not None:
log.info(f'Creating new SLACK message for {title} in channel', channel)
else:
if recipient_email is not None:
channel = userid
log.info(
f'Creating new SLACK message for {title} for user {recipient_email}'
)
else:
log.error(f'Cannot identify assignee email')
return None
text = title
if template is not None:
properties = {'channel': channel, 'message': message}
# create Slack message structure in Snowflake javascript UDF
payload = message_template(locals())
if payload is not None:
if 'blocks' in payload:
blocks = json.dumps(payload['blocks'])
if 'attachments' in payload:
attachments = json.dumps(payload['attachments'])
if 'text' in payload:
text = payload['text']
else:
log.error(f'Payload is empty for template {template}')
return None
else:
# does not have template, will send just simple message
if message is not None:
text = message
response = None
if file_content is not None:
if template is not None:
response = sc.api_call(
"chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments,
)
file_descriptor = sc.api_call(
"files.upload",
content=file_content,
title=text,
channels=channel,
iletype=file_type,
filename=file_name,
)
if file_descriptor['ok'] is True:
file = file_descriptor["file"]
file_url = file["url_private"]
else:
log.error(f"Slack file upload error", file_descriptor['error'])
else:
response = sc.api_call(
"chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments,
)
if response is not None:
log.debug(f'Slack response', response)
if response['ok'] is False:
log.error(f"Slack handler error", response['error'])
return None
if 'message' in response:
del response['message']
return response
def ingest(table_name, options):
ingest_type = 'client' if table_name.endswith(
'_CLIENT_CONNECTION') else 'device'
landing_table = f'data.{table_name}'
timestamp = datetime.utcnow()
api_token = options['api_token']
whitelist = set(options['network_id_whitelist'])
organizations = get_data(f"https://api.meraki.com/api/v0/organizations",
api_token)
for organization in organizations:
organization_id = organization.get('id')
log.debug(f'Processing Meraki organization id {organization_id}')
if not organization_id:
continue
networks = get_data(
f"https://api.meraki.com/api/v0/organizations/{organization_id}/networks",
api_token,
)
network_ids = {network.get('id') for network in networks}
if whitelist:
network_ids = network_ids.intersection(whitelist)
for network in network_ids:
log.debug(f'Processing Meraki network {network}')
try:
devices = get_data(
f"https://api.meraki.com/api/v0/networks/{network}/devices",
api_token,
)
except requests.exceptions.HTTPError as e:
log.error(f"{network} not accessible, ")
log.error(e)
continue
if ingest_type == 'device':
db.insert(
landing_table,
values=[(
timestamp,
device,
device.get('serial'),
device.get('address'),
device.get('name'),
device.get('networkId'),
device.get('model'),
device.get('mac'),
device.get('lanIp'),
device.get('wan1Ip'),
device.get('wan2Ip'),
device.get('tags'),
device.get('lng'),
device.get('lat'),
) for device in devices],
select=db.derive_insert_select(
LANDING_TABLE_COLUMNS_DEVICE),
columns=db.derive_insert_columns(
LANDING_TABLE_COLUMNS_DEVICE),
)
log.info(f'Inserted {len(devices)} rows ({landing_table}).')
yield len(devices)
else:
for device in devices:
serial_number = device['serial']
try:
clients = get_data(
f"https://api.meraki.com/api/v0/devices/{serial_number}/clients",
api_token,
)
except requests.exceptions.HTTPError as e:
log.error(f"{network} not accessible, ")
log.error(e)
continue
db.insert(
landing_table,
values=[
(
timestamp,
client,
client.get('id'),
client.get('mac'),
client.get('description'),
client.get('mdnsName'),
client.get('dhcpHostname'),
client.get('ip'),
client.get('switchport'),
# vlan sometimes set to ''
client.get('vlan') or None,
client.get('usage', {}).get('sent') or None,
client.get('usage', {}).get('recv') or None,
serial_number,
) for client in clients
],
select=db.derive_insert_select(
LANDING_TABLE_COLUMNS_CLIENT),
columns=db.derive_insert_columns(
LANDING_TABLE_COLUMNS_CLIENT),
)
log.info(
f'Inserted {len(clients)} rows ({landing_table}).')
yield len(clients)
def handle(
alert,
type='ses',
recipient_email=None,
sender_email=None,
text=None,
html=None,
subject=None,
cc=None,
bcc=None,
reply_to=None,
charset="UTF-8",
):
# check if recipient email is not empty
if recipient_email is None:
log.error(f'Cannot identify recipient email')
return None
if text is None:
log.error(f'SES Message is empty')
return None
if cc is None:
ccs = []
else:
ccs = cc.split(",")
if bcc is None:
bccs = []
else:
bccs = bcc.split(",")
if reply_to is None:
replyTo = []
else:
replyTo = reply_to.split(",")
destination = {
'ToAddresses': [recipient_email],
'CcAddresses': ccs,
'BccAddresses': bccs,
}
body = {'Text': {'Charset': charset, 'Data': text}}
if html is not None:
body.update(Html={'Charset': charset, 'Data': html})
message = {'Body': body, 'Subject': {'Charset': charset, 'Data': subject}}
log.debug(f'SES message for recipient with email {recipient_email}', message)
client = boto3.client('ses', region_name=REGION)
# Try to send the email.
try:
# Provide the contents of the email.
response = client.send_email(
Destination=destination,
Message=message,
Source=sender_email,
ReplyToAddresses=replyTo,
)
# Display an error if something goes wrong.
except ClientError as e:
log.error(f'Failed to send email {e}')
return None
else:
log.debug("SES Email sent!")
return response
