def handle(alert,
type='sms',
recipient_phone=None,
sender_phone=None,
message=None):
if not os.environ.get('TWILIO_API_SID'):
log.info(f"No TWILIO_API_SID in env, skipping handler.")
return None
twilio_sid = os.environ["TWILIO_API_SID"]
twilio_token = vault.decrypt_if_encrypted(os.environ['TWILIO_API_TOKEN'])
# check if phone is not empty if yes notification will be delivered to twilio
if recipient_phone is None:
log.error(f'Cannot identify assignee phone number')
return None
if message is None:
log.error(f'SMS Message is empty')
return None
log.debug(
f'Twilio message for recipient with phone number {recipient_phone}',
message)
client = Client(twilio_sid, twilio_token)
response = client.messages.create(body=message,
from_=sender_phone,
to=recipient_phone)
return response
def main():
for pipe in db.get_pipes('data'):
metadata = yaml.load(pipe['comment'])
if metadata and metadata.get('type') != 'Azure':
log.info(f"{pipe['name']} is not an Azure pipe, and will be skipped.")
continue
blob_name = metadata['blob']
account_name = metadata['account']
pipe_name = pipe['name']
table = metadata['target']
sas_token_envar = 'AZURE_SAS_TOKEN_' + metadata.get('suffix', '')
if sas_token_envar in environ:
encrypted_sas_token = environ.get(sas_token_envar)
elif 'encrypted_sas_token' in metadata:
encrypted_sas_token = metadata['encrypted_sas_token']
else:
log.info(f"{pipe['name']} has no azure auth")
continue
sas_token = vault.decrypt_if_encrypted(encrypted_sas_token)
log.info(f"Now working on pipe {pipe_name}")
endpoint_suffix = metadata.get('endpoint_suffix', 'core.windows.net')
block_blob_service = BlockBlobService(
account_name=account_name,
sas_token=sas_token,
endpoint_suffix=endpoint_suffix
)
files = block_blob_service.list_blobs(blob_name)
newest_time = get_timestamp(table)
new_files = []
if newest_time:
for file in files:
if file.properties.creation_time > newest_time:
new_files.append(StagedFile(file.name, None))
else:
for file in files:
new_files.append(StagedFile(file.name, None))
log.info(new_files)
# Proxy object that abstracts the Snowpipe REST API
ingest_manager = SimpleIngestManager(account=environ.get('SNOWFLAKE_ACCOUNT'),
host=f'{environ.get("SNOWFLAKE_ACCOUNT")}.snowflakecomputing.com',
user=environ.get('SA_USER'),
pipe=f'SNOWALERT.DATA.{pipe_name}',
private_key=load_pkb_rsa(PRIVATE_KEY, PRIVATE_KEY_PASSWORD))
if len(new_files) > 0:
try:
response = ingest_manager.ingest_files(new_files)
log.info(response)
except Exception as e:
log.error(e)
return
def connection_run(connection_table, run_now=False):
table_name = connection_table['name']
table_comment = connection_table['comment']
log.info(f"-- START DC {table_name} --")
try:
metadata = {'START_TIME': datetime.utcnow()}
options = yaml.safe_load(table_comment) or {}
if 'schedule' in options:
schedule = options['schedule']
now = datetime.now()
if not run_now and not time_to_run(schedule, now):
log.info(f'not scheduled: {schedule} at {now}')
log.info(f"-- END DC --")
return
if 'module' not in options:
log.info(f'no module in options')
log.info(f"-- END DC --")
return
module = options['module']
metadata.update({
'RUN_ID': RUN_ID,
'TYPE': module,
'LANDING_TABLE': table_name
})
connector = importlib.import_module(f"connectors.{module}")
for module_option in connector.CONNECTION_OPTIONS:
name = module_option['name']
if module_option.get('secret') and name in options:
options[name] = vault.decrypt_if_encrypted(options[name])
if module_option.get('type') == 'json':
options[name] = json.loads(options[name])
if module_option.get('type') == 'list':
if type(options[name]) is str:
options[name] = options[name].split(',')
if module_option.get('type') == 'int':
options[name] = int(options[name])
if callable(getattr(connector, 'ingest', None)):
db.record_metadata(metadata, table=DC_METADATA_TABLE)
result = do_ingest(connector, table_name, options)
if result is not None:
metadata['INGEST_COUNT'] = result
else:
metadata['INGESTED'] = result
db.record_metadata(metadata, table=DC_METADATA_TABLE)
except Exception as e:
log.error(f"Error loading logs into {table_name}: ", e)
db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e)
log.info(f"-- END DC --")
def main():
# Set your Azure Active Directory application credentials.
# Application must have permission for Microsoft.Graph AuditLog.Read.All
# and RBAC role "Storage Blob Contributor" to the storage account.
tenant_id = vault.decrypt_if_encrypted(envar='AAD_TENANT_ID')
client_id = vault.decrypt_if_encrypted(envar='AAD_CLIENT_ID')
client_secret = vault.decrypt_if_encrypted(envar='AAD_CLIENT_SECRET')
storage_account = vault.decrypt_if_encrypted(envar='AAD_STORAGE_ACCOUNT')
if not (tenant_id and client_id and client_secret and storage_account):
print('[aad_auditlogs] missing required env var')
return
save_aad_auditlogs("directoryAudits", tenant_id, client_id, client_secret,
storage_account, "logs-audit")
# AAD signIns report is only available for Azure AD Premium P1 or higher and will return an error for non-premium
# AAD tenants.
save_aad_auditlogs("signIns", tenant_id, client_id, client_secret,
storage_account, "logs-signin")
def connection_run(connection_table):
table_name = connection_table['name']
table_comment = connection_table['comment']
log.info(f"-- START DC {table_name} --")
try:
metadata = {'START_TIME': datetime.utcnow()}
options = yaml.load(table_comment) or {}
if 'module' in options:
module = options['module']
metadata.update({
'RUN_ID': RUN_ID,
'TYPE': module,
'LANDING_TABLE': table_name,
'INGEST_COUNT': 0,
})
connector = importlib.import_module(f"connectors.{module}")
for module_option in connector.CONNECTION_OPTIONS:
name = module_option['name']
if module_option.get('secret') and name in options:
options[name] = vault.decrypt_if_encrypted(options[name])
if module_option.get('type') == 'json':
options[name] = json.loads(options[name])
if module_option.get('type') == 'list':
if type(options[name]) is str:
options[name] = options[name].split(',')
if module_option.get('type') == 'int':
options[name] = int(options[name])
if callable(getattr(connector, 'ingest', None)):
ingested = connector.ingest(table_name, options)
if isinstance(ingested, int):
metadata['INGEST_COUNT'] += ingested
elif isinstance(ingested, GeneratorType):
for n in ingested:
metadata['INGEST_COUNT'] += n
else:
metadata['INGESTED'] = ingested
db.record_metadata(metadata, table=DC_METADATA_TABLE)
except Exception as e:
log.error(f"Error loading logs into {table_name}: ", e)
db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e)
log.info(f"-- END DC --")
def handle(alert,
type='msteams',
webhook=None,
title=None,
color=None,
message=None):
""" Handler for the MS Teams integration utilizing the pymsteams library """
if not webhook and not os.environ.get('MSTEAMS_WEBHOOK'):
# log.info(f"No Webhook is provided nor there is a MSTEAMS_WEBHOOK in env, skipping handler.")
return None
webhook = webhook or vault.decrypt_if_encrypted(
os.environ['MSTEAMS_WEBHOOK'])
if message is None:
log.error('Message is empty')
return None
# You must create the connectorcard object with the Microsoft Webhook URL
m = connectorcard(webhook)
if title:
m.title(f'SnowAlert: {title}')
else:
m.title('SnowAlert')
if color:
# setting a hex color for the message
m.color(color)
# Add text to the message.
if message:
m.text(message)
log.debug('Microsoft Teams message for via webhook', message)
# send the message.
m.send()
if m.last_http_status.status_code != 300:
log.error(f"MS Teams handler error", m.last_http_status.text)
return None
return m.last_http_status
def main(connection_table="%_CONNECTION"):
for table in db.fetch(f"SHOW TABLES LIKE '{connection_table}' IN data"):
table_name = table['name']
table_comment = table['comment']
log.info(f"-- START DC {table_name} --")
try:
options = yaml.load(table_comment) or {}
if 'module' in options:
module = options['module']
metadata = {
'RUN_ID': RUN_ID,
'TYPE': module,
'START_TIME': datetime.utcnow(),
'LANDING_TABLE': table_name,
'INGEST_COUNT': 0
}
connector = importlib.import_module(f"connectors.{module}")
for module_option in connector.CONNECTION_OPTIONS:
name = module_option['name']
if module_option.get('secret') and name in options:
options[name] = vault.decrypt_if_encrypted(
options[name])
if callable(getattr(connector, 'ingest', None)):
ingested = connector.ingest(table_name, options)
if isinstance(ingested, int):
metadata['INGEST_COUNT'] += ingested
elif isinstance(ingested, GeneratorType):
for n in ingested:
metadata['INGEST_COUNT'] += n
else:
metadata['INGESTED'] = ingested
db.record_metadata(metadata, table=DC_METADATA_TABLE)
except Exception as e:
log.error(f"Error loading logs into {table_name}: ", e)
db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e)
log.info(f"-- END DC --")
def ingest(table_name, options):
base_name = re.sub(r'_CONNECTION$', '', table_name)
storage_account = options['storage_account']
sas_token = vault.decrypt_if_encrypted(options['sas_token'])
suffix = options['suffix']
container_name = options['container_name']
snowflake_account = options['snowflake_account']
sa_user = options['sa_user']
database = options['database']
block_blob_service = BlockBlobService(account_name=storage_account,
sas_token=sas_token,
endpoint_suffix=suffix)
db.execute(f"select SYSTEM$PIPE_FORCE_RESUME('DATA.{base_name}_PIPE');")
last_loaded = db.fetch_latest(f'data.{table_name}', 'loaded_on')
log.info(f"Last loaded time is {last_loaded}")
blobs = block_blob_service.list_blobs(container_name)
new_files = [
StagedFile(b.name, None) for b in blobs
if (last_loaded is None or b.properties.creation_time > last_loaded)
]
log.info(f"Found {len(new_files)} files to ingest")
# Proxy object that abstracts the Snowpipe REST API
ingest_manager = SimpleIngestManager(
account=snowflake_account,
host=f'{snowflake_account}.snowflakecomputing.com',
user=sa_user,
pipe=f'{database}.data.{base_name}_PIPE',
private_key=load_pkb_rsa(PRIVATE_KEY, PRIVATE_KEY_PASSWORD))
if len(new_files) > 0:
for file_group in groups_of(4999, new_files):
response = ingest_manager.ingest_files(file_group)
log.info(response)
yield len(file_group)
def handle(
alert,
summary=None,
source=None,
dedup_key=None,
severity=None,
custom_details=None,
pd_api_token=None,
):
if 'PD_API_TOKEN' not in os.environ and pd_api_token is None:
log.error(f"No PD_API_TOKEN in env, skipping handler.")
return None
pd_token_ct = pd_api_token or os.environ['PD_API_TOKEN']
pd_token = vault.decrypt_if_encrypted(pd_token_ct)
pds = EventsAPISession(pd_token)
summary = summary or alert['DESCRIPTION']
source = source or alert['DETECTOR']
severity = severity or alert['SEVERITY']
if severity not in severityDictionary:
log.warn(
f"Set severity to {severityDictionary[-1]}, "
f"supplied {severity} is not in allowed values: {severityDictionary}"
)
severity = severityDictionary[-1]
custom_details = custom_details or alert
try:
response = pds.trigger(
summary, source, dedup_key, severity, custom_details=alert
)
log.info(f"triggered PagerDuty alert \"{summary}\" at severity {severity}")
return response
except PDClientError as e:
log.error(f"Cannot trigger PagerDuty alert: {e.msg}")
return None
def handle(alert,
recipient_email=None,
channel=None,
template=None,
message=None):
if 'SLACK_API_TOKEN' not in os.environ:
log.info(f"No SLACK_API_TOKEN in env, skipping handler.")
return None
slack_token = vault.decrypt_if_encrypted(os.environ['SLACK_API_TOKEN'])
sc = SlackClient(slack_token)
# otherwise we will retrieve email from assignee and use it to identify Slack user
# Slack user id will be assigned as a channel
title = alert['TITLE']
if recipient_email is not None:
result = sc.api_call("users.lookupByEmail", email=recipient_email)
# log.info(f'Slack user info for {email}', result)
if result['ok'] is True and 'error' not in result:
user = result['user']
userid = user['id']
else:
log.error(
f'Cannot identify Slack user for email {recipient_email}')
return None
# check if channel exists, if yes notification will be delivered to the channel
if channel is not None:
log.info(f'Creating new SLACK message for {title} in channel', channel)
else:
if recipient_email is not None:
channel = userid
log.info(
f'Creating new SLACK message for {title} for user {recipient_email}'
)
else:
log.error(f'Cannot identify assignee email')
return None
blocks = None
attachments = None
text = title
if template is not None:
properties = {'channel': channel, 'message': message}
# create Slack message structure in Snowflake javascript UDF
try:
payload = message_template(locals())
except Exception:
return None
if payload is not None:
if 'blocks' in payload:
blocks = json.dumps(payload['blocks'])
if 'attachments' in payload:
attachments = json.dumps(payload['attachments'])
if 'text' in payload:
text = payload['text']
else:
log.error(f'Payload is empty for template {template}')
return None
else:
# does not have template, will send just simple message
if message is not None:
text = message
response = sc.api_call("chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments)
log.debug(f'Slack response', response)
if response['ok'] is False:
log.error(f"Slack handler error", response['error'])
return None
if 'message' in response:
del response['message']
return response
def handle(
alert,
type='smtp',
sender_email=None,
recipient_email=None,
text=None,
html=None,
subject=None,
reply_to=None,
cc=None,
bcc=None,
host=HOST,
port=PORT,
user=USER,
password=PASSWORD,
use_ssl=USE_SSL,
use_tls=USE_TLS,
):
user = vault.decrypt_if_encrypted(user)
password = vault.decrypt_if_encrypted(password)
sender_email = sender_email or user
if recipient_email is None:
log.error(f"param 'recipient_email' required")
return None
if text is None:
log.error(f"param 'text' required")
return None
# Create the base MIME message.
if html is None:
message = MIMEMultipart()
else:
message = MIMEMultipart('alternative')
# Add HTML/plain-text parts to MIMEMultipart message
# The email client will try to render the last part first
# Turn these into plain/html MIMEText objects
textPart = MIMEText(text, 'plain')
message.attach(textPart)
if html is not None:
htmlPart = MIMEText(html, 'html')
message.attach(htmlPart)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = recipient_email
recipients = recipient_email.split(',')
if cc is not None:
message['Cc'] = cc
recipients = recipients + cc.split(',')
if bcc is not None:
recipients = recipients + bcc.split(',')
if reply_to is not None:
message.add_header('reply-to', reply_to)
if use_ssl is True:
context = ssl.create_default_context()
if use_tls is True:
smtpserver = smtplib.SMTP(host, port)
smtpserver.starttls(context=context)
else:
smtpserver = smtplib.SMTP_SSL(host, port, context=context)
else:
smtpserver = smtplib.SMTP(host, port)
if user and password:
smtpserver.login(user, password)
result = smtpserver.sendmail(sender_email, recipients, message.as_string())
smtpserver.close()
return result
Sources: {SOURCES}
Actor: {ACTOR}
Object: {OBJECT}
Action: {ACTION}
Title: {TITLE}
Event Time: {EVENT_TIME}
Alert Time: {ALERT_TIME}
Description: {{quote}}
{DESCRIPTION}
{{quote}}
Detector: {DETECTOR}
Event Data: {{code}}{EVENT_DATA}{{code}}
Severity: {SEVERITY}
"""
password = vault.decrypt_if_encrypted(environ.get('JIRA_PASSWORD'))
user = environ.get('JIRA_USER')
if user and password:
jira = JIRA(URL, basic_auth=(user, password))
def jira_ticket_body(alert):
alert['SOURCES'] = ', '.join(alert['SOURCES'])
escaped_locals_strings = {k: escape_jira_strings(v) for k, v in alert.items()}
sources = escaped_locals_strings['SOURCES']
escaped_locals_strings[
'SOURCES'
] = f'[{sources}|{link_search_todos(f"Sources: {sources}")}]'
jira_body = {**JIRA_TICKET_BODY_DEFAULTS, **escaped_locals_strings}
ticket_body = JIRA_TICKET_BODY_FMT.format(**jira_body)
def handle(
alert,
type='smtp',
sender_email=None,
recipient_email=None,
text=None,
html=None,
subject=None,
reply_to=None,
cc=None,
bcc=None,
):
if not os.environ.get('SMTP_SERVER'):
log.info("No SMTP_SERVER in env, skipping handler.")
return None
smtp_server = os.environ['SMTP_SERVER']
if 'SMTP_PORT' in os.environ:
smtp_port = os.environ['SMTP_PORT']
else:
smtp_port = 587
if 'SMTP_USE_SSL' in os.environ:
smtp_use_ssl = os.environ['SMTP_USE_SSL']
else:
smtp_use_ssl = True
if 'SMTP_USE_TLS' in os.environ:
smtp_use_tls = os.environ['SMTP_USE_TLS']
else:
smtp_use_tls = True
smtp_user = vault.decrypt_if_encrypted(os.environ['SMTP_USER'])
smtp_password = vault.decrypt_if_encrypted(os.environ['SMTP_PASSWORD'])
if recipient_email is None:
log.error(f"Cannot identify recipient email")
return None
if text is None:
log.error(f"SES Message is empty")
return None
# Create the base MIME message.
if html is None:
message = MIMEMultipart()
else:
message = MIMEMultipart('alternative')
# Add HTML/plain-text parts to MIMEMultipart message
# The email client will try to render the last part first
# Turn these into plain/html MIMEText objects
textPart = MIMEText(text, 'plain')
message.attach(textPart)
if html is not None:
htmlPart = MIMEText(html, 'html')
message.attach(htmlPart)
message['Subject'] = subject
message['From'] = sender_email
message['To'] = recipient_email
recipients = recipient_email.split(',')
if cc is not None:
message['Cc'] = cc
recipients = recipients + cc.split(',')
if bcc is not None:
recipients = recipients + bcc.split(',')
if reply_to is not None:
message.add_header('reply-to', reply_to)
if smtp_use_ssl is True:
context = ssl.create_default_context()
if smtp_use_tls is True:
smtpserver = smtplib.SMTP(smtp_server, smtp_port)
smtpserver.starttls(context=context)
else:
smtpserver = smtplib.SMTP_SSL(smtp_server,
smtp_port,
context=context)
else:
smtpserver = smtplib.SMTP(smtp_server, smtp_port)
smtpserver.login(smtp_user, smtp_password)
smtpserver.sendmail(sender_email, recipients, message.as_string())
smtpserver.close()
def handle(
alert,
recipient_email=None,
channel=None,
template=None,
message=None,
file_content=None,
file_type=None,
file_name=None,
blocks=None,
attachments=None,
api_token=API_TOKEN,
slack_api_token=None,
):
slack_token_ct = slack_api_token or api_token
slack_token = vault.decrypt_if_encrypted(slack_token_ct)
sc = SlackClient(slack_token)
# otherwise we will retrieve email from assignee and use it to identify Slack user
# Slack user id will be assigned as a channel
title = alert['TITLE']
if recipient_email is not None:
result = sc.api_call("users.lookupByEmail", email=recipient_email)
# log.info(f'Slack user info for {email}', result)
if result['ok'] is True and 'error' not in result:
user = result['user']
userid = user['id']
else:
log.error(
f'Cannot identify Slack user for email {recipient_email}')
return None
# check if channel exists, if yes notification will be delivered to the channel
if channel is not None:
log.info(f'Creating new SLACK message for {title} in channel', channel)
else:
if recipient_email is not None:
channel = userid
log.info(
f'Creating new SLACK message for {title} for user {recipient_email}'
)
else:
log.error(f'Cannot identify assignee email')
return None
text = title
if template is not None:
properties = {'channel': channel, 'message': message}
# create Slack message structure in Snowflake javascript UDF
payload = message_template(locals())
if payload is not None:
if 'blocks' in payload:
blocks = json.dumps(payload['blocks'])
if 'attachments' in payload:
attachments = json.dumps(payload['attachments'])
if 'text' in payload:
text = payload['text']
else:
log.error(f'Payload is empty for template {template}')
return None
else:
# does not have template, will send just simple message
if message is not None:
text = message
response = None
if file_content is not None:
if template is not None:
response = sc.api_call(
"chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments,
)
file_descriptor = sc.api_call(
"files.upload",
content=file_content,
title=text,
channels=channel,
iletype=file_type,
filename=file_name,
)
if file_descriptor['ok'] is True:
file = file_descriptor["file"]
file_url = file["url_private"]
else:
log.error(f"Slack file upload error", file_descriptor['error'])
else:
response = sc.api_call(
"chat.postMessage",
channel=channel,
text=text,
blocks=blocks,
attachments=attachments,
)
if response is not None:
log.debug(f'Slack response', response)
if response['ok'] is False:
log.error(f"Slack handler error", response['error'])
return None
if 'message' in response:
del response['message']
return response
Actor: {ACTOR}
Object: {OBJECT}
Action: {ACTION}
Title: {TITLE}
Event Time: {EVENT_TIME}
Alert Time: {ALERT_TIME}
Description: {{quote}}
{DESCRIPTION}
{{quote}}
Detector: {DETECTOR}
Event Data: {{code}}{EVENT_DATA}{{code}}
Severity: {SEVERITY}
"""
password = vault.decrypt_if_encrypted(
environ.get('SA_JIRA_API_TOKEN', environ.get('JIRA_API_TOKEN'))
or environ.get('SA_JIRA_PASSWORD', environ.get('JIRA_PASSWORD')))
user = environ.get('SA_JIRA_USER', environ.get('JIRA_USER'))
jira_server = URL if URL.startswith('https://') else f'https://{URL}'
if user and password:
jira = JIRA(jira_server, basic_auth=(user, password))
def jira_ticket_body(alert, project):
sources = alert['SOURCES']
alert['SOURCES'] = ', '.join(sources) if isinstance(sources,
list) else sources
escaped_locals_strings = {
k: escape_jira_strings(v)
def handle(alert, assignee='', payload={}):
host = env.get('SA_SN_API_HOST')
if not host:
log.info('skipping service-now handler, missing host')
return
username = vault.decrypt_if_encrypted(envar='SA_SN_API_USER')
password = vault.decrypt_if_encrypted(envar='SA_SN_API_PASS')
client_id = env.get('SA_SN_OAUTH_CLIENT_ID')
client_secret = vault.decrypt_if_encrypted(
envar='SA_SN_OAUTH_CLIENT_SECRET')
refresh_token = vault.decrypt_if_encrypted(
envar='SA_SN_OAUTH_REFRESH_TOKEN')
if client_id:
oauth_return_params = {
'grant_type': 'refresh_token',
'client_id': client_id,
'client_secret': client_secret,
'refresh_token': refresh_token,
}
oauthresp = requests.post(
f'https://{host}/oauth_token.do',
data=oauth_return_params,
)
result = oauthresp.json()
access_token = result.get('access_token')
if not access_token:
log.info('skipping service-now handler, bad oauth')
raise RuntimeError(result)
else:
access_token = None
if not (username and password) and not access_token:
log.info('skipping service-now handler, no authorization')
return
title = alert.get('TITLE', 'SnowAlert Generate Incident')
description = alert.get('DESCRIPTION', '')
endpoint = env.get('SA_SN_API_ENDPOINT', '/now/table/incident')
api_url = f'https://{host}/api{endpoint}'
fp = env.get('SA_SN_FIELD_PREFIX', '')
response = requests.post(
api_url,
auth=Bearer(access_token) if access_token else (username, password),
json=payload or {
f'{fp}contact_type': 'Integration',
f'{fp}impact': '2',
f'{fp}urgency': '2',
f'{fp}category': 'IT Security',
f'{fp}subcategory': 'Remediation',
f'{fp}assignment_group': 'Security Compliance',
f'{fp}short_description': title,
f'{fp}description': description,
f'{fp}assigned_to': assignee,
},
)
if response.status_code != 201:
log.info(
f'URL: {api_url}',
f'Status Code: {response.status_code}',
f'Response Length: {len(response.text)}',
f'Response Headers: {response.headers}',
)
raise RuntimeError(response)
return response
