def handle(alert, type='sms', recipient_phone=None, sender_phone=None, message=None): if not os.environ.get('TWILIO_API_SID'): log.info(f"No TWILIO_API_SID in env, skipping handler.") return None twilio_sid = os.environ["TWILIO_API_SID"] twilio_token = vault.decrypt_if_encrypted(os.environ['TWILIO_API_TOKEN']) # check if phone is not empty if yes notification will be delivered to twilio if recipient_phone is None: log.error(f'Cannot identify assignee phone number') return None if message is None: log.error(f'SMS Message is empty') return None log.debug( f'Twilio message for recipient with phone number {recipient_phone}', message) client = Client(twilio_sid, twilio_token) response = client.messages.create(body=message, from_=sender_phone, to=recipient_phone) return response
def main(): for pipe in db.get_pipes('data'): metadata = yaml.load(pipe['comment']) if metadata and metadata.get('type') != 'Azure': log.info(f"{pipe['name']} is not an Azure pipe, and will be skipped.") continue blob_name = metadata['blob'] account_name = metadata['account'] pipe_name = pipe['name'] table = metadata['target'] sas_token_envar = 'AZURE_SAS_TOKEN_' + metadata.get('suffix', '') if sas_token_envar in environ: encrypted_sas_token = environ.get(sas_token_envar) elif 'encrypted_sas_token' in metadata: encrypted_sas_token = metadata['encrypted_sas_token'] else: log.info(f"{pipe['name']} has no azure auth") continue sas_token = vault.decrypt_if_encrypted(encrypted_sas_token) log.info(f"Now working on pipe {pipe_name}") endpoint_suffix = metadata.get('endpoint_suffix', 'core.windows.net') block_blob_service = BlockBlobService( account_name=account_name, sas_token=sas_token, endpoint_suffix=endpoint_suffix ) files = block_blob_service.list_blobs(blob_name) newest_time = get_timestamp(table) new_files = [] if newest_time: for file in files: if file.properties.creation_time > newest_time: new_files.append(StagedFile(file.name, None)) else: for file in files: new_files.append(StagedFile(file.name, None)) log.info(new_files) # Proxy object that abstracts the Snowpipe REST API ingest_manager = SimpleIngestManager(account=environ.get('SNOWFLAKE_ACCOUNT'), host=f'{environ.get("SNOWFLAKE_ACCOUNT")}.snowflakecomputing.com', user=environ.get('SA_USER'), pipe=f'SNOWALERT.DATA.{pipe_name}', private_key=load_pkb_rsa(PRIVATE_KEY, PRIVATE_KEY_PASSWORD)) if len(new_files) > 0: try: response = ingest_manager.ingest_files(new_files) log.info(response) except Exception as e: log.error(e) return
def connection_run(connection_table, run_now=False): table_name = connection_table['name'] table_comment = connection_table['comment'] log.info(f"-- START DC {table_name} --") try: metadata = {'START_TIME': datetime.utcnow()} options = yaml.safe_load(table_comment) or {} if 'schedule' in options: schedule = options['schedule'] now = datetime.now() if not run_now and not time_to_run(schedule, now): log.info(f'not scheduled: {schedule} at {now}') log.info(f"-- END DC --") return if 'module' not in options: log.info(f'no module in options') log.info(f"-- END DC --") return module = options['module'] metadata.update({ 'RUN_ID': RUN_ID, 'TYPE': module, 'LANDING_TABLE': table_name }) connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted(options[name]) if module_option.get('type') == 'json': options[name] = json.loads(options[name]) if module_option.get('type') == 'list': if type(options[name]) is str: options[name] = options[name].split(',') if module_option.get('type') == 'int': options[name] = int(options[name]) if callable(getattr(connector, 'ingest', None)): db.record_metadata(metadata, table=DC_METADATA_TABLE) result = do_ingest(connector, table_name, options) if result is not None: metadata['INGEST_COUNT'] = result else: metadata['INGESTED'] = result db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")
def main(): # Set your Azure Active Directory application credentials. # Application must have permission for Microsoft.Graph AuditLog.Read.All # and RBAC role "Storage Blob Contributor" to the storage account. tenant_id = vault.decrypt_if_encrypted(envar='AAD_TENANT_ID') client_id = vault.decrypt_if_encrypted(envar='AAD_CLIENT_ID') client_secret = vault.decrypt_if_encrypted(envar='AAD_CLIENT_SECRET') storage_account = vault.decrypt_if_encrypted(envar='AAD_STORAGE_ACCOUNT') if not (tenant_id and client_id and client_secret and storage_account): print('[aad_auditlogs] missing required env var') return save_aad_auditlogs("directoryAudits", tenant_id, client_id, client_secret, storage_account, "logs-audit") # AAD signIns report is only available for Azure AD Premium P1 or higher and will return an error for non-premium # AAD tenants. save_aad_auditlogs("signIns", tenant_id, client_id, client_secret, storage_account, "logs-signin")
def connection_run(connection_table): table_name = connection_table['name'] table_comment = connection_table['comment'] log.info(f"-- START DC {table_name} --") try: metadata = {'START_TIME': datetime.utcnow()} options = yaml.load(table_comment) or {} if 'module' in options: module = options['module'] metadata.update({ 'RUN_ID': RUN_ID, 'TYPE': module, 'LANDING_TABLE': table_name, 'INGEST_COUNT': 0, }) connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted(options[name]) if module_option.get('type') == 'json': options[name] = json.loads(options[name]) if module_option.get('type') == 'list': if type(options[name]) is str: options[name] = options[name].split(',') if module_option.get('type') == 'int': options[name] = int(options[name]) if callable(getattr(connector, 'ingest', None)): ingested = connector.ingest(table_name, options) if isinstance(ingested, int): metadata['INGEST_COUNT'] += ingested elif isinstance(ingested, GeneratorType): for n in ingested: metadata['INGEST_COUNT'] += n else: metadata['INGESTED'] = ingested db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")
def handle(alert, type='msteams', webhook=None, title=None, color=None, message=None): """ Handler for the MS Teams integration utilizing the pymsteams library """ if not webhook and not os.environ.get('MSTEAMS_WEBHOOK'): # log.info(f"No Webhook is provided nor there is a MSTEAMS_WEBHOOK in env, skipping handler.") return None webhook = webhook or vault.decrypt_if_encrypted( os.environ['MSTEAMS_WEBHOOK']) if message is None: log.error('Message is empty') return None # You must create the connectorcard object with the Microsoft Webhook URL m = connectorcard(webhook) if title: m.title(f'SnowAlert: {title}') else: m.title('SnowAlert') if color: # setting a hex color for the message m.color(color) # Add text to the message. if message: m.text(message) log.debug('Microsoft Teams message for via webhook', message) # send the message. m.send() if m.last_http_status.status_code != 300: log.error(f"MS Teams handler error", m.last_http_status.text) return None return m.last_http_status
def main(connection_table="%_CONNECTION"): for table in db.fetch(f"SHOW TABLES LIKE '{connection_table}' IN data"): table_name = table['name'] table_comment = table['comment'] log.info(f"-- START DC {table_name} --") try: options = yaml.load(table_comment) or {} if 'module' in options: module = options['module'] metadata = { 'RUN_ID': RUN_ID, 'TYPE': module, 'START_TIME': datetime.utcnow(), 'LANDING_TABLE': table_name, 'INGEST_COUNT': 0 } connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted( options[name]) if callable(getattr(connector, 'ingest', None)): ingested = connector.ingest(table_name, options) if isinstance(ingested, int): metadata['INGEST_COUNT'] += ingested elif isinstance(ingested, GeneratorType): for n in ingested: metadata['INGEST_COUNT'] += n else: metadata['INGESTED'] = ingested db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")
def ingest(table_name, options): base_name = re.sub(r'_CONNECTION$', '', table_name) storage_account = options['storage_account'] sas_token = vault.decrypt_if_encrypted(options['sas_token']) suffix = options['suffix'] container_name = options['container_name'] snowflake_account = options['snowflake_account'] sa_user = options['sa_user'] database = options['database'] block_blob_service = BlockBlobService(account_name=storage_account, sas_token=sas_token, endpoint_suffix=suffix) db.execute(f"select SYSTEM$PIPE_FORCE_RESUME('DATA.{base_name}_PIPE');") last_loaded = db.fetch_latest(f'data.{table_name}', 'loaded_on') log.info(f"Last loaded time is {last_loaded}") blobs = block_blob_service.list_blobs(container_name) new_files = [ StagedFile(b.name, None) for b in blobs if (last_loaded is None or b.properties.creation_time > last_loaded) ] log.info(f"Found {len(new_files)} files to ingest") # Proxy object that abstracts the Snowpipe REST API ingest_manager = SimpleIngestManager( account=snowflake_account, host=f'{snowflake_account}.snowflakecomputing.com', user=sa_user, pipe=f'{database}.data.{base_name}_PIPE', private_key=load_pkb_rsa(PRIVATE_KEY, PRIVATE_KEY_PASSWORD)) if len(new_files) > 0: for file_group in groups_of(4999, new_files): response = ingest_manager.ingest_files(file_group) log.info(response) yield len(file_group)
def handle( alert, summary=None, source=None, dedup_key=None, severity=None, custom_details=None, pd_api_token=None, ): if 'PD_API_TOKEN' not in os.environ and pd_api_token is None: log.error(f"No PD_API_TOKEN in env, skipping handler.") return None pd_token_ct = pd_api_token or os.environ['PD_API_TOKEN'] pd_token = vault.decrypt_if_encrypted(pd_token_ct) pds = EventsAPISession(pd_token) summary = summary or alert['DESCRIPTION'] source = source or alert['DETECTOR'] severity = severity or alert['SEVERITY'] if severity not in severityDictionary: log.warn( f"Set severity to {severityDictionary[-1]}, " f"supplied {severity} is not in allowed values: {severityDictionary}" ) severity = severityDictionary[-1] custom_details = custom_details or alert try: response = pds.trigger( summary, source, dedup_key, severity, custom_details=alert ) log.info(f"triggered PagerDuty alert \"{summary}\" at severity {severity}") return response except PDClientError as e: log.error(f"Cannot trigger PagerDuty alert: {e.msg}") return None
def handle(alert, recipient_email=None, channel=None, template=None, message=None): if 'SLACK_API_TOKEN' not in os.environ: log.info(f"No SLACK_API_TOKEN in env, skipping handler.") return None slack_token = vault.decrypt_if_encrypted(os.environ['SLACK_API_TOKEN']) sc = SlackClient(slack_token) # otherwise we will retrieve email from assignee and use it to identify Slack user # Slack user id will be assigned as a channel title = alert['TITLE'] if recipient_email is not None: result = sc.api_call("users.lookupByEmail", email=recipient_email) # log.info(f'Slack user info for {email}', result) if result['ok'] is True and 'error' not in result: user = result['user'] userid = user['id'] else: log.error( f'Cannot identify Slack user for email {recipient_email}') return None # check if channel exists, if yes notification will be delivered to the channel if channel is not None: log.info(f'Creating new SLACK message for {title} in channel', channel) else: if recipient_email is not None: channel = userid log.info( f'Creating new SLACK message for {title} for user {recipient_email}' ) else: log.error(f'Cannot identify assignee email') return None blocks = None attachments = None text = title if template is not None: properties = {'channel': channel, 'message': message} # create Slack message structure in Snowflake javascript UDF try: payload = message_template(locals()) except Exception: return None if payload is not None: if 'blocks' in payload: blocks = json.dumps(payload['blocks']) if 'attachments' in payload: attachments = json.dumps(payload['attachments']) if 'text' in payload: text = payload['text'] else: log.error(f'Payload is empty for template {template}') return None else: # does not have template, will send just simple message if message is not None: text = message response = sc.api_call("chat.postMessage", channel=channel, text=text, blocks=blocks, attachments=attachments) log.debug(f'Slack response', response) if response['ok'] is False: log.error(f"Slack handler error", response['error']) return None if 'message' in response: del response['message'] return response
def handle( alert, type='smtp', sender_email=None, recipient_email=None, text=None, html=None, subject=None, reply_to=None, cc=None, bcc=None, host=HOST, port=PORT, user=USER, password=PASSWORD, use_ssl=USE_SSL, use_tls=USE_TLS, ): user = vault.decrypt_if_encrypted(user) password = vault.decrypt_if_encrypted(password) sender_email = sender_email or user if recipient_email is None: log.error(f"param 'recipient_email' required") return None if text is None: log.error(f"param 'text' required") return None # Create the base MIME message. if html is None: message = MIMEMultipart() else: message = MIMEMultipart('alternative') # Add HTML/plain-text parts to MIMEMultipart message # The email client will try to render the last part first # Turn these into plain/html MIMEText objects textPart = MIMEText(text, 'plain') message.attach(textPart) if html is not None: htmlPart = MIMEText(html, 'html') message.attach(htmlPart) message['Subject'] = subject message['From'] = sender_email message['To'] = recipient_email recipients = recipient_email.split(',') if cc is not None: message['Cc'] = cc recipients = recipients + cc.split(',') if bcc is not None: recipients = recipients + bcc.split(',') if reply_to is not None: message.add_header('reply-to', reply_to) if use_ssl is True: context = ssl.create_default_context() if use_tls is True: smtpserver = smtplib.SMTP(host, port) smtpserver.starttls(context=context) else: smtpserver = smtplib.SMTP_SSL(host, port, context=context) else: smtpserver = smtplib.SMTP(host, port) if user and password: smtpserver.login(user, password) result = smtpserver.sendmail(sender_email, recipients, message.as_string()) smtpserver.close() return result
Sources: {SOURCES} Actor: {ACTOR} Object: {OBJECT} Action: {ACTION} Title: {TITLE} Event Time: {EVENT_TIME} Alert Time: {ALERT_TIME} Description: {{quote}} {DESCRIPTION} {{quote}} Detector: {DETECTOR} Event Data: {{code}}{EVENT_DATA}{{code}} Severity: {SEVERITY} """ password = vault.decrypt_if_encrypted(environ.get('JIRA_PASSWORD')) user = environ.get('JIRA_USER') if user and password: jira = JIRA(URL, basic_auth=(user, password)) def jira_ticket_body(alert): alert['SOURCES'] = ', '.join(alert['SOURCES']) escaped_locals_strings = {k: escape_jira_strings(v) for k, v in alert.items()} sources = escaped_locals_strings['SOURCES'] escaped_locals_strings[ 'SOURCES' ] = f'[{sources}|{link_search_todos(f"Sources: {sources}")}]' jira_body = {**JIRA_TICKET_BODY_DEFAULTS, **escaped_locals_strings} ticket_body = JIRA_TICKET_BODY_FMT.format(**jira_body)
def handle( alert, type='smtp', sender_email=None, recipient_email=None, text=None, html=None, subject=None, reply_to=None, cc=None, bcc=None, ): if not os.environ.get('SMTP_SERVER'): log.info("No SMTP_SERVER in env, skipping handler.") return None smtp_server = os.environ['SMTP_SERVER'] if 'SMTP_PORT' in os.environ: smtp_port = os.environ['SMTP_PORT'] else: smtp_port = 587 if 'SMTP_USE_SSL' in os.environ: smtp_use_ssl = os.environ['SMTP_USE_SSL'] else: smtp_use_ssl = True if 'SMTP_USE_TLS' in os.environ: smtp_use_tls = os.environ['SMTP_USE_TLS'] else: smtp_use_tls = True smtp_user = vault.decrypt_if_encrypted(os.environ['SMTP_USER']) smtp_password = vault.decrypt_if_encrypted(os.environ['SMTP_PASSWORD']) if recipient_email is None: log.error(f"Cannot identify recipient email") return None if text is None: log.error(f"SES Message is empty") return None # Create the base MIME message. if html is None: message = MIMEMultipart() else: message = MIMEMultipart('alternative') # Add HTML/plain-text parts to MIMEMultipart message # The email client will try to render the last part first # Turn these into plain/html MIMEText objects textPart = MIMEText(text, 'plain') message.attach(textPart) if html is not None: htmlPart = MIMEText(html, 'html') message.attach(htmlPart) message['Subject'] = subject message['From'] = sender_email message['To'] = recipient_email recipients = recipient_email.split(',') if cc is not None: message['Cc'] = cc recipients = recipients + cc.split(',') if bcc is not None: recipients = recipients + bcc.split(',') if reply_to is not None: message.add_header('reply-to', reply_to) if smtp_use_ssl is True: context = ssl.create_default_context() if smtp_use_tls is True: smtpserver = smtplib.SMTP(smtp_server, smtp_port) smtpserver.starttls(context=context) else: smtpserver = smtplib.SMTP_SSL(smtp_server, smtp_port, context=context) else: smtpserver = smtplib.SMTP(smtp_server, smtp_port) smtpserver.login(smtp_user, smtp_password) smtpserver.sendmail(sender_email, recipients, message.as_string()) smtpserver.close()
def handle( alert, recipient_email=None, channel=None, template=None, message=None, file_content=None, file_type=None, file_name=None, blocks=None, attachments=None, api_token=API_TOKEN, slack_api_token=None, ): slack_token_ct = slack_api_token or api_token slack_token = vault.decrypt_if_encrypted(slack_token_ct) sc = SlackClient(slack_token) # otherwise we will retrieve email from assignee and use it to identify Slack user # Slack user id will be assigned as a channel title = alert['TITLE'] if recipient_email is not None: result = sc.api_call("users.lookupByEmail", email=recipient_email) # log.info(f'Slack user info for {email}', result) if result['ok'] is True and 'error' not in result: user = result['user'] userid = user['id'] else: log.error( f'Cannot identify Slack user for email {recipient_email}') return None # check if channel exists, if yes notification will be delivered to the channel if channel is not None: log.info(f'Creating new SLACK message for {title} in channel', channel) else: if recipient_email is not None: channel = userid log.info( f'Creating new SLACK message for {title} for user {recipient_email}' ) else: log.error(f'Cannot identify assignee email') return None text = title if template is not None: properties = {'channel': channel, 'message': message} # create Slack message structure in Snowflake javascript UDF payload = message_template(locals()) if payload is not None: if 'blocks' in payload: blocks = json.dumps(payload['blocks']) if 'attachments' in payload: attachments = json.dumps(payload['attachments']) if 'text' in payload: text = payload['text'] else: log.error(f'Payload is empty for template {template}') return None else: # does not have template, will send just simple message if message is not None: text = message response = None if file_content is not None: if template is not None: response = sc.api_call( "chat.postMessage", channel=channel, text=text, blocks=blocks, attachments=attachments, ) file_descriptor = sc.api_call( "files.upload", content=file_content, title=text, channels=channel, iletype=file_type, filename=file_name, ) if file_descriptor['ok'] is True: file = file_descriptor["file"] file_url = file["url_private"] else: log.error(f"Slack file upload error", file_descriptor['error']) else: response = sc.api_call( "chat.postMessage", channel=channel, text=text, blocks=blocks, attachments=attachments, ) if response is not None: log.debug(f'Slack response', response) if response['ok'] is False: log.error(f"Slack handler error", response['error']) return None if 'message' in response: del response['message'] return response
Actor: {ACTOR} Object: {OBJECT} Action: {ACTION} Title: {TITLE} Event Time: {EVENT_TIME} Alert Time: {ALERT_TIME} Description: {{quote}} {DESCRIPTION} {{quote}} Detector: {DETECTOR} Event Data: {{code}}{EVENT_DATA}{{code}} Severity: {SEVERITY} """ password = vault.decrypt_if_encrypted( environ.get('SA_JIRA_API_TOKEN', environ.get('JIRA_API_TOKEN')) or environ.get('SA_JIRA_PASSWORD', environ.get('JIRA_PASSWORD'))) user = environ.get('SA_JIRA_USER', environ.get('JIRA_USER')) jira_server = URL if URL.startswith('https://') else f'https://{URL}' if user and password: jira = JIRA(jira_server, basic_auth=(user, password)) def jira_ticket_body(alert, project): sources = alert['SOURCES'] alert['SOURCES'] = ', '.join(sources) if isinstance(sources, list) else sources escaped_locals_strings = { k: escape_jira_strings(v)
def handle(alert, assignee='', payload={}): host = env.get('SA_SN_API_HOST') if not host: log.info('skipping service-now handler, missing host') return username = vault.decrypt_if_encrypted(envar='SA_SN_API_USER') password = vault.decrypt_if_encrypted(envar='SA_SN_API_PASS') client_id = env.get('SA_SN_OAUTH_CLIENT_ID') client_secret = vault.decrypt_if_encrypted( envar='SA_SN_OAUTH_CLIENT_SECRET') refresh_token = vault.decrypt_if_encrypted( envar='SA_SN_OAUTH_REFRESH_TOKEN') if client_id: oauth_return_params = { 'grant_type': 'refresh_token', 'client_id': client_id, 'client_secret': client_secret, 'refresh_token': refresh_token, } oauthresp = requests.post( f'https://{host}/oauth_token.do', data=oauth_return_params, ) result = oauthresp.json() access_token = result.get('access_token') if not access_token: log.info('skipping service-now handler, bad oauth') raise RuntimeError(result) else: access_token = None if not (username and password) and not access_token: log.info('skipping service-now handler, no authorization') return title = alert.get('TITLE', 'SnowAlert Generate Incident') description = alert.get('DESCRIPTION', '') endpoint = env.get('SA_SN_API_ENDPOINT', '/now/table/incident') api_url = f'https://{host}/api{endpoint}' fp = env.get('SA_SN_FIELD_PREFIX', '') response = requests.post( api_url, auth=Bearer(access_token) if access_token else (username, password), json=payload or { f'{fp}contact_type': 'Integration', f'{fp}impact': '2', f'{fp}urgency': '2', f'{fp}category': 'IT Security', f'{fp}subcategory': 'Remediation', f'{fp}assignment_group': 'Security Compliance', f'{fp}short_description': title, f'{fp}description': description, f'{fp}assigned_to': assignee, }, ) if response.status_code != 201: log.info( f'URL: {api_url}', f'Status Code: {response.status_code}', f'Response Length: {len(response.text)}', f'Response Headers: {response.headers}', ) raise RuntimeError(response) return response