def create_alerts(ctx, rule_name: str) -> Tuple[int, int]: metadata: Dict[str, Any] = { 'QUERY_NAME': rule_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), } try: insert_count, update_count = db.insert_alerts_query_run( rule_name, GROUPING_CUTOFF) metadata['ROW_COUNT'] = { 'INSERTED': insert_count, 'UPDATED': update_count, } except Exception as e: log_failure(ctx, rule_name, e) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) return 0, 0 log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) QUERY_HISTORY.append(metadata) log.info(f"{rule_name} done.") return insert_count, update_count
def main(rule_name=None): RUN_METADATA = { 'RUN_ID': RUN_ID, 'RUN_TYPE': 'ALERT QUERY', 'START_TIME': datetime.datetime.utcnow(), } ctx = db.connect_and_execute("ALTER SESSION SET USE_CACHED_RESULT=FALSE;") if rule_name: create_alerts(ctx, rule_name) else: for rule_name in db.load_rules(ctx, ALERT_QUERY_POSTFIX): create_alerts(ctx, rule_name) RUN_METADATA['ROW_COUNT'] = { 'INSERTED': sum(q['ROW_COUNT']['INSERTED'] for q in QUERY_HISTORY), 'UPDATED': sum(q['ROW_COUNT']['UPDATED'] for q in QUERY_HISTORY), } log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) try: if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Alert Query Runner' }], 1) except Exception as e: log.error("Cloudwatch metric logging failed: ", e)
def run_suppression(squelch_name): metadata = { 'QUERY_NAME': squelch_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), 'ROW_COUNT': { 'SUPPRESSED': 0 } } ctx = db.connect() log.info(f"{squelch_name} processing...") try: query = VIOLATION_SUPPRESSION_QUERY.format(squelch_name=squelch_name) num_violations_suppressed = next(db.fetch(query))['number of rows updated'] log.info(f"{squelch_name} updated {num_violations_suppressed} rows.") metadata['ROW_COUNT']['SUPPRESSED'] = num_violations_suppressed log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) RULE_METADATA_RECORDS.append(metadata) except Exception as e: log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) log.error("Suppression query {squelch_name} execution failed.", e) print(f"Suppression query {squelch_name} executed")
def main(): RUN_METADATA = { 'RUN_TYPE': 'ALERT SUPPRESSION', 'START_TIME': datetime.datetime.utcnow(), 'RUN_ID': RUN_ID, } ctx = db.connect() for squelch_name in db.load_rules(ctx, ALERT_SQUELCH_POSTFIX): run_suppressions(squelch_name) num_rows_updated = next(db.fetch( ctx, SET_SUPPRESSED_FALSE))['number of rows updated'] log.info( f'All suppressions done, {num_rows_updated} remaining alerts marked suppressed=FALSE.' ) RUN_METADATA['ROW_COUNT'] = { 'PASSED': num_rows_updated, 'SUPPRESSED': sum(m['ROW_COUNT']['SUPPRESSED'] for m in METADATA_HISTORY), } log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) try: if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Alert Suppression Runner' }], 1) except Exception as e: log.error("Cloudwatch metric logging failed: ", e)
def run_query(query_name): metadata = {} metadata['QUERY_NAME'] = query_name metadata['RUN_ID'] = RUN_ID metadata['ATTEMPTS'] = 1 metadata['START_TIME'] = datetime.datetime.utcnow() ctx = db.insert_violations_query_run(query_name) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
def main(): RUN_METADATA = {} RUN_METADATA['RUN_TYPE'] = 'ALERT SUPPRESSION' RUN_METADATA['START_TIME'] = datetime.datetime.utcnow() RUN_METADATA['RUN_ID'] = RUN_ID ctx = db.connect() for squelch_name in db.load_rules(ctx, ALERT_SQUELCH_POSTFIX): run_suppressions(squelch_name) flag_remaining_alerts(ctx) log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{'Name': 'Component', 'Value': 'Alert Suppression Runner'}], 1)
def main(): # Force warehouse resume so query runner doesn't have a bunch of queries waiting for warehouse resume RUN_METADATA = {} RUN_METADATA['RUN_TYPE'] = 'VIOLATION QUERY' RUN_METADATA['START_TIME'] = datetime.datetime.utcnow() RUN_METADATA['RUN_ID'] = RUN_ID ctx = db.connect_and_execute("ALTER SESSION SET use_cached_result=FALSE;") for query_name in db.load_rules(ctx, VIOLATION_QUERY_POSTFIX): run_query(query_name) log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Violation Query Runner' }], 1)
def run_suppressions(squelch_name): print(f"Received suppression {squelch_name}") metadata = {} metadata['QUERY_NAME'] = squelch_name metadata['RUN_ID'] = RUN_ID metadata['ATTEMPTS'] = 1 metadata['START_TIME'] = datetime.datetime.utcnow() ctx = db.connect() try: do_suppression(squelch_name, ctx) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) except Exception as e: log_failure(ctx, squelch_name, e) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) pass print(f"Suppression query {squelch_name} executed. ")
def main(): RUN_METADATA = { 'RUN_TYPE': 'VIOLATION SUPPRESSION', 'START_TIME': datetime.datetime.utcnow(), 'RUN_ID': RUN_ID, } ctx = db.connect() for squelch_name in db.load_rules(ctx, VIOLATION_SQUELCH_POSTFIX): run_suppression(squelch_name) num_violations_passed = next(db.fetch(SET_SUPPRESSED_FALSE))['number of rows updated'] RUN_METADATA['ROW_COUNT'] = { 'SUPPRESSED': sum(rmr['ROW_COUNT']['SUPPRESSED'] for rmr in RULE_METADATA_RECORDS), 'PASSED': num_violations_passed, } log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{'Name': 'Component', 'Value': 'Violation Suppression Runner'}], 1)
def main(rule_name=None): # Force warehouse resume so query runner doesn't have a bunch of queries waiting for warehouse resume RUN_METADATA = {} RUN_METADATA['RUN_TYPE'] = 'ALERT QUERY' RUN_METADATA['START_TIME'] = datetime.datetime.utcnow() RUN_METADATA['RUN_ID'] = RUN_ID ctx = db.connect_and_execute("ALTER SESSION SET USE_CACHED_RESULT=FALSE;") if rule_name: query_for_alerts(ctx, rule_name) else: for query_name in db.load_rules(ctx, ALERT_QUERY_POSTFIX): query_for_alerts(ctx, query_name) log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE) if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Alert Query Runner' }], 1)
def run_suppression(squelch_name): metadata = {} metadata['QUERY_NAME'] = squelch_name metadata['RUN_ID'] = RUN_ID metadata['ATTEMPTS'] = 1 metadata['START_TIME'] = datetime.datetime.utcnow() ctx = db.connect() print(f"Received suppression {squelch_name}") try: ctx.cursor().execute(f""" MERGE INTO {VIOLATIONS_TABLE} t USING(SELECT result:EVENT_HASH AS event_hash FROM {RULES_SCHEMA}.{squelch_name}) s ON t.result:EVENT_HASH=s.event_hash WHEN MATCHED THEN UPDATE SET t.suppressed='true', t.suppression_rule='{squelch_name}'; """) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) except Exception as e: log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) log.error("Suppression query {squelch_name} execution failed.", e) pass print(f"Suppression query {squelch_name} executed")
def snowalert_query(ctx, query_name: str): log.info(f"{query_name} processing...") metadata: Dict[str, Any] = {} metadata['QUERY_NAME'] = query_name metadata['RUN_ID'] = RUN_ID metadata['ATTEMPTS'] = 1 metadata['START_TIME'] = datetime.datetime.utcnow() attempt = 0 results: List[Tuple[str]] = [] while attempt <= 1: try: attempt += 1 query = f''' SELECT OBJECT_CONSTRUCT(*) FROM {RULES_SCHEMA}.{query_name} WHERE event_time > {GROUPING_CUTOFF} ''' results = ctx.cursor().execute(query).fetchall() except Exception as e: if attempt > 1: log_failure(ctx, query_name, e) log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) return results else: log.info(f"Query {query_name} failed to run, retrying...") metadata['ATTEMPTS'] += 1 log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) QUERY_HISTORY.append(metadata) log.info(f"{query_name} done.") return results
def run_suppressions(squelch_name): log.info(f"{squelch_name} processing...") metadata = { 'QUERY_NAME': squelch_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), } ctx = db.connect() try: suppression_count = run_suppression_query(squelch_name) log.info(f"{squelch_name} updated {suppression_count} rows.") metadata['ROW_COUNT'] = {'SUPPRESSED': suppression_count} log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE) except Exception as e: log_failure(ctx, squelch_name, e) metadata['ROW_COUNT'] = {'SUPPRESSED': 0} log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e) METADATA_HISTORY.append(metadata) log.info(f"{squelch_name} done.")