def create_alerts(ctx, rule_name: str) -> Tuple[int, int]:
metadata: Dict[str, Any] = {
'QUERY_NAME': rule_name,
'RUN_ID': RUN_ID,
'ATTEMPTS': 1,
'START_TIME': datetime.datetime.utcnow(),
}
try:
insert_count, update_count = db.insert_alerts_query_run(
rule_name, GROUPING_CUTOFF)
metadata['ROW_COUNT'] = {
'INSERTED': insert_count,
'UPDATED': update_count,
}
except Exception as e:
log_failure(ctx, rule_name, e)
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e)
return 0, 0
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
QUERY_HISTORY.append(metadata)
log.info(f"{rule_name} done.")
return insert_count, update_count
def main(rule_name=None):
RUN_METADATA = {
'RUN_ID': RUN_ID,
'RUN_TYPE': 'ALERT QUERY',
'START_TIME': datetime.datetime.utcnow(),
}
ctx = db.connect_and_execute("ALTER SESSION SET USE_CACHED_RESULT=FALSE;")
if rule_name:
create_alerts(ctx, rule_name)
else:
for rule_name in db.load_rules(ctx, ALERT_QUERY_POSTFIX):
create_alerts(ctx, rule_name)
RUN_METADATA['ROW_COUNT'] = {
'INSERTED': sum(q['ROW_COUNT']['INSERTED'] for q in QUERY_HISTORY),
'UPDATED': sum(q['ROW_COUNT']['UPDATED'] for q in QUERY_HISTORY),
}
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
try:
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{
'Name': 'Component',
'Value': 'Alert Query Runner'
}], 1)
except Exception as e:
log.error("Cloudwatch metric logging failed: ", e)
def run_suppression(squelch_name):
metadata = {
'QUERY_NAME': squelch_name,
'RUN_ID': RUN_ID,
'ATTEMPTS': 1,
'START_TIME': datetime.datetime.utcnow(),
'ROW_COUNT': {
'SUPPRESSED': 0
}
}
ctx = db.connect()
log.info(f"{squelch_name} processing...")
try:
query = VIOLATION_SUPPRESSION_QUERY.format(squelch_name=squelch_name)
num_violations_suppressed = next(db.fetch(query))['number of rows updated']
log.info(f"{squelch_name} updated {num_violations_suppressed} rows.")
metadata['ROW_COUNT']['SUPPRESSED'] = num_violations_suppressed
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
RULE_METADATA_RECORDS.append(metadata)
except Exception as e:
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e)
log.error("Suppression query {squelch_name} execution failed.", e)
print(f"Suppression query {squelch_name} executed")
def main():
RUN_METADATA = {
'RUN_TYPE': 'ALERT SUPPRESSION',
'START_TIME': datetime.datetime.utcnow(),
'RUN_ID': RUN_ID,
}
ctx = db.connect()
for squelch_name in db.load_rules(ctx, ALERT_SQUELCH_POSTFIX):
run_suppressions(squelch_name)
num_rows_updated = next(db.fetch(
ctx, SET_SUPPRESSED_FALSE))['number of rows updated']
log.info(
f'All suppressions done, {num_rows_updated} remaining alerts marked suppressed=FALSE.'
)
RUN_METADATA['ROW_COUNT'] = {
'PASSED': num_rows_updated,
'SUPPRESSED':
sum(m['ROW_COUNT']['SUPPRESSED'] for m in METADATA_HISTORY),
}
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
try:
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{
'Name': 'Component',
'Value': 'Alert Suppression Runner'
}], 1)
except Exception as e:
log.error("Cloudwatch metric logging failed: ", e)
def run_query(query_name):
metadata = {}
metadata['QUERY_NAME'] = query_name
metadata['RUN_ID'] = RUN_ID
metadata['ATTEMPTS'] = 1
metadata['START_TIME'] = datetime.datetime.utcnow()
ctx = db.insert_violations_query_run(query_name)
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
def main():
RUN_METADATA = {}
RUN_METADATA['RUN_TYPE'] = 'ALERT SUPPRESSION'
RUN_METADATA['START_TIME'] = datetime.datetime.utcnow()
RUN_METADATA['RUN_ID'] = RUN_ID
ctx = db.connect()
for squelch_name in db.load_rules(ctx, ALERT_SQUELCH_POSTFIX):
run_suppressions(squelch_name)
flag_remaining_alerts(ctx)
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{'Name': 'Component', 'Value': 'Alert Suppression Runner'}], 1)
def main():
# Force warehouse resume so query runner doesn't have a bunch of queries waiting for warehouse resume
RUN_METADATA = {}
RUN_METADATA['RUN_TYPE'] = 'VIOLATION QUERY'
RUN_METADATA['START_TIME'] = datetime.datetime.utcnow()
RUN_METADATA['RUN_ID'] = RUN_ID
ctx = db.connect_and_execute("ALTER SESSION SET use_cached_result=FALSE;")
for query_name in db.load_rules(ctx, VIOLATION_QUERY_POSTFIX):
run_query(query_name)
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{
'Name': 'Component',
'Value': 'Violation Query Runner'
}], 1)
def run_suppressions(squelch_name):
print(f"Received suppression {squelch_name}")
metadata = {}
metadata['QUERY_NAME'] = squelch_name
metadata['RUN_ID'] = RUN_ID
metadata['ATTEMPTS'] = 1
metadata['START_TIME'] = datetime.datetime.utcnow()
ctx = db.connect()
try:
do_suppression(squelch_name, ctx)
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
except Exception as e:
log_failure(ctx, squelch_name, e)
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e)
pass
print(f"Suppression query {squelch_name} executed. ")
def main():
RUN_METADATA = {
'RUN_TYPE': 'VIOLATION SUPPRESSION',
'START_TIME': datetime.datetime.utcnow(),
'RUN_ID': RUN_ID,
}
ctx = db.connect()
for squelch_name in db.load_rules(ctx, VIOLATION_SQUELCH_POSTFIX):
run_suppression(squelch_name)
num_violations_passed = next(db.fetch(SET_SUPPRESSED_FALSE))['number of rows updated']
RUN_METADATA['ROW_COUNT'] = {
'SUPPRESSED': sum(rmr['ROW_COUNT']['SUPPRESSED'] for rmr in RULE_METADATA_RECORDS),
'PASSED': num_violations_passed,
}
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{'Name': 'Component', 'Value': 'Violation Suppression Runner'}], 1)
def main(rule_name=None):
# Force warehouse resume so query runner doesn't have a bunch of queries waiting for warehouse resume
RUN_METADATA = {}
RUN_METADATA['RUN_TYPE'] = 'ALERT QUERY'
RUN_METADATA['START_TIME'] = datetime.datetime.utcnow()
RUN_METADATA['RUN_ID'] = RUN_ID
ctx = db.connect_and_execute("ALTER SESSION SET USE_CACHED_RESULT=FALSE;")
if rule_name:
query_for_alerts(ctx, rule_name)
else:
for query_name in db.load_rules(ctx, ALERT_QUERY_POSTFIX):
query_for_alerts(ctx, query_name)
log.metadata_record(ctx, RUN_METADATA, table=RUN_METADATA_TABLE)
if CLOUDWATCH_METRICS:
log.metric('Run', 'SnowAlert', [{
'Name': 'Component',
'Value': 'Alert Query Runner'
}], 1)
def run_suppression(squelch_name):
metadata = {}
metadata['QUERY_NAME'] = squelch_name
metadata['RUN_ID'] = RUN_ID
metadata['ATTEMPTS'] = 1
metadata['START_TIME'] = datetime.datetime.utcnow()
ctx = db.connect()
print(f"Received suppression {squelch_name}")
try:
ctx.cursor().execute(f"""
MERGE INTO {VIOLATIONS_TABLE} t
USING(SELECT result:EVENT_HASH AS event_hash FROM {RULES_SCHEMA}.{squelch_name}) s
ON t.result:EVENT_HASH=s.event_hash
WHEN MATCHED THEN UPDATE
SET t.suppressed='true', t.suppression_rule='{squelch_name}';
""")
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
except Exception as e:
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e)
log.error("Suppression query {squelch_name} execution failed.", e)
pass
print(f"Suppression query {squelch_name} executed")
def snowalert_query(ctx, query_name: str):
log.info(f"{query_name} processing...")
metadata: Dict[str, Any] = {}
metadata['QUERY_NAME'] = query_name
metadata['RUN_ID'] = RUN_ID
metadata['ATTEMPTS'] = 1
metadata['START_TIME'] = datetime.datetime.utcnow()
attempt = 0
results: List[Tuple[str]] = []
while attempt <= 1:
try:
attempt += 1
query = f'''
SELECT OBJECT_CONSTRUCT(*) FROM {RULES_SCHEMA}.{query_name}
WHERE event_time > {GROUPING_CUTOFF}
'''
results = ctx.cursor().execute(query).fetchall()
except Exception as e:
if attempt > 1:
log_failure(ctx, query_name, e)
log.metadata_record(ctx,
metadata,
table=QUERY_METADATA_TABLE,
e=e)
return results
else:
log.info(f"Query {query_name} failed to run, retrying...")
metadata['ATTEMPTS'] += 1
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
QUERY_HISTORY.append(metadata)
log.info(f"{query_name} done.")
return results
def run_suppressions(squelch_name):
log.info(f"{squelch_name} processing...")
metadata = {
'QUERY_NAME': squelch_name,
'RUN_ID': RUN_ID,
'ATTEMPTS': 1,
'START_TIME': datetime.datetime.utcnow(),
}
ctx = db.connect()
try:
suppression_count = run_suppression_query(squelch_name)
log.info(f"{squelch_name} updated {suppression_count} rows.")
metadata['ROW_COUNT'] = {'SUPPRESSED': suppression_count}
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE)
except Exception as e:
log_failure(ctx, squelch_name, e)
metadata['ROW_COUNT'] = {'SUPPRESSED': 0}
log.metadata_record(ctx, metadata, table=QUERY_METADATA_TABLE, e=e)
METADATA_HISTORY.append(metadata)
log.info(f"{squelch_name} done.")