def main(rule_name=None): RUN_METADATA = { 'RUN_ID': RUN_ID, 'RUN_TYPE': 'ALERT QUERY', 'START_TIME': datetime.datetime.utcnow(), } if rule_name: metadata = [create_alerts(rule_name)] else: rules = list(db.load_rules(ALERT_QUERY_POSTFIX)) metadata = Pool(POOLSIZE).map(create_alerts, rules) RUN_METADATA['ROW_COUNT'] = { 'INSERTED': sum(q['ROW_COUNT']['INSERTED'] for q in metadata), 'UPDATED': sum(q['ROW_COUNT']['UPDATED'] for q in metadata), } db.record_metadata(RUN_METADATA, table=RUN_METADATA_TABLE) try: if CLOUDWATCH_METRICS: log.metric( 'Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Alert Query Runner' }], 1, ) except Exception as e: log.error("Cloudwatch metric logging failed: ", e)
def run_suppression(squelch_name): metadata = { 'QUERY_NAME': squelch_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), 'ROW_COUNT': { 'SUPPRESSED': 0 } } log.info(f"{squelch_name} processing...") try: query = VIOLATION_SUPPRESSION_QUERY.format(squelch_name=squelch_name) num_violations_suppressed = next( db.fetch(query))['number of rows updated'] log.info(f"{squelch_name} updated {num_violations_suppressed} rows.") metadata['ROW_COUNT']['SUPPRESSED'] = num_violations_suppressed db.record_metadata(metadata, table=QUERY_METADATA_TABLE) RULE_METADATA_RECORDS.append(metadata) except Exception as e: db.record_metadata(metadata, table=QUERY_METADATA_TABLE, e=e) log.error("Suppression query {squelch_name} execution failed.", e) print(f"Suppression query {squelch_name} executed")
def main(): RUN_METADATA = { 'RUN_TYPE': 'VIOLATION SUPPRESSION', 'START_TIME': datetime.datetime.utcnow(), 'RUN_ID': RUN_ID, } for squelch_name in db.load_rules(VIOLATION_SQUELCH_POSTFIX): run_suppression(squelch_name) num_violations_passed = next( db.fetch(SET_SUPPRESSED_FALSE))['number of rows updated'] RUN_METADATA['ROW_COUNT'] = { 'SUPPRESSED': sum(rmr['ROW_COUNT']['SUPPRESSED'] for rmr in RULE_METADATA_RECORDS), 'PASSED': num_violations_passed, } db.record_metadata(RUN_METADATA, table=RUN_METADATA_TABLE) if CLOUDWATCH_METRICS: log.metric('Run', 'SnowAlert', [{ 'Name': 'Component', 'Value': 'Violation Suppression Runner' }], 1)
def create_alerts(rule_name: str) -> Dict[str, Any]: metadata: Dict[str, Any] = { 'QUERY_NAME': rule_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), 'ROW_COUNT': { 'INSERTED': 0, 'UPDATED': 0, } } try: db.execute(RUN_ALERT_QUERY.format( query_name=rule_name, from_time_sql="DATEADD(minute, -90, CURRENT_TIMESTAMP())", to_time_sql="CURRENT_TIMESTAMP()", ), fix_errors=False) insert_count, update_count = merge_alerts(rule_name, GROUPING_CUTOFF) metadata['ROW_COUNT'] = { 'INSERTED': insert_count, 'UPDATED': update_count, } db.execute(f"DROP TABLE results.RUN_{RUN_ID}_{rule_name}") except Exception as e: db.record_metadata(metadata, table=QUERY_METADATA_TABLE, e=e) return metadata db.record_metadata(metadata, table=QUERY_METADATA_TABLE) log.info(f"{rule_name} done.") return metadata
def main(squelch_name=None): RUN_METADATA = { 'RUN_TYPE': 'ALERT SUPPRESSION', 'START_TIME': datetime.datetime.utcnow(), 'RUN_ID': RUN_ID, } rules = ( db.load_rules(ALERT_SQUELCH_POSTFIX) if squelch_name is None else [squelch_name] ) for squelch_name in rules: run_suppressions(squelch_name) num_rows_updated = next(db.fetch(SET_SUPPRESSED_FALSE))['number of rows updated'] log.info( f'All suppressions done, {num_rows_updated} remaining alerts marked suppressed=FALSE.' ) RUN_METADATA['ROW_COUNT'] = { 'PASSED': num_rows_updated, 'SUPPRESSED': sum(m['ROW_COUNT']['SUPPRESSED'] for m in METADATA_HISTORY), } db.record_metadata(RUN_METADATA, table=RUN_METADATA_TABLE) try: if CLOUDWATCH_METRICS: log.metric( 'Run', 'SnowAlert', [{'Name': 'Component', 'Value': 'Alert Suppression Runner'}], 1, ) except Exception as e: log.error("Cloudwatch metric logging failed: ", e)
def create_alerts(rule_name: str) -> Dict[str, Any]: metadata: Dict[str, Any] = { 'QUERY_NAME': rule_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), 'ROW_COUNT': { 'INSERTED': 0, 'UPDATED': 0, } } try: insert_count, update_count = db.insert_alerts_query_run(rule_name, GROUPING_CUTOFF) metadata['ROW_COUNT'] = { 'INSERTED': insert_count, 'UPDATED': update_count, } except Exception as e: log_failure(rule_name, e) db.record_metadata(metadata, table=QUERY_METADATA_TABLE, e=e) return metadata db.record_metadata(metadata, table=QUERY_METADATA_TABLE) log.info(f"{rule_name} done.") return metadata
def connection_run(connection_table): table_name = connection_table['name'] table_comment = connection_table['comment'] log.info(f"-- START DC {table_name} --") try: metadata = {'START_TIME': datetime.utcnow()} options = yaml.load(table_comment) or {} if 'module' in options: module = options['module'] metadata.update({ 'RUN_ID': RUN_ID, 'TYPE': module, 'LANDING_TABLE': table_name, 'INGEST_COUNT': 0, }) connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted(options[name]) if module_option.get('type') == 'json': options[name] = json.loads(options[name]) if module_option.get('type') == 'list': if type(options[name]) is str: options[name] = options[name].split(',') if module_option.get('type') == 'int': options[name] = int(options[name]) if callable(getattr(connector, 'ingest', None)): ingested = connector.ingest(table_name, options) if isinstance(ingested, int): metadata['INGEST_COUNT'] += ingested elif isinstance(ingested, GeneratorType): for n in ingested: metadata['INGEST_COUNT'] += n else: metadata['INGESTED'] = ingested db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")
def main(connection_table="%_CONNECTION"): for table in db.fetch(f"SHOW TABLES LIKE '{connection_table}' IN data"): table_name = table['name'] table_comment = table['comment'] log.info(f"-- START DC {table_name} --") try: options = yaml.load(table_comment) or {} if 'module' in options: module = options['module'] metadata = { 'RUN_ID': RUN_ID, 'TYPE': module, 'START_TIME': datetime.utcnow(), 'LANDING_TABLE': table_name, 'INGEST_COUNT': 0 } connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted( options[name]) if callable(getattr(connector, 'ingest', None)): ingested = connector.ingest(table_name, options) if isinstance(ingested, int): metadata['INGEST_COUNT'] += ingested elif isinstance(ingested, GeneratorType): for n in ingested: metadata['INGEST_COUNT'] += n else: metadata['INGESTED'] = ingested db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")
def run_suppressions(squelch_name): log.info(f"{squelch_name} processing...") metadata = { 'QUERY_NAME': squelch_name, 'RUN_ID': RUN_ID, 'ATTEMPTS': 1, 'START_TIME': datetime.datetime.utcnow(), } try: suppression_count = run_suppression_query(squelch_name) log.info(f"{squelch_name} updated {suppression_count} rows.") metadata['ROW_COUNT'] = {'SUPPRESSED': suppression_count} db.record_metadata(metadata, table=QUERY_METADATA_TABLE) except Exception as e: metadata['ROW_COUNT'] = {'SUPPRESSED': 0} db.record_metadata(metadata, table=QUERY_METADATA_TABLE, e=e) METADATA_HISTORY.append(metadata) log.info(f"{squelch_name} done.")
def connection_run(connection_table, run_now=False): table_name = connection_table['name'] table_comment = connection_table['comment'] log.info(f"-- START DC {table_name} --") try: metadata = {'START_TIME': datetime.utcnow()} options = yaml.safe_load(table_comment) or {} if 'schedule' in options: schedule = options['schedule'] now = datetime.now() if not run_now and not time_to_run(schedule, now): log.info(f'not scheduled: {schedule} at {now}') log.info(f"-- END DC --") return if 'module' not in options: log.info(f'no module in options') log.info(f"-- END DC --") return module = options['module'] metadata.update({ 'RUN_ID': RUN_ID, 'TYPE': module, 'LANDING_TABLE': table_name }) connector = importlib.import_module(f"connectors.{module}") for module_option in connector.CONNECTION_OPTIONS: name = module_option['name'] if module_option.get('secret') and name in options: options[name] = vault.decrypt_if_encrypted(options[name]) if module_option.get('type') == 'json': options[name] = json.loads(options[name]) if module_option.get('type') == 'list': if type(options[name]) is str: options[name] = options[name].split(',') if module_option.get('type') == 'int': options[name] = int(options[name]) if callable(getattr(connector, 'ingest', None)): db.record_metadata(metadata, table=DC_METADATA_TABLE) result = do_ingest(connector, table_name, options) if result is not None: metadata['INGEST_COUNT'] = result else: metadata['INGESTED'] = result db.record_metadata(metadata, table=DC_METADATA_TABLE) except Exception as e: log.error(f"Error loading logs into {table_name}: ", e) db.record_metadata(metadata, table=DC_METADATA_TABLE, e=e) log.info(f"-- END DC --")