def get_eligible_alerts(listing_events: list, conditions: list):
"""
This function creates the same rules used to store the conditions bloom filters using the listing events,
and finds all alerts that a given listing event attend.
:param listing_events: A list of listing event structure.
:return: A list of tuples containing the ID of the listing event and what alert_ids are eligible to be triggered
"""
db = get_db_instance()
result = []
is_valid_key = is_valid_key_builder(conditions)
bfs = _get_all_bf_alerts(db)
i = 0
for listing_event in listing_events:
listing_rules = []
for key, value in listing_event.items():
if not is_valid_key(key):
continue
rules = _build_rules(key, value)
for rule in rules:
listing_rules.append(rule)
print(f"handling listing event number {i}")
eligible_alerts = _get_alerts_that_apply_to_all_listing_rules(listing_rules, bfs)
result.append((listing_event["id"], eligible_alerts))
i = i + 1
filtered_results = [item for item in result if len(item[1]) > 0]
print(f"Eligible alerts found {filtered_results}")
return filtered_results
def create_alerts_bloom_filters(n=41, p=0.05):
"""
This method creates a bloom filter for each alert saved on the database.
The bloom filters contain the conditions each alert is related to.
:param n: number of items expected to be stored in filter for Bloom filters
:param p: False Positive probability in decimal for Bloom Filters
:return None
"""
db = get_db_instance()
condition_types = ["boolean", "integer"]
alerts = [
result for condition_type in condition_types
for result in db.fetch_data(
_alert_condition_query_generator(condition_type))
]
sorted_alerts = sorted(alerts, key=lambda x: x.alert_id)
prev_alert_id = None
bf = BloomFilter(n, p)
for alert in sorted_alerts:
add_bloom_filter_rule(bf, alert.name, alert.value)
if prev_alert_id != alert.alert_id and prev_alert_id is not None:
save_bloom_filter_rule(alert.alert_id, bf)
bf = BloomFilter(n, p)
prev_alert_id = alert.alert_id
def setup(num_of_alerts, conditions):
"""
This function will create the database snapshot for this POC
"""
db = get_db_instance()
_drop_tables(db)
_create_tables(db)
_insert_alerts_and_conditions(db, num_of_alerts, conditions)
def main():
db = get_db_instance()
try:
should_save = True
num_of_conditions = 5
num_of_condition_types = 3
num_of_alerts = 300000
number_of_listing_events = 21
db.connect()
conditions = [
generate_conditions(i, num_of_conditions)
for i in range(num_of_condition_types)
]
setup(num_of_alerts, conditions)
print("CREATING BLOOM FILTERS")
create_alerts_bloom_filters()
listing_events = generate_listing_events(number_of_listing_events,
conditions)
t_start = datetime.now()
print("starting to find matching alerts")
get_eligible_alerts(listing_events, conditions)
print("finished finding matching alerts")
t_end = datetime.now()
time_elapsed = t_end - t_start
if should_save:
row = [
num_of_alerts, num_of_conditions, number_of_listing_events,
time_elapsed
]
save_results(row)
except (Exception, psycopg2.DatabaseError) as error:
print(error)
finally:
if db.is_connected():
db.close_connection()
print('Database connection closed.')
from bloomFilter.create_filter_rule import create_filter_rule
from database import get_db_instance
db = get_db_instance()
def add_bloom_filter_rule(bf, key, value):
rule = create_filter_rule(key, value)
bf.add(rule)
return bf
def save_bloom_filter_rule(alert_id, bf):
insert_or_update_bloom_filter(db, alert_id, bf.bit_array.to01())
def insert_or_update_bloom_filter(db, alert_id, filter):
q = f"""
INSERT INTO bloomFiltersConditions VALUES ({alert_id}, '{filter}')
ON CONFLICT (alert_id) DO UPDATE SET filter = '{filter}'
"""
db.exec_commit_query(q)