def auto_ignore_check_rule(self, key):
rule = self.autoignore_rules.get(key, None)
if not rule:
return
if rule["interval"] <= 0:
return
thres_num_peers_seen = rule["thres_num_peers_seen"]
thres_num_ases_infected = rule["thres_num_ases_infected"]
interval = rule["interval"]
log.debug("Checking autoignore rule {}".format(rule))
try:
# fetch ongoing hijack events
query = (
"SELECT time_started, time_last, num_peers_seen, "
"num_asns_inf, key, prefix, hijack_as, type, time_detected "
"FROM hijacks WHERE active = true")
entries = self.ro_db.execute(query)
# check which of them should be auto-ignored
time_now = int(time.time())
for entry in entries:
prefix = entry[5]
best_node_match = self.find_best_prefix_node(prefix)
if not best_node_match:
continue
if best_node_match["rule_key"] != key:
continue
log.debug("Matched prefix {}".format(prefix))
time_last_updated = max(int(entry[1].timestamp()),
int(entry[8].timestamp()))
num_peers_seen = int(entry[2])
num_asns_inf = int(entry[3])
hij_key = entry[4]
hijack_as = entry[6]
hij_type = entry[7]
if ((time_now - time_last_updated > interval)
and (num_peers_seen < thres_num_peers_seen)
and (num_asns_inf < thres_num_ases_infected)):
redis_hijack_key = redis_key(prefix, hijack_as,
hij_type)
# if ongoing, clear redis
if self.redis.sismember("persistent-keys", hij_key):
purge_redis_eph_pers_keys(self.redis,
redis_hijack_key,
hij_key)
self.wo_db.execute(
"UPDATE hijacks SET active=false, dormant=false, under_mitigation=false, seen=false, ignored=true WHERE key=%s;",
(hij_key, ),
)
log.debug("Ignored hijack {}".format(entry))
except Exception:
log.exception("exception")
finally:
self.rule_timer_threads[key] = Timer(
interval=self.autoignore_rules[key]["interval"],
function=self.auto_ignore_check_rule,
args=[key],
)
self.rule_timer_threads[key].start()
log.debug("Started timer {} - {}".format(
self.rule_timer_threads[key], self.autoignore_rules[key]))
def handle_bgp_update(self, message: Dict) -> NoReturn:
"""
Callback function that runs the main logic of
detecting hijacks for every bgp update.
"""
# log.debug('{}'.format(message))
if isinstance(message, dict):
monitor_event = message
else:
monitor_event = json.loads(message.payload)
monitor_event["path"] = monitor_event["as_path"]
monitor_event["timestamp"] = datetime(
*map(int, re.findall(
r"\d+", monitor_event["timestamp"]))).timestamp()
raw = monitor_event.copy()
# mark the initial redis hijack key since it may change upon
# outdated checks
if "hij_key" in monitor_event:
monitor_event["initial_redis_hijack_key"] = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
is_hijack = False
if monitor_event["type"] == "A":
monitor_event["path"] = Detection.Worker.__clean_as_path(
monitor_event["path"])
prefix_node = self.prefix_tree.search_best(
monitor_event["prefix"])
if prefix_node:
monitor_event["matched_prefix"] = prefix_node.prefix
try:
path_hijacker = -1
pol_hijacker = -1
hij_dimensions = [
"-",
"-",
"-",
"-",
] # prefix, path, dplane, policy
hij_dimension_index = 0
for func_dim in self.__hijack_dimension_checker_gen():
if hij_dimension_index == 0:
# prefix dimension
for func_pref in func_dim():
hij_dimensions[
hij_dimension_index] = func_pref(
monitor_event, prefix_node)
if hij_dimensions[
hij_dimension_index] != "-":
break
elif hij_dimension_index == 1:
# path type dimension
for func_path in func_dim(
len(monitor_event["path"])):
(
path_hijacker,
hij_dimensions[hij_dimension_index],
) = func_path(monitor_event, prefix_node)
if hij_dimensions[
hij_dimension_index] != "-":
break
elif hij_dimension_index == 2:
# data plane dimension
for func_dplane in func_dim():
hij_dimensions[
hij_dimension_index] = func_dplane(
monitor_event, prefix_node)
if hij_dimensions[
hij_dimension_index] != "-":
break
elif hij_dimension_index == 3:
# policy dimension
for func_pol in func_dim(
len(monitor_event["path"])):
(
pol_hijacker,
hij_dimensions[hij_dimension_index],
) = func_pol(monitor_event, prefix_node)
if hij_dimensions[
hij_dimension_index] != "-":
break
hij_dimension_index += 1
# check if dimension combination in hijack combinations
# and commit hijack
if hij_dimensions in HIJACK_DIM_COMBINATIONS:
is_hijack = True
# show pol hijacker only if the path hijacker is uncertain
hijacker = path_hijacker
if path_hijacker == -1 and pol_hijacker != -1:
hijacker = pol_hijacker
self.commit_hijack(monitor_event, hijacker,
hij_dimensions)
except Exception:
log.exception("exception")
if (not is_hijack and "hij_key" in monitor_event) or (
is_hijack and "hij_key" in monitor_event
and monitor_event["initial_redis_hijack_key"] !=
monitor_event["final_redis_hijack_key"]):
redis_hijack_key = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
purge_redis_eph_pers_keys(self.redis, redis_hijack_key,
monitor_event["hij_key"])
self.mark_outdated(monitor_event["hij_key"],
redis_hijack_key)
elif not is_hijack:
self.gen_implicit_withdrawal(monitor_event)
self.mark_handled(raw)
elif monitor_event["type"] == "W":
self.producer.publish(
{
"prefix": monitor_event["prefix"],
"peer_asn": monitor_event["peer_asn"],
"timestamp": monitor_event["timestamp"],
"key": monitor_event["key"],
},
exchange=self.update_exchange,
routing_key="withdraw",
priority=0,
)
def handle_bgp_update(self, message: Dict) -> NoReturn:
"""
Callback function that runs the main logic of
detecting hijacks for every bgp update.
"""
# log.debug('{}'.format(message))
if isinstance(message, dict):
monitor_event = message
else:
message.ack()
monitor_event = message.payload
monitor_event["path"] = monitor_event["as_path"]
monitor_event["timestamp"] = datetime(
*map(int, re.findall(r"\d+", monitor_event["timestamp"]))
).timestamp()
raw = monitor_event.copy()
# mark the initial redis hijack key since it may change upon
# outdated checks
if "hij_key" in monitor_event:
monitor_event["initial_redis_hijack_key"] = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
is_hijack = False
if monitor_event["type"] == "A":
monitor_event["path"] = Detection.Worker.__clean_as_path(
monitor_event["path"]
)
ip_version = get_ip_version(monitor_event["prefix"])
if monitor_event["prefix"] in self.prefix_tree[ip_version]:
prefix_node = self.prefix_tree[ip_version][monitor_event["prefix"]]
monitor_event["matched_prefix"] = prefix_node["prefix"]
try:
path_hijacker = -1
pol_hijacker = -1
hij_dimensions = [
"-",
"-",
"-",
"-",
] # prefix, path, dplane, policy
hij_dimension_index = 0
for func_dim in self.__hijack_dimension_checker_gen():
if hij_dimension_index == 0:
# prefix dimension
for func_pref in func_dim():
hij_dimensions[hij_dimension_index] = func_pref(
monitor_event, prefix_node
)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 1:
# path type dimension
for func_path in func_dim(len(monitor_event["path"])):
(
path_hijacker,
hij_dimensions[hij_dimension_index],
) = func_path(monitor_event, prefix_node)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 2:
# data plane dimension
for func_dplane in func_dim():
hij_dimensions[hij_dimension_index] = func_dplane(
monitor_event, prefix_node
)
if hij_dimensions[hij_dimension_index] != "-":
break
elif hij_dimension_index == 3:
# policy dimension
for func_pol in func_dim(len(monitor_event["path"])):
(
pol_hijacker,
hij_dimensions[hij_dimension_index],
) = func_pol(monitor_event, prefix_node)
if hij_dimensions[hij_dimension_index] != "-":
break
hij_dimension_index += 1
# check if dimension combination in hijack combinations
# and commit hijack
if hij_dimensions in HIJACK_DIM_COMBINATIONS:
is_hijack = True
# show pol hijacker only if the path hijacker is uncertain
hijacker = path_hijacker
if path_hijacker == -1 and pol_hijacker != -1:
hijacker = pol_hijacker
self.commit_hijack(monitor_event, hijacker, hij_dimensions)
except Exception:
log.exception("exception")
outdated_hijack = None
if not is_hijack and "hij_key" in monitor_event:
try:
# outdated hijack, benign from now on
redis_hijack_key = redis_key(
monitor_event["prefix"],
monitor_event["hijack_as"],
monitor_event["hij_type"],
)
outdated_hijack = self.redis.get(redis_hijack_key)
purge_redis_eph_pers_keys(
self.redis, redis_hijack_key, monitor_event["hij_key"]
)
# mark in DB only if it is the first time this hijack was purged (pre-existent in redis)
if outdated_hijack:
self.mark_outdated(
monitor_event["hij_key"], redis_hijack_key
)
except Exception:
log.exception("exception")
elif (
is_hijack
and "hij_key" in monitor_event
and monitor_event["initial_redis_hijack_key"]
!= monitor_event["final_redis_hijack_key"]
):
try:
outdated_hijack = self.redis.get(
monitor_event["initial_redis_hijack_key"]
)
# outdated hijack, but still a hijack; need key change
purge_redis_eph_pers_keys(
self.redis,
monitor_event["initial_redis_hijack_key"],
monitor_event["hij_key"],
)
# mark in DB only if it is the first time this hijack was purged (pre-existsent in redis)
if outdated_hijack:
self.mark_outdated(
monitor_event["hij_key"],
monitor_event["initial_redis_hijack_key"],
)
except Exception:
log.exception("exception")
elif not is_hijack:
self.gen_implicit_withdrawal(monitor_event)
self.mark_handled(raw)
if outdated_hijack:
try:
outdated_hijack = json.loads(outdated_hijack)
outdated_hijack["end_tag"] = "outdated"
mail_log.info(
"{}".format(
json.dumps(
hijack_log_field_formatter(outdated_hijack),
indent=4,
)
),
extra={
"community_annotation": outdated_hijack.get(
"community_annotation", "NA"
)
},
)
hij_log.info(
"{}".format(
json.dumps(hijack_log_field_formatter(outdated_hijack))
),
extra={
"community_annotation": outdated_hijack.get(
"community_annotation", "NA"
)
},
)
except Exception:
log.exception("exception")
elif monitor_event["type"] == "W":
self.producer.publish(
{
"prefix": monitor_event["prefix"],
"peer_asn": monitor_event["peer_asn"],
"timestamp": monitor_event["timestamp"],
"key": monitor_event["key"],
},
exchange=self.update_exchange,
routing_key="withdraw",
priority=0,
serializer="ujson",
)
def handle_bgp_update(self, message: Dict) -> NoReturn:
"""
Callback function that runs the main logic of detecting hijacks for every bgp update.
"""
# log.debug('{}'.format(message))
if isinstance(message, dict):
monitor_event = message
else:
monitor_event = json.loads(message.payload)
monitor_event['path'] = monitor_event['as_path']
monitor_event['timestamp'] = datetime(
*map(int, re.findall(
'\d+', monitor_event['timestamp']))).timestamp()
if not self.redis.exists(
monitor_event['key']) or 'hij_key' in monitor_event:
raw = monitor_event.copy()
# mark the initial redis hijack key since it may change upon outdated checks
if 'hij_key' in monitor_event:
monitor_event['initial_redis_hijack_key'] = redis_key(
monitor_event['prefix'], monitor_event['hijack_as'],
monitor_event['hij_type'])
is_hijack = False
# ignore withdrawals for now
if monitor_event['type'] == 'A':
monitor_event['path'] = Detection.Worker.__clean_as_path(
monitor_event['path'])
prefix_node = self.prefix_tree.search_best(
monitor_event['prefix'])
if prefix_node is not None:
monitor_event['matched_prefix'] = prefix_node.prefix
try:
for func in self.__detection_generator(
len(monitor_event['path'])):
if func(monitor_event, prefix_node):
is_hijack = True
break
except Exception:
log.exception('exception')
if ((not is_hijack and 'hij_key' in monitor_event)
or (is_hijack and 'hij_key' in monitor_event
and monitor_event['initial_redis_hijack_key']
!= monitor_event['final_redis_hijack_key'])):
redis_hijack_key = redis_key(
monitor_event['prefix'],
monitor_event['hijack_as'],
monitor_event['hij_type'])
purge_redis_eph_pers_keys(self.redis, redis_hijack_key,
monitor_event['hij_key'])
self.mark_outdated(monitor_event['hij_key'],
redis_hijack_key)
elif not is_hijack:
self.mark_handled(raw)
elif monitor_event['type'] == 'W':
self.producer.publish(
{
'prefix': monitor_event['prefix'],
'peer_asn': monitor_event['peer_asn'],
'timestamp': monitor_event['timestamp'],
'key': monitor_event['key']
},
exchange=self.update_exchange,
routing_key='withdraw',
priority=0)
self.redis.set(monitor_event['key'], '', ex=60 * 60)
else:
log.debug('already handled {}'.format(monitor_event['key']))