def auto_ignore_check_rule(self, key): rule = self.autoignore_rules.get(key, None) if not rule: return if rule["interval"] <= 0: return thres_num_peers_seen = rule["thres_num_peers_seen"] thres_num_ases_infected = rule["thres_num_ases_infected"] interval = rule["interval"] log.debug("Checking autoignore rule {}".format(rule)) try: # fetch ongoing hijack events query = ( "SELECT time_started, time_last, num_peers_seen, " "num_asns_inf, key, prefix, hijack_as, type, time_detected " "FROM hijacks WHERE active = true") entries = self.ro_db.execute(query) # check which of them should be auto-ignored time_now = int(time.time()) for entry in entries: prefix = entry[5] best_node_match = self.find_best_prefix_node(prefix) if not best_node_match: continue if best_node_match["rule_key"] != key: continue log.debug("Matched prefix {}".format(prefix)) time_last_updated = max(int(entry[1].timestamp()), int(entry[8].timestamp())) num_peers_seen = int(entry[2]) num_asns_inf = int(entry[3]) hij_key = entry[4] hijack_as = entry[6] hij_type = entry[7] if ((time_now - time_last_updated > interval) and (num_peers_seen < thres_num_peers_seen) and (num_asns_inf < thres_num_ases_infected)): redis_hijack_key = redis_key(prefix, hijack_as, hij_type) # if ongoing, clear redis if self.redis.sismember("persistent-keys", hij_key): purge_redis_eph_pers_keys(self.redis, redis_hijack_key, hij_key) self.wo_db.execute( "UPDATE hijacks SET active=false, dormant=false, under_mitigation=false, seen=false, ignored=true WHERE key=%s;", (hij_key, ), ) log.debug("Ignored hijack {}".format(entry)) except Exception: log.exception("exception") finally: self.rule_timer_threads[key] = Timer( interval=self.autoignore_rules[key]["interval"], function=self.auto_ignore_check_rule, args=[key], ) self.rule_timer_threads[key].start() log.debug("Started timer {} - {}".format( self.rule_timer_threads[key], self.autoignore_rules[key]))
def handle_bgp_update(self, message: Dict) -> NoReturn: """ Callback function that runs the main logic of detecting hijacks for every bgp update. """ # log.debug('{}'.format(message)) if isinstance(message, dict): monitor_event = message else: monitor_event = json.loads(message.payload) monitor_event["path"] = monitor_event["as_path"] monitor_event["timestamp"] = datetime( *map(int, re.findall( r"\d+", monitor_event["timestamp"]))).timestamp() raw = monitor_event.copy() # mark the initial redis hijack key since it may change upon # outdated checks if "hij_key" in monitor_event: monitor_event["initial_redis_hijack_key"] = redis_key( monitor_event["prefix"], monitor_event["hijack_as"], monitor_event["hij_type"], ) is_hijack = False if monitor_event["type"] == "A": monitor_event["path"] = Detection.Worker.__clean_as_path( monitor_event["path"]) prefix_node = self.prefix_tree.search_best( monitor_event["prefix"]) if prefix_node: monitor_event["matched_prefix"] = prefix_node.prefix try: path_hijacker = -1 pol_hijacker = -1 hij_dimensions = [ "-", "-", "-", "-", ] # prefix, path, dplane, policy hij_dimension_index = 0 for func_dim in self.__hijack_dimension_checker_gen(): if hij_dimension_index == 0: # prefix dimension for func_pref in func_dim(): hij_dimensions[ hij_dimension_index] = func_pref( monitor_event, prefix_node) if hij_dimensions[ hij_dimension_index] != "-": break elif hij_dimension_index == 1: # path type dimension for func_path in func_dim( len(monitor_event["path"])): ( path_hijacker, hij_dimensions[hij_dimension_index], ) = func_path(monitor_event, prefix_node) if hij_dimensions[ hij_dimension_index] != "-": break elif hij_dimension_index == 2: # data plane dimension for func_dplane in func_dim(): hij_dimensions[ hij_dimension_index] = func_dplane( monitor_event, prefix_node) if hij_dimensions[ hij_dimension_index] != "-": break elif hij_dimension_index == 3: # policy dimension for func_pol in func_dim( len(monitor_event["path"])): ( pol_hijacker, hij_dimensions[hij_dimension_index], ) = func_pol(monitor_event, prefix_node) if hij_dimensions[ hij_dimension_index] != "-": break hij_dimension_index += 1 # check if dimension combination in hijack combinations # and commit hijack if hij_dimensions in HIJACK_DIM_COMBINATIONS: is_hijack = True # show pol hijacker only if the path hijacker is uncertain hijacker = path_hijacker if path_hijacker == -1 and pol_hijacker != -1: hijacker = pol_hijacker self.commit_hijack(monitor_event, hijacker, hij_dimensions) except Exception: log.exception("exception") if (not is_hijack and "hij_key" in monitor_event) or ( is_hijack and "hij_key" in monitor_event and monitor_event["initial_redis_hijack_key"] != monitor_event["final_redis_hijack_key"]): redis_hijack_key = redis_key( monitor_event["prefix"], monitor_event["hijack_as"], monitor_event["hij_type"], ) purge_redis_eph_pers_keys(self.redis, redis_hijack_key, monitor_event["hij_key"]) self.mark_outdated(monitor_event["hij_key"], redis_hijack_key) elif not is_hijack: self.gen_implicit_withdrawal(monitor_event) self.mark_handled(raw) elif monitor_event["type"] == "W": self.producer.publish( { "prefix": monitor_event["prefix"], "peer_asn": monitor_event["peer_asn"], "timestamp": monitor_event["timestamp"], "key": monitor_event["key"], }, exchange=self.update_exchange, routing_key="withdraw", priority=0, )
def handle_bgp_update(self, message: Dict) -> NoReturn: """ Callback function that runs the main logic of detecting hijacks for every bgp update. """ # log.debug('{}'.format(message)) if isinstance(message, dict): monitor_event = message else: message.ack() monitor_event = message.payload monitor_event["path"] = monitor_event["as_path"] monitor_event["timestamp"] = datetime( *map(int, re.findall(r"\d+", monitor_event["timestamp"])) ).timestamp() raw = monitor_event.copy() # mark the initial redis hijack key since it may change upon # outdated checks if "hij_key" in monitor_event: monitor_event["initial_redis_hijack_key"] = redis_key( monitor_event["prefix"], monitor_event["hijack_as"], monitor_event["hij_type"], ) is_hijack = False if monitor_event["type"] == "A": monitor_event["path"] = Detection.Worker.__clean_as_path( monitor_event["path"] ) ip_version = get_ip_version(monitor_event["prefix"]) if monitor_event["prefix"] in self.prefix_tree[ip_version]: prefix_node = self.prefix_tree[ip_version][monitor_event["prefix"]] monitor_event["matched_prefix"] = prefix_node["prefix"] try: path_hijacker = -1 pol_hijacker = -1 hij_dimensions = [ "-", "-", "-", "-", ] # prefix, path, dplane, policy hij_dimension_index = 0 for func_dim in self.__hijack_dimension_checker_gen(): if hij_dimension_index == 0: # prefix dimension for func_pref in func_dim(): hij_dimensions[hij_dimension_index] = func_pref( monitor_event, prefix_node ) if hij_dimensions[hij_dimension_index] != "-": break elif hij_dimension_index == 1: # path type dimension for func_path in func_dim(len(monitor_event["path"])): ( path_hijacker, hij_dimensions[hij_dimension_index], ) = func_path(monitor_event, prefix_node) if hij_dimensions[hij_dimension_index] != "-": break elif hij_dimension_index == 2: # data plane dimension for func_dplane in func_dim(): hij_dimensions[hij_dimension_index] = func_dplane( monitor_event, prefix_node ) if hij_dimensions[hij_dimension_index] != "-": break elif hij_dimension_index == 3: # policy dimension for func_pol in func_dim(len(monitor_event["path"])): ( pol_hijacker, hij_dimensions[hij_dimension_index], ) = func_pol(monitor_event, prefix_node) if hij_dimensions[hij_dimension_index] != "-": break hij_dimension_index += 1 # check if dimension combination in hijack combinations # and commit hijack if hij_dimensions in HIJACK_DIM_COMBINATIONS: is_hijack = True # show pol hijacker only if the path hijacker is uncertain hijacker = path_hijacker if path_hijacker == -1 and pol_hijacker != -1: hijacker = pol_hijacker self.commit_hijack(monitor_event, hijacker, hij_dimensions) except Exception: log.exception("exception") outdated_hijack = None if not is_hijack and "hij_key" in monitor_event: try: # outdated hijack, benign from now on redis_hijack_key = redis_key( monitor_event["prefix"], monitor_event["hijack_as"], monitor_event["hij_type"], ) outdated_hijack = self.redis.get(redis_hijack_key) purge_redis_eph_pers_keys( self.redis, redis_hijack_key, monitor_event["hij_key"] ) # mark in DB only if it is the first time this hijack was purged (pre-existent in redis) if outdated_hijack: self.mark_outdated( monitor_event["hij_key"], redis_hijack_key ) except Exception: log.exception("exception") elif ( is_hijack and "hij_key" in monitor_event and monitor_event["initial_redis_hijack_key"] != monitor_event["final_redis_hijack_key"] ): try: outdated_hijack = self.redis.get( monitor_event["initial_redis_hijack_key"] ) # outdated hijack, but still a hijack; need key change purge_redis_eph_pers_keys( self.redis, monitor_event["initial_redis_hijack_key"], monitor_event["hij_key"], ) # mark in DB only if it is the first time this hijack was purged (pre-existsent in redis) if outdated_hijack: self.mark_outdated( monitor_event["hij_key"], monitor_event["initial_redis_hijack_key"], ) except Exception: log.exception("exception") elif not is_hijack: self.gen_implicit_withdrawal(monitor_event) self.mark_handled(raw) if outdated_hijack: try: outdated_hijack = json.loads(outdated_hijack) outdated_hijack["end_tag"] = "outdated" mail_log.info( "{}".format( json.dumps( hijack_log_field_formatter(outdated_hijack), indent=4, ) ), extra={ "community_annotation": outdated_hijack.get( "community_annotation", "NA" ) }, ) hij_log.info( "{}".format( json.dumps(hijack_log_field_formatter(outdated_hijack)) ), extra={ "community_annotation": outdated_hijack.get( "community_annotation", "NA" ) }, ) except Exception: log.exception("exception") elif monitor_event["type"] == "W": self.producer.publish( { "prefix": monitor_event["prefix"], "peer_asn": monitor_event["peer_asn"], "timestamp": monitor_event["timestamp"], "key": monitor_event["key"], }, exchange=self.update_exchange, routing_key="withdraw", priority=0, serializer="ujson", )
def handle_bgp_update(self, message: Dict) -> NoReturn: """ Callback function that runs the main logic of detecting hijacks for every bgp update. """ # log.debug('{}'.format(message)) if isinstance(message, dict): monitor_event = message else: monitor_event = json.loads(message.payload) monitor_event['path'] = monitor_event['as_path'] monitor_event['timestamp'] = datetime( *map(int, re.findall( '\d+', monitor_event['timestamp']))).timestamp() if not self.redis.exists( monitor_event['key']) or 'hij_key' in monitor_event: raw = monitor_event.copy() # mark the initial redis hijack key since it may change upon outdated checks if 'hij_key' in monitor_event: monitor_event['initial_redis_hijack_key'] = redis_key( monitor_event['prefix'], monitor_event['hijack_as'], monitor_event['hij_type']) is_hijack = False # ignore withdrawals for now if monitor_event['type'] == 'A': monitor_event['path'] = Detection.Worker.__clean_as_path( monitor_event['path']) prefix_node = self.prefix_tree.search_best( monitor_event['prefix']) if prefix_node is not None: monitor_event['matched_prefix'] = prefix_node.prefix try: for func in self.__detection_generator( len(monitor_event['path'])): if func(monitor_event, prefix_node): is_hijack = True break except Exception: log.exception('exception') if ((not is_hijack and 'hij_key' in monitor_event) or (is_hijack and 'hij_key' in monitor_event and monitor_event['initial_redis_hijack_key'] != monitor_event['final_redis_hijack_key'])): redis_hijack_key = redis_key( monitor_event['prefix'], monitor_event['hijack_as'], monitor_event['hij_type']) purge_redis_eph_pers_keys(self.redis, redis_hijack_key, monitor_event['hij_key']) self.mark_outdated(monitor_event['hij_key'], redis_hijack_key) elif not is_hijack: self.mark_handled(raw) elif monitor_event['type'] == 'W': self.producer.publish( { 'prefix': monitor_event['prefix'], 'peer_asn': monitor_event['peer_asn'], 'timestamp': monitor_event['timestamp'], 'key': monitor_event['key'] }, exchange=self.update_exchange, routing_key='withdraw', priority=0) self.redis.set(monitor_event['key'], '', ex=60 * 60) else: log.debug('already handled {}'.format(monitor_event['key']))