def parse_registry_data(irr_org_file=None, irr_mnt_file=None, irr_ro_file=None, rpki_roa_file=None): logger.info("loading metadata...") funcs = [annotate_if_direct] if irr_org_file is not None and irr_mnt_file is not None: relations_dict = dict() fill_relation_struct(irr_org_file, relations_dict, "organisations") fill_relation_struct(irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, roa_rad_tree)) funcs.append(annotate_with_type) return funcs
def test_annotate_if_valid_rpki_ok1(): file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file") rpki_rad_tree = Radix() fill_roa_struct(file, rpki_rad_tree) input_dict = {"timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": {"prefix": "212.234.194.0/24", "asn": 16071, "as_path": "13030 3491 4651 9737 23969"}, "conflict_with": {"prefix": "212.234.194.0/24", "asn": 16072}, "asn": 16072} expected = {"timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": {"prefix": "212.234.194.0/24", "asn": 16071, "valid": ["roa"], "as_path": "13030 3491 4651 9737 23969"}, "conflict_with": {"prefix": "212.234.194.0/24", "asn": 16072, "valid": ["roa"]}, "asn": 16072} annotate_if_roa(rpki_rad_tree, input_dict) assert input_dict == expected
def test_fill_roa_struct(): file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file") rad_tree = Radix() fill_roa_struct(file, rad_tree) res = list() for node in rad_tree: res.append((node.prefix, node.data)) assert res == [('86.63.224.0/19', {35238: 19}), ('91.91.0.0/16', {35238: 16}), ('192.113.0.0/16', {16074: 16}), ('194.209.159.0/24', {16079: 32}), ('212.234.194.0/24', {16071: 28, 16072: 24})]
def detect_hijacks(collector, files, irr_org_file=None, irr_mnt_file=None, irr_ro_file=None, rpki_roa_file=None, opener=default_opener, format=mabo_format, is_watched=None): """ Detect BGP hijacks from `files' and annotate them using metadata. :param collector: Name of the collector the BGP files come from :param files: List of BGP files to process :param irr_org_file: CSV file containing irr,organisation,asn :param irr_mrt_file: CSV file containing irr,maintainer,asn :param irr_ro_file: CSV file containing irr,prefix,asn :param rpki_roa_file: CSV file containing asn,prefix,max_length,valid :param opener: Function to use in order to open the files :param format: Format of the BGP data in the files :return: Generator of hijacks (conflicts with annotation) """ logger.info("loading metadata...") funcs = [annotate_if_direct] if irr_org_file is not None and irr_mnt_file is not None: relations_dict = dict() fill_relation_struct(irr_org_file, relations_dict, "organisations") fill_relation_struct(irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, roa_rad_tree)) funcs.append(annotate_with_type) logger.info("starting hijacks detection...") for conflict in detect_conflicts(collector, files, opener=opener, format=format, is_watched=is_watched): for f in funcs: f(conflict) yield conflict
def test_fill_roa_struct(): file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file") rad_tree = Radix() fill_roa_struct(file, rad_tree) res = list() for node in rad_tree: res.append((node.prefix, node.data)) assert res == [('86.63.224.0/19', { 35238: 19 }), ('91.91.0.0/16', { 35238: 16 }), ('192.113.0.0/16', { 16074: 16 }), ('194.209.159.0/24', { 16079: 32 }), ('212.234.194.0/24', { 16071: 28, 16072: 24 })]
def test_annotate_if_valid_rpki_ok1(): file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file") rpki_rad_tree = Radix() fill_roa_struct(file, rpki_rad_tree) input_dict = { "timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": { "prefix": "212.234.194.0/24", "asn": 16071, "as_path": "13030 3491 4651 9737 23969" }, "conflict_with": { "prefix": "212.234.194.0/24", "asn": 16072 }, "asn": 16072 } expected = { "timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": { "prefix": "212.234.194.0/24", "asn": 16071, "valid": ["roa"], "as_path": "13030 3491 4651 9737 23969" }, "conflict_with": { "prefix": "212.234.194.0/24", "asn": 16072, "valid": ["roa"] }, "asn": 16072 } annotate_if_roa(rpki_rad_tree, input_dict) assert input_dict == expected
args = parser.parse_args() logging.basicConfig(level=logging.INFO) kwargs = kafka_input(args.collector, broker=args.our_servers.split(",")) logger.info("loading metadata...") funcs = [annotate_if_direct] if args.irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(args.irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if args.rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(args.rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, ro_rad_tree)) if args.irr_org_file is not None and args.irr_mnt_file: relations_dict = dict() fill_relation_struct(args.irr_org_file, relations_dict, "organisations") fill_relation_struct(args.irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if args.as_rel_file is not None and args.ppdc_ases_file is not None and args.as2org_file is not None: a, b,c,d = caida_filter_annaunce(args.as_rel_file, args.ppdc_ases_file, args.as2org_file) funcs.append(partial(is_legittimate, a, b, c,d)) if args.from_timestamp is None: consumer = KafkaConsumer("conflicts",