def parse_registry_data(irr_org_file=None, irr_mnt_file=None, irr_ro_file=None, rpki_roa_file=None): logger.info("loading metadata...") funcs = [annotate_if_direct] if irr_org_file is not None and irr_mnt_file is not None: relations_dict = dict() fill_relation_struct(irr_org_file, relations_dict, "organisations") fill_relation_struct(irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, roa_rad_tree)) funcs.append(annotate_with_type) return funcs
def test_annotate_if_valid_ok(): file = os.path.join(PATH, "conflict_annotation", "inputs", "ro_file") ro_rad_tree = Radix() fill_ro_struct(file, ro_rad_tree) input_dict = {"timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": {"prefix": "60.145.0.0/28", "asn": 17676, "as_path": "13030 3491 4651 9737 23969"}, "conflict_with": {"prefix": "1.0.128.0/17", "asn": 9737}, "asn": 9737} expected = {"timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": {"prefix": "60.145.0.0/28", "asn": 17676, "valid": ['jpnic'], "as_path": "13030 3491 4651 9737 23969"}, "conflict_with": {"prefix": "1.0.128.0/17", "asn": 9737}, "asn": 9737} annotate_if_route_objects(ro_rad_tree, input_dict) assert input_dict == expected
def test_fill_ro_struct(): file = os.path.join(PATH, "conflict_annotation", "inputs", "ro_file") rad_tree = Radix() fill_ro_struct(file, rad_tree) res = list() for node in rad_tree: res.append((node.prefix, node.data)) assert res == [('60.136.0.0/16', {17676: {'jpnic'}}), ('60.145.0.0/16', {17676: {'jpnic'}}), ('60.145.0.0/24', {17676: {'jpnic'}}), ('192.0.0.0/16', {17: {'jpnic'}})]
def detect_hijacks(collector, files, irr_org_file=None, irr_mnt_file=None, irr_ro_file=None, rpki_roa_file=None, opener=default_opener, format=mabo_format, is_watched=None): """ Detect BGP hijacks from `files' and annotate them using metadata. :param collector: Name of the collector the BGP files come from :param files: List of BGP files to process :param irr_org_file: CSV file containing irr,organisation,asn :param irr_mrt_file: CSV file containing irr,maintainer,asn :param irr_ro_file: CSV file containing irr,prefix,asn :param rpki_roa_file: CSV file containing asn,prefix,max_length,valid :param opener: Function to use in order to open the files :param format: Format of the BGP data in the files :return: Generator of hijacks (conflicts with annotation) """ logger.info("loading metadata...") funcs = [annotate_if_direct] if irr_org_file is not None and irr_mnt_file is not None: relations_dict = dict() fill_relation_struct(irr_org_file, relations_dict, "organisations") fill_relation_struct(irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, roa_rad_tree)) funcs.append(annotate_with_type) logger.info("starting hijacks detection...") for conflict in detect_conflicts(collector, files, opener=opener, format=format, is_watched=is_watched): for f in funcs: f(conflict) yield conflict
def test_fill_ro_struct(): file = os.path.join(PATH, "conflict_annotation", "inputs", "ro_file") rad_tree = Radix() fill_ro_struct(file, rad_tree) res = list() for node in rad_tree: res.append((node.prefix, node.data)) assert res == [('60.136.0.0/16', { 17676: {'jpnic'} }), ('60.145.0.0/16', { 17676: {'jpnic'} }), ('60.145.0.0/24', { 17676: {'jpnic'} }), ('192.0.0.0/16', { 17: {'jpnic'} })]
def test_annotate_if_valid_both(): file = os.path.join(PATH, "conflict_annotation", "inputs", "ro_file") ro_rad_tree = Radix() fill_ro_struct(file, ro_rad_tree) input_dict = { "timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": { "prefix": "60.145.0.0/28", "asn": 17676, "as_path": "13030 3491 4651 9737 23969" }, "conflict_with": { "prefix": "192.0.0.0", "asn": 17 }, "asn": 17676 } expected = { "timestamp": 1445817600.0, "collector": "rrc01", "peer_as": 13030, "peer_ip": "195.66.224.175", "type": "F", "announce": { "prefix": "60.145.0.0/28", "asn": 17676, "valid": ['jpnic'], "as_path": "13030 3491 4651 9737 23969" }, "conflict_with": { "prefix": "192.0.0.0", "asn": 17, "valid": ['jpnic'] }, "asn": 17676 } annotate_if_route_objects(ro_rad_tree, input_dict) assert input_dict == expected
help="TXT file containing AS relation") parser.add_argument("--ppdc-ases-file") parser.add_argument("--as2org-file", help="TXT file containing AS to organizations") args = parser.parse_args() logging.basicConfig(level=logging.INFO) kwargs = kafka_input(args.collector, broker=args.our_servers.split(",")) logger.info("loading metadata...") funcs = [annotate_if_direct] if args.irr_ro_file is not None: ro_rad_tree = Radix() fill_ro_struct(args.irr_ro_file, ro_rad_tree) funcs.append(partial(annotate_if_route_objects, ro_rad_tree)) if args.rpki_roa_file is not None: roa_rad_tree = Radix() fill_roa_struct(args.rpki_roa_file, roa_rad_tree) funcs.append(partial(annotate_if_roa, ro_rad_tree)) if args.irr_org_file is not None and args.irr_mnt_file: relations_dict = dict() fill_relation_struct(args.irr_org_file, relations_dict, "organisations") fill_relation_struct(args.irr_mnt_file, relations_dict, "maintainers") funcs.append(partial(annotate_if_relation, relations_dict)) if args.as_rel_file is not None and args.ppdc_ases_file is not None and args.as2org_file is not None: