def parse_registry_data(irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None):
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict, "organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
return funcs
def test_annotate_if_valid_rpki_ok1():
file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file")
rpki_rad_tree = Radix()
fill_roa_struct(file, rpki_rad_tree)
input_dict = {"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {"prefix": "212.234.194.0/24",
"asn": 16071,
"as_path": "13030 3491 4651 9737 23969"},
"conflict_with": {"prefix": "212.234.194.0/24",
"asn": 16072},
"asn": 16072}
expected = {"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {"prefix": "212.234.194.0/24",
"asn": 16071, "valid": ["roa"],
"as_path": "13030 3491 4651 9737 23969"},
"conflict_with": {"prefix": "212.234.194.0/24",
"asn": 16072, "valid": ["roa"]},
"asn": 16072}
annotate_if_roa(rpki_rad_tree, input_dict)
assert input_dict == expected
def test_fill_roa_struct():
file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file")
rad_tree = Radix()
fill_roa_struct(file, rad_tree)
res = list()
for node in rad_tree:
res.append((node.prefix, node.data))
assert res == [('86.63.224.0/19', {35238: 19}),
('91.91.0.0/16', {35238: 16}),
('192.113.0.0/16', {16074: 16}),
('194.209.159.0/24', {16079: 32}),
('212.234.194.0/24', {16071: 28, 16072: 24})]
def detect_hijacks(collector,
files,
irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None,
opener=default_opener,
format=mabo_format,
is_watched=None):
"""
Detect BGP hijacks from `files' and annotate them using metadata.
:param collector: Name of the collector the BGP files come from
:param files: List of BGP files to process
:param irr_org_file: CSV file containing irr,organisation,asn
:param irr_mrt_file: CSV file containing irr,maintainer,asn
:param irr_ro_file: CSV file containing irr,prefix,asn
:param rpki_roa_file: CSV file containing asn,prefix,max_length,valid
:param opener: Function to use in order to open the files
:param format: Format of the BGP data in the files
:return: Generator of hijacks (conflicts with annotation)
"""
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict, "organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
logger.info("starting hijacks detection...")
for conflict in detect_conflicts(collector,
files,
opener=opener,
format=format,
is_watched=is_watched):
for f in funcs:
f(conflict)
yield conflict
def detect_hijacks(collector, files,
irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None,
opener=default_opener,
format=mabo_format, is_watched=None):
"""
Detect BGP hijacks from `files' and annotate them using metadata.
:param collector: Name of the collector the BGP files come from
:param files: List of BGP files to process
:param irr_org_file: CSV file containing irr,organisation,asn
:param irr_mrt_file: CSV file containing irr,maintainer,asn
:param irr_ro_file: CSV file containing irr,prefix,asn
:param rpki_roa_file: CSV file containing asn,prefix,max_length,valid
:param opener: Function to use in order to open the files
:param format: Format of the BGP data in the files
:return: Generator of hijacks (conflicts with annotation)
"""
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict,
"organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
logger.info("starting hijacks detection...")
for conflict in detect_conflicts(collector, files,
opener=opener, format=format,
is_watched=is_watched):
for f in funcs:
f(conflict)
yield conflict
def test_fill_roa_struct():
file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file")
rad_tree = Radix()
fill_roa_struct(file, rad_tree)
res = list()
for node in rad_tree:
res.append((node.prefix, node.data))
assert res == [('86.63.224.0/19', {
35238: 19
}), ('91.91.0.0/16', {
35238: 16
}), ('192.113.0.0/16', {
16074: 16
}), ('194.209.159.0/24', {
16079: 32
}), ('212.234.194.0/24', {
16071: 28,
16072: 24
})]
def test_annotate_if_valid_rpki_ok1():
file = os.path.join(PATH, "conflict_annotation", "inputs", "roa_file")
rpki_rad_tree = Radix()
fill_roa_struct(file, rpki_rad_tree)
input_dict = {
"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {
"prefix": "212.234.194.0/24",
"asn": 16071,
"as_path": "13030 3491 4651 9737 23969"
},
"conflict_with": {
"prefix": "212.234.194.0/24",
"asn": 16072
},
"asn": 16072
}
expected = {
"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {
"prefix": "212.234.194.0/24",
"asn": 16071,
"valid": ["roa"],
"as_path": "13030 3491 4651 9737 23969"
},
"conflict_with": {
"prefix": "212.234.194.0/24",
"asn": 16072,
"valid": ["roa"]
},
"asn": 16072
}
annotate_if_roa(rpki_rad_tree, input_dict)
assert input_dict == expected
args = parser.parse_args()
logging.basicConfig(level=logging.INFO)
kwargs = kafka_input(args.collector, broker=args.our_servers.split(","))
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if args.irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(args.irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if args.rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(args.rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, ro_rad_tree))
if args.irr_org_file is not None and args.irr_mnt_file:
relations_dict = dict()
fill_relation_struct(args.irr_org_file, relations_dict,
"organisations")
fill_relation_struct(args.irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if args.as_rel_file is not None and args.ppdc_ases_file is not None and args.as2org_file is not None:
a, b,c,d = caida_filter_annaunce(args.as_rel_file, args.ppdc_ases_file, args.as2org_file)
funcs.append(partial(is_legittimate, a, b, c,d))
if args.from_timestamp is None:
consumer = KafkaConsumer("conflicts",
