def test_annotate_if_relation_ko():
relations_dict = defaultdict(set)
file = os.path.join(PATH, "conflict_annotation", "inputs", "maintainers_file")
fill_relation_struct(file, relations_dict, "maintainers")
file = os.path.join(PATH, "conflict_annotation", "inputs", "organisations_file")
fill_relation_struct(file, relations_dict, "organisations")
input_dict = {"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {"prefix": "60.145.0.0/28",
"asn": 202214,
"as_path": "13030 3491 4651 9737 23969"},
"conflict_with": {"prefix": "1.0.128.0/17",
"asn": 12322},
"asn": 12322}
annotate_if_relation(relations_dict, input_dict)
expected = {"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {"prefix": "60.145.0.0/28",
"asn": 202214,
"as_path": "13030 3491 4651 9737 23969"},
"conflict_with": {"prefix": "1.0.128.0/17",
"asn": 12322},
"asn": 12322}
assert input_dict == expected
def parse_registry_data(irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None):
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict, "organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
return funcs
def detect_hijacks(collector,
files,
irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None,
opener=default_opener,
format=mabo_format,
is_watched=None):
"""
Detect BGP hijacks from `files' and annotate them using metadata.
:param collector: Name of the collector the BGP files come from
:param files: List of BGP files to process
:param irr_org_file: CSV file containing irr,organisation,asn
:param irr_mrt_file: CSV file containing irr,maintainer,asn
:param irr_ro_file: CSV file containing irr,prefix,asn
:param rpki_roa_file: CSV file containing asn,prefix,max_length,valid
:param opener: Function to use in order to open the files
:param format: Format of the BGP data in the files
:return: Generator of hijacks (conflicts with annotation)
"""
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict, "organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
logger.info("starting hijacks detection...")
for conflict in detect_conflicts(collector,
files,
opener=opener,
format=format,
is_watched=is_watched):
for f in funcs:
f(conflict)
yield conflict
def detect_hijacks(collector, files,
irr_org_file=None,
irr_mnt_file=None,
irr_ro_file=None,
rpki_roa_file=None,
opener=default_opener,
format=mabo_format, is_watched=None):
"""
Detect BGP hijacks from `files' and annotate them using metadata.
:param collector: Name of the collector the BGP files come from
:param files: List of BGP files to process
:param irr_org_file: CSV file containing irr,organisation,asn
:param irr_mrt_file: CSV file containing irr,maintainer,asn
:param irr_ro_file: CSV file containing irr,prefix,asn
:param rpki_roa_file: CSV file containing asn,prefix,max_length,valid
:param opener: Function to use in order to open the files
:param format: Format of the BGP data in the files
:return: Generator of hijacks (conflicts with annotation)
"""
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if irr_org_file is not None and irr_mnt_file is not None:
relations_dict = dict()
fill_relation_struct(irr_org_file, relations_dict,
"organisations")
fill_relation_struct(irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, roa_rad_tree))
funcs.append(annotate_with_type)
logger.info("starting hijacks detection...")
for conflict in detect_conflicts(collector, files,
opener=opener, format=format,
is_watched=is_watched):
for f in funcs:
f(conflict)
yield conflict
def test_annotate_if_relation_ko():
relations_dict = defaultdict(set)
file = os.path.join(PATH, "conflict_annotation", "inputs",
"maintainers_file")
fill_relation_struct(file, relations_dict, "maintainers")
file = os.path.join(PATH, "conflict_annotation", "inputs",
"organisations_file")
fill_relation_struct(file, relations_dict, "organisations")
input_dict = {
"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {
"prefix": "60.145.0.0/28",
"asn": 202214,
"as_path": "13030 3491 4651 9737 23969"
},
"conflict_with": {
"prefix": "1.0.128.0/17",
"asn": 12322
},
"asn": 12322
}
annotate_if_relation(relations_dict, input_dict)
expected = {
"timestamp": 1445817600.0,
"collector": "rrc01",
"peer_as": 13030,
"peer_ip": "195.66.224.175",
"type": "F",
"announce": {
"prefix": "60.145.0.0/28",
"asn": 202214,
"as_path": "13030 3491 4651 9737 23969"
},
"conflict_with": {
"prefix": "1.0.128.0/17",
"asn": 12322
},
"asn": 12322
}
assert input_dict == expected
def test_fill_relation_struct():
file = os.path.join(PATH, "conflict_annotation", "inputs", "maintainers_file")
relations_dict = defaultdict(set)
fill_relation_struct(file, relations_dict, "maintainers")
assert relations_dict == {"maintainers": {'AFRINIC-FAKE': {37554, 202214}},
"maintainers_reverse": {37554: {'AFRINIC-FAKE'}, 202214: {'AFRINIC-FAKE'}}}
file = os.path.join(PATH, "conflict_annotation", "inputs", "organisations_file")
relations_dict = defaultdict(set)
fill_relation_struct(file, relations_dict, "organisations")
assert relations_dict == {"organisations_reverse": {30896: {'ORG-ACL2-AFRINIC', 'ORG-ACL1-AFRINIC'},
21242: {'ORG-AC5-AFRINIC'},
37572: {'ORG-ACFC1-AFRINIC'},
17676: {'FAKE'}, 9737: {'FAKE'}},
"organisations": {'ORG-ACL2-AFRINIC': {30896}, 'ORG-ACL1-AFRINIC': {30896},
'ORG-AC5-AFRINIC': {21242}, 'ORG-ACFC1-AFRINIC': {37572},
'FAKE': {17676, 9737}}}
file = os.path.join(PATH, "conflict_annotation", "inputs", "maintainers_file")
fill_relation_struct(file, relations_dict, "maintainers")
assert relations_dict == {"maintainers": {'AFRINIC-FAKE': {37554, 202214}},
"maintainers_reverse": {37554: {'AFRINIC-FAKE'}, 202214: {'AFRINIC-FAKE'}},
"organisations_reverse": {30896: {'ORG-ACL2-AFRINIC', 'ORG-ACL1-AFRINIC'},
21242: {'ORG-AC5-AFRINIC'},
37572: {'ORG-ACFC1-AFRINIC'},
17676: {'FAKE'}, 9737: {'FAKE'}},
"organisations": {'ORG-ACL2-AFRINIC': {30896}, 'ORG-ACL1-AFRINIC': {30896},
'ORG-AC5-AFRINIC': {21242}, 'ORG-ACFC1-AFRINIC': {37572},
'FAKE': {17676, 9737}}}
def test_fill_relation_struct():
file = os.path.join(PATH, "conflict_annotation", "inputs",
"maintainers_file")
relations_dict = defaultdict(set)
fill_relation_struct(file, relations_dict, "maintainers")
assert relations_dict == {
"maintainers": {
'AFRINIC-FAKE': {37554, 202214}
},
"maintainers_reverse": {
37554: {'AFRINIC-FAKE'},
202214: {'AFRINIC-FAKE'}
}
}
file = os.path.join(PATH, "conflict_annotation", "inputs",
"organisations_file")
relations_dict = defaultdict(set)
fill_relation_struct(file, relations_dict, "organisations")
assert relations_dict == {
"organisations_reverse": {
30896: {'ORG-ACL2-AFRINIC', 'ORG-ACL1-AFRINIC'},
21242: {'ORG-AC5-AFRINIC'},
37572: {'ORG-ACFC1-AFRINIC'},
17676: {'FAKE'},
9737: {'FAKE'}
},
"organisations": {
'ORG-ACL2-AFRINIC': {30896},
'ORG-ACL1-AFRINIC': {30896},
'ORG-AC5-AFRINIC': {21242},
'ORG-ACFC1-AFRINIC': {37572},
'FAKE': {17676, 9737}
}
}
file = os.path.join(PATH, "conflict_annotation", "inputs",
"maintainers_file")
fill_relation_struct(file, relations_dict, "maintainers")
assert relations_dict == {
"maintainers": {
'AFRINIC-FAKE': {37554, 202214}
},
"maintainers_reverse": {
37554: {'AFRINIC-FAKE'},
202214: {'AFRINIC-FAKE'}
},
"organisations_reverse": {
30896: {'ORG-ACL2-AFRINIC', 'ORG-ACL1-AFRINIC'},
21242: {'ORG-AC5-AFRINIC'},
37572: {'ORG-ACFC1-AFRINIC'},
17676: {'FAKE'},
9737: {'FAKE'}
},
"organisations": {
'ORG-ACL2-AFRINIC': {30896},
'ORG-ACL1-AFRINIC': {30896},
'ORG-AC5-AFRINIC': {21242},
'ORG-ACFC1-AFRINIC': {37572},
'FAKE': {17676, 9737}
}
}
logger.info("loading metadata...")
funcs = [annotate_if_direct]
if args.irr_ro_file is not None:
ro_rad_tree = Radix()
fill_ro_struct(args.irr_ro_file, ro_rad_tree)
funcs.append(partial(annotate_if_route_objects, ro_rad_tree))
if args.rpki_roa_file is not None:
roa_rad_tree = Radix()
fill_roa_struct(args.rpki_roa_file, roa_rad_tree)
funcs.append(partial(annotate_if_roa, ro_rad_tree))
if args.irr_org_file is not None and args.irr_mnt_file:
relations_dict = dict()
fill_relation_struct(args.irr_org_file, relations_dict,
"organisations")
fill_relation_struct(args.irr_mnt_file, relations_dict, "maintainers")
funcs.append(partial(annotate_if_relation, relations_dict))
if args.as_rel_file is not None and args.ppdc_ases_file is not None and args.as2org_file is not None:
a, b,c,d = caida_filter_annaunce(args.as_rel_file, args.ppdc_ases_file, args.as2org_file)
funcs.append(partial(is_legittimate, a, b, c,d))
if args.from_timestamp is None:
consumer = KafkaConsumer("conflicts",
metadata_broker_list=args.our_servers.split(","),
group_id="detector",
auto_commit_enable=False)
offset, = consumer.get_partition_offsets("conflicts", PARTITIONS[args.collector], -1, 1)
consumer.set_topic_partitions({("conflicts", PARTITIONS[args.collector]): offset - 1})
last_message = next(iter(consumer))
