def obtenerRutas(target_as,
time_init,
time_end,
collector,
obtenerNombre=True):
global PrefijosAnunciados
stream = pybgpstream.BGPStream(
from_time=time_init,
until_time=time_end,
filter='type ribs and collector %s and path %s' %
(collector, target_as))
ASs = list()
for elem in stream:
if int(elem.fields["as-path"].split(' ')[-1]) == target_as:
aspath = list()
for as_path in elem.fields["as-path"].split(' '):
if as_path not in aspath:
aspath.append(as_path)
nextHop = elem.fields["next-hop"].strip()
prefix = elem.fields["prefix"].strip()
if target_as not in PrefijosAnunciados.keys():
PrefijosAnunciados[target_as] = set()
if PrefijosAnunciados[target_as] is None:
PrefijosAnunciados[target_as] = set()
_set = PrefijosAnunciados[target_as]
if prefix not in _set:
_set.add(prefix)
_as = AS(aspath, nextHop, prefix, obtenerNombre)
ASs.append(_as)
#pprint.pprint(elem.fields)
return ASs
def crear_stream(objetivo):
return pybgpstream.BGPStream(
from_time='2012-01-05 19:59:00 UTC', until_time='2012-01-05 20:01:00 UTC',
collectors=['route-views.saopaulo'],
record_type='ribs',
filter='path %s' % objetivo
)
def process_ribs(ts, collectors, countries, catalog):
date = ts[0:4] + '-' + ts[4:6] + '-' + ts[6:8]
print("* Processing RIBs from " + ts + " (" + ", ".join(collectors) + ")")
computed_lines = 0
stream = pybgpstream.BGPStream(
from_time=date + " 07:50:00",
until_time=date + " 08:10:00",
collectors=collectors,
record_type="ribs",
)
result = RoutingDatabase(countries, catalog)
for rec in stream.records():
for elem in rec:
prefix = elem.fields["prefix"]
path = elem.fields["as-path"].split()
if '.' in prefix:
v = 'ipv4'
else:
v = 'ipv6'
prefix_cc = catalog.get_pfx(v, prefix)
if prefix_cc in countries:
result.add_path(prefix, prefix_cc, path, v)
computed_lines += 1
if computed_lines % 1000 == 0:
print('\r* ' + str(computed_lines) + " rows computed",
end="",
flush=True)
print('\r* ' + str(computed_lines) + " total rows computed\n",
end="",
flush=True)
return result
def run_bgpstream(prefix, start, end, out_file):
"""
Retrieve all records related to a certain prefix for a certain time period
and save them on a .csv file
https://bgpstream.caida.org/docs/api/pybgpstream/_pybgpstream.html
:param prefix: <str> input prefix
:param start: <int> start timestamp in UNIX epochs
:param end: <int> end timestamp in UNIX epochs
:param out_file: <str> .csv file to store information
format: PREFIX|ORIGIN_AS|PEER_AS|AS_PATH|PROJECT|COLLECTOR|TYPE|COMMUNITIES|TIME
:return: -
"""
# create a new bgpstream instance and a reusable bgprecord instance
# More to information about filter: https://github.com/CAIDA/bgpstream/blob/master/FILTERING
stream = pybgpstream.BGPStream(
from_time=start,
until_time=end, # filter based on timing
collectors=[], # empty=use all RRC from RIPE and RouteViews
record_type="updates", # filter record type
filter="prefix any " + str(prefix), # filter prefix
)
# set the csv writer
with open(out_file, "w") as f:
csv_writer = csv.writer(f, delimiter="|")
for elem in stream:
if (elem.status != "valid") or (elem.record.rec.type != "update"):
continue
if elem.type in ["A", "W"]:
elem_csv_list = []
if elem.type == "A":
elem_csv_list = [
str(elem.fields["prefix"]),
str(elem.fields["as-path"].split(" ")[-1]),
str(elem.peer_asn),
str(elem.fields["as-path"]),
str(elem.project),
str(elem.collector),
str(elem.type),
community_list(elem.fields["communities"]),
str(elem.time),
]
else:
elem_csv_list = [
str(elem.fields["prefix"]),
"",
str(elem.peer_asn),
"",
str(elem.project),
str(elem.collector),
str(elem.type),
ujson.dumps([]),
str(elem.time),
]
csv_writer.writerow(elem_csv_list)
def obtenerRIB(time_init, time_end, collector):
stream = pybgpstream.BGPStream(from_time=time_init,
until_time=time_end,
filter='type ribs and collector %s' %
(collector))
ASs = list()
for elem in stream:
aspath = elem.fields["as-path"].split(' ')
nextHop = elem.fields["next-hop"].strip()
prefix = elem.fields["prefix"].strip()
_as = AS(aspath, nextHop, prefix, False)
ASs.append(_as)
return ASs
def main(args):
start_date_time = dt.datetime.combine(args.start_date, args.time)
days_collectors = it.product(range(args.ndays), COLLECTORS)
tqdm_total = args.ndays*len(COLLECTORS)
unique_paths = set()
for nday, collector in tqdm(days_collectors, desc='Days and collectors', total=tqdm_total):
date_time = start_date_time + dt.timedelta(days=nday)
date_time_str = date_time.strftime(r'%Y-%m-%d %H:%M:%S')
print(f'Collecting data from {collector} at {date_time_str}', file=sys.stderr)
stream = pybgpstream.BGPStream(
from_time=date_time_str, until_time=date_time_str,
collectors=[collector],
record_type="ribs",
)
n_paths_for_pair = 0
for elem in stream:
path_str = elem.fields["as-path"]
path_clean_str = parse_path(path_str, args.not_collapse_prepending_asns)
if args.verbose and path_clean_str != path_str:
print(f'Cleaned {path_str} to {path_clean_str}', file=sys.stderr)
if not args.not_only_unique_paths:
if path_clean_str in unique_paths:
if args.verbose:
print(f'Found repeated path {path_clean_str}', file=sys.stderr)
continue
unique_paths.add(path_clean_str)
elem_time = dt.datetime.fromtimestamp(elem.time)
if args.path_only:
print(f'{path_clean_str}')
else:
print(f'{elem_time}|{elem.collector}|{path_clean_str}')
n_paths_for_pair += 1
print(f'Added {n_paths_for_pair} paths', file=sys.stderr)
def main():
stream = pybgpstream.BGPStream(
from_time="2015-01-05 20:00:00",
until_time="2015-01-05 20:00:10 UTC",
collectors=['route-views.kixp'],
record_type="ribs",
)
# Creamos un archivo csv para la salida: { ip_addr, isn }
fd = open('full_rib.csv', 'w')
for entrada in stream:
origen = entrada.fields['as-path'].split(' ')[0]
prefijo = entrada.fields['prefix']
fd.write('%s,%s\n' % (prefijo, origen))
fd.close()
print("Fin del procesamiento")
def run_bgpstream(args):
(collector, start_time, end_time, data_type) = args
# initialize and configure BGPStream
stream = pybgpstream.BGPStream(
collector=collector,
from_time=start_time,
until_time=end_time-1,
record_type=data_type
)
# per-peer data
peers_data = {}
# loop over all records in the stream
for rec in stream.records():
# to track the peers that have elems in this record
peer_signatures = set()
# loop over all elems in the record
for elem in rec:
# create a peer signature for this elem
sig = peer_signature(rec, elem)
peer_signatures.add(sig)
# if this is the first time we have ever seen this peer, create
# an empty result: (elem_cnt, peer_record_cnt, coll_record_cnt)
if sig not in peers_data:
peers_data[sig] = [0, 0, 0]
peers_data[sig][0] += 1 # increment elem cnt for this peer
# done with elems, increment the 'coll_record_cnt' field for just
# one peer that was present in this record (allows a true, per-collector
# count of records since each record can contain elems for many peers)
if len(peer_signatures):
first = True
for sig in peer_signatures: # increment peer_record_cnt for all
if first:
peers_data[sig][2] += 1 # increment the coll_record_cnt
first = False
peers_data[sig][1] += 1
# the time in the output row is truncated down to a multiple of
# RESULT_GRANULARITY so that slices can be merged correctly
start_time = \
int(math.floor(start_time/RESULT_GRANULARITY) * RESULT_GRANULARITY)
# for each peer that we processed data for, create an output row
return [((start_time, collector, p), (peers_data[p])) for p in peers_data]
def run(self):
"""Fetch BGP data from collectors and push to wikibase. """
today = arrow.now().replace(hour=0, minute=0)
start = today.shift(hours=-1)
end = today.shift(hours=1)
stream = pybgpstream.BGPStream(
from_time=int(start.timestamp()),
until_time=int(end.timestamp()),
record_type="ribs",
)
rtree = radix.Radix()
sys.stderr.write(f'\nReading BGP data:\n')
for i, elem in enumerate(stream):
# Extract the prefix and origin ASN
msg = elem.fields
prefix = msg['prefix']
origin_asn_str = msg['as-path'].split(' ')[-1]
origin_asns = []
if '{' in origin_asn_str:
origin_asns = origin_asn_str[1:-1].split(',')
else:
origin_asns = [origin_asn_str]
# Store origin ASN in radix tree
rnode = rtree.search_exact(prefix)
if rnode is None:
rnode = rtree.add(prefix)
rnode.data['origin'] = defaultdict(set)
for asn in origin_asns:
rnode.data['origin'][asn].add(elem.collector)
sys.stderr.write(f'\rProcessed {i+1} BGP messages')
sys.stderr.write(f'\nPushing data to IYP...\n')
# Push all prefixes data to IYP
for i, rnode in enumerate(rtree):
data = rnode.data['origin']
self.update_entry(rnode.prefix, data)
sys.stderr.write(f'\rProcessed {i+1} prefixes')
# Definitions
TIME_INIT = "2018-03-01"
TIME_END = "2018-03-01 00:05"
COLLECTOR = 'route-views.wide'
# AS target
# ASN: 8966
# ASname: Etisalat
targetAS = 8966
#---------------------------------------------------------
# Query configuration
stream = pybgpstream.BGPStream(
from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s and path %s" % (COLLECTOR, targetAS))
#---------------------------------------------------------
# Execution & parse of the query
index = 0
ASPATHs = []
ASPATHS_v = []
for elem in stream:
if str(elem.fields['as-path'].split(' ')[-1]) == str(targetAS):
if elem.fields['as-path'] not in ASPATHs:
index += 1
ASPATHs.append(elem.fields['as-path'])
ASPATHS_v.append(
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
import pybgpstream
# create a new bgpstream instance
# configure the stream to retrieve RIB records from the RRC06 collector
# from 2015-04-01 00:00:00 GMT to 2015-04-01 00:05:00 GMT
stream = pybgpstream.BGPStream(
collector="rrc06",
record_type="ribs",
from_time="2015-04-01 00:00:00",
until_time="2015-04-01 00:05:00",
)
as_topology = set()
rib_entries = 0
# Process data
for elem in stream:
rib_entries += 1
# get the AS path
path = elem.fields['as-path']
# get the list of ASes in the path
ases = path.split(" ")
for i in range(0, len(ases) - 1):
# avoid multiple prepended ASes
# Youtube and Telecom Pakistan
# 24 February 2008, 20:07
# TIME_INIT = "2008-02-24 20:00"
# TIME_END = "2008-02-24 23:00"
# COLLECTOR_WIDE = 'route-views.isc'
TIME_INIT = "2015-01-14"
TIME_END = "2015-01-14 00:05"
COLLECTOR_WIDE = 'route-views.wide'
#---------------------------------------------------------
# Query configuration
stream = pybgpstream.BGPStream(from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s" %
(COLLECTOR_WIDE))
#---------------------------------------------------------
TARGET_AS_BY_PREFIX = {}
for elem in stream:
AS_ID = str(elem.fields['as-path'].split(' ')[-1])
network = str(elem.fields['prefix'])
if network in TARGET_AS_BY_PREFIX:
# if network == '208.65.153.0/24':
# print network + ' - ' + AS_ID + ' - ' + str(TARGET_AS_BY_PREFIX[network])
if AS_ID not in TARGET_AS_BY_PREFIX[network]:
print "SECUESTRO - " + network + ' - ' + AS_ID + ' - ' + str(
TARGET_AS_BY_PREFIX[network])
TARGET_AS_BY_PREFIX[network].append(AS_ID)
import pybgpstream
time_init = '2019-06-03 01:00:00'
time_before_hijack = '2019-06-03 02:00:00'
time_end = '2019-06-03 03:00:00'
collector = 'route-views.chicago'
stream = {
'before':
pybgpstream.BGPStream(from_time=time_init,
until_time=time_before_hijack,
filter='type ribs and collector %s' % (collector)),
'after':
pybgpstream.BGPStream(from_time=time_before_hijack,
until_time=time_end,
filter='type ribs and collector %s' % (collector))
}
as_numbers_per_prefix = {'before': {}, 'after': {}}
def to_binary(mask):
_ip, length = mask.split('/')
return ''.join([bin(int(x) + 256)[3:]
for x in _ip.split('.')])[0:int(length)]
def dest_as_number(elem):
return elem.fields['as-path'].split(' ')[-1]
# DOCUMENTATION, EVEN IF THE UNIVERSITY OF CALIFORNIA HAS BEEN ADVISED OF
# THE POSSIBILITY OF SUCH DAMAGE. THE UNIVERSITY OF CALIFORNIA SPECIFICALLY
# DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE
# SOFTWARE PROVIDED HEREUNDER IS ON AN “AS IS” BASIS, AND THE UNIVERSITY OF
# CALIFORNIA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES,
# ENHANCEMENTS, OR MODIFICATIONS.
#!/usr/bin/env python
import pybgpstream
# Create pybgpstream instance
stream = pybgpstream.BGPStream(
from_time="2017-07-07 00:00:00",
until_time="2017-07-07 00:10:00 UTC",
collectors=["route-views.sg", "route-views.eqix"],
record_type="updates",
)
# This should limit BGPStream to download the full first BGP dump
stream.add_rib_period_filter(86400)
for elem in stream:
# record fields can be accessed directly from elem
# print (elem.fields)
# Get next-hop
if "next-hop" in elem.fields:
next_hop = elem.fields["next-hop"]
# Get asns
gobgp_do(["neighbor"])
now = time.time()
record_types = ["updates"]
if CONFIG["ribs"]:
record_types.append("ribs")
from_time = int(now / 28800) * 28800
else:
# grab updates from 15 mins ago
from_time = int(now - 900)
log("Starting PyBGPStream: from_time=%d, config: %s" % (from_time, CONFIG))
stream = pybgpstream.BGPStream(
record_types=record_types,
from_time=from_time,
until_time=0,
projects=CONFIG["projects"],
collectors=CONFIG["collectors"],
filter="peer %d" % CONFIG["peer_asn"],
)
if CONFIG["ribs"]:
stream.add_rib_period_filter(-1)
stats = [
{
"*": 0,
"A": 0,
"R": 0,
"W": 0,
"S": 0, # unused
},
{
def get_his_info(self):
events = pd.read_csv(
'../events_data/edge_events_from_bgpObservatory1.csv')
begin_minute = 120
until_minute = 60
label_range = [1, 2, 3, 0] # 0 means all
Window_size = 2 # min
total_num = events.shape[0] # event_total_num in this list
duration = 0
all_features = pd.DataFrame()
for i in range(total_num):
event_after = False
VICTIM_AS = str(events['victim_AS'].iloc[i])
begin_timestamp = events['timestamp'].iloc[i] - begin_minute * 60
until_timestamp = events['timestamp'].iloc[i] + until_minute * 60
begin_time = str(pd.to_datetime(begin_timestamp, unit='s'))
until_time = str(pd.to_datetime(until_timestamp, unit='s'))
start_time = events['timestamp'].iloc[i]
label = events['label'].iloc[i]
# duration=events['Duration'].iloc[i]
count_2 = 2
count_3 = 3
step = 0
stream = pybgpstream.BGPStream(
# Consider this time interval:
# Sat, 01 Aug 2015 7:50:00 GMT - 08:10:00 GMT
from_time=begin_time,
until_time=until_time,
collectors=[
"rrc00", 'rrc01', 'rrc02', 'rrc03', 'rrc04', 'rrc05',
'rrc06', 'rrc07', 'rrc08', 'rrc09', 'rrc10', 'rrc11',
'rrc12', 'rrc13', 'rrc14', 'rrc15', 'rrc16', 'rrc18',
'rrc19', 'rrc20', 'rrc21', 'rrc22', 'rrc23'
],
record_type="updates", # announcement、rib field中才有as-path，
# withdrawal中只有prefix
filter='ipversion 4'
# filter='ipversion 4 and path "_{:}$"'.format(VICTIM_AS),
)
# <prefix, origin-ASns-set > dictionary
path = set()
victim_prefix = set()
MOAS = defaultdict(set)
old_time = 0
first = True
peer = set() # victime's peer
peer_num = 0
peer_increase = defaultdict(int)
features = pd.DataFrame() # [timestamp][feature][values]
# prefix_origin = defaultdict(set) #prefix origin pairs
temp = list
# feature
MPL = defaultdict(int) # MAX path length in time t
PL = defaultdict(dd) # each number of path len in time t.
MOAS_AS = defaultdict(
ds) # Number of ASes that conflict with victime AS
old_AS_Path = defaultdict(list)
new_AS_Path = defaultdict(list)
diff_AS_Path = defaultdict(dd)
diff_AS_Path_num = defaultdict(dd) # AS-PATH edit distance set
withdraw_num = defaultdict(int)
new_sub_prefix_num = defaultdict(
int) # Number of new sub-prefixes belongs to Victim AS
new_sub_prefix = defaultdict(set)
own_ann_num = defaultdict(
int) # Number of Announcements from victim AS
MOAS_ann_num = defaultdict(
int) # Number of announcements from origin conflict AS
ann_shorter = defaultdict(dd) # AS-PATH length decrease set
ann_longer = defaultdict(dd) # AS-PATH length increase set
Diff_Ann = defaultdict(int)
duplicate_ann = defaultdict(int)
withdraw_unique_prefix = defaultdict(set)
# IGP_num=defaultdict(int)
# EGP_num=defaultdict(int)
# incomplete_packets=defaultdict(int)
new_MOAS = defaultdict(int)
avg_edit_distance = 0
avg_PL = 0
# 标签
labels = defaultdict(dd)
for rec in tqdm(stream.records()):
for elem in rec:
# Get the prefix
if (first == True):
old_time = elem.time
first = False
pfx = elem.fields["prefix"]
# Get the list of ASes in the AS path
# print(elem)
if (elem.type == 'A'):
ases = elem.fields["as-path"].split(" ")
len_path = len(ases) # AS-PATH len
if len_path > 0:
# Get the origin ASn (rightmost)
origin = ases[-1]
if (origin == VICTIM_AS):
own_ann_num[old_time] += 1
if pfx not in victim_prefix:
for father_pfx in victim_prefix: # if it's the new_subprefix
if self.is_sub_pfx(father_pfx, pfx):
new_sub_prefix_num[old_time] += 1
new_sub_prefix[old_time].add(pfx)
break
victim_prefix.add(pfx)
peer = ases[0]
if peer not in new_AS_Path.keys():
peer_num += 1
peer_increase[old_time] += 1
new_AS_Path[peer] = ases
path_str = 'len_path' + str(len_path)
PL[old_time][path_str] += 1
if (len_path > MPL[old_time]):
MPL[old_time] = len_path
else:
if ases != new_AS_Path[
peer]: # if path change, calculate it's edit distance
Diff_Ann[old_time] += 1
old_AS_Path[peer] = new_AS_Path[peer]
new_AS_Path[peer] = ases
num, len_cut = self.edit_distance(
old_AS_Path[peer],
new_AS_Path[peer])
if (len_cut > 0):
ann_shorter_str = 'ann_shorter_' + str(
len_cut)
ann_shorter[old_time][
ann_shorter_str] += 1
else:
ann_longer_str = 'ann_longer_' + str(
-len_cut)
ann_longer[old_time][
ann_longer_str] += 1
diff_num = 'diff_' + str(num)
# diff_peer = 'diff_peer_' + str(peer)
diff_AS_Path_num[old_time][
diff_num] += 1
# diff_AS_Path[old_time][diff_peer] = num
path_str = 'len_path' + str(len_path)
PL[old_time][path_str] += 1
if (len_path > MPL[elem.time]):
MPL[old_time] = len_path
else:
duplicate_ann[old_time] += 1
# print(elem.fields["as-path"])
# print(pfx)
# Insert the origin ASn in the set of
# origins for the prefix
else:
if pfx in victim_prefix:
MOAS_ann_num[old_time] += 1
if origin not in MOAS:
new_MOAS[old_time] += 1
MOAS[old_time].add(origin)
MOAS_AS[old_time][pfx].add(origin)
elif (elem.type == 'W'):
if pfx in victim_prefix:
withdraw_num[old_time] += 1
withdraw_unique_prefix[old_time].add(pfx)
if (elem.time >= (old_time + 30 * Window_size)):
if (abs(old_time - start_time) < 30): # label our date
labels[old_time]['label_1'] = label
event_after = True
# print(abs(old_time-start_time))
if event_after:
labels[old_time]['label_0'] = label
if (count_2 > 0):
labels[old_time]['label_2'] = label
count_2 -= 1
if (count_3 > 0):
labels[old_time]['label_3'] = label
count_3 -= 1
else:
labels[old_time]['label_0'] = 0
labels[old_time]['label_1'] = 0
labels[old_time]['label_2'] = 0
labels[old_time]['label_3'] = 0
df = pd.DataFrame(
{
'time':
pd.to_datetime(old_time, unit='s'),
'MPL':
MPL[old_time],
'MOAS_prefix_num':
len(MOAS_AS[old_time]),
'MOAS_AS': [MOAS_AS[old_time]],
'MOAS': [MOAS[old_time]],
'new_MOAS':
new_MOAS[old_time],
'MOAS_num':
len(MOAS[old_time]),
'withdraw_num':
withdraw_num[old_time],
'peer_increase':
peer_increase[old_time],
'peer_num':
peer_num,
'new_prefix_num':
new_sub_prefix_num[old_time],
'MOAS_Ann_num':
MOAS_ann_num[old_time],
'own_Ann_num':
own_ann_num[old_time],
'new_sub_prefix': [new_sub_prefix[old_time]],
'Victim_AS':
VICTIM_AS,
'Diff_Ann':
Diff_Ann[old_time],
'duplicate_ann':
duplicate_ann[old_time],
'withdraw_unique_prefix_num':
len(withdraw_unique_prefix[old_time]),
'withdraw_unique_prefix':
[withdraw_unique_prefix[old_time]],
},
index=[old_time])
d1 = pd.DataFrame(diff_AS_Path_num[old_time],
index=[old_time])
# d2=pd.DataFrame(diff_AS_Path[old_time],index=[old_time])
d2 = pd.DataFrame(labels[old_time], index=[old_time])
d3 = pd.DataFrame(PL[old_time], index=[old_time])
d5 = pd.DataFrame(ann_shorter[old_time],
index=[old_time])
d6 = pd.DataFrame(ann_longer[old_time],
index=[old_time])
# df2=pd.concat([d1,d3],axis=1)
d4 = pd.concat([df, d1, d2, d3, d5, d6], axis=1)
print(d4)
features = features.append(d4)
old_time = elem.time
print(features)
print(features['label_0'])
# print(victim_prefix)
# Print the list of MOAS prefix and their origin ASns
for pfx in victim_prefix:
print('victim_prefix', pfx)
for time in MOAS_AS:
for pfx in MOAS_AS[time]:
print((time, pfx, ','.join(MOAS_AS[time][pfx])))
for time in MPL:
print('MPL', MPL[time])
for time in PL:
print('path_legth_num', PL[time])
for time in diff_AS_Path:
print('edit_distance', diff_AS_Path[time])
all_features = all_features.append((features))
features.to_csv('../datasets/features' + VICTIM_AS + '_' +
str(begin_time) + '.csv')
network = str(IPNetwork(ipAddr).network)
return network + '/' + str(subnetmask)
#---------------------------------------------------------
COLLECTOR = 'route-views.isc'
#---------------------------------------------------------
# Previous RIB stream
TIME_INIT = "2008-02-24 18"
TIME_END = "2008-02-24 18:15"
previous_stream = pybgpstream.BGPStream(from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s" %
(COLLECTOR))
#---------------------------------------------------------
# Current RIB stream
TIME_INIT = "2008-02-24 20"
TIME_END = "2008-02-24 20:15"
current_stream = pybgpstream.BGPStream(from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s" %
(COLLECTOR))
#---------------------------------------------------------
# Post RIB stream
def main(date, ixp, dst, subfolder, delegated_src):
if date == '00000000':
date = datetime.today().strftime('%Y%m%d')
if not delegated_src:
url = "https://ftp.ripe.net/pub/stats/ripencc/nro-stats/{d}/combined-stat".format(
d=date)
delpath = "{dir}/delegated-{date}.csv".format(dir=dst, date=date)
print("* Downloading delegated from {url}".format(url=url))
urllib.request.urlretrieve(url, delpath)
print("- DONE!")
else:
delpath = "{dir}/delegated-{date}.csv".format(dir=delegated_src,
date=date)
year = date[0:4]
month = date[4:6]
day = date[6:8]
catalog = ResourceCatalog()
catalog.load_delegated(delpath)
if subfolder:
outfile = "{dir}/{ixp}/bgp-table-{ixp}-{date}.csv".format(dir=dst,
ixp=ixp,
date=date)
os.makedirs(os.path.dirname(outfile), exist_ok=True)
else:
outfile = "{dir}/bgp-table-{ixp}-{date}.csv".format(dir=dst,
ixp=ixp,
date=date)
with open(os.path.join(sys.path[0],
'../ixp-data.json')) as json_file, open(
outfile, 'w', newline='') as fcsv:
writer = csv.writer(fcsv)
writer.writerow(["prefix", "prefix_cc", "as_path", "as_path_cc"])
ixpdata = json.load(json_file)
print("* {ixp} selected".format(ixp=ixp))
if ixp not in ixpdata:
raise Exception("IXP not found")
selected = ixpdata[ixp]
if selected['source'] == 'pch':
url = "https://www.pch.net/resources/Routing_Data/IPv%s_daily_snapshots/{year}/{month}/route-collector.{ixp}.pch.net/route-collector.{ixp}.pch.net-ipv%s_bgp_routes.{year}.{month}.{day}.gz"
process_pch(url.format(year=year, month=month, day=day, ixp=ixp),
'4', catalog, writer)
process_pch(url.format(year=year, month=month, day=day, ixp=ixp),
'6', catalog, writer)
elif selected['source'] == 'lacnic':
url = "https://ixpdata.labs.lacnic.net/raw-data/{path}/{y}/{m}/{d}/rib.{y}{m}{d}.{t}.bz2".format(
path=selected['path'],
y=year,
m=month,
d=day,
t=selected['time'])
stream = pybgpstream.BGPStream(data_interface="singlefile")
stream.set_data_interface_option("singlefile", "rib-file", url)
for rec in stream.records():
for elem in rec:
prefix = elem.fields["prefix"]
if '.' in prefix:
v = 'ipv4'
else:
v = 'ipv6'
path = getpath(elem.fields["as-path"])
if path != False:
cc_path = [catalog.get_asn(a) for a in path]
cc_prefix = catalog.get_pfx(v, prefix)
writer.writerow([
prefix, cc_prefix, " ".join(path),
" ".join(cc_path)
])
# #https://stackoverflow.com/questions/5574702/how-to-print-to-stderr-in-python
# #print(*args, file=sys.stderr, **kwargs) # python3
# print >>sys.stderr, s
from_day = datetime.strptime(c.start_ymd, "%Y%m%d")
hhmm = int(c.start_hhmm[0:2])*60 + int(c.start_hhmm[2:])
from_time = from_day + timedelta(seconds=hhmm)
print("from_time = %s (%s)" % ( from_time, type(from_time)))
until_time = from_time + timedelta(minutes=30*c.n_bins*c.n_days) # duration
print("until_time = %s" % until_time)
stream = pybgpstream.BGPStream(
from_time = "%s" % from_time,
until_time = "%s UTC" % until_time,
collectors=["route-views.sg", "route-views.eqix"],
record_type="ribs",
)
### stream.add_filter('routeviews')
ofn = c.bgp_fn
print("fn = %s" % ofn)
print("====================================")
timestamp = arrow.get(from_time).timestamp
# fudge by 12 minutes to allow for creating of ribs
now = time.time()
fudge_secs = 12*60
if now - timestamp < fudge_secs:
#!/usr/bin/env python
# Craig Tomkow
import pybgpstream
# 2019-11-23 00:00:00 UTC
bgp_stream = pybgpstream.BGPStream(
#from_time="2019-01-01 00:00:00 UTC",
#until_time="2019-01-01 00:05:00 UTC",
data_interface="singlefile"
#filter="prefix exact 128.189.0.0/16 and elemtype withdrawals"
)
for elem in bgp_stream:
#if elem.type == "W":
print(elem)
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
#
import pybgpstream
# configure the stream to retrieve Updates records from the RRC06 collector
# The commented out add_filter lines are the old way, the parse_filter_string
# way is the new method for applying filters
# Wed Apr 1 00:02:50 UTC 2015 -> Wed Apr 1 00:04:30
stream = pybgpstream.BGPStream(
filter='collector rrc06 and type updates',
from_time="2015-04-01 00:02:50",
until_time="2015-04-01 00:04:30",
)
# equivalent version:
# stream = pybgpstream.BGPStream(
# collector="rrc06",
# record_type="updates",
# from_time="2015-04-01 00:02:50",
# until_time="2015-04-01 00:04:30",
# )
# print the stream
for rec in stream.records():
print(rec.status, rec.project + "." + rec.collector, rec.time)
for elem in rec:
'''
IMPORTANT: Load first
export LD_LIBRARY_PATH=/usr/local/lib
'''
import pybgpstream
from netaddr import *
import numpy as np
TIME_INIT = "2018-04-24 11:00"
TIME_END = "2018-04-24 13:00"
COLLECTOR = 'route-views2'
AS1 = 10297
AS2 = 16509
streamAs1 = pybgpstream.BGPStream(
from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s and path %s" % (COLLECTOR, AS1))
streamAs2 = pybgpstream.BGPStream(
from_time=TIME_INIT,
until_time=TIME_END,
filter="type ribs and collector %s and path %s" % (COLLECTOR, AS2))
prefixAs1_set = set()
for elem in streamAs1:
prefixAs1_set.add(elem.fields["prefix"])
prefixAs2_set = set()
for elem in streamAs2:
prefixAs2_set.add(elem.fields["prefix"])
def remove_prepend(as_path):
return list(OrderedDict.fromkeys(as_path))
time_init = "2017-03-01"
time_end = "2017-03-01 00:01"
collector = 'route-views.saopaulo' # route-views.chicago
target_as = 262907 # 27747 (Telecentro)
stream = pybgpstream.BGPStream(
from_time=time_init,
until_time=time_end,
filter="type ribs and collector %s and path %s" % (
collector,
target_as
)
)
providers = set()
for elem in stream:
if int(elem.fields["as-path"].split(' ')[-1]) == target_as:
as_path = remove_prepend(elem.fields["as-path"].split(' '))
if len(as_path) > 1:
providers.add(as_path[-2]) # provider
pprint.pprint(providers)
# POSSIBILITY OF SUCH DAMAGE.
#
"""
This is a sample script that uses the high-level pybgpstream module.
You can use this script as a starting point when creating new applications that use PyBGPStream
"""
import pybgpstream
# create and configure the stream
stream = pybgpstream.BGPStream(
# from_time="2017-07-07 00:00:00",
# from_time=1499385600,
from_time=1499385600.0,
until_time="2017-07-07 00:10:00 UTC",
collectors=["route-views.sg", "route-views.eqix"],
record_type="updates",
filter="peer 11666 and prefix more 210.180.0.0/16"
)
# add any additional (or dynamic) filters
# e.g. from peer AS 11666 regarding the more-specifics of 210.180.0.0/16:
# stream.parse_filter_string("peer 11666 and prefix more 210.180.0.0/16")
# or using the old filter interface:
# stream.add_filter("peer-asn", "11666")
# stream.add_filter("prefix-more", "210.180.0.0/16")
# read elems
for elem in stream:
# record fields can be accessed directly from elem