def __init__(self, bid, bstart, blength,
energies=None,
photons=None,
events = None,
filename=None,
instrument="gbm",
fnyquist = 4096.0,
norm='leahy',
fluence = None,
epeak = None,
ttrig = None):
### set burst ID
self.bid = bid
### if photons and filename aren't given, then data comes from procdata file
if photons is None and not filename:
filename = "tte_bn" + str(bid) + "_procdata.dat"
Burst.__init__(self, bstart, blength,
energies,
photons,
events,
filename,
instrument,
fnyquist,
norm,
fluence = fluence,
epeak = epeak,
ttrig = ttrig)
return
def read_data(self, filename, filetype='ascii', det="combined"):
Burst.read_data(self, filename, type=filetype)
evt = self.photons[det]
self.photons = np.array([x.time for x in evt.photons])
self.energies = np.array([x.energy for x in evt.photons])
return
def party_burstiness(features, weeks, party, dic_tweets):
bt = Burst()
b_index = dict()
for f in features:
qtd_r = list()
qtd_d = list()
for w in range(1, weeks + 1):
party_week = party + "_" + str(w)
if party_week in dic_tweets:
qtd_r.append(dic_tweets[party_week][f])
qtd_d.append(sum(dic_tweets[party_week].values()))
else:
qtd_r.append(0)
qtd_d.append(0)
r = np.array(qtd_r)
d = np.array(qtd_d)
n = len(r)
q, d, r, p = bt.burst_detection(r, d, n, 2, 1, 2)
bursts = bt.enumerate_bursts(q, 'burstLabel')
b_index[f] = bursts
return b_index
def run_peer(queue, win_size, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, socket_rib_name, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo=False, nb_bits_aspath=33, \
run_encoding_threshold=1000000, global_rib_enabled=True, silent=False):
global peer_logger
import socket
try:
os.nice(-20)
except OSError:
peer_logger.info('Cannot change the nice.')
# Create the topologies for this peer
G = ASTopology(1, silent) # Main topology
G_W = ASTopology(
nb_withdrawals_burst_start,
silent) # Subset of the topology with the withdraws in the queue
# Current burst (if any)
current_burst = None
# Last time the peer wrote the rib and queue size in the log file
last_log_write = 0
# Socket connected to the global RIB
socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Exit properly when receiving SIGINT
def signal_handler(signal, frame):
if current_burst is not None:
current_burst.stop(bgp_msg.time)
socket.close()
peer_logger.info('Received SIGTERM. Exiting.')
sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
peer_id = None
peer_as = None
# Create the RIB for this peer
rib = RIBPeer()
encoding = None
# This function create and initialize the encoding
def init_encoding():
encoding = Encoding(peer_id,
G,
'encoding',
nb_bits_aspath,
5,
output=True)
encoding.compute_encoding()
peer_logger.info(
str(int(bgp_msg.time)) + '\t' + str(len(rib)) + '\t' +
str(len(W_queue)) + '\t' + 'Encoding computed!')
if global_rib_enabled:
for p in rib.rib:
send_fake_update(p, peer_ip, bgp_msg.time, rib, encoding,
socket)
return encoding
#A_queue = BGPMessagesQueue(win_size) # Queue of Updates
W_queue = BGPMessagesQueue(win_size) # Queue of Withdraws
last_ts = 0
while True:
while True:
bgp_msg = queue.get()
if bgp_msg is not None:
if peer_id is None:
peer_id = bgp_msg.peer_id
peer_as = bgp_msg.peer_as
peer_as_set = set()
last_ts = bgp_msg.time
peer_ip = peer_id.split('-')[-1]
peer_handler = logging.handlers.RotatingFileHandler(
log_dir + '/peer_' + str(peer_id),
maxBytes=200000000000000,
backupCount=5)
peer_handler.setFormatter(formatter)
peer_logger.removeHandler(handler)
peer_logger.addHandler(peer_handler)
peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
str(str(peer_as)) + ')_started.')
if bgp_msg.as_path is not None and len(
bgp_msg.as_path) > 0:
if peer_as != bgp_msg.as_path[0]:
peer_logger.warning(
'Peer AS ' + str(peer_as) + ' and first AS ' +
str(bgp_msg.as_path[0]) +
' in AS path does not match. Setting first AS as peer AS.'
)
peer_as = bgp_msg.as_path[0]
# Make the connection with the global RIB
rib_global_socket_address = '/tmp/' + socket_rib_name
socket.connect(rib_global_socket_address)
peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
str(str(peer_as)) +
') connected with the global RIB.')
if peer_id != bgp_msg.peer_id:
peer_logger.critical(
'Received a bgp_message with peer_id: ' +
str(bgp_msg.peer_id))
if bgp_msg.mtype == 'A':
# Update the set set of peer_as (useful when doing the naive solution)
if len(bgp_msg.as_path) > 0:
peer_as_set.add(bgp_msg.as_path[0])
# Update the RIB for this peer
old_as_path = rib.update(bgp_msg)
# Remove the old as-path in the main graph for this prefix
G.remove(old_as_path, bgp_msg.prefix)
# Add the new as-path in both the main graph and the graph of advertisments
G.add(bgp_msg.as_path, bgp_msg.prefix)
# Update the encoding, and send the fake advertisement to the global RIB
if encoding is not None:
encoding.advertisement(old_as_path, bgp_msg.as_path)
if global_rib_enabled:
send_fake_update(bgp_msg.prefix, peer_ip,
bgp_msg.time, rib, encoding,
socket)
elif len(rib.rib) > run_encoding_threshold:
encoding = init_encoding()
elif bgp_msg.mtype == 'W':
# Create the encoding if not done yet
if encoding is None:
encoding = init_encoding()
# Update the RIB for this peer
bgp_msg.as_path = rib.withdraw(bgp_msg)
# Remove the old as-path in the main graph for this prefix
G.remove(bgp_msg.as_path, bgp_msg.prefix)
# Add the withdrawn as-path in the graph of withdraws
G_W.add(bgp_msg.as_path)
# Update the queue of withdraws
if bgp_msg.as_path != []:
W_queue.append(bgp_msg)
# Update the encoding
encoding.withdraw(bgp_msg.as_path)
# Send the withdrawal to the global RIB
if global_rib_enabled:
send_fake_update(bgp_msg.prefix, peer_ip, bgp_msg.time,
None, None, socket)
elif bgp_msg.mtype == 'CLOSE':
# CLOSE this peer. Clear all the topologies, ribs, queues, bursts, etc
if current_burst is not None:
best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
peer_as_set)
current_burst.fd_predicted.write(
'PREDICTION_END_CLOSE|' + bpa_algo + '|' +
str(len(current_burst)) + '|' +
str(best_fm_score) + '|' + str(best_TP) + '|' +
str(best_FN) + '|' + str(best_FP) + '\n')
current_burst.fd_predicted.write(
'PREDICTION_END_EDGE|')
res = ''
depth = 9999999999
for e in best_edge_set:
depth = min(G_W.get_depth(e[0], e[1]), depth)
res += str(e[0]) + '-' + str(e[1]) + ','
current_burst.fd_predicted.write(res[:len(res) - 1] +
'|' + str(depth) +
'\n')
#G_W.draw_graph(peer_as)
current_burst.stop(bgp_msg.time)
current_burst = None
# Withdraw all the routes advertised by this peer
if global_rib_enabled:
for p in rib.rib:
send_fake_update(p, peer_ip, -1, None, None,
socket)
peer_logger.info('Received CLOSE. CLEANING the peer.')
# Stop this peer
os.kill(os.getpid(), signal.SIGTERM)
else:
peer_logger.info(bgp_msg)
# Make sure to compute start en end time of burst with a second granularity (only if ther is a burst)
if current_burst is not None:
while (last_ts != bgp_msg.time):
last_ts += 1
# Update the graph of withdraws
for w in W_queue.refresh_iter(last_ts):
current_burst.deleted_from_W_queue.append(w)
# Remove the current burst (if any) if it the size of the withdraws is lower than w_threshold (meaning it has finished)
if len(
W_queue
) < nb_withdrawals_burst_end: #current_burst.is_expired(bgp_msg.time):
# Execute BPA at the end of the burst if the burst is large enough
best_edge_set, best_fm_score, best_TP, best_FN, best_FP = burst_prediction(
current_burst, G, G_W, W_queue, p_w, r_w,
bpa_algo, peer_as_set)
current_burst.fd_predicted.write(
'PREDICTION_END|' + bpa_algo + '|' +
str(len(current_burst)) + '|' +
str(best_fm_score) + '|' + str(best_TP) + '|' +
str(best_FN) + '|' + str(best_FP) + '\n')
current_burst.fd_predicted.write(
'PREDICTION_END_EDGE|')
# Print some information about the prediction on the prediction file
res = ''
depth = 9999999999
for e in best_edge_set:
res += str(e[0]) + '-' + str(e[1]) + ','
depth = min(G_W.get_depth(e[0], e[1]), depth)
current_burst.fd_predicted.write(res[:len(res) -
1] + '|' +
str(depth) + '\n')
#G_W.draw_graph(peer_as, G, current_burst, outfile='as_graph_'+str(current_burst.start_time)+'.dot', threshold=500)
# Update the graph of withdrawals
for w in current_burst.deleted_from_W_queue:
G_W.remove(w.as_path)
current_burst.stop(bgp_msg.time)
current_burst = None
break
else:
current_burst.last_ts = last_ts
# Update the graph of withdraws.
if current_burst is None:
for w in W_queue.refresh_iter(bgp_msg.time):
G_W.remove(w.as_path)
# Update the last timestamp seen
last_ts = bgp_msg.time
# Add the updates in the real prefixes set of the burst, if any
if current_burst is not None: #and not silent:
if bgp_msg.as_path != []:
old_as_path = bgp_msg.as_path if bgp_msg.mtype == 'W' else old_as_path
current_burst.add_real_prefix(bgp_msg.time,
bgp_msg.prefix,
bgp_msg.mtype,
old_as_path)
# If we are not in the burst yet, we create the burst
if current_burst is None and len(
W_queue) >= nb_withdrawals_burst_start:
current_burst = Burst(peer_id, bgp_msg.time, win_size,
burst_outdir, encoding, W_queue,
silent)
next_bpa_execution = min_bpa_burst_size
# Print some log ...
if (bgp_msg.time > last_log_write
) or bgp_msg.time - last_log_write >= 3600:
peer_logger.info(
str(int(bgp_msg.time)) + '\t' + str(len(rib)) + '\t' +
str(len(W_queue)))
last_log_write = bgp_msg.time
# Execute BPA if there is a burst and
# i) the current burst is greater than the minimum required
# ii) we have wait the number of withdrawals required per cycle or the queue is empty
if current_burst is not None:
total_current_burst_size = len(
current_burst) + nb_withdrawals_burst_start
if total_current_burst_size >= min_bpa_burst_size and total_current_burst_size > next_bpa_execution: #\
if nb_withdraws_per_cycle > 0 and total_current_burst_size < 12505:
next_bpa_execution += nb_withdraws_per_cycle
else:
next_bpa_execution = 999999999999
break
#print ('Queue size: '+str(len(rib))+'\t'+str(len(W_queue))+'\t'+str(len(current_burst)+nb_withdrawals_burst_start))
if current_burst is not None:
# Compute the set of edges with the highest FM score
best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
peer_as_set)
# Load that set in the burst
if not silent:
burst_add_edge(current_burst, rib, encoding, bgp_msg.time,
best_edge_set, G, G_W, W_queue, silent)
# Inform the global RIB about the set of failed links
for e in best_edge_set:
depth_set = set()
if G_W.has_edge(e[0], e[1]):
depth_set = depth_set.union(
G_W[e[0]][e[1]]['depth'].keys())
if G.has_edge(e[0], e[1]):
depth_set = depth_set.union(G[e[0]][e[1]]['depth'].keys())
for d in depth_set:
if encoding.is_encoded(d, e[0], e[1]):
vmac_partial = ''
bitmask_partial = ''
for i in range(2, encoding.max_depth + 2):
if i == d:
vmac_partial += encoding.mapping[
i].get_mapping_string(e[0])
bitmask_partial += '1' * encoding.mapping[
i].nb_bytes
elif i == d + 1:
vmac_partial += encoding.mapping[
i].get_mapping_string(e[1])
bitmask_partial += '1' * encoding.mapping[
i].nb_bytes
else:
if i in encoding.mapping:
vmac_partial += '0' * encoding.mapping[
i].nb_bytes
bitmask_partial += '0' * encoding.mapping[
i].nb_bytes
if global_rib_enabled:
socket.send('FR|' + peer_ip + '|' + vmac_partial +
'|' + bitmask_partial + '|' + str(d) +
'|' + str(last_ts) + '\n')
# Print information about the perdiction in the predicted file
current_burst.fd_predicted.write('PREDICTION|' + bpa_algo + '|' +
str(len(current_burst)) + '|' +
str(best_fm_score) + '|' +
str(best_TP) + '|' +
str(best_FP) + '|' +
str(best_FN) + '\n')
current_burst.fd_predicted.write('PREDICTION_EDGE|')
res = ''
depth = 9999999999
for e in best_edge_set:
depth = min(G_W.get_depth(e[0], e[1]), depth)
res += str(e[0]) + '-' + str(e[1]) + ','
current_burst.fd_predicted.write(res[:len(res) - 1] + '|' +
str(depth) + '\n')
from burst import Burst import numpy as np import matplotlib.pyplot as plt import sys import numpy.ma as ma # 00143: Norris Fit out_file = '/home/thomas/git/MSDataSci/Thesis/out_files/00143_b536_3_0.out' burst_file = './sample_data/cat64ms.00143' burst = Burst(burst_file) burst.parse_batse_file() data = burst.burst_data burst_meta = burst.header_data four_channel_data = np.sum(data, axis=1) trig = burst_meta['trig#'] nlasc = burst_meta['nlasc'] npts = burst_meta['npts'] bg_height = 1047.2220 bg_slope = 0.025935641 start = 44.965676 ampl = 2823.6814 tau1 = 8.1735936 tau2 = 2.2154609 taupk = 48.006471 time = (np.linspace((-nlasc), (npts - nlasc), num=npts) * 0.064) - 2
logger.setLevel(logging.DEBUG)
# 配置得放到config.from_object前面
NAME = 'normal_demo'
# ADMIN_ADDRESS = ('127.0.0.1', 7778)
GROUP_CONFIG = {
1: {
'count': 2,
},
10: {
'count': 2,
},
}
GROUP_ROUTER = lambda box: 1 if box.cmd == 1 else 10
app = Burst()
app.config.from_object(__name__)
@app.create_worker
def create_worker(worker):
logger.error('create_worker: %r', worker)
@app.before_first_request
def before_first_request(request):
logger.error('before_first_request')
@app.before_request
def before_request(request):
'/' + '-' * 80,
'[%(levelname)s][%(asctime)s][%(process)d:%(thread)d][%(filename)s:%(lineno)d %(funcName)s]:',
'%(message)s',
'-' * 80 + '/',
))
logger = logging.getLogger('burst')
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
GROUP_CONFIG = {
1: {
'count': 10,
},
}
app = Burst()
app.config.from_object(__name__)
@app.route(1)
def index(request):
return dict(
ret=10
)
app.run()
import mpfit # import numpy.oldnumeric as Numeric from burst import Burst import numpy as np import matplotlib.pyplot as plt import sys # 00143: Norris Fit out_file = '/home/thomas/git/MSDataSci/Thesis/out_files/00143_b536_3_0.out' burst_file = './sample_data/cat64ms.00143' burst = Burst(burst_file) burst.parse_batse_file() data = burst.burst_data burst_meta = burst.header_data four_channel_data = np.sum(data, axis=1) trig = burst_meta['trig#'] nlasc = burst_meta['nlasc'] npts = burst_meta['npts'] bg_height = 1047.2220 bg_slope = 0.025935641 start = 44.965676 ampl = 2823.6814 tau1 = 4.1735936 tau2 = 2.2154609 taupk = 48.006471
def pe_time_series_selection_to_sunburst_and_transaction_table(
figure, selectedData, time_resolution, time_span, data_store,
param_store):
"""Selecting specific points from the time series chart updates the
account burst and the detail labels. Reminder to self: When you
think selectedData input is broken, remember that unaltered
default action in the graph is to zoom, not to select.
Note: all of the necessary information is in figure but that
doesn't seem to trigger reliably. Adding selectedData as a second
Input causes reliable triggering.
"""
params: Params() = Params.from_json(param_store)
preventupdate_if_empty(data_store)
trans, atree, eras, dstore = Datastore.get_parts(data_store,
params.pe_roots)
if not time_resolution:
time_resolution = params.init_time_res
if not time_span:
time_span = params.init_time_span
if len(trans) == 0:
app.logger.error(
"Tried to make burst figure from transactions, but no transactions provided."
)
raise PreventUpdate()
unit = params.unit
ts_label = CONST["time_span_lookup"].get(time_span)["label"]
min_period_start: np.datetime64 = None
max_period_end: np.datetime64 = None
selected_accounts = []
selected_trans = pd.DataFrame()
desc_account_count = 0
colormap = {}
# Get the names and colors of all accounts in the Input figure.
# If anything is clicked, set the selection dates, accounts, and transactions.
if figure:
for trace in figure.get("data"):
account = trace.get("name")
points = trace.get("selectedpoints")
colormap[account] = trace.get("marker").get("color")
if not points:
continue
selected_accounts.append(account)
for point in points:
point_x = trace["x"][point]
period_start, period_end = period_to_date_range(
time_resolution, point_x, eras)
if min_period_start is None:
min_period_start = period_start
else:
min_period_start = min(min_period_start, period_start)
if max_period_end is None:
max_period_end = period_end
else:
max_period_end = max(max_period_end, period_end)
desc_accounts = atree.get_descendent_ids(account)
desc_account_count = desc_account_count + len(desc_accounts)
subtree_accounts = [account] + desc_accounts
new_trans = (trans.loc[trans["account"].isin(
subtree_accounts)].loc[trans["date"] >= period_start].loc[
trans["date"] <= period_end])
new_trans = Ledger.positize(
new_trans) # each top-level account should net positive
if len(selected_trans) > 0:
selected_trans = selected_trans.append(new_trans)
else:
selected_trans = new_trans
selected_count = len(selected_trans)
if selected_count > 0 and len(selected_accounts) > 0:
# If there are selected data, describe the contents of the sunburst
# TODO: desc_account_count is still wrong.
description = Burst.pretty_account_label(
selected_accounts,
desc_account_count,
selected_count,
)
else:
# If no trans are selected, show everything. Note that we
# could logically get here even if valid accounts are
# selected, in which case it would be confusing to get back
# all trans instead of none, but this should never happen haha
# because any clickable bar must have $$, and so, trans
description = f"Click a bar in the graph to filter from {len(trans):,d} records"
selected_trans = trans
min_period_start = trans["date"].min()
max_period_end = trans["date"].max()
title = f"{ts_label} {unit} from {pretty_date(min_period_start)} to {pretty_date(max_period_end)}"
pe_selection_store = {
"start": min_period_start,
"end": max_period_end,
"count": len(selected_trans),
"accounts": selected_accounts,
}
duration = round(
pd.to_timedelta((max_period_end - min_period_start), unit="ms") /
np.timedelta64(1, "M"))
factor = Ledger.prorate_factor(time_span, duration=duration)
try:
sun_fig = Burst.from_trans(atree, selected_trans, time_span, unit,
factor, colormap, title)
except LError as E:
text = f"Failed to generate sunburst. Error: {E}"
app.logger.warning(text)
description = text
return (sun_fig, title, description, pe_selection_store)
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
# 配置得放到config.from_object前面
NAME = "normal_demo"
# ADMIN_ADDRESS = ('127.0.0.1', 7778)
GROUP_CONFIG = {1: {"count": 2}, 10: {"count": 2}}
GROUP_ROUTER = lambda box: 1 if box.cmd == 1 else 10
PROXY_CLIENT_TIMEOUT = 60
# PROXY_MSG_QUEUE_MAX_SIZE = 1
STOP_TIMEOUT = 10
# WORKER_AUTO_RSP_ON_FAIL = False
app = Burst()
app.config.from_object(__name__)
@app.create_worker
def create_worker(worker):
logger.error("create_worker: %r", worker)
@app.before_first_request
def before_first_request(request):
logger.error("before_first_request")
@app.before_request
def before_request(request):
LOG_FORMAT = '\n'.join((
'/' + '-' * 80,
'[%(levelname)s][%(asctime)s][%(process)d:%(thread)d][%(filename)s:%(lineno)d %(funcName)s]:',
'%(message)s',
'-' * 80 + '/',
))
logger = logging.getLogger('burst')
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
GROUP_CONFIG = {
1: {
'count': 10,
},
}
app = Burst()
app.config.from_object(__name__)
@app.route(1)
def index(request):
return dict(ret=10)
app.run()
def run_peer(logger, queue_server_peer, queue_peer_server, FR_queue, win_size, peer_id, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, max_depth, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo='bpa-multiple', nb_bits_aspath=12, \
run_encoding_threshold=1000000, silent=False):
import socket
try:
os.nice(-20)
except OSError:
#peer_logger.info('Cannot change the nice.')
print "nochangenice"
# Create the topologies for this peer
G = ASTopology(1, silent) # Main topology
G_W = ASTopology(
nb_withdrawals_burst_start,
silent) # Subset of the topology with the withdraws in the queue
# Current burst (if any)
current_burst = None
# Last time the peer wrote the rib and queue size in the log file
last_log_write = 0
# Socket connected to the global RIB
#socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Exit properly when receiving SIGINT
#def signal_handler(signal, frame):
#if current_burst is not None:
#current_burst.stop(bgp_msg.time)
#socket.close()
#peer_logger.info('Received SIGTERM. Exiting.')
#sys.exit(0)
#signal.signal(signal.SIGTERM, signal_handler)
peer_id = peer_id
peer_as = None
peer_as_set = None
# Create the RIB for this peer
rib = RIBPeer()
encoding = None
# This function create and initialize the encoding
def init_encoding():
encoding = Encoding(peer_id,
G,
'encoding',
nb_bits_aspath,
5,
max_depth,
output=True)
encoding.compute_encoding()
peer_logger.info(
str(int(bgp_msg['time'])) + '\t' + str(len(rib)) + '\t' +
str(len(W_queue)) + '\t' + 'Encoding computed!')
return encoding
#A_queue = BGPMessagesQueue(win_size) # Queue of Updates
W_queue = BGPMessagesQueue(win_size) # Queue of Withdraws
last_ts = 0
routes_without_as_path_encoding = []
peer_handler = logging.handlers.RotatingFileHandler(
LOG_DIRNAME + '/peer_' + str(peer_id),
maxBytes=200000000000000,
backupCount=5)
peer_handler.setFormatter(formatter)
peer_logger.removeHandler(handler)
peer_logger.addHandler(peer_handler)
peer_logger.info('Peer_' + str(peer_id) + '_(AS' + str(str(peer_as)) +
')_started.')
while True:
while True:
bgp_msg = queue_server_peer.get()
if bgp_msg is not None:
if 'announce' in bgp_msg:
prefix = bgp_msg['announce'].prefix
as_path = bgp_msg['announce'].as_path
#as_path = [65000] + as_path
# Update the set set of peer_as (useful when doing the naive solution)
#if len(as_path) > 0:
#peer_as_set.add(as_path[0])
# Update the RIB for this peer
old_as_path = rib.update(prefix, as_path)
# Remove the old as-path in the main graph for this prefix
G.remove(old_as_path, prefix)
# Add the new as-path in both the main graph and the graph of advertisments
G.add(as_path, prefix)
if encoding is not None:
encoding.advertisement(old_as_path, as_path)
bgp_msg['announce'] = add_as_path_encoding_to_route(
bgp_msg['announce'], rib, encoding)
elif len(rib.rib) > run_encoding_threshold:
encoding = init_encoding()
bgp_msg['announce'] = add_as_path_encoding_to_route(
bgp_msg['announce'], rib, encoding)
else:
routes_without_as_path_encoding.append(bgp_msg)
queue_peer_server.put(bgp_msg)
if 'withdraw' in bgp_msg:
prefix = bgp_msg['withdraw'].prefix
#Withdraws get sent directly to the route server
queue_peer_server.put(bgp_msg)
# Create the encoding if not done yet
if encoding is None:
encoding = init_encoding()
# Update the RIB for this peer
as_path = rib.withdraw(prefix)
# Remove the old as-path in the main graph for this prefix
G.remove(as_path, prefix)
# Add the withdrawn as-path in the graph of withdraws
G_W.add(as_path)
# Update the queue of withdraws
if as_path != []:
bgp_msg['withdraw'].as_path = as_path
W_queue.append(bgp_msg)
# Update the encoding
encoding.withdraw(as_path)
#elif'state' in bgp_msg['neighbor'] and bgp_msg['neighbor']['state'] == 'down':
#queue_peer_server.put(bgp_msg)
# CLOSE this peer. Clear all the topologies, ribs, queues, bursts, etc
#if current_burst is not None:
# best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo, peer_as_set)
#current_burst.fd_predicted.write('PREDICTION_END_CLOSE|'+bpa_algo+'|'+str(len(current_burst))+'|'+str(best_fm_score)+'|'+str(best_TP)+'|'+str(best_FN)+'|'+str(best_FP)+'\n')
#current_burst.fd_predicted.write('PREDICTION_END_EDGE|')
#res = ''
#depth = 9999999999
#for e in best_edge_set:
#depth = min(G_W.get_depth(e[0], e[1]), depth)
#res += str(e[0])+'-'+str(e[1])+','
#current_burst.fd_predicted.write(res[:len(res)-1]+'|'+str(depth)+'\n')
#G_W.draw_graph(peer_as)
#current_burst.stop(bgp_msg['time'])
#current_burst = None
# Withdraw all the routes advertised by this peer
#peer_logger.info('Received CLOSE. CLEANING the peer.')
# Stop this peer
#os.kill(os.getpid(), signal.SIGTERM)
#else:
#peer_logger.info(bgp_msg)
#Ceck for routes without encoding, check for encdoding, send modified routes to route_server
if len(routes_without_as_path_encoding) > 0:
if encoding is not None:
for unsent_bgp_msg in routes_without_as_path_encoding:
if 'withdraw' in bgp_msg:
if unsent_bgp_msg[
'announce'].prefix == bgp_msg[
'withdraw'].prefix:
routes_without_as_path_encoding.pop(
unsent_bgp_msg)
continue
unsent_bgp_msg[
'announce'] = add_as_path_encoding_to_route(
unsent_bgp_msg['announce'], rib, encoding)
queue_peer_server.put(unsent_bgp_msg)
routes_without_as_path_encoding = []
# Make sure to compute start en end time of burst with a second granularity (only if ther is a burst)
if current_burst is not None:
while (last_ts != bgp_msg['time']):
last_ts += 1
# Update the graph of withdraws
for w in W_queue.refresh_iter(last_ts):
current_burst.deleted_from_W_queue.append(w)
# Remove the current burst (if any) if it the size of the withdraws is lower than w_threshold (meaning it has finished)
if len(
W_queue
) < nb_withdrawals_burst_end: #current_burst.is_expired(bgp_msg.time):
# Execute BPA at the end of the burst if the burst is large enough
print "burst is over"
best_edge_set, best_fm_score, best_TP, best_FN, best_FP = burst_prediction(
current_burst, G, G_W, W_queue, p_w, r_w,
bpa_algo, peer_as_set)
current_burst.fd_predicted.write(
'PREDICTION_END|' + bpa_algo + '|' +
str(len(current_burst)) + '|' +
str(best_fm_score) + '|' + str(best_TP) + '|' +
str(best_FN) + '|' + str(best_FP) + '\n')
current_burst.fd_predicted.write(
'PREDICTION_END_EDGE|')
# Print some information about the prediction on the prediction file
res = ''
depth = 9999999999
for e in best_edge_set:
res += str(e[0]) + '-' + str(e[1]) + ','
depth = min(G_W.get_depth(e[0], e[1]), depth)
current_burst.fd_predicted.write(res[:len(res) -
1] + '|' +
str(depth) + '\n')
#G_W.draw_graph(peer_as, G, current_burst, outfile='as_graph_'+str(current_burst.start_time)+'.dot', threshold=500)
# Update the graph of withdrawals
for w in current_burst.deleted_from_W_queue:
G_W.remove(w['withdraw'].as_path)
current_burst.stop(bgp_msg['time'])
current_burst = None
break
else:
current_burst.last_ts = last_ts
# Update the graph of withdraws.)
if current_burst is None:
for w in W_queue.refresh_iter(bgp_msg['time']):
G_W.remove(w['withdraw'].as_path)
# Update the last timestamp seen
last_ts = bgp_msg['time']
# Add the updates in the real prefixes set of the burst, if any
if current_burst is not None: #and not silent:
if 'announce' in bgp_msg:
current_burst.add_real_prefix(
bgp_msg['time'], bgp_msg['announce'].prefix, 'A',
bgp_msg['announce'].as_path)
if 'withdraw' in bgp_msg:
current_burst.add_real_prefix(
bgp_msg['time'], bgp_msg['withdraw'].prefix, 'W',
bgp_msg['withdraw'].as_path)
# If we are not in the burst yet, we create the burst
if current_burst is None and len(
W_queue) >= nb_withdrawals_burst_start:
print "SWIFT STARTING BURST!!!"
burst_start_time = W_queue[100]['time'] if len(
W_queue) > 100 else W_queue[0]['time']
current_burst = Burst(peer_id, bgp_msg['time'], win_size,
burst_outdir, encoding,
burst_start_time, silent)
next_bpa_execution = min_bpa_burst_size
# Print some log ...
if (bgp_msg['time'] > last_log_write
) or bgp_msg['time'] - last_log_write >= 3600:
peer_logger.info(
str(int(bgp_msg['time'])) + '\t' + str(len(rib)) +
'\t' + str(len(W_queue)))
last_log_write = bgp_msg['time']
# Execute BPA if there is a burst and
# i) the current burst is greater than the minimum required
# ii) we have wait the number of withdrawals required per cycle or the queue is empty
if current_burst is not None:
total_current_burst_size = len(
current_burst) + nb_withdrawals_burst_start
if total_current_burst_size >= min_bpa_burst_size and total_current_burst_size > next_bpa_execution:
if nb_withdraws_per_cycle > 0 and total_current_burst_size < 12505:
next_bpa_execution += nb_withdraws_per_cycle
else:
next_bpa_execution = 999999999999
break
#print ('Queue size: '+str(len(rib))+'\t'+str(len(W_queue))+'\t'+str(len(current_burst)+nb_withdrawals_burst_start))
if current_burst is not None:
logger.info("Burst detected starting preditcion")
# Compute the set of edges with the highest FM score
best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
peer_as_set)
# Load that set in the burst
if not silent:
burst_add_edge(current_burst, rib, encoding, bgp_msg['time'],
best_edge_set, G, G_W, silent)
logger.info("prediction finished")
# Inform the global RIB about the set of failed links
for e in best_edge_set:
depth_set = set()
if G_W.has_edge(e[0], e[1]):
depth_set = depth_set.union(
G_W[e[0]][e[1]]['depth'].keys())
if G.has_edge(e[0], e[1]):
depth_set = depth_set.union(G[e[0]][e[1]]['depth'].keys())
for d in depth_set:
if encoding.is_encoded(d, e[0], e[1]):
vmac_partial = ''
bitmask_partial = ''
for i in range(2, encoding.max_depth + 2):
if i == d:
vmac_partial += encoding.mapping[
i].get_mapping_string(e[0])
bitmask_partial += '1' * encoding.mapping[
i].nb_bytes
elif i == d + 1:
vmac_partial += encoding.mapping[
i].get_mapping_string(e[1])
bitmask_partial += '1' * encoding.mapping[
i].nb_bytes
else:
if i in encoding.mapping:
vmac_partial += '0' * encoding.mapping[
i].nb_bytes
bitmask_partial += '0' * encoding.mapping[
i].nb_bytes
FR_message = {
'FR': {
'peer_id': peer_id,
'as_path_vmac': vmac_partial,
'as_path_bitmask': bitmask_partial,
'depth': d
}
}
FR_queue.put(FR_message)
print "FR_message:", FR_message, "best_edge", e
# Print information about the perdiction in the predicted file
current_burst.fd_predicted.write('PREDICTION|' + bpa_algo + '|' +
str(len(current_burst)) + '|' +
str(best_fm_score) + '|' +
str(best_TP) + '|' +
str(best_FP) + '|' +
str(best_FN) + '\n')
current_burst.fd_predicted.write('PREDICTION_EDGE|')
res = ''
depth = 9999999999
for e in best_edge_set:
depth = min(G_W.get_depth(e[0], e[1]), depth)
res += str(e[0]) + '-' + str(e[1]) + ','
current_burst.fd_predicted.write(res[:len(res) - 1] + '|' +
str(depth) + '\n')
def run_peer_bpavalidation(queue, win_size, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo=False, nb_bits_aspath=33, \
run_encoding_threshold=1000000, global_rib_enabled=True, silent=False):
import socket
try:
os.nice(-20)
except OSError:
peer_logger.info('Cannot change the nice.')
# Last time the peer wrote the rib and queue size in the log file
last_log_write = 0
# Socket connected to the global RIB
socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
# Current burst (if any)
current_burst = None
# Exit properly when receiving SIGINT
def signal_handler(signal, frame):
if current_burst is not None:
current_burst.stop(bgp_msg.time)
socket.close()
peer_logger.info('Received SIGTERM. Exiting.')
sys.exit(0)
signal.signal(signal.SIGTERM, signal_handler)
peer_id = None
peer_as = None
# Create the RIB for this peer
rib = RIBPeer()
U_queue = BGPMessagesQueue(
win_size) # Queue of Updates (advertisement or withdrawals)
last_ts = 0
while True:
while True:
bgp_msg = queue.get()
if bgp_msg is not None:
if peer_id is None:
peer_id = bgp_msg.peer_id
peer_as = bgp_msg.peer_as
peer_as_set = set()
last_ts = bgp_msg.time
last_log_write = bgp_msg.time
peer_ip = peer_id.split('-')[-1]
peer_handler = logging.handlers.RotatingFileHandler(
log_dir + '/peer_' + str(peer_id),
maxBytes=200000000000000,
backupCount=5)
peer_handler.setFormatter(formatter)
peer_logger.removeHandler(handler)
peer_logger.addHandler(peer_handler)
peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
str(str(peer_as)) + ')_started.')
if bgp_msg.as_path is not None and len(
bgp_msg.as_path) > 0:
if peer_as != bgp_msg.as_path[0]:
peer_logger.warning(
'Peer AS ' + str(peer_as) + ' and first AS ' +
str(bgp_msg.as_path[0]) +
' in AS path does not match. Setting first AS as peer AS.'
)
peer_as = bgp_msg.as_path[0]
if peer_id != bgp_msg.peer_id:
peer_logger.critical(
'Received a bgp_message with peer_id: ' +
str(bgp_msg.peer_id))
if bgp_msg.mtype == 'A':
# Update the RIB for this peer
bgp_msg.as_path = rib.update(bgp_msg)
# Update the queue of updates
if bgp_msg.as_path != []:
U_queue.append(bgp_msg)
elif bgp_msg.mtype == 'W':
# Update the RIB for this peer
bgp_msg.as_path = rib.withdraw(bgp_msg)
# Update the queue of withdraws
if bgp_msg.as_path != []:
U_queue.append(bgp_msg)
else:
peer_logger.info(bgp_msg)
# Print size of hte queue for each second
while last_log_write < bgp_msg.time:
# Refresh the queue
U_queue.refresh(last_log_write)
peer_logger.info(
str(int(last_log_write)) + ' ' + str(len(rib)) + ' ' +
str(len(U_queue)))
last_log_write += 1
# Stop the burst if the size of the queue is lower than the threshold
if current_burst is not None:
if len(U_queue) < nb_withdrawals_burst_end:
current_burst.stop(bgp_msg.time)
current_burst = None
else:
current_burst.add_real_prefix(bgp_msg.time,
bgp_msg.prefix,
bgp_msg.mtype,
bgp_msg.as_path)
# Create a burst if the size of the is higher than the threshold
if current_burst is None:
if len(U_queue) > nb_withdrawals_burst_start:
burst_start_time = U_queue[100].time if len(
U_queue) > 100 else U_queue[0].time
current_burst = Burst(peer_id, bgp_msg.time, win_size,
burst_outdir, burst_start_time,
silent)