コード例 #1
0
ファイル: gbmburst.py プロジェクト: dhuppenkothen/UTools
    def __init__(self, bid, bstart, blength,
                 energies=None,
                 photons=None,
                 events = None,
                 filename=None,
                 instrument="gbm",
                 fnyquist = 4096.0,
                 norm='leahy',
                 fluence = None,
                 epeak = None,
                 ttrig = None):

        ### set burst ID
        self.bid = bid

        ### if photons and filename aren't given, then data comes from procdata file
        if photons is None and not filename:
           filename = "tte_bn" + str(bid) + "_procdata.dat"
       
        Burst.__init__(self, bstart, blength,
                 energies,
                 photons,
                 events,
                 filename,
                 instrument,
                 fnyquist,
                 norm,
                 fluence = fluence,
                 epeak = epeak,
                 ttrig = ttrig)
        
        return
コード例 #2
0
ファイル: gbmburst.py プロジェクト: dhuppenkothen/UTools
    def read_data(self, filename, filetype='ascii', det="combined"):

        Burst.read_data(self, filename, type=filetype)

        evt = self.photons[det]
        self.photons = np.array([x.time for x in evt.photons])
        self.energies = np.array([x.energy for x in evt.photons])
        return
コード例 #3
0
ファイル: metrics.py プロジェクト: lucasant10/Twitter
def party_burstiness(features, weeks, party, dic_tweets):
    bt = Burst()
    b_index = dict()
    for f in features:
        qtd_r = list()
        qtd_d = list()
        for w in range(1, weeks + 1):
            party_week = party + "_" + str(w)
            if party_week in dic_tweets:
                qtd_r.append(dic_tweets[party_week][f])
                qtd_d.append(sum(dic_tweets[party_week].values()))
            else:
                qtd_r.append(0)
                qtd_d.append(0)
        r = np.array(qtd_r)
        d = np.array(qtd_d)
        n = len(r)
        q, d, r, p = bt.burst_detection(r, d, n, 2, 1, 2)
        bursts = bt.enumerate_bursts(q, 'burstLabel')
        b_index[f] = bursts
    return b_index
コード例 #4
0
ファイル: peer.py プロジェクト: chuzirui/swift
def run_peer(queue, win_size, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, socket_rib_name, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo=False, nb_bits_aspath=33, \
run_encoding_threshold=1000000, global_rib_enabled=True, silent=False):

    global peer_logger

    import socket

    try:
        os.nice(-20)
    except OSError:
        peer_logger.info('Cannot change the nice.')

    # Create the topologies for this peer
    G = ASTopology(1, silent)  # Main topology
    G_W = ASTopology(
        nb_withdrawals_burst_start,
        silent)  # Subset of the topology with the withdraws in the queue

    # Current burst (if any)
    current_burst = None

    # Last time the peer wrote the rib and queue size in the log file
    last_log_write = 0

    # Socket connected to the global RIB
    socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

    # Exit properly when receiving SIGINT
    def signal_handler(signal, frame):
        if current_burst is not None:
            current_burst.stop(bgp_msg.time)

        socket.close()

        peer_logger.info('Received SIGTERM. Exiting.')

        sys.exit(0)

    signal.signal(signal.SIGTERM, signal_handler)

    peer_id = None
    peer_as = None

    # Create the RIB for this peer
    rib = RIBPeer()

    encoding = None

    # This function create and initialize the encoding
    def init_encoding():
        encoding = Encoding(peer_id,
                            G,
                            'encoding',
                            nb_bits_aspath,
                            5,
                            output=True)
        encoding.compute_encoding()
        peer_logger.info(
            str(int(bgp_msg.time)) + '\t' + str(len(rib)) + '\t' +
            str(len(W_queue)) + '\t' + 'Encoding computed!')

        if global_rib_enabled:
            for p in rib.rib:
                send_fake_update(p, peer_ip, bgp_msg.time, rib, encoding,
                                 socket)

        return encoding

    #A_queue = BGPMessagesQueue(win_size) # Queue of Updates
    W_queue = BGPMessagesQueue(win_size)  # Queue of Withdraws

    last_ts = 0

    while True:

        while True:
            bgp_msg = queue.get()

            if bgp_msg is not None:
                if peer_id is None:
                    peer_id = bgp_msg.peer_id
                    peer_as = bgp_msg.peer_as
                    peer_as_set = set()
                    last_ts = bgp_msg.time
                    peer_ip = peer_id.split('-')[-1]

                    peer_handler = logging.handlers.RotatingFileHandler(
                        log_dir + '/peer_' + str(peer_id),
                        maxBytes=200000000000000,
                        backupCount=5)
                    peer_handler.setFormatter(formatter)
                    peer_logger.removeHandler(handler)
                    peer_logger.addHandler(peer_handler)

                    peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
                                     str(str(peer_as)) + ')_started.')

                    if bgp_msg.as_path is not None and len(
                            bgp_msg.as_path) > 0:
                        if peer_as != bgp_msg.as_path[0]:
                            peer_logger.warning(
                                'Peer AS ' + str(peer_as) + ' and first AS ' +
                                str(bgp_msg.as_path[0]) +
                                ' in AS path does not match. Setting first AS as peer AS.'
                            )
                            peer_as = bgp_msg.as_path[0]

                    # Make the connection with the global RIB
                    rib_global_socket_address = '/tmp/' + socket_rib_name
                    socket.connect(rib_global_socket_address)
                    peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
                                     str(str(peer_as)) +
                                     ') connected with the global RIB.')

                if peer_id != bgp_msg.peer_id:
                    peer_logger.critical(
                        'Received a bgp_message with peer_id: ' +
                        str(bgp_msg.peer_id))

                if bgp_msg.mtype == 'A':
                    # Update the set set of peer_as (useful when doing the naive solution)
                    if len(bgp_msg.as_path) > 0:
                        peer_as_set.add(bgp_msg.as_path[0])

                    # Update the RIB for this peer
                    old_as_path = rib.update(bgp_msg)

                    # Remove the old as-path in the main graph for this prefix
                    G.remove(old_as_path, bgp_msg.prefix)

                    # Add the new as-path in both the main graph and the graph of advertisments
                    G.add(bgp_msg.as_path, bgp_msg.prefix)

                    # Update the encoding, and send the fake advertisement to the global RIB
                    if encoding is not None:
                        encoding.advertisement(old_as_path, bgp_msg.as_path)
                        if global_rib_enabled:
                            send_fake_update(bgp_msg.prefix, peer_ip,
                                             bgp_msg.time, rib, encoding,
                                             socket)
                    elif len(rib.rib) > run_encoding_threshold:
                        encoding = init_encoding()

                elif bgp_msg.mtype == 'W':
                    # Create the encoding if not done yet
                    if encoding is None:
                        encoding = init_encoding()

                    # Update the RIB for this peer
                    bgp_msg.as_path = rib.withdraw(bgp_msg)

                    # Remove the old as-path in the main graph for this prefix
                    G.remove(bgp_msg.as_path, bgp_msg.prefix)

                    # Add the withdrawn as-path in the graph of withdraws
                    G_W.add(bgp_msg.as_path)

                    # Update the queue of withdraws
                    if bgp_msg.as_path != []:
                        W_queue.append(bgp_msg)

                    # Update the encoding
                    encoding.withdraw(bgp_msg.as_path)

                    # Send the withdrawal to the global RIB
                    if global_rib_enabled:
                        send_fake_update(bgp_msg.prefix, peer_ip, bgp_msg.time,
                                         None, None, socket)

                elif bgp_msg.mtype == 'CLOSE':

                    # CLOSE this peer. Clear all the topologies, ribs, queues, bursts, etc
                    if current_burst is not None:
                        best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
                            current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
                            peer_as_set)
                        current_burst.fd_predicted.write(
                            'PREDICTION_END_CLOSE|' + bpa_algo + '|' +
                            str(len(current_burst)) + '|' +
                            str(best_fm_score) + '|' + str(best_TP) + '|' +
                            str(best_FN) + '|' + str(best_FP) + '\n')
                        current_burst.fd_predicted.write(
                            'PREDICTION_END_EDGE|')
                        res = ''
                        depth = 9999999999
                        for e in best_edge_set:
                            depth = min(G_W.get_depth(e[0], e[1]), depth)
                            res += str(e[0]) + '-' + str(e[1]) + ','

                        current_burst.fd_predicted.write(res[:len(res) - 1] +
                                                         '|' + str(depth) +
                                                         '\n')

                        #G_W.draw_graph(peer_as)

                        current_burst.stop(bgp_msg.time)
                        current_burst = None

                    # Withdraw all the routes advertised by this peer
                    if global_rib_enabled:
                        for p in rib.rib:
                            send_fake_update(p, peer_ip, -1, None, None,
                                             socket)

                    peer_logger.info('Received CLOSE. CLEANING the peer.')

                    # Stop this peer
                    os.kill(os.getpid(), signal.SIGTERM)
                else:
                    peer_logger.info(bgp_msg)

                # Make sure to compute start en end time of burst with a second granularity (only if ther is a burst)
                if current_burst is not None:
                    while (last_ts != bgp_msg.time):
                        last_ts += 1

                        # Update the graph of withdraws
                        for w in W_queue.refresh_iter(last_ts):
                            current_burst.deleted_from_W_queue.append(w)

                        # Remove the current burst (if any) if it the size of the withdraws is lower than w_threshold (meaning it has finished)
                        if len(
                                W_queue
                        ) < nb_withdrawals_burst_end:  #current_burst.is_expired(bgp_msg.time):
                            # Execute BPA at the end of the burst if the burst is large enough
                            best_edge_set, best_fm_score, best_TP, best_FN, best_FP = burst_prediction(
                                current_burst, G, G_W, W_queue, p_w, r_w,
                                bpa_algo, peer_as_set)
                            current_burst.fd_predicted.write(
                                'PREDICTION_END|' + bpa_algo + '|' +
                                str(len(current_burst)) + '|' +
                                str(best_fm_score) + '|' + str(best_TP) + '|' +
                                str(best_FN) + '|' + str(best_FP) + '\n')
                            current_burst.fd_predicted.write(
                                'PREDICTION_END_EDGE|')

                            # Print some information about the prediction on the prediction file
                            res = ''
                            depth = 9999999999
                            for e in best_edge_set:
                                res += str(e[0]) + '-' + str(e[1]) + ','
                                depth = min(G_W.get_depth(e[0], e[1]), depth)
                            current_burst.fd_predicted.write(res[:len(res) -
                                                                 1] + '|' +
                                                             str(depth) + '\n')

                            #G_W.draw_graph(peer_as, G, current_burst, outfile='as_graph_'+str(current_burst.start_time)+'.dot', threshold=500)

                            # Update the graph of withdrawals
                            for w in current_burst.deleted_from_W_queue:
                                G_W.remove(w.as_path)

                            current_burst.stop(bgp_msg.time)
                            current_burst = None
                            break
                        else:
                            current_burst.last_ts = last_ts

                # Update the graph of withdraws.
                if current_burst is None:
                    for w in W_queue.refresh_iter(bgp_msg.time):
                        G_W.remove(w.as_path)

                # Update the last timestamp seen
                last_ts = bgp_msg.time

                # Add the updates in the real prefixes set of the burst, if any
                if current_burst is not None:  #and not silent:
                    if bgp_msg.as_path != []:
                        old_as_path = bgp_msg.as_path if bgp_msg.mtype == 'W' else old_as_path
                        current_burst.add_real_prefix(bgp_msg.time,
                                                      bgp_msg.prefix,
                                                      bgp_msg.mtype,
                                                      old_as_path)

                # If we are not in the burst yet, we create the burst
                if current_burst is None and len(
                        W_queue) >= nb_withdrawals_burst_start:
                    current_burst = Burst(peer_id, bgp_msg.time, win_size,
                                          burst_outdir, encoding, W_queue,
                                          silent)
                    next_bpa_execution = min_bpa_burst_size

                # Print some log ...
                if (bgp_msg.time > last_log_write
                    ) or bgp_msg.time - last_log_write >= 3600:
                    peer_logger.info(
                        str(int(bgp_msg.time)) + '\t' + str(len(rib)) + '\t' +
                        str(len(W_queue)))
                    last_log_write = bgp_msg.time

                # Execute BPA if there is a burst and
                # i) the current burst is greater than the minimum required
                # ii) we have wait the number of withdrawals required per cycle or the queue is empty
                if current_burst is not None:
                    total_current_burst_size = len(
                        current_burst) + nb_withdrawals_burst_start
                    if total_current_burst_size >= min_bpa_burst_size and total_current_burst_size > next_bpa_execution:  #\
                        if nb_withdraws_per_cycle > 0 and total_current_burst_size < 12505:
                            next_bpa_execution += nb_withdraws_per_cycle
                        else:
                            next_bpa_execution = 999999999999
                        break

        #print ('Queue size: '+str(len(rib))+'\t'+str(len(W_queue))+'\t'+str(len(current_burst)+nb_withdrawals_burst_start))

        if current_burst is not None:

            # Compute the set of edges with the highest FM score
            best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
                current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
                peer_as_set)
            # Load that set in the burst
            if not silent:
                burst_add_edge(current_burst, rib, encoding, bgp_msg.time,
                               best_edge_set, G, G_W, W_queue, silent)

            # Inform the global RIB about the set of failed links
            for e in best_edge_set:
                depth_set = set()
                if G_W.has_edge(e[0], e[1]):
                    depth_set = depth_set.union(
                        G_W[e[0]][e[1]]['depth'].keys())
                if G.has_edge(e[0], e[1]):
                    depth_set = depth_set.union(G[e[0]][e[1]]['depth'].keys())

                for d in depth_set:
                    if encoding.is_encoded(d, e[0], e[1]):

                        vmac_partial = ''
                        bitmask_partial = ''

                        for i in range(2, encoding.max_depth + 2):
                            if i == d:
                                vmac_partial += encoding.mapping[
                                    i].get_mapping_string(e[0])
                                bitmask_partial += '1' * encoding.mapping[
                                    i].nb_bytes
                            elif i == d + 1:
                                vmac_partial += encoding.mapping[
                                    i].get_mapping_string(e[1])
                                bitmask_partial += '1' * encoding.mapping[
                                    i].nb_bytes
                            else:
                                if i in encoding.mapping:
                                    vmac_partial += '0' * encoding.mapping[
                                        i].nb_bytes
                                    bitmask_partial += '0' * encoding.mapping[
                                        i].nb_bytes

                        if global_rib_enabled:
                            socket.send('FR|' + peer_ip + '|' + vmac_partial +
                                        '|' + bitmask_partial + '|' + str(d) +
                                        '|' + str(last_ts) + '\n')

            # Print information about the perdiction in the predicted file
            current_burst.fd_predicted.write('PREDICTION|' + bpa_algo + '|' +
                                             str(len(current_burst)) + '|' +
                                             str(best_fm_score) + '|' +
                                             str(best_TP) + '|' +
                                             str(best_FP) + '|' +
                                             str(best_FN) + '\n')
            current_burst.fd_predicted.write('PREDICTION_EDGE|')
            res = ''
            depth = 9999999999
            for e in best_edge_set:
                depth = min(G_W.get_depth(e[0], e[1]), depth)
                res += str(e[0]) + '-' + str(e[1]) + ','
            current_burst.fd_predicted.write(res[:len(res) - 1] + '|' +
                                             str(depth) + '\n')
コード例 #5
0
from burst import Burst
import numpy as np
import matplotlib.pyplot as plt
import sys
import numpy.ma as ma

# 00143: Norris Fit
out_file = '/home/thomas/git/MSDataSci/Thesis/out_files/00143_b536_3_0.out'
burst_file = './sample_data/cat64ms.00143'

burst = Burst(burst_file)
burst.parse_batse_file()

data = burst.burst_data
burst_meta = burst.header_data

four_channel_data = np.sum(data, axis=1)
trig = burst_meta['trig#']
nlasc = burst_meta['nlasc']
npts = burst_meta['npts']

bg_height = 1047.2220
bg_slope = 0.025935641
start = 44.965676
ampl = 2823.6814
tau1 = 8.1735936
tau2 = 2.2154609
taupk = 48.006471

time = (np.linspace((-nlasc), (npts - nlasc), num=npts) * 0.064) - 2
コード例 #6
0
ファイル: server.py プロジェクト: xubingyue/burst
logger.setLevel(logging.DEBUG)

# 配置得放到config.from_object前面
NAME = 'normal_demo'
# ADMIN_ADDRESS = ('127.0.0.1', 7778)
GROUP_CONFIG = {
    1: {
        'count': 2,
    },
    10: {
        'count': 2,
    },
}
GROUP_ROUTER = lambda box: 1 if box.cmd == 1 else 10

app = Burst()
app.config.from_object(__name__)


@app.create_worker
def create_worker(worker):
    logger.error('create_worker: %r', worker)


@app.before_first_request
def before_first_request(request):
    logger.error('before_first_request')


@app.before_request
def before_request(request):
コード例 #7
0
ファイル: server.py プロジェクト: boyxiaolong/burst
    '/' + '-' * 80,
    '[%(levelname)s][%(asctime)s][%(process)d:%(thread)d][%(filename)s:%(lineno)d %(funcName)s]:',
    '%(message)s',
    '-' * 80 + '/',
))

logger = logging.getLogger('burst')
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)

GROUP_CONFIG = {
    1: {
        'count': 10,
    },
}

app = Burst()
app.config.from_object(__name__)


@app.route(1)
def index(request):
    return dict(
        ret=10
    )


app.run()
コード例 #8
0
import mpfit
# import numpy.oldnumeric as Numeric
from burst import Burst
import numpy as np
import matplotlib.pyplot as plt
import sys


# 00143: Norris Fit
out_file = '/home/thomas/git/MSDataSci/Thesis/out_files/00143_b536_3_0.out'
burst_file = './sample_data/cat64ms.00143'

burst = Burst(burst_file)
burst.parse_batse_file()

data       = burst.burst_data
burst_meta = burst.header_data 

four_channel_data = np.sum(data, axis=1)
trig  = burst_meta['trig#']
nlasc = burst_meta['nlasc']
npts  = burst_meta['npts']

bg_height = 1047.2220
bg_slope = 0.025935641
start = 44.965676
ampl = 2823.6814
tau1 = 4.1735936
tau2 = 2.2154609
taupk = 48.006471
コード例 #9
0
ファイル: periodic.py プロジェクト: saufrecht/ledger-explorer
def pe_time_series_selection_to_sunburst_and_transaction_table(
        figure, selectedData, time_resolution, time_span, data_store,
        param_store):
    """Selecting specific points from the time series chart updates the
    account burst and the detail labels.  Reminder to self: When you
    think selectedData input is broken, remember that unaltered
    default action in the graph is to zoom, not to select.

    Note: all of the necessary information is in figure but that
    doesn't seem to trigger reliably.  Adding selectedData as a second
    Input causes reliable triggering.

    """
    params: Params() = Params.from_json(param_store)
    preventupdate_if_empty(data_store)
    trans, atree, eras, dstore = Datastore.get_parts(data_store,
                                                     params.pe_roots)
    if not time_resolution:
        time_resolution = params.init_time_res
    if not time_span:
        time_span = params.init_time_span
    if len(trans) == 0:
        app.logger.error(
            "Tried to make burst figure from transactions, but no transactions provided."
        )
        raise PreventUpdate()
    unit = params.unit

    ts_label = CONST["time_span_lookup"].get(time_span)["label"]
    min_period_start: np.datetime64 = None
    max_period_end: np.datetime64 = None
    selected_accounts = []
    selected_trans = pd.DataFrame()
    desc_account_count = 0
    colormap = {}

    # Get the names and colors of all accounts in the Input figure.
    # If anything is clicked, set the selection dates, accounts, and transactions.
    if figure:
        for trace in figure.get("data"):
            account = trace.get("name")
            points = trace.get("selectedpoints")
            colormap[account] = trace.get("marker").get("color")
            if not points:
                continue
            selected_accounts.append(account)
            for point in points:
                point_x = trace["x"][point]
                period_start, period_end = period_to_date_range(
                    time_resolution, point_x, eras)
                if min_period_start is None:
                    min_period_start = period_start
                else:
                    min_period_start = min(min_period_start, period_start)
                if max_period_end is None:
                    max_period_end = period_end
                else:
                    max_period_end = max(max_period_end, period_end)
                desc_accounts = atree.get_descendent_ids(account)
                desc_account_count = desc_account_count + len(desc_accounts)
                subtree_accounts = [account] + desc_accounts
                new_trans = (trans.loc[trans["account"].isin(
                    subtree_accounts)].loc[trans["date"] >= period_start].loc[
                        trans["date"] <= period_end])
                new_trans = Ledger.positize(
                    new_trans)  # each top-level account should net positive
                if len(selected_trans) > 0:
                    selected_trans = selected_trans.append(new_trans)
                else:
                    selected_trans = new_trans
    selected_count = len(selected_trans)

    if selected_count > 0 and len(selected_accounts) > 0:
        # If there are selected data, describe the contents of the sunburst
        # TODO: desc_account_count is still wrong.
        description = Burst.pretty_account_label(
            selected_accounts,
            desc_account_count,
            selected_count,
        )
    else:
        # If no trans are selected, show everything.  Note that we
        # could logically get here even if valid accounts are
        # selected, in which case it would be confusing to get back
        # all trans instead of none, but this should never happen haha
        # because any clickable bar must have $$, and so, trans
        description = f"Click a bar in the graph to filter from {len(trans):,d} records"
        selected_trans = trans
        min_period_start = trans["date"].min()
        max_period_end = trans["date"].max()

    title = f"{ts_label} {unit} from {pretty_date(min_period_start)} to {pretty_date(max_period_end)}"
    pe_selection_store = {
        "start": min_period_start,
        "end": max_period_end,
        "count": len(selected_trans),
        "accounts": selected_accounts,
    }

    duration = round(
        pd.to_timedelta((max_period_end - min_period_start), unit="ms") /
        np.timedelta64(1, "M"))
    factor = Ledger.prorate_factor(time_span, duration=duration)
    try:
        sun_fig = Burst.from_trans(atree, selected_trans, time_span, unit,
                                   factor, colormap, title)
    except LError as E:
        text = f"Failed to generate sunburst.  Error: {E}"
        app.logger.warning(text)
        description = text

    return (sun_fig, title, description, pe_selection_store)
コード例 #10
0
ファイル: server.py プロジェクト: dantezhu/burst
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)


# 配置得放到config.from_object前面
NAME = "normal_demo"
# ADMIN_ADDRESS = ('127.0.0.1', 7778)
GROUP_CONFIG = {1: {"count": 2}, 10: {"count": 2}}
GROUP_ROUTER = lambda box: 1 if box.cmd == 1 else 10
PROXY_CLIENT_TIMEOUT = 60
# PROXY_MSG_QUEUE_MAX_SIZE = 1

STOP_TIMEOUT = 10
# WORKER_AUTO_RSP_ON_FAIL = False

app = Burst()
app.config.from_object(__name__)


@app.create_worker
def create_worker(worker):
    logger.error("create_worker: %r", worker)


@app.before_first_request
def before_first_request(request):
    logger.error("before_first_request")


@app.before_request
def before_request(request):
コード例 #11
0
ファイル: server.py プロジェクト: xubingyue/burst
LOG_FORMAT = '\n'.join((
    '/' + '-' * 80,
    '[%(levelname)s][%(asctime)s][%(process)d:%(thread)d][%(filename)s:%(lineno)d %(funcName)s]:',
    '%(message)s',
    '-' * 80 + '/',
))

logger = logging.getLogger('burst')
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter(LOG_FORMAT))
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)

GROUP_CONFIG = {
    1: {
        'count': 10,
    },
}

app = Burst()
app.config.from_object(__name__)


@app.route(1)
def index(request):
    return dict(ret=10)


app.run()
コード例 #12
0
def run_peer(logger, queue_server_peer, queue_peer_server, FR_queue, win_size, peer_id, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, max_depth, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo='bpa-multiple', nb_bits_aspath=12, \
run_encoding_threshold=1000000, silent=False):

    import socket

    try:
        os.nice(-20)
    except OSError:
        #peer_logger.info('Cannot change the nice.')
        print "nochangenice"

    # Create the topologies for this peer
    G = ASTopology(1, silent)  # Main topology
    G_W = ASTopology(
        nb_withdrawals_burst_start,
        silent)  # Subset of the topology with the withdraws in the queue

    # Current burst (if any)
    current_burst = None

    # Last time the peer wrote the rib and queue size in the log file
    last_log_write = 0

    # Socket connected to the global RIB
    #socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

    # Exit properly when receiving SIGINT
    #def signal_handler(signal, frame):
    #if current_burst is not None:
    #current_burst.stop(bgp_msg.time)

    #socket.close()

    #peer_logger.info('Received SIGTERM. Exiting.')

    #sys.exit(0)

    #signal.signal(signal.SIGTERM, signal_handler)

    peer_id = peer_id
    peer_as = None
    peer_as_set = None

    # Create the RIB for this peer
    rib = RIBPeer()

    encoding = None

    # This function create and initialize the encoding
    def init_encoding():
        encoding = Encoding(peer_id,
                            G,
                            'encoding',
                            nb_bits_aspath,
                            5,
                            max_depth,
                            output=True)
        encoding.compute_encoding()
        peer_logger.info(
            str(int(bgp_msg['time'])) + '\t' + str(len(rib)) + '\t' +
            str(len(W_queue)) + '\t' + 'Encoding computed!')

        return encoding

    #A_queue = BGPMessagesQueue(win_size) # Queue of Updates
    W_queue = BGPMessagesQueue(win_size)  # Queue of Withdraws

    last_ts = 0

    routes_without_as_path_encoding = []

    peer_handler = logging.handlers.RotatingFileHandler(
        LOG_DIRNAME + '/peer_' + str(peer_id),
        maxBytes=200000000000000,
        backupCount=5)
    peer_handler.setFormatter(formatter)
    peer_logger.removeHandler(handler)
    peer_logger.addHandler(peer_handler)

    peer_logger.info('Peer_' + str(peer_id) + '_(AS' + str(str(peer_as)) +
                     ')_started.')

    while True:

        while True:
            bgp_msg = queue_server_peer.get()

            if bgp_msg is not None:

                if 'announce' in bgp_msg:

                    prefix = bgp_msg['announce'].prefix
                    as_path = bgp_msg['announce'].as_path
                    #as_path = [65000] + as_path

                    # Update the set set of peer_as (useful when doing the naive solution)
                    #if len(as_path) > 0:
                    #peer_as_set.add(as_path[0])

                    # Update the RIB for this peer
                    old_as_path = rib.update(prefix, as_path)

                    # Remove the old as-path in the main graph for this prefix
                    G.remove(old_as_path, prefix)

                    # Add the new as-path in both the main graph and the graph of advertisments
                    G.add(as_path, prefix)

                    if encoding is not None:
                        encoding.advertisement(old_as_path, as_path)
                        bgp_msg['announce'] = add_as_path_encoding_to_route(
                            bgp_msg['announce'], rib, encoding)
                    elif len(rib.rib) > run_encoding_threshold:
                        encoding = init_encoding()
                        bgp_msg['announce'] = add_as_path_encoding_to_route(
                            bgp_msg['announce'], rib, encoding)
                    else:
                        routes_without_as_path_encoding.append(bgp_msg)

                    queue_peer_server.put(bgp_msg)

                if 'withdraw' in bgp_msg:

                    prefix = bgp_msg['withdraw'].prefix

                    #Withdraws get sent directly to the route server
                    queue_peer_server.put(bgp_msg)

                    # Create the encoding if not done yet
                    if encoding is None:
                        encoding = init_encoding()

                    # Update the RIB for this peer
                    as_path = rib.withdraw(prefix)

                    # Remove the old as-path in the main graph for this prefix
                    G.remove(as_path, prefix)

                    # Add the withdrawn as-path in the graph of withdraws
                    G_W.add(as_path)

                    # Update the queue of withdraws
                    if as_path != []:
                        bgp_msg['withdraw'].as_path = as_path
                        W_queue.append(bgp_msg)

                    # Update the encoding
                    encoding.withdraw(as_path)

                #elif'state' in bgp_msg['neighbor'] and bgp_msg['neighbor']['state'] == 'down':

                #queue_peer_server.put(bgp_msg)

                # CLOSE this peer. Clear all the topologies, ribs, queues, bursts, etc
                #if current_burst is not None:
                # best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo, peer_as_set)
                #current_burst.fd_predicted.write('PREDICTION_END_CLOSE|'+bpa_algo+'|'+str(len(current_burst))+'|'+str(best_fm_score)+'|'+str(best_TP)+'|'+str(best_FN)+'|'+str(best_FP)+'\n')
                #current_burst.fd_predicted.write('PREDICTION_END_EDGE|')
                #res = ''
                #depth = 9999999999
                #for e in best_edge_set:
                #depth = min(G_W.get_depth(e[0], e[1]), depth)
                #res += str(e[0])+'-'+str(e[1])+','

                #current_burst.fd_predicted.write(res[:len(res)-1]+'|'+str(depth)+'\n')

                #G_W.draw_graph(peer_as)

                #current_burst.stop(bgp_msg['time'])
                #current_burst = None

                # Withdraw all the routes advertised by this peer

                #peer_logger.info('Received CLOSE. CLEANING the peer.')

                # Stop this peer
                #os.kill(os.getpid(), signal.SIGTERM)
                #else:
                #peer_logger.info(bgp_msg)

                #Ceck for routes without encoding, check for encdoding, send modified routes to route_server
                if len(routes_without_as_path_encoding) > 0:
                    if encoding is not None:
                        for unsent_bgp_msg in routes_without_as_path_encoding:
                            if 'withdraw' in bgp_msg:
                                if unsent_bgp_msg[
                                        'announce'].prefix == bgp_msg[
                                            'withdraw'].prefix:
                                    routes_without_as_path_encoding.pop(
                                        unsent_bgp_msg)
                                    continue
                            unsent_bgp_msg[
                                'announce'] = add_as_path_encoding_to_route(
                                    unsent_bgp_msg['announce'], rib, encoding)
                            queue_peer_server.put(unsent_bgp_msg)

                        routes_without_as_path_encoding = []

                # Make sure to compute start en end time of burst with a second granularity (only if ther is a burst)
                if current_burst is not None:
                    while (last_ts != bgp_msg['time']):
                        last_ts += 1

                        # Update the graph of withdraws
                        for w in W_queue.refresh_iter(last_ts):
                            current_burst.deleted_from_W_queue.append(w)

                        # Remove the current burst (if any) if it the size of the withdraws is lower than w_threshold (meaning it has finished)
                        if len(
                                W_queue
                        ) < nb_withdrawals_burst_end:  #current_burst.is_expired(bgp_msg.time):
                            # Execute BPA at the end of the burst if the burst is large enough
                            print "burst is over"
                            best_edge_set, best_fm_score, best_TP, best_FN, best_FP = burst_prediction(
                                current_burst, G, G_W, W_queue, p_w, r_w,
                                bpa_algo, peer_as_set)
                            current_burst.fd_predicted.write(
                                'PREDICTION_END|' + bpa_algo + '|' +
                                str(len(current_burst)) + '|' +
                                str(best_fm_score) + '|' + str(best_TP) + '|' +
                                str(best_FN) + '|' + str(best_FP) + '\n')
                            current_burst.fd_predicted.write(
                                'PREDICTION_END_EDGE|')

                            # Print some information about the prediction on the prediction file
                            res = ''
                            depth = 9999999999
                            for e in best_edge_set:
                                res += str(e[0]) + '-' + str(e[1]) + ','
                                depth = min(G_W.get_depth(e[0], e[1]), depth)
                            current_burst.fd_predicted.write(res[:len(res) -
                                                                 1] + '|' +
                                                             str(depth) + '\n')

                            #G_W.draw_graph(peer_as, G, current_burst, outfile='as_graph_'+str(current_burst.start_time)+'.dot', threshold=500)

                            # Update the graph of withdrawals
                            for w in current_burst.deleted_from_W_queue:
                                G_W.remove(w['withdraw'].as_path)

                            current_burst.stop(bgp_msg['time'])
                            current_burst = None
                            break
                        else:
                            current_burst.last_ts = last_ts

                # Update the graph of withdraws.)
                if current_burst is None:
                    for w in W_queue.refresh_iter(bgp_msg['time']):
                        G_W.remove(w['withdraw'].as_path)

                # Update the last timestamp seen
                last_ts = bgp_msg['time']

                # Add the updates in the real prefixes set of the burst, if any
                if current_burst is not None:  #and not silent:
                    if 'announce' in bgp_msg:
                        current_burst.add_real_prefix(
                            bgp_msg['time'], bgp_msg['announce'].prefix, 'A',
                            bgp_msg['announce'].as_path)
                    if 'withdraw' in bgp_msg:
                        current_burst.add_real_prefix(
                            bgp_msg['time'], bgp_msg['withdraw'].prefix, 'W',
                            bgp_msg['withdraw'].as_path)

                # If we are not in the burst yet, we create the burst
                if current_burst is None and len(
                        W_queue) >= nb_withdrawals_burst_start:
                    print "SWIFT STARTING BURST!!!"
                    burst_start_time = W_queue[100]['time'] if len(
                        W_queue) > 100 else W_queue[0]['time']
                    current_burst = Burst(peer_id, bgp_msg['time'], win_size,
                                          burst_outdir, encoding,
                                          burst_start_time, silent)
                    next_bpa_execution = min_bpa_burst_size

                # Print some log ...
                if (bgp_msg['time'] > last_log_write
                    ) or bgp_msg['time'] - last_log_write >= 3600:
                    peer_logger.info(
                        str(int(bgp_msg['time'])) + '\t' + str(len(rib)) +
                        '\t' + str(len(W_queue)))
                    last_log_write = bgp_msg['time']

                # Execute BPA if there is a burst and
                # i) the current burst is greater than the minimum required
                # ii) we have wait the number of withdrawals required per cycle or the queue is empty
                if current_burst is not None:
                    total_current_burst_size = len(
                        current_burst) + nb_withdrawals_burst_start
                    if total_current_burst_size >= min_bpa_burst_size and total_current_burst_size > next_bpa_execution:
                        if nb_withdraws_per_cycle > 0 and total_current_burst_size < 12505:
                            next_bpa_execution += nb_withdraws_per_cycle
                        else:
                            next_bpa_execution = 999999999999
                        break

        #print ('Queue size: '+str(len(rib))+'\t'+str(len(W_queue))+'\t'+str(len(current_burst)+nb_withdrawals_burst_start))

        if current_burst is not None:

            logger.info("Burst detected starting preditcion")

            # Compute the set of edges with the highest FM score
            best_edge_set, best_fm_score, best_TP, best_FP, best_FN = burst_prediction(
                current_burst, G, G_W, W_queue, p_w, r_w, bpa_algo,
                peer_as_set)
            # Load that set in the burst
            if not silent:
                burst_add_edge(current_burst, rib, encoding, bgp_msg['time'],
                               best_edge_set, G, G_W, silent)

            logger.info("prediction finished")

            # Inform the global RIB about the set of failed links
            for e in best_edge_set:
                depth_set = set()
                if G_W.has_edge(e[0], e[1]):
                    depth_set = depth_set.union(
                        G_W[e[0]][e[1]]['depth'].keys())
                if G.has_edge(e[0], e[1]):
                    depth_set = depth_set.union(G[e[0]][e[1]]['depth'].keys())

                for d in depth_set:
                    if encoding.is_encoded(d, e[0], e[1]):

                        vmac_partial = ''
                        bitmask_partial = ''

                        for i in range(2, encoding.max_depth + 2):
                            if i == d:
                                vmac_partial += encoding.mapping[
                                    i].get_mapping_string(e[0])
                                bitmask_partial += '1' * encoding.mapping[
                                    i].nb_bytes
                            elif i == d + 1:
                                vmac_partial += encoding.mapping[
                                    i].get_mapping_string(e[1])
                                bitmask_partial += '1' * encoding.mapping[
                                    i].nb_bytes
                            else:
                                if i in encoding.mapping:
                                    vmac_partial += '0' * encoding.mapping[
                                        i].nb_bytes
                                    bitmask_partial += '0' * encoding.mapping[
                                        i].nb_bytes

                        FR_message = {
                            'FR': {
                                'peer_id': peer_id,
                                'as_path_vmac': vmac_partial,
                                'as_path_bitmask': bitmask_partial,
                                'depth': d
                            }
                        }
                        FR_queue.put(FR_message)

                        print "FR_message:", FR_message, "best_edge", e

            # Print information about the perdiction in the predicted file
            current_burst.fd_predicted.write('PREDICTION|' + bpa_algo + '|' +
                                             str(len(current_burst)) + '|' +
                                             str(best_fm_score) + '|' +
                                             str(best_TP) + '|' +
                                             str(best_FP) + '|' +
                                             str(best_FN) + '\n')
            current_burst.fd_predicted.write('PREDICTION_EDGE|')
            res = ''
            depth = 9999999999
            for e in best_edge_set:
                depth = min(G_W.get_depth(e[0], e[1]), depth)
                res += str(e[0]) + '-' + str(e[1]) + ','
            current_burst.fd_predicted.write(res[:len(res) - 1] + '|' +
                                             str(depth) + '\n')
コード例 #13
0
def run_peer_bpavalidation(queue, win_size, nb_withdrawals_burst_start, \
nb_withdrawals_burst_end, min_bpa_burst_size, burst_outdir, \
nb_withdraws_per_cycle=100, p_w=1, r_w=1, bpa_algo=False, nb_bits_aspath=33, \
run_encoding_threshold=1000000, global_rib_enabled=True, silent=False):

    import socket

    try:
        os.nice(-20)
    except OSError:
        peer_logger.info('Cannot change the nice.')

    # Last time the peer wrote the rib and queue size in the log file
    last_log_write = 0

    # Socket connected to the global RIB
    socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

    # Current burst (if any)
    current_burst = None

    # Exit properly when receiving SIGINT
    def signal_handler(signal, frame):
        if current_burst is not None:
            current_burst.stop(bgp_msg.time)

        socket.close()
        peer_logger.info('Received SIGTERM. Exiting.')
        sys.exit(0)

    signal.signal(signal.SIGTERM, signal_handler)

    peer_id = None
    peer_as = None

    # Create the RIB for this peer
    rib = RIBPeer()

    U_queue = BGPMessagesQueue(
        win_size)  # Queue of Updates (advertisement or withdrawals)

    last_ts = 0

    while True:

        while True:
            bgp_msg = queue.get()

            if bgp_msg is not None:
                if peer_id is None:
                    peer_id = bgp_msg.peer_id
                    peer_as = bgp_msg.peer_as
                    peer_as_set = set()
                    last_ts = bgp_msg.time
                    last_log_write = bgp_msg.time
                    peer_ip = peer_id.split('-')[-1]

                    peer_handler = logging.handlers.RotatingFileHandler(
                        log_dir + '/peer_' + str(peer_id),
                        maxBytes=200000000000000,
                        backupCount=5)
                    peer_handler.setFormatter(formatter)
                    peer_logger.removeHandler(handler)
                    peer_logger.addHandler(peer_handler)

                    peer_logger.info('Peer_' + str(peer_id) + '_(AS' +
                                     str(str(peer_as)) + ')_started.')

                    if bgp_msg.as_path is not None and len(
                            bgp_msg.as_path) > 0:
                        if peer_as != bgp_msg.as_path[0]:
                            peer_logger.warning(
                                'Peer AS ' + str(peer_as) + ' and first AS ' +
                                str(bgp_msg.as_path[0]) +
                                ' in AS path does not match. Setting first AS as peer AS.'
                            )
                            peer_as = bgp_msg.as_path[0]

                if peer_id != bgp_msg.peer_id:
                    peer_logger.critical(
                        'Received a bgp_message with peer_id: ' +
                        str(bgp_msg.peer_id))

                if bgp_msg.mtype == 'A':
                    # Update the RIB for this peer
                    bgp_msg.as_path = rib.update(bgp_msg)

                    # Update the queue of updates
                    if bgp_msg.as_path != []:
                        U_queue.append(bgp_msg)

                elif bgp_msg.mtype == 'W':
                    # Update the RIB for this peer
                    bgp_msg.as_path = rib.withdraw(bgp_msg)

                    # Update the queue of withdraws
                    if bgp_msg.as_path != []:
                        U_queue.append(bgp_msg)

                else:
                    peer_logger.info(bgp_msg)

                # Print size of hte queue for each second
                while last_log_write < bgp_msg.time:
                    # Refresh the queue
                    U_queue.refresh(last_log_write)
                    peer_logger.info(
                        str(int(last_log_write)) + ' ' + str(len(rib)) + ' ' +
                        str(len(U_queue)))
                    last_log_write += 1

                # Stop the burst if the size of the queue is lower than the threshold
                if current_burst is not None:
                    if len(U_queue) < nb_withdrawals_burst_end:
                        current_burst.stop(bgp_msg.time)
                        current_burst = None
                    else:
                        current_burst.add_real_prefix(bgp_msg.time,
                                                      bgp_msg.prefix,
                                                      bgp_msg.mtype,
                                                      bgp_msg.as_path)

                # Create a burst if the size of the is higher than the threshold
                if current_burst is None:
                    if len(U_queue) > nb_withdrawals_burst_start:
                        burst_start_time = U_queue[100].time if len(
                            U_queue) > 100 else U_queue[0].time
                        current_burst = Burst(peer_id, bgp_msg.time, win_size,
                                              burst_outdir, burst_start_time,
                                              silent)