self.as_nodes = None
return None # i.e. return an ASN_Graph
for msm_id in reqd_msms:
no_asnf = no_whoisf = False
asns_fn = c.asns_fn(msm_id)
print("asns file >%s<" % asns_fn)
if not os.path.isfile(asns_fn):
print("No asns file; run pypy3 bulk-bgp-lookup.py <<<")
exit()
dgs_stem = c.dgs_stem
# full_graphs adds "-asn" to dgs_stem !
print("dgs_stem = >%s<" % dgs_stem)
in_graphs_fn = c.msm_graphs_fn(msm_id)
print("graph_fn = %s" % in_graphs_fn)
n_bins_to_read = c.n_bins # 3 ######################## c.n_bins
dgs = dgs_ld.load_graphs(
in_graphs_fn, # Load a DestGraphs file (dgs)
mx_depth,
mn_trpkts,
n_bins_to_read)
# dgs.ba is an array of BinGraphs
# each BinGraph has a .pops dictionary, keys are IPprefixes
print("BinGraphs: %s %s %d traces, %d,%d" %
(dgs.msm_id, dgs.dest, dgs.n_traces, mx_depth, mn_trpkts))
tb = timebins.TimeBins(dgs.start_dt, dgs.end_dt)
def __init__(self, msm_id, node_dir, n_bins, asn_graph,
n_asns, no_asn_nodes, mx_allowed_depth, mn_allowed_trpkts):
write_ne_data = False # Write fie of counts
if write_ne_data:
ne_data = open("%s/%s-ne-data.txt" % (c.start_ymd, msm_id),
"w", encoding='utf-8')
ne_data.write("ne-data %s/%d\n" % (c.start_ymd, msm_id))
# build_graphs only allows a single ymd !!
ne_data.write("bn nodes all_nodes s_nodes edges all_edges\n")
self.msm_id = msm_id
self.mn_allowed_trpkts = mn_allowed_trpkts
self.fn = None # graphs- filename
self.n_traces = self.n_succ_traces = self.n_dup_traces = \
self.t_addrs = self.t_hops = self.t_hops_deleted = 0
self.node_dir = node_dir; self.n_bins = n_bins
print(" GraphInfo node_dir len = %d" % len(self.node_dir))
self.asn_graph = asn_graph; self.no_asn_nodes = no_asn_nodes
self.n_asns = n_asns
self.sub_root_icounts = {} # in_count (for all bins)
self.stats = {}
self.p_counts = np.zeros(self.n_bins+1) # Edges in bins 0..(c.n_bins)
self.stats['edges_counts'] = self.p_counts
self.n_traces = np.zeros(self.n_bins) # Totals from BinGraph lines
self.stats["n_traces"] = self.n_traces
self.n_succ_traces = np.zeros(self.n_bins)
self.stats["tr_success"] = self.n_succ_traces
self.n_dup_traces = np.zeros(self.n_bins)
self.stats["n_dup_traces"] = self.n_dup_traces
self.t_addrs = np.zeros(self.n_bins)
self.stats["t_addrs"] = self.t_addrs
self.t_hops = np.zeros(self.n_bins)
self.stats["t_hops"] = self.t_hops
self.t_hops_deleted = np.zeros(self.n_bins)
self.stats["t_hops_deleted"] = self.t_hops_deleted
self.dest = "" # Dest IP address for traces
self.distal_nodes = {} # sn_nodes not in all_nodes
self.sn_nodes = {} # Nodes from s_nodes lines
self.all_s_nodes = {} # Edges (from s_nodes lines)
self.all_edges = {} # Edges from s_nodes lines
self.all_nodes = {} # Nodes from Node lines
self.nodes_tot = np.zeros(self.n_bins) # nodes per bin
# All nodes in graph (from s_node_nodes)
self.stats["nodes"] = self.nodes_tot
self.nodes_distal = np.zeros(self.n_bins) # Nodes with no s_node lines
self.stats["nodes_distal"] = self.nodes_distal
self.nodes_internal = np.zeros(self.n_bins) # Nodes in s_nodes lines
self.stats["nodes_internal"] = self.nodes_internal
self.nodes_1hop = np.zeros(self.n_bins) # 1 hop before dest
self.stats["nodes_1hop"] = self.nodes_1hop
self.trpkts_tot = np.zeros(self.n_bins) # trs from probes
self.stats["trpkts_tot"] = self.trpkts_tot
#lsp_trpkts = [39, 60, 93, 141, 213, 327, 501, 768, 1176]
self.trpkts_part = []
for k in range(0,len(self.trpart_sizes)):
self.trpkts_part.append(np.zeros(self.n_bins)) # trs in each part
self.start_msm_id() # Initialise depth and tr variables
self.stats["depth_16 (%)"] = self.depth_16
self.stats["depth_19 (%)"] = self.depth_19
self.stats["depth_25 (%)"] = self.depth_25
self.stats["depth_32 (%)"] = self.depth_32
self.stats["depth_max"] = self.mx_depth
self.stats["trpkts_3 (%)"] = self.trpkts_3
self.stats["trpkts_9 (%)"] = self.trpkts_9
self.stats["trpkts_27 (%)"] = self.trpkts_27
self.stats["trpkts_38 (%)"] = self.trpkts_81
self.trs_dest = np.zeros(self.n_bins) # trs arriving at dest
self.stats["trpkts_dest"] = self.trs_dest
self.stats["asns"] = self.n_asns
self.n_subroots = np.zeros(self.n_bins) # subroots for each bn
self.stats["subroots"] = self.n_subroots
self.n_subroot_trs = np.zeros(self.n_bins)
self.stats["trpkts_subroot"] = self.n_subroot_trs
self.tot_edges = np.zeros(self.n_bins) # edges per bin
self.stats['edges'] = self.tot_edges
self.bin_same_edges = np.zeros(self.n_bins)
self.bin_inter_edges = np.zeros(self.n_bins)
self.stats['edges_same'] = self.bin_same_edges
self.stats['edges_inter'] = self.bin_inter_edges
self.same_counts = np.zeros(self.n_bins+1)
self.inter_counts = np.zeros(self.n_bins+1)
# same_counts[] and inter_counts[] are np arrays
# indicating the number of bins their edges were actaully present.
# Computed by the asn_edges() function (below)
# Used by cum-edge-presence-v-timebins.py
self.distal_mx_depth = np.zeros(self.n_bins)
self.stats['distal_mx_depth'] = self.distal_mx_depth
self.distal_min_ntrs = np.zeros(self.n_bins)
self.stats['distal_min_trs'] = self.distal_min_ntrs
self.same_asn_edges = np.zeros(self.n_bins) # Edges in same ASN
self.stats['edges_same'] = self.same_asn_edges
self.inter_asn_edges = np.zeros(self.n_bins) # Edges between ASNs
self.stats['edges_inter'] = self.inter_asn_edges
self.stats_fn = c.stats_fn(c.msm_id)
#?sf = open(self.stats_fn, "r")
#?for line in sf:
###self.last_bn = 0#; self.ne = 3 #################################
#info_f.write("msm_id %d\n" % msm_id)
self.fn = c.msm_graphs_fn(self.msm_id)
print(" >>> %s / %d: %s" % (c.start_ymd, msm_id, self.fn))
f = open(self.fn, "r")
gf_version = 1
bn = -1
mxa_depth = mna_trpkts = 0
if mx_allowed_depth:
mxa_depth = mx_allowed_depth # True
if mn_allowed_trpkts:
mna_trpkts = mn_allowed_trpkts # True
print(" mxa_depth %d, mna_trpkts %d" % (mxa_depth, mna_trpkts))
s_nodes = {} # Nodes found in s_nodes lines
trpkts_tot = 0
mx_depth_seen = 0; mx_n_depth_seen = 0
for line in f: # Read in the graphs file
la = line.strip().split(maxsplit=1)
if la[0] == "BinGraph":
if bn >= 0:
self.trpkts_tot[bn] = trpkts_tot
self.n_subroots[bn] = n_subroots
self.n_subroot_trs[bn] = n_subroot_trpkts
self.tot_edges[bn] = len(self.edges)
self.same_asn_edges[bn] = bin_same_edges
self.inter_asn_edges[bn] = bin_inter_edges
if write_ne_data:
ne_data.write("%2s %6d %6d %6d %6d %6d\n" % (
bn, len(self.nodes), len(self.all_nodes),
len(s_nodes),
len(self.edges), len(self.all_edges)))
#else:
# print(" ##$$ %d nodes, %d all_nodes, %d s_nodes, %d edges %d all_edges" % (
# len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges)))
self.end_bin(msm_id, bn)
#!!!self.nodes_tot[bn] = len(s_nodes)
self.nodes_tot[bn] = len(self.nodes)
if self.trs_dest[bn] != 0: # Some trpkts reached dest
self.nodes_tot[bn] += 1
self.nodes_internal[bn] = len(self.nodes)
ext_nodes = 0
for nn in s_nodes:
if nn not in self.nodes:
ext_nodes += 1
if nn not in self.all_s_nodes:
self.all_s_nodes[nn] = s_nodes[nn]
self.nodes_distal[bn] = ext_nodes
ila = list(map(int, la[1].split()))
bn = ila[6]
#if bn == 5: # Testing, testing . . .
# break
self.n_traces[bn] = ila[0]
self.n_succ_traces[bn] = ila[1]
self.n_dup_traces[bn] = ila[2]
self.t_addrs[bn] = ila[3]
self.t_hops[bn] = ila[4]
self.t_hops_deleted[bn] = ila[5]
self.start_bin(int(bn))
roots_line = f.readline()
rla = roots_line.strip().split()
self.dest = rla[1] # Save the dest
self.nodes = {} # Dictionary, all nodes seen >> in bin bn <<
self.edges = {} # Ditto for edges (use np arrays for the bins)
self.paths = {} # Ditto for paths
trpkts_tot = 0; s_nodes = {}
node_lines = n_subroots = n_subroot_trpkts = 0
bin_same_edges = bin_inter_edges = 0
for r in rla[2:]: # rla[0] = mx_counts, rla[1] = dest
if not r in self.sub_root_icounts: # r = subroot name
self.sub_root_icounts[r] = np.zeros(self.n_bins+1)
n_subroots += 1
elif la[0] == "Node":
nv = la[1].split()
name = nv[0]; subnet = int(nv[1])
# 'subtree' as found when building the graph
icount = int(nv[2]); depth = int(nv[3])
if depth > mx_depth_seen:
mx_depth_seen = depth
#print("NO: depth %d, la %s\n rla %s" % (
# depth, la, rla))
if depth >= len(self.node_depths):
self.node_depths = np.append(self.node_depths,
np.zeros(depth+10-len(self.node_depths)))
self.node_depths[depth] += 1
if icount < len(self.node_trpkts):
self.node_trpkts[icount] += 1
trpkts_tot += icount
self.trparts_bin[
(np.abs(self.trpart_sizes-icount)).argmin()] += 1
if name in self.sub_root_icounts and \
icount >= self.mn_allowed_trpkts:
self.sub_root_icounts[name][bn] = icount
n_subroot_trpkts += icount
fails = -1 # Not in v1 graphs file
if gf_version != 1:
fails = int(nv[3])
s_nodes_line = f.readline() # s_nodes line
sna = s_nodes_line.split() # name, in_count pairs
for j in range(0, len(sna)-1, 2): # name.count pairs, + depth
src_name = sna[j]; in_pkts = int(sna[j+1])
# <?> if src_name not in self.all_nodes:
###if src_name not in self.sn_nodes:
if src_name not in self.sn_nodes:
# <?> self.all_nodes[src_name] = NodeInfo(
self.sn_nodes[src_name] = NodeInfo(
bn, self.dest, src_name, False,
self.node_dir, in_pkts, fails, [], [],
self.n_bins, no_asn_nodes)
#print("bn %2d src_name %s, icounts %s <<< new" % (
# bn, src_name,
# self.sn_nodes[src_name].icounts.astype(int)))
else:
# <?> self.all_nodes[src_name].icounts[bn] = in_pkts
self.sn_nodes[src_name].icounts[bn] = in_pkts
#print("bn %2d src_name %s, in_pkts %s" % (
# bn, src_name,
# self.sn_nodes[src_name].icounts.astype(int)))
if name not in self.nodes: # Nodes in this bin
self.nodes[name] = True
if name not in self.all_nodes: # Get info about nodes
self.all_nodes[name] = NodeInfo(bn, self.dest, name,
True, self.node_dir, icount, fails, rla, sna,
self.n_bins, no_asn_nodes)
else:
self.all_nodes[name].update(
bn, icount, fails, rla, sna)
#print("-2- %s" % (self.all_nodes[name]))
if name == self.dest:
n_1hop = 0
for j in range(0, len(sna)-1, 2):
count = int(sna[j+1])
if count >= mn_allowed_trpkts:
n_1hop += 1 # Edges to be drawn on graphs
# There are also _lots_ of edges with count < 27 !!!
self.nodes_1hop[bn] = n_1hop
self.trs_dest[bn] = icount
#print("s_nodes: >%s<" % sna)
n_depth = int(sna.pop()) # Depth of s_node edges
#print(" depth %d" % n_depth)
for j in range(0, len(sna), 2): # Get info about edges
src = sna[j]; count = int(sna[j+1])
# "distal" (probe) nodes only appear in s_nodes lines!
# "internal" nodes are sources for nodes nearer to dest
if src in self.nodes: # Not an s_node
# We only want edges between internal nodes
# i.e. nodes in this bin's graph
if not src in s_nodes:
s_nodes[src] = True
e_key = "%s %s" % (src, name)
in_all_edges = False
if not e_key in self.all_edges: # Edges in all bins
e = self.all_edges[e_key] = Edge( src, name,
self.node_dir, self.n_bins, asn_graph,
no_asn_nodes, n_depth)
else:
e = self.all_edges[e_key]
in_all_edges = True
e.set_icount(bn, count)
if e_key not in self.edges: # Edges in this bin
self.edges[e_key] = e
if e.inter_as:
bin_inter_edges += 1
else:
bin_same_edges += 1
elif la[0] == "DestGraphs":
nv = la[1].split() # One or more white-space chars
self.dest = nv[1]
if len(nv) > 7: # nv[7] = min_tr_pkts (last in v1 headers)
gf_version = 2
self.end_msm_id() # EOF reached
self.trpkts_tot[bn] = trpkts_tot # Copied from if bn >= 0 above
self.n_subroots[bn] = n_subroots
self.n_subroot_trs[bn] = n_subroot_trpkts
self.tot_edges[bn] = len(self.edges)
self.same_asn_edges[bn] = bin_same_edges
self.inter_asn_edges[bn] = bin_inter_edges
if write_ne_data:
ne_data.write("%2s %6d %6d %6d %6d %6d\n" % (
bn, len(self.nodes), len(self.all_nodes),
len(s_nodes),
len(self.edges), len(self.all_edges)))
ne_data.close()
#else:
# print(" ##$$ %d nodes, %d all_nodes, %d s_nodes, %d edges %d all_edges" % (
# len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges)))
self.end_bin(msm_id, bn)
#!!self.nodes_tot[bn] = len(s_nodes)
self.nodes_tot[bn] = len(self.nodes)
if self.trs_dest[bn] != 0: # Some trpkts reached dest
self.nodes_tot[bn] += 1
#$self.nodes_internal[bn] = len(nodes_this_bin)
self.nodes_internal[bn] = len(self.nodes)
ext_nodes = 0
for nn in s_nodes:
#$if nn not in nodes_this_bin:
if nn not in self.nodes:
ext_nodes += 1 # Distal nodes in this bin
if nn not in self.all_s_nodes:
self.all_s_nodes[nn] = s_nodes[nn]
self.nodes_distal[bn] = ext_nodes
print("<<< bn %d, %d edges. %d in all_edges" % (
bn, len(self.edges), len(self.all_edges)))
print("mx_depth_seen = %d" % mx_depth_seen)
self.end_bin(msm_id, bn) # For last bin
self.ext_nodes_count = 0
for sn in self.all_s_nodes:
if sn not in self.all_nodes:
self.ext_nodes_count += 1
print(" ##$$ %d all_nodes, %d ext_nodes_count, %d all_edges $$##" % (
len(self.all_nodes), self.ext_nodes_count, len(self.all_edges)))
# >>> total nodes = len(self.all_nodes) + self.ext_nodes_count <<<
# Delete subroots with max_icount < mn_allowed_trpkts
if self.mn_allowed_trpkts != 0:
subrs_to_delete = []
for sr_key in self.sub_root_icounts.keys():
sr_icounts = np.array(self.sub_root_icounts[sr_key])
mx_icount = np.max(sr_icounts)
if mx_icount < self.mn_allowed_trpkts:
subrs_to_delete.append(sr_key)
print("About to delete %d subroots with max_icount < %d" % (
len(subrs_to_delete), self.mn_allowed_trpkts))
for sr_key in subrs_to_delete:
self.sub_root_icounts.pop(sr_key)
f.close()
#info_f.write("End (MxDepth %d)\n" % mx_depth_seen)
print("EOF reached")
self.test_n_d_overlap()