self.as_nodes = None return None # i.e. return an ASN_Graph for msm_id in reqd_msms: no_asnf = no_whoisf = False asns_fn = c.asns_fn(msm_id) print("asns file >%s<" % asns_fn) if not os.path.isfile(asns_fn): print("No asns file; run pypy3 bulk-bgp-lookup.py <<<") exit() dgs_stem = c.dgs_stem # full_graphs adds "-asn" to dgs_stem ! print("dgs_stem = >%s<" % dgs_stem) in_graphs_fn = c.msm_graphs_fn(msm_id) print("graph_fn = %s" % in_graphs_fn) n_bins_to_read = c.n_bins # 3 ######################## c.n_bins dgs = dgs_ld.load_graphs( in_graphs_fn, # Load a DestGraphs file (dgs) mx_depth, mn_trpkts, n_bins_to_read) # dgs.ba is an array of BinGraphs # each BinGraph has a .pops dictionary, keys are IPprefixes print("BinGraphs: %s %s %d traces, %d,%d" % (dgs.msm_id, dgs.dest, dgs.n_traces, mx_depth, mn_trpkts)) tb = timebins.TimeBins(dgs.start_dt, dgs.end_dt)
def __init__(self, msm_id, node_dir, n_bins, asn_graph, n_asns, no_asn_nodes, mx_allowed_depth, mn_allowed_trpkts): write_ne_data = False # Write fie of counts if write_ne_data: ne_data = open("%s/%s-ne-data.txt" % (c.start_ymd, msm_id), "w", encoding='utf-8') ne_data.write("ne-data %s/%d\n" % (c.start_ymd, msm_id)) # build_graphs only allows a single ymd !! ne_data.write("bn nodes all_nodes s_nodes edges all_edges\n") self.msm_id = msm_id self.mn_allowed_trpkts = mn_allowed_trpkts self.fn = None # graphs- filename self.n_traces = self.n_succ_traces = self.n_dup_traces = \ self.t_addrs = self.t_hops = self.t_hops_deleted = 0 self.node_dir = node_dir; self.n_bins = n_bins print(" GraphInfo node_dir len = %d" % len(self.node_dir)) self.asn_graph = asn_graph; self.no_asn_nodes = no_asn_nodes self.n_asns = n_asns self.sub_root_icounts = {} # in_count (for all bins) self.stats = {} self.p_counts = np.zeros(self.n_bins+1) # Edges in bins 0..(c.n_bins) self.stats['edges_counts'] = self.p_counts self.n_traces = np.zeros(self.n_bins) # Totals from BinGraph lines self.stats["n_traces"] = self.n_traces self.n_succ_traces = np.zeros(self.n_bins) self.stats["tr_success"] = self.n_succ_traces self.n_dup_traces = np.zeros(self.n_bins) self.stats["n_dup_traces"] = self.n_dup_traces self.t_addrs = np.zeros(self.n_bins) self.stats["t_addrs"] = self.t_addrs self.t_hops = np.zeros(self.n_bins) self.stats["t_hops"] = self.t_hops self.t_hops_deleted = np.zeros(self.n_bins) self.stats["t_hops_deleted"] = self.t_hops_deleted self.dest = "" # Dest IP address for traces self.distal_nodes = {} # sn_nodes not in all_nodes self.sn_nodes = {} # Nodes from s_nodes lines self.all_s_nodes = {} # Edges (from s_nodes lines) self.all_edges = {} # Edges from s_nodes lines self.all_nodes = {} # Nodes from Node lines self.nodes_tot = np.zeros(self.n_bins) # nodes per bin # All nodes in graph (from s_node_nodes) self.stats["nodes"] = self.nodes_tot self.nodes_distal = np.zeros(self.n_bins) # Nodes with no s_node lines self.stats["nodes_distal"] = self.nodes_distal self.nodes_internal = np.zeros(self.n_bins) # Nodes in s_nodes lines self.stats["nodes_internal"] = self.nodes_internal self.nodes_1hop = np.zeros(self.n_bins) # 1 hop before dest self.stats["nodes_1hop"] = self.nodes_1hop self.trpkts_tot = np.zeros(self.n_bins) # trs from probes self.stats["trpkts_tot"] = self.trpkts_tot #lsp_trpkts = [39, 60, 93, 141, 213, 327, 501, 768, 1176] self.trpkts_part = [] for k in range(0,len(self.trpart_sizes)): self.trpkts_part.append(np.zeros(self.n_bins)) # trs in each part self.start_msm_id() # Initialise depth and tr variables self.stats["depth_16 (%)"] = self.depth_16 self.stats["depth_19 (%)"] = self.depth_19 self.stats["depth_25 (%)"] = self.depth_25 self.stats["depth_32 (%)"] = self.depth_32 self.stats["depth_max"] = self.mx_depth self.stats["trpkts_3 (%)"] = self.trpkts_3 self.stats["trpkts_9 (%)"] = self.trpkts_9 self.stats["trpkts_27 (%)"] = self.trpkts_27 self.stats["trpkts_38 (%)"] = self.trpkts_81 self.trs_dest = np.zeros(self.n_bins) # trs arriving at dest self.stats["trpkts_dest"] = self.trs_dest self.stats["asns"] = self.n_asns self.n_subroots = np.zeros(self.n_bins) # subroots for each bn self.stats["subroots"] = self.n_subroots self.n_subroot_trs = np.zeros(self.n_bins) self.stats["trpkts_subroot"] = self.n_subroot_trs self.tot_edges = np.zeros(self.n_bins) # edges per bin self.stats['edges'] = self.tot_edges self.bin_same_edges = np.zeros(self.n_bins) self.bin_inter_edges = np.zeros(self.n_bins) self.stats['edges_same'] = self.bin_same_edges self.stats['edges_inter'] = self.bin_inter_edges self.same_counts = np.zeros(self.n_bins+1) self.inter_counts = np.zeros(self.n_bins+1) # same_counts[] and inter_counts[] are np arrays # indicating the number of bins their edges were actaully present. # Computed by the asn_edges() function (below) # Used by cum-edge-presence-v-timebins.py self.distal_mx_depth = np.zeros(self.n_bins) self.stats['distal_mx_depth'] = self.distal_mx_depth self.distal_min_ntrs = np.zeros(self.n_bins) self.stats['distal_min_trs'] = self.distal_min_ntrs self.same_asn_edges = np.zeros(self.n_bins) # Edges in same ASN self.stats['edges_same'] = self.same_asn_edges self.inter_asn_edges = np.zeros(self.n_bins) # Edges between ASNs self.stats['edges_inter'] = self.inter_asn_edges self.stats_fn = c.stats_fn(c.msm_id) #?sf = open(self.stats_fn, "r") #?for line in sf: ###self.last_bn = 0#; self.ne = 3 ################################# #info_f.write("msm_id %d\n" % msm_id) self.fn = c.msm_graphs_fn(self.msm_id) print(" >>> %s / %d: %s" % (c.start_ymd, msm_id, self.fn)) f = open(self.fn, "r") gf_version = 1 bn = -1 mxa_depth = mna_trpkts = 0 if mx_allowed_depth: mxa_depth = mx_allowed_depth # True if mn_allowed_trpkts: mna_trpkts = mn_allowed_trpkts # True print(" mxa_depth %d, mna_trpkts %d" % (mxa_depth, mna_trpkts)) s_nodes = {} # Nodes found in s_nodes lines trpkts_tot = 0 mx_depth_seen = 0; mx_n_depth_seen = 0 for line in f: # Read in the graphs file la = line.strip().split(maxsplit=1) if la[0] == "BinGraph": if bn >= 0: self.trpkts_tot[bn] = trpkts_tot self.n_subroots[bn] = n_subroots self.n_subroot_trs[bn] = n_subroot_trpkts self.tot_edges[bn] = len(self.edges) self.same_asn_edges[bn] = bin_same_edges self.inter_asn_edges[bn] = bin_inter_edges if write_ne_data: ne_data.write("%2s %6d %6d %6d %6d %6d\n" % ( bn, len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges))) #else: # print(" ##$$ %d nodes, %d all_nodes, %d s_nodes, %d edges %d all_edges" % ( # len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges))) self.end_bin(msm_id, bn) #!!!self.nodes_tot[bn] = len(s_nodes) self.nodes_tot[bn] = len(self.nodes) if self.trs_dest[bn] != 0: # Some trpkts reached dest self.nodes_tot[bn] += 1 self.nodes_internal[bn] = len(self.nodes) ext_nodes = 0 for nn in s_nodes: if nn not in self.nodes: ext_nodes += 1 if nn not in self.all_s_nodes: self.all_s_nodes[nn] = s_nodes[nn] self.nodes_distal[bn] = ext_nodes ila = list(map(int, la[1].split())) bn = ila[6] #if bn == 5: # Testing, testing . . . # break self.n_traces[bn] = ila[0] self.n_succ_traces[bn] = ila[1] self.n_dup_traces[bn] = ila[2] self.t_addrs[bn] = ila[3] self.t_hops[bn] = ila[4] self.t_hops_deleted[bn] = ila[5] self.start_bin(int(bn)) roots_line = f.readline() rla = roots_line.strip().split() self.dest = rla[1] # Save the dest self.nodes = {} # Dictionary, all nodes seen >> in bin bn << self.edges = {} # Ditto for edges (use np arrays for the bins) self.paths = {} # Ditto for paths trpkts_tot = 0; s_nodes = {} node_lines = n_subroots = n_subroot_trpkts = 0 bin_same_edges = bin_inter_edges = 0 for r in rla[2:]: # rla[0] = mx_counts, rla[1] = dest if not r in self.sub_root_icounts: # r = subroot name self.sub_root_icounts[r] = np.zeros(self.n_bins+1) n_subroots += 1 elif la[0] == "Node": nv = la[1].split() name = nv[0]; subnet = int(nv[1]) # 'subtree' as found when building the graph icount = int(nv[2]); depth = int(nv[3]) if depth > mx_depth_seen: mx_depth_seen = depth #print("NO: depth %d, la %s\n rla %s" % ( # depth, la, rla)) if depth >= len(self.node_depths): self.node_depths = np.append(self.node_depths, np.zeros(depth+10-len(self.node_depths))) self.node_depths[depth] += 1 if icount < len(self.node_trpkts): self.node_trpkts[icount] += 1 trpkts_tot += icount self.trparts_bin[ (np.abs(self.trpart_sizes-icount)).argmin()] += 1 if name in self.sub_root_icounts and \ icount >= self.mn_allowed_trpkts: self.sub_root_icounts[name][bn] = icount n_subroot_trpkts += icount fails = -1 # Not in v1 graphs file if gf_version != 1: fails = int(nv[3]) s_nodes_line = f.readline() # s_nodes line sna = s_nodes_line.split() # name, in_count pairs for j in range(0, len(sna)-1, 2): # name.count pairs, + depth src_name = sna[j]; in_pkts = int(sna[j+1]) # <?> if src_name not in self.all_nodes: ###if src_name not in self.sn_nodes: if src_name not in self.sn_nodes: # <?> self.all_nodes[src_name] = NodeInfo( self.sn_nodes[src_name] = NodeInfo( bn, self.dest, src_name, False, self.node_dir, in_pkts, fails, [], [], self.n_bins, no_asn_nodes) #print("bn %2d src_name %s, icounts %s <<< new" % ( # bn, src_name, # self.sn_nodes[src_name].icounts.astype(int))) else: # <?> self.all_nodes[src_name].icounts[bn] = in_pkts self.sn_nodes[src_name].icounts[bn] = in_pkts #print("bn %2d src_name %s, in_pkts %s" % ( # bn, src_name, # self.sn_nodes[src_name].icounts.astype(int))) if name not in self.nodes: # Nodes in this bin self.nodes[name] = True if name not in self.all_nodes: # Get info about nodes self.all_nodes[name] = NodeInfo(bn, self.dest, name, True, self.node_dir, icount, fails, rla, sna, self.n_bins, no_asn_nodes) else: self.all_nodes[name].update( bn, icount, fails, rla, sna) #print("-2- %s" % (self.all_nodes[name])) if name == self.dest: n_1hop = 0 for j in range(0, len(sna)-1, 2): count = int(sna[j+1]) if count >= mn_allowed_trpkts: n_1hop += 1 # Edges to be drawn on graphs # There are also _lots_ of edges with count < 27 !!! self.nodes_1hop[bn] = n_1hop self.trs_dest[bn] = icount #print("s_nodes: >%s<" % sna) n_depth = int(sna.pop()) # Depth of s_node edges #print(" depth %d" % n_depth) for j in range(0, len(sna), 2): # Get info about edges src = sna[j]; count = int(sna[j+1]) # "distal" (probe) nodes only appear in s_nodes lines! # "internal" nodes are sources for nodes nearer to dest if src in self.nodes: # Not an s_node # We only want edges between internal nodes # i.e. nodes in this bin's graph if not src in s_nodes: s_nodes[src] = True e_key = "%s %s" % (src, name) in_all_edges = False if not e_key in self.all_edges: # Edges in all bins e = self.all_edges[e_key] = Edge( src, name, self.node_dir, self.n_bins, asn_graph, no_asn_nodes, n_depth) else: e = self.all_edges[e_key] in_all_edges = True e.set_icount(bn, count) if e_key not in self.edges: # Edges in this bin self.edges[e_key] = e if e.inter_as: bin_inter_edges += 1 else: bin_same_edges += 1 elif la[0] == "DestGraphs": nv = la[1].split() # One or more white-space chars self.dest = nv[1] if len(nv) > 7: # nv[7] = min_tr_pkts (last in v1 headers) gf_version = 2 self.end_msm_id() # EOF reached self.trpkts_tot[bn] = trpkts_tot # Copied from if bn >= 0 above self.n_subroots[bn] = n_subroots self.n_subroot_trs[bn] = n_subroot_trpkts self.tot_edges[bn] = len(self.edges) self.same_asn_edges[bn] = bin_same_edges self.inter_asn_edges[bn] = bin_inter_edges if write_ne_data: ne_data.write("%2s %6d %6d %6d %6d %6d\n" % ( bn, len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges))) ne_data.close() #else: # print(" ##$$ %d nodes, %d all_nodes, %d s_nodes, %d edges %d all_edges" % ( # len(self.nodes), len(self.all_nodes), len(s_nodes), len(self.edges), len(self.all_edges))) self.end_bin(msm_id, bn) #!!self.nodes_tot[bn] = len(s_nodes) self.nodes_tot[bn] = len(self.nodes) if self.trs_dest[bn] != 0: # Some trpkts reached dest self.nodes_tot[bn] += 1 #$self.nodes_internal[bn] = len(nodes_this_bin) self.nodes_internal[bn] = len(self.nodes) ext_nodes = 0 for nn in s_nodes: #$if nn not in nodes_this_bin: if nn not in self.nodes: ext_nodes += 1 # Distal nodes in this bin if nn not in self.all_s_nodes: self.all_s_nodes[nn] = s_nodes[nn] self.nodes_distal[bn] = ext_nodes print("<<< bn %d, %d edges. %d in all_edges" % ( bn, len(self.edges), len(self.all_edges))) print("mx_depth_seen = %d" % mx_depth_seen) self.end_bin(msm_id, bn) # For last bin self.ext_nodes_count = 0 for sn in self.all_s_nodes: if sn not in self.all_nodes: self.ext_nodes_count += 1 print(" ##$$ %d all_nodes, %d ext_nodes_count, %d all_edges $$##" % ( len(self.all_nodes), self.ext_nodes_count, len(self.all_edges))) # >>> total nodes = len(self.all_nodes) + self.ext_nodes_count <<< # Delete subroots with max_icount < mn_allowed_trpkts if self.mn_allowed_trpkts != 0: subrs_to_delete = [] for sr_key in self.sub_root_icounts.keys(): sr_icounts = np.array(self.sub_root_icounts[sr_key]) mx_icount = np.max(sr_icounts) if mx_icount < self.mn_allowed_trpkts: subrs_to_delete.append(sr_key) print("About to delete %d subroots with max_icount < %d" % ( len(subrs_to_delete), self.mn_allowed_trpkts)) for sr_key in subrs_to_delete: self.sub_root_icounts.pop(sr_key) f.close() #info_f.write("End (MxDepth %d)\n" % mx_depth_seen) print("EOF reached") self.test_n_d_overlap()