def test_shortest_vf_route(self): s, t = 0, 1 # shortest vf: N0 - N5 - N1 shortest_vf = [0, 5, 1] sh = vft.get_shortest_vf_route(self.sample_graph, s, t) self.assertEqual(sh, shortest_vf) s, t = 0, 2 shortest_vf = [0, 5, 6, 2] sh = vft.get_shortest_vf_route(self.sample_graph, s, t) self.assertEqual(sh, shortest_vf)
def test_all_shortest_vf_route(self): self.sample_graph.add_vertex('N8') self.sample_graph.add_edges([['N8', 'N5'], ['N6', 'N8']]) self.prelabeled[8] = self.prelabeled[4] self.sample_graph.delete_edges([ self.sample_graph.get_eid(5, 6), ]) s, t = 0, 2 shortest_vf1 = [0, 5, 4, 6, 2] shortest_vf2 = [0, 5, 8, 6, 2] sh = vft.get_shortest_vf_route(self.sample_graph, s, t, _all=True) self.assertEqual(len(sh), 2) self.assertIn(shortest_vf1, sh) self.assertIn(shortest_vf2, sh)
def test_lp_choose_peer_if_no_customer_cone(self): self.sample_graph.add_vertices(['N8', 'N9']) self.prelabeled[8] = self.prelabeled[7] self.prelabeled[9] = self.prelabeled[7] * 2 self.closenesses[8] = self.closenesses[7] self.closenesses[9] = self.closenesses[7] * 2 self.sample_graph.add_edges([['N9', 'N7'], ['N8', 'N7'], ['N6', 'N8'], ['N9', 'N3']]) s = self.sample_graph.vs.find('N7').index t = self.sample_graph.vs.find('N3').index sh = vft.get_shortest_vf_route(self.sample_graph, s, t, _all=True, mode='lp') self.assertEqual(len(sh), 1) self.assertEqual(sh[0], [7, 8, 6, 3])
def test_all_shortest_lp_route(self): self.sample_graph.add_vertices(['N8', 'N9']) self.sample_graph.add_edges([['N8', 'N5'], ['N2', 'N8'], ['N9', 'N5'], ['N2', 'N9']]) self.prelabeled[8] = 0.5 self.prelabeled[9] = 0.5 s, t = 1, 2 shortest_lp = [1, 5, 8, 2] shortest_lp2 = [1, 5, 9, 2] sh = vft.get_shortest_vf_route(self.sample_graph, s, t, _all=True, mode='lp') self.assertEqual(len(sh), 2) self.assertIn(shortest_lp, sh) self.assertIn(shortest_lp2, sh)
def trace_dir(g, meta, vf_g, arguments): N = g.vcount() vf_sum_lp_w = sum([e['LPweight'] for e in vf_g.es]) out_instead_core = 0 core_instead_out = 0 in_customer = 0 in_customer_but_use_up = 0 progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) for m in meta: progress += 1 progress.show_progress() trace = m[helpers.TRACE] first_edge = [trace[0], trace[1]] first_hop_type = vft.edge_dir(g, first_edge, vfmode=vft.CLOSENESS) s_idx = vft.node_to_nodeid(g, trace[0]) t_idx = vft.node_to_nodeid(g, trace[-1]) sh_vf = vf_g.get_all_shortest_paths(s_idx + N, t_idx + N) if len(sh_vf) > 0: in_customer += 1 if first_hop_type != LinkDir.D: in_customer_but_use_up += 1 if first_hop_type == LinkDir.D: # 'remove' all downward path: for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT): down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True) vf_g.es[down_edge]['LPweight_old'] = vf_g.es[down_edge][ 'LPweight'] vf_g.es[down_edge]['LPweight'] = vf_sum_lp_w + 1 lp_sh = vft.get_shortest_vf_route(g, s_idx, t_idx, mode='lp', vf_g=vf_g, vfmode=vft.CLOSENESS) pretty_plotter.pretty_plot(g, trace, lp_sh, []) first_new_edge = [lp_sh[0], lp_sh[1]] first_new_hop_type = vft.edge_dir(g, first_new_edge, vfmode=vft.CLOSENESS) if first_hop_type != first_new_hop_type: out_instead_core += 1 print '' print "Original trace: %s" % trace print 'Original trace dir: %s' % vft.trace_to_string(g, trace) print 'Original closeness: %s' % [ g.vs.find(x)['closeness'] for x in trace ] print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh] print 'LP dir: %s' % vft.trace_to_string(g, lp_sh) print 'LP closeness: %s' % [ g.vs.find(x)['closeness'] for x in lp_sh ] # raw_input() for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT): down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True) vf_g.es[down_edge]['LPweight'] = vf_g.es[down_edge][ 'LPweight_old'] elif first_hop_type == LinkDir.U: lp_sh = vft.get_shortest_vf_route(g, s_idx, t_idx, mode='lp', vf_g=vf_g, vfmode=vft.CLOSENESS) lp_dir = vft.trace_to_string(g, lp_sh) # if lp_dir[1:].startswith('U') or lp_dir[1:].startswith('D'): # continue first_new_edge = [lp_sh[0], lp_sh[1]] first_new_hop_type = vft.edge_dir(g, first_new_edge, vfmode=vft.CLOSENESS) if first_hop_type != first_new_hop_type: core_instead_out += 1 print '' print "Original trace: %s" % trace print 'Original trace dir: %s' % vft.trace_to_string(g, trace) print 'Original closeness: %s' % [ g.vs.find(x)['closeness'] for x in trace ] print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh] print 'LP dir: %s' % vft.trace_to_string(g, lp_sh) print 'LP closeness: %s' % [ g.vs.find(x)['closeness'] for x in lp_sh ] pretty_plotter.pretty_plot(g, trace, lp_sh, []) # raw_input() logger.info('Core instead down: %d' % core_instead_out) logger.info('Down instead core: %d' % out_instead_core) logger.info('In customer cone: %d' % in_customer) logger.info('In customer but use UP: %d' % in_customer_but_use_up)
def purify(g, traceroutes, flags, show_progress=False): results = list() # remove traces with unknown nodes traceroutes = vft.trace_in_vertex_id(g, traceroutes) # generate valley-free graph if flags[FLAG_PRELABELED]: logger.info('Generate VF_G_PRE') vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED) else: logger.info('Skip prelabeled graph') if flags[FLAG_DEGREE]: logger.info('Generate VF_G_DEGREE') vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE) else: logger.info('Skip degree graph') if flags[FLAG_CLOSENESS]: logger.info('Generate VF_G_CLOSENESS') vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS) else: logger.info('Skip closeness graph') progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() logger.debug('Current trace: %s' % ([g.vs[x]['name'] for x in trace])) if len(trace) == 1: continue s, t = trace[0], trace[-1] is_vf_prelabeled = -1 is_lp_prelabeled_hard = -1 is_lp_prelabeled_soft = -1 is_vf_degree = -1 is_lp_degree_hard = -1 is_lp_degree_soft = -1 is_vf_closeness = -1 is_lp_closeness_hard = -1 is_lp_closeness_soft = -1 trace_len = len(trace) sh_len = g.shortest_paths(s, t, mode=i.OUT)[0][0] sh_len += 1 # convert hop count to node Counter if flags[FLAG_PRELABELED]: is_vf_prelabeled = vft.is_valley_free(g, trace, vft.PRELABELED) is_vf_prelabeled = int(is_vf_prelabeled) if is_vf_prelabeled: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_pre, first_edge=True, vfmode=vft.PRELABELED) is_lp_prelabeled_soft = 1 if lp_soft else 0 else: is_lp_prelabeled_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_pre, first_edge=False, vfmode=vft.PRELABELED) is_lp_prelabeled_hard = 1 if lp_hard else 0 else: is_lp_prelabeled_hard = -1 if flags[FLAG_DEGREE]: is_vf_degree = vft.is_valley_free(g, trace, vft.DEGREE) is_vf_degree = int(is_vf_degree) if is_vf_degree: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_degree, first_edge=True, vfmode=vft.DEGREE) is_lp_degree_soft = 1 if lp_soft else 0 else: is_lp_degree_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_degree, first_edge=False, vfmode=vft.DEGREE) is_lp_degree_hard = 1 if lp_hard else 0 else: is_lp_degree_hard = -1 if flags[FLAG_CLOSENESS]: is_vf_closeness = vft.is_valley_free(g, trace, vft.CLOSENESS) is_vf_closeness = int(is_vf_closeness) if is_vf_closeness: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_closeness, first_edge=True, vfmode=vft.CLOSENESS) is_lp_closeness_soft = 1 if lp_soft else 0 else: is_lp_closeness_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_closeness, first_edge=False, vfmode=vft.CLOSENESS) is_lp_closeness_hard = 1 if lp_hard else 0 else: is_lp_closeness_hard = -1 if False: sh_vf_len = vft.get_shortest_vf_route(g, s, t, mode='vf', vf_g=vf_g_pre, _all=True, vfmode=vft.PRELABELED) # ugy tunik, mintha nem mindig lenne pontos? fentartassal kezelendo # ez az ertek azert is kerult bele, hogy ellenorizzuk in_vf_prediction = 1 if sh_vf_len and trace in sh_vf_len else 0 else: sh_vf_len = -1 in_vf_prediction = -1 sh_vf_len = len(sh_vf_len[0]) if isinstance(sh_vf_len, list) else -1 percentage_stretch = trace_len / float(sh_len) named_trace = [g.vs[_id]["name"] for _id in trace] result = { helpers.TRACE: named_trace, helpers.TRACE_LEN: trace_len, helpers.SH_LEN: sh_len, helpers.SH_VF_LEN: sh_vf_len, helpers.IS_VF_PRELABELED: is_vf_prelabeled, helpers.IS_VF_DEGREE: is_vf_degree, helpers.IS_VF_CLOSENESS: is_vf_closeness, helpers.HOP_STRETCH: trace_len - sh_len, helpers.PERC_STRETCH: percentage_stretch, helpers.IN_VF_PRED: in_vf_prediction, helpers.IS_LP_SOFT_PRELABELED: is_lp_prelabeled_soft, helpers.IS_LP_HARD_PRELABELED: is_lp_prelabeled_hard, helpers.IS_LP_SOFT_DEGREE: is_lp_degree_soft, helpers.IS_LP_HARD_DEGREE: is_lp_degree_hard, helpers.IS_LP_SOFT_CLOSENESS: is_lp_closeness_soft, helpers.IS_LP_HARD_CLOSENESS: is_lp_closeness_hard, } results.append(result) # print >> sys.stderr, ('TRACE\tTRACE_LEN\tSH_LEN\tSH_VF_LEN\tIS_VF', # '\tSTRETCH\tIN_VF_PREDICTION\tIS_LP_F\tIS_LP_ALL') # for result in results: # result = [str(r) for r in result] # print >> sys.stderr, '\t'.join(result) # statistic = statistics.purify(g, results, # 'nc+ec+tc+rt+vf+vf_closeness+pred+lp_soft_prelabeled+lp_hard_prelabeled+lp_soft_degree+lp_hard_degree+lp_soft_closeness+lp_hard_closeness'.split('+')) # statistics.stat_printer(statistic) return results
def main(): parser = argparse.ArgumentParser( description=('Syntetic route generator'), formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('network') parser.add_argument('meta') parser.add_argument('all_trace_out', metavar='all-trace-out') parser.add_argument('syntetic_out', metavar='syntetic-out') parser.add_argument('--trace-count', '-tc', type=int, dest='trace_count', default=5000) parser.add_argument('--random-sample', dest='random_sample', action='store_true') parser.add_argument('--closeness-error', '-ce', type=float, dest='closeness_error', default=0.0) parser.add_argument('--core-limit-percentile', '-cl', type=int, dest='core_limit', default=0) parser.add_argument('--toggle-node-error-mode', action='store_true') arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS) - 1, arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) show_progress = arguments.progressbar g = helpers.load_network(arguments.network) g = g.simplify() meta = helpers.load_from_json(arguments.meta) if arguments.random_sample: random.shuffle(meta) meta = meta[:arguments.trace_count] N = g.vcount() cl = sorted([x['closeness'] for x in g.vs], reverse=True) logger.info('Min closeness: %s' % np.min(cl)) logger.info('Max closeness: %s' % np.max(cl)) logger.info('Mean closenss: %s' % np.mean(cl)) logger.info('10%% closeness: %s' % np.percentile(cl, 10)) logger.info('90%% closeness: %s' % np.percentile(cl, 90)) logger.info('Core limit: [r]%d%%[/]' % arguments.core_limit) change_probability = 100 * arguments.closeness_error logger.info('Change probability: [r]%6.2f%%[/]' % change_probability) core_limit = np.percentile(cl, arguments.core_limit) logger.info('Core limit in closeness: [bb]%f[/]' % core_limit) if arguments.toggle_node_error_mode: logger.info('[r]Node error mode[/]') msg = ( "If given node's closeness >= core_limit then the new ", "closeness in this node is ", #"rand(closeness_error ... old closeness)" "OLD_CLOSENSS * +/- closeness_error%") logger.info(''.join(msg)) logger.info('Minimum node closeness: [g]%f[/]' % arguments.closeness_error) for n in g.vs: if n['closeness'] < core_limit: continue # sign = -1 if random.uniform(-1, 1) < 0 else 1 # n['closeness'] = n['closeness'] * (1 + sign * arguments.closeness_error) new_closeness = random.uniform(arguments.closeness_error, n['closeness']) n['closeness'] = new_closeness g_labeled = vft.label_graph_edges(g, vfmode=vft.CLOSENESS) peer_edge_count = len([x for x in g_labeled.es if x['dir'] == LinkDir.P]) logger.info('Peer edge count: %d' % peer_edge_count) changed_edges = [] if not arguments.toggle_node_error_mode: msg = ("If the closeness values of the endpoints in given edge is ", "larger than the core_limit and ", "random(0,1) < closeness_error then change the direction ", "for this edge") logger.info(''.join(msg)) changed_u = changed_d = 0 changed_edges = [] changed_edgess = [] for edge in g_labeled.es: s, t = edge.source, edge.target s_cl = g_labeled.vs[s]['closeness'] t_cl = g_labeled.vs[t]['closeness'] if (s_cl < core_limit or t_cl < core_limit): continue if random.uniform(0, 1) > arguments.closeness_error: continue # if abs(s_cl - t_cl) / min(s_cl, t_cl) > 0.02: continue new_edge_dir = LinkDir.U if random.uniform(0, 1) > 0.5 else LinkDir.D if new_edge_dir != edge['dir']: if edge['dir'] == LinkDir.U: changed_u += 1 else: changed_d += 1 edge['dir'] = new_edge_dir changed_edges.append(edge) changed_edgess.append((edge.source, edge.target)) # if edge['dir'] == LinkDir.U: # changed_u += 1 # changed_edgess.append((edge.source, edge.target)) # edge['dir'] = LinkDir.D # changed_edges.append(edge) # elif edge['dir'] == LinkDir.D: # changed_d += 1 # changed_edgess.append((edge.source, edge.target)) # edge['dir'] = LinkDir.U # changed_edges.append(edge) logger.info('E count: %d' % g_labeled.ecount()) logger.info('Changed U: %d' % changed_u) logger.info('Changed D: %d' % changed_d) logger.info('Changed: %d' % (changed_d + changed_u)) changed_e = [(g_labeled.vs[e.source]['name'], g_labeled.vs[e.target]['name']) for e in changed_edges] changed_e = changed_e + [(x[1], x[0]) for x in changed_e] changed_e = set(changed_e) vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS, labeled_graph=g_labeled) # e_colors = [] # for e in vf_g_closeness.es: # if e.source < N and e.target < N: col = 'grey' # elif e.source < N and e.target >= N: col = 'blue' # elif e.source >= N and e.target >= N: col = 'red' # else: col = 'cyan' # e_colors.append(col) # igraph.plot(vf_g_closeness, "/tmp/closeness.pdf", # vertex_label=vf_g_closeness.vs['name'], # vertex_size=0.2, # edge_color=e_colors) pairs = [(g.vs.find(x[helpers.TRACE][0]).index, g.vs.find(x[helpers.TRACE][-1]).index, tuple(x[helpers.TRACE])) for x in meta] # pairs = list(set(pairs)) # random.shuffle(pairs) # visited = set() # pairs2 = [] # for x in pairs: # k = (x[0], x[1]) # if k in visited: continue # visited.add(k) # visited.add((k[1], k[0])) # pairs2.append(x) # pairs = pairs2 traces = [x[2] for x in pairs] stretches = [] syntetic_traces = [] sh_traces = [] base_traces = [] original_traces = [] bad = 0 progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15) for s, t, trace_original in pairs: progress += 1 progress.show_progress() trace_original_idx = [g.vs.find(x).index for x in trace_original] logger.debug('Original trace: %s -- %s -- %s', [g.vs[x]['name'] for x in trace_original_idx], vft.trace_to_string(g, trace_original_idx, vft.CLOSENESS), [g.vs[x]['closeness'] for x in trace_original_idx]) sh_routes = g.get_all_shortest_paths(s, t) sh_len = len(sh_routes[0]) sh_routes_named = [[g.vs[y]['name'] for y in x] for x in sh_routes] sh_trace_name = random.choice(sh_routes_named) base_trace_name = random.choice(sh_routes_named) candidates = vf_g_closeness.get_all_shortest_paths(s + N, t + N) candidates = [vft.vf_route_converter(x, N) for x in candidates] # candidates = [] if len(candidates) == 0: candidates = vft.get_shortest_vf_route(g_labeled, s, t, mode='vf', vf_g=vf_g_closeness, _all=True, vfmode=vft.CLOSENESS) if len(candidates) == 0: s_name, t_name = g.vs[s]['name'], g.vs[t]['name'] logger.debug("!!!No syntetic route from %s to %s" % (s_name, t_name)) continue logger.debug('Candidates from %s to %s:' % (g.vs[s]['name'], g.vs[t]['name'])) for c in candidates: logger.debug('%s -- %s -- %s' % ([g.vs[x]['name'] for x in c], vft.trace_to_string(g_labeled, c, vft.PRELABELED), [g.vs[x]['closeness'] for x in c])) chosen_one = random.choice(candidates) chosen_one_name = [g.vs[x]['name'] for x in chosen_one] # print chosen_one # print trace_original # pretty_plotter.pretty_plot(g, trace_original_idx, # chosen_one, changed_edgess, # spec_color=(0, 0, 0, 155)) hop_stretch = len(chosen_one) - sh_len stretches.append(hop_stretch) trace_original_e = zip(trace_original, trace_original[1:]) chosen_one_e = zip(chosen_one_name, chosen_one_name[1:]) trace_affected = any([x in changed_e for x in trace_original_e]) chosen_affected = any([x in changed_e for x in chosen_one_e]) logger.debug('Trace affected: %s' % trace_affected) logger.debug('Chosen affected: %s' % chosen_affected) # if hop_stretch > 2: # logger.debug('Base: %s' % trace_to_string(g_labeled, base_trace_name)) # logger.debug('SH: %s' % trace_to_string(g_labeled, sh_trace_name)) # logger.debug('Trace: %s' % trace_to_string(g_labeled, trace_original)) # logger.debug('Syntetic: %s' % trace_to_string(g_labeled, chosen_one_name)) if trace_affected or chosen_affected or hop_stretch > 2: # pretty_plotter.pretty_plot_all(g, traces, # chosen_one, changed_edgess, # spec_color=(0, 0, 0, 255)) bad += 1 syntetic_traces.append(chosen_one_name) sh_traces.append(sh_trace_name) base_traces.append(base_trace_name) original_traces.append(trace_original) logger.debug('From %s to %s chosen one %s' % (g.vs[s]['name'], g.vs[t]['name'], chosen_one_name)) result = zip(base_traces, sh_traces, original_traces, syntetic_traces) helpers.save_to_json(arguments.all_trace_out, result) helpers.save_to_json(arguments.syntetic_out, syntetic_traces) print 'Bad: %d' % bad c = collections.Counter(stretches) trace_count = len(syntetic_traces) logger.info('Stretch dist:') for k in c: logger.info('\t%d: %5.2f%%[%d]' % (k, 100 * c[k] / float(trace_count), c[k])) logger.info('Valid route count: %d' % trace_count) logger.info('Route count parameter: %d' % arguments.trace_count) logger.info('Generated valid pair count: %d' % len(pairs))