def watts_converter(network, show_progress=False): logger.info('Get giant component') g = network.clusters().giant() logger.info('Calculate closeness') closenesses = [] progress = progressbar1.DummyProgressBar(end=g.vcount(), width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15) for x in g.vs: progress += 1 progress.show_progress() closenesses.append(g.closeness(x, mode=igraph.ALL)) g.vs['closeness'] = closenesses logger.info('Add edges') new_edge_list = [] watts_graph = g.copy() watts_graph.delete_edges(watts_graph.es) watts_graph.to_directed() progress = progressbar1.DummyProgressBar(end=g.ecount(), width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=g.ecount(), width=15) for e in g.es: progress += 1 progress.show_progress() s, t = e.source, e.target if g.vs[s]['closeness'] > g.vs[t]['closeness']: t, s = s, t new_edge_list.append((s, t)) watts_graph.add_edges(new_edge_list) for e in watts_graph.es: e['provider'] = False logger.info('Get provider edges') progress = progressbar1.DummyProgressBar(end=g.vcount(), width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15) for n in g.vs: progress += 1 progress.show_progress() provider_eid = get_provider_edge_highest_closeness(g, watts_graph, n) if provider_eid is None: continue watts_graph.es[provider_eid]['provider'] = True return watts_graph
def main(): parser = argparse.ArgumentParser( description='Save closeness values for all node', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('network') parser.add_argument('out') parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS), arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) g = helpers.load_network(arguments.network) logger.info('Graph loaded from: %s' % arguments.network) logger.info('Graph vertex count: %d' % g.vcount()) if 'closeness' not in g.vs: logger.info('Calculate closeness values') progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15) for n in g.vs: progress += 1 progress.show_progress() closeness = g.closeness(n, mode=igraph.OUT) n['closeness'] = closeness g.save(arguments.out)
def purify(g, sample_size, show_progress): nodes = range(0, g.vcount()) samples = [random.sample(nodes, 4) for idx in range(0, sample_size)] progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(samples), width=15) logger.info('Avg distance calculation') avg_len = g.average_path_length() logger.info('Avg len: %d' % avg_len) hbcs = [] for quadruplet in samples: progress += 1 progress.show_progress() hbc = hyperbolicity(g, quadruplet) hbcs.append(hbc) avg_hbc = sum(hbcs) / float(len(hbcs)) logger.info('Avg hbc num: %f' % avg_hbc) logger.info('Avg hbc: %f' % (avg_hbc / float(avg_len)))
def filter(g, traceroutes, filters=['sh', 'loop', 'ex', 'vf', 'lp'], first_edge=True): logger.info('Traceroutes: %d', len(traceroutes)) # remove empty traces traceroutes = [x for x in traceroutes if len(x) > 0] logger.info('Non empty traceroutes: %d', (len(traceroutes))) traceroutes = [x for x in traceroutes if len(x) > 1] logger.info('Larger than one hop traceroutes: %d', (len(traceroutes))) # remove traces with unknown nodes traceroutes, _ = vft.trace_clean(g, traceroutes) logger.info('Ignored: %d', _) traceroutes = vft.trace_in_vertex_id(g, traceroutes) logger.info('Trace count: %d', len(traceroutes)) progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) good_traceroutes = traceroutes[:] if 'sh' in filters: logger.debug('Remove short traces') good_traceroutes = [x for x in good_traceroutes if len(x) >= 3] logger.debug('Remained: %d', len(good_traceroutes)) if 'loop' in filters: logger.debug('Remove traces with loops') good_traceroutes = [ x for x in good_traceroutes if len(set(x)) == len(x) ] logger.debug('Remained: %d' % len(good_traceroutes)) if 'ex' in filters: logger.debug('Remove non existent traces') good_traceroutes = [ x for x in good_traceroutes if vft.trace_exists(g, x) ] logger.debug('Remained: %d', len(good_traceroutes)) if 'vf' in filters: logger.debug('Remove non vf traces') good_traceroutes = [ x for x in good_traceroutes if vft.is_valley_free(g, x) ] logger.debug('Remained: %d' % len(good_traceroutes)) if 'lp' in filters: logger.debug('Remove non lp traces') good_traceroutes = [ x for x in good_traceroutes if vft.is_local_preferenced(g, x, first_edge=first_edge) ] logger.debug('Remained: %d' % len(good_traceroutes)) # convert back node ids to node names good_traceroutes = [[g.vs[id]["name"] for id in trace] for trace in good_traceroutes] logger.debug(len(good_traceroutes)) return good_traceroutes
def upwalker_counter(g, meta, vf_g, arguments): N = g.vcount() progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) # trace_up_map = {} # random_up_map = {} trace_up_counter = [] random_up_counter = [] for m in meta: progress += 1 progress.show_progress() trace = m[helpers.TRACE] trace_dir = vft.trace_to_string(g, trace) trace_up_count = trace_dir.count('U') trace_up_counter.append(trace_up_count) # trace_up_map[trace_up_count] = trace_up_map.get(trace_up_count, 0) + # 1 s, t = trace[0], trace[-1] s_idx, t_idx = vft.node_to_nodeid(g, s), vft.node_to_nodeid(g, t) random_vf_route = helpers.random_route_walk(vf_g, s_idx, t_idx + N, len(trace), named=False, weight_field='VFweight') random_vf_route = vft.vf_route_converter(random_vf_route, N) random_vf_dir = vft.trace_to_string(g, random_vf_route) random_vf_count = random_vf_dir.count('U') random_up_counter.append(random_vf_count) # random_up_map[random_vf_count] = random_up_map.get(random_vf_count, # 0) + 1 real_counter = collections.Counter(trace_up_counter) real_up = ' '.join( ['%s: %s' % (k, real_counter[k]) for k in sorted(list(real_counter))]) random_counter = collections.Counter(random_up_counter) random_up = ' '.join([ '%s: %s' % (k, random_counter[k]) for k in sorted(list(random_counter)) ]) logger.info('') logger.info('Real trace UP counter: %s' % real_up) logger.info('Random vf trace up counter: %s' % random_up) helpers.save_to_json(arguments.out, { 'REAL': dict(real_counter), 'RANDOM': dict(random_counter) }) keys = sorted(set(list(real_counter) + list(random_counter))) logger.info('IDX;REAL;RANDOM') for k in keys: logger.info('%s;%d;%d' % (k, real_counter[k], random_counter[k]))
def ba_gen(arguments): g = helpers.load_network(arguments.input_graph) if arguments.use_degree_seq: method = 'Degree Seq.' generated_graph = _degree_graph(g) else: method = 'Barabasi-Albert' generated_graph = _ba_graph(g) logger.info('Used method: [b]%s[/]' % method) logger.debug('Graph created. Fill up closeness values') N = generated_graph.vcount() progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=N, width=15) for vs in generated_graph.vs: progress += 1 progress.show_progress() vs['name'] = 'BA%s' % vs.index vs['closeness'] = generated_graph.closeness(vs, mode=igraph.OUT) logger.info('Save to %s' % arguments.ba_output) generated_graph.save(arguments.ba_output)
def main(): parser = argparse.ArgumentParser( description='SANDBOX mode. Write something useful here', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('network') parser.add_argument('--sample-size', type=int, default=1000, dest='sample_size') parser.add_argument('--node-drop', type=float, default=.1, dest='node_drop') parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS), arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) g = helpers.load_network(arguments.network) g = g.components().giant() logger.info('Graph loaded from: %s' % arguments.network) logger.info('Graph vertex count: %d' % g.vcount()) end = int(g.vcount() * arguments.node_drop) logger.info('Remaining node count: %d' % end) try: nodes = [(x.index, x['closeness']) for x in g.vs] except KeyError: logger.info('Calculate closeness values') progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15) for n in g.vs: progress += 1 progress.show_progress() closeness = g.closeness(n) n['closeness'] = closeness g.save('with_closeness.gml') nodes = [(x.index, x['closeness']) for x in g.vs] nodes = sorted(nodes, reverse=True, key=lambda x: x[1]) top_nodes = [x[0] for x in nodes[:end]] delete_nodes = [x.index for x in g.vs if x.index not in top_nodes] g.delete_vertices(delete_nodes) logger.info('Left nodes: %d' % g.vcount()) purify(g, arguments.sample_size, arguments.progressbar)
def ba_generator(ba_graph, sh_paths, stretch, vf_g, progressbar=False): vf_count = 0 trace_count = 0 lp_count = 0 progress = progressbar1.DummyProgressBar(end=10, width=15) if progressbar: progress = progressbar1.AnimatedProgressBar(end=len(sh_paths), width=15) for (s, t), shl in sh_paths: progress += 1 progress.show_progress() logger.debug('SH from %s to %s is %d' % (s, t, shl)) random_route = helpers.random_route_walk(ba_graph, s, t, shl + stretch) logger.debug('Random route: %s' % random_route) real_stretch = len(random_route) - shl if real_stretch != stretch: continue trace_count += 1 is_vf = vft.is_valley_free(ba_graph, random_route, vfmode=vft.CLOSENESS) logger.debug( 'Trace edge dir: %s' % vft.trace_to_string(ba_graph, random_route, vfmode=vft.CLOSENESS)) logger.debug('Is VF: %s' % is_vf) if is_vf: is_lp = vft.is_local_preferenced(ba_graph, random_route, first_edge=True, vfmode=vft.CLOSENESS, vf_g=vf_g) else: is_lp = 0 logger.debug('Is LP: %s' % is_lp) vf_count += int(is_vf) lp_count += int(is_lp) logger.info('Stretch %d' % stretch) logger.info('Trace count: %d' % trace_count) logger.info('VF count: %d' % vf_count) logger.info('LP count: %d' % lp_count) return (stretch, trace_count, vf_count, lp_count)
def upwalker_deepness(g, meta, vf_g, arguments): N = g.vcount() progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) graph_max = max([x['closeness'] for x in g.vs]) with open(arguments.out, 'w') as f: f.write( 'TRACE_MIN\tTRACE_AVG\tTRACE_MEDIAN\tTRACE_MAX\tSH_MIN\tSH_AVG\tSH_MEDIAN\tSH_MAX\tTRACE_MAX\tGRAPH_MAX\tSH_MAX\tSTRETCH\tLENGTH\n' ) for m in meta: progress += 1 progress.show_progress() trace = m[helpers.TRACE] s, t = trace[0], trace[-1] sh_trace = g.get_all_shortest_paths(s, t) sh_trace = random.choice(sh_trace) sh_closeness = [g.vs[x]['closeness'] for x in sh_trace] trace_closeness = [g.vs.find(x)['closeness'] for x in trace] stretch = m[helpers.HOP_STRETCH] if m[ helpers.HOP_STRETCH] else len(trace) - len(sh_trace) trace_max = max(trace_closeness) sh_max = max(sh_trace) with open('{}_trace{}'.format(arguments.out, len(trace)), 'a') as fo: for x in trace_closeness: fo.write('{} '.format(x / float(graph_max))) fo.write('\n') with open('{}_sh{}'.format(arguments.out, len(sh_trace)), 'a') as fo: for x in sh_closeness: fo.write('{} '.format(x / float(graph_max))) fo.write('\n') with open(arguments.out, 'a') as f: f.write( "{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format( min(trace_closeness), np.average(trace_closeness), np.median(trace_closeness), np.max(trace_closeness), min(sh_closeness), np.average(sh_closeness), np.median(sh_closeness), np.max(sh_closeness), trace_max, graph_max, sh_max, stretch, len(trace)))
def main(): parser = argparse.ArgumentParser(description='SANDBOX mode. Write something useful here', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('network') parser.add_argument('meta') parser.add_argument('--top-node-ratio', type=float, default=.1, dest='top_node_ratio') parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS), arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) g = helpers.load_network(arguments.network) meta = helpers.load_from_json(arguments.meta) logger.info('Graph loaded from: %s' % arguments.network) logger.info('Graph vertex count: %d' % g.vcount()) end = int(g.vcount() * arguments.top_node_ratio) logger.info('Top node count: %d' % end) try: nodes = [(x.index, x['closeness']) for x in g.vs] except KeyError: logger.info('Calculate closeness values') progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15) for n in g.vs: progress += 1 progress.show_progress() closeness = g.closeness(n) n['closeness'] = closeness g.save('%s_with_closeness.gml' % arguments.network) nodes = [(x.index, x['closeness']) for x in g.vs] nodes = sorted(nodes, reverse=True, key=lambda x: x[1]) top_nodes = set([x[0] for x in nodes[:end]]) purify(g, meta, top_nodes, arguments.progressbar)
def watts_trace_gen(g, traceroutes, show_progress=False): watts_traceroutes = [] progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() s, t = trace[0], trace[-1] logger.debug('Get route from {s} to {t}'.format(s=s, t=t)) logger.debug('Original trace[[g]%d[/]]: %s' % (len(trace), trace)) try: watts_trace = get_watts_route(g, s, t) watts_trace = [g.vs[x]['name'] for x in watts_trace] watts_traceroutes.append(watts_trace) logger.debug('Watts trace[[p]%d[/]]: %s' % (len(watts_trace), watts_trace)) except RuntimeError: logger.warning('[b]No watts route[/]') return watts_traceroutes
def purify(g, out, count=1000, show_progress=False): logger.info('Started') nodes = range(0, g.vcount()) endpoints = [random.sample(nodes, 2) for idx in range(0, count)] progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar( end=len(endpoints), width=15) traces = [] for endpoint in endpoints: progress += 1 progress.show_progress() src, dst = endpoint trace = g.get_shortest_paths(src, dst)[0] if len(trace) > 0: traces.append([g.vs[x]['name'] for x in trace]) logger.info('Len last trace: %d' % len(trace)) helpers.save_to_json(out, traces)
def main(): parser = argparse.ArgumentParser( description='Pretty plot', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('network') parser.add_argument('meta') # parser.add_argument('output') parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS), arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) g = helpers.load_network(arguments.network) meta = helpers.load_from_json(arguments.meta) edges = [(e.source, e.target) for e in g.es] rand_edges = random.sample(edges, 40) progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) for m in meta[:15]: progress += 1 progress.show_progress() trace = m[helpers.TRACE] if len(trace) < 3: continue s, t = trace[0], trace[-1] sh = g.get_shortest_paths(s, t)[0] pretty_plot(g, trace, sh, rand_edges)
def purify(labeled_g, out, network_path, extra_hop=0): vs = [x.index for x in labeled_g.vs] ## Jus like in R # print '================' # for x in orig_vs: # shp = labeled_g.get_all_shortest_paths(x, orig_vs, mode=i.ALL) # res = [] # mes = 0 # for p in shp: # mes += 1 # # print [labeled_g.vs[u]['name'] for u in p] # vf_indicator = 1 if vft.is_valley_free(labeled_g, p) else 0 # # if vf_indicator == 0: # # print [labeled_g.vs[u]['name'] for u in [p[0], p[-1]]] # # print [labeled_g.degree(u) for u in p] # # print vft.trace_to_string(labeled_g, p) # # print vf_indicator == 1 # res.append(vf_indicator) # # raw_input() # # print mes # print np.mean(res) # raw_input() # print '///////////////////////////' pairs = random_pairs(vs, NODE_PAIRS) print 'Random pairs: %d' % len(pairs) probed_pairs = 0 all_vf = 0 all_nonvf = 0 all_vf_closeness = 0 all_nonvf_closeness = 0 results = [] results2 = [] results3 = [] short_results = list() short_results2 = list() short_results3 = list() all_short_vf = 0 all_short_nonvf = 0 all_short_vf_closeness = 0 all_short_nonvf_closeness = 0 long_results = list() long_results2 = list() long_results3 = list() all_long_vf = 0 all_long_nonvf = 0 all_long_vf_closeness = 0 all_long_nonvf_closeness = 0 results_closeness = [] results3_closeness = [] progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15) for s, t in pairs: progress += 1 progress.show_progress() for x in range(0, labeled_g.vcount()): labeled_g.vs[x]['traces'] = dict() # all_path = labeled_g.get_all_shortest_paths(s, t, mode=i.ALL) sh_len = labeled_g.shortest_paths(s, t, mode=i.ALL)[0][0] sh_len += 1 # convert to hop count all_path = helpers.dfs_mark(copy.deepcopy(labeled_g), s, t, sh_len + extra_hop) if all_path is None or len(all_path) < 1: print 'No path between %s %s' % (s, t) continue probed_pairs += 1 vf_indicator = [ 1 if vft.is_valley_free(labeled_g, x) else 0 for x in all_path ] vf_closeness_indicator = [ 1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS) else 0 for x in all_path ] vf_count = sum(vf_indicator) vf_closeness_count = sum(vf_closeness_indicator) nonvf = len(all_path) - vf_count nonvf_closeness = len(all_path) - vf_closeness_count all_vf += vf_count all_nonvf += nonvf all_vf_closeness += vf_closeness_count all_nonvf_closeness += nonvf_closeness long_path = [x for x in all_path if len(x) == sh_len + extra_hop] short_path = [x for x in all_path if len(x) < sh_len + extra_hop] tmp = [ 1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_PRELABELED) else 0 for x in all_path if len(x) < sh_len + extra_hop ] short_vf_count = sum(tmp) short_nonvf = len(tmp) - short_vf_count tmp = [ 1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS) else 0 for x in all_path if len(x) < sh_len + extra_hop ] short_vf_closeness_count = sum(tmp) short_nonvf_closeness = len(tmp) - short_vf_closeness_count tmp = [ 1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_PRELABELED) else 0 for x in all_path if len(x) >= sh_len + extra_hop ] long_vf_count = sum(tmp) long_nonvf = len(tmp) - long_vf_count tmp = [ 1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS) else 0 for x in all_path if len(x) >= sh_len + extra_hop ] long_vf_closeness_count = sum(tmp) long_nonvf_closeness = len(tmp) - long_vf_closeness_count if len(all_path) > 0: results.append(vf_count / float(len(all_path))) results_closeness.append(vf_closeness_count / float(len(all_path))) else: results.append(0) results_closeness.append(0) results3.append([vf_count, nonvf]) results3_closeness.append([vf_closeness_count, nonvf_closeness]) if len(all_path) > 1: results2.append(vf_count / float(len(all_path))) all_long_vf += long_vf_count all_long_nonvf += long_nonvf all_long_vf_closeness += long_vf_closeness_count all_long_nonvf_closeness += long_nonvf_closeness all_short_vf += short_vf_count all_short_nonvf += short_nonvf all_short_vf_closeness += short_vf_closeness_count all_short_nonvf_closeness += short_nonvf_closeness if len(long_path) > 0: long_results.append(long_vf_count / float(len(long_path))) long_results3.append(long_vf_closeness_count / float(len(long_path))) else: long_results.append(0) long_results3.append(0) if len(long_path) > 1: long_results2.append(long_vf_count / float(len(long_path))) if len(short_path) > 0: short_results.append(short_vf_count / float(len(short_path))) short_results3.append(short_vf_closeness_count / float(len(short_path))) else: short_results.append(0) short_results3.append(0) print with open(out, 'w') as f: f.write('%s\n' % network_path) f.write('Probed pairs: %d\n' % probed_pairs) f.write('VF count: %d\n' % all_vf) f.write('Non vf count: %d\n' % all_nonvf) f.write('VF perc: %f\n' % (all_vf / float(all_vf + all_nonvf))) f.write('Mean VF prob: %f\n' % np.mean(results)) f.write('Mean VF2 prob: %f\n' % np.mean(results2)) f.write('\n') f.write('VF CLOSENESS count: %d\n' % all_vf_closeness) f.write('Non vf CLOSENESS count: %d\n' % all_nonvf_closeness) f.write( 'VF CLOSENESS perc: %f\n' % (all_vf_closeness / float(all_vf_closeness + all_nonvf_closeness))) f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(results_closeness)) f.write('\n') f.write('==========\n') f.write('VF count: %d\n' % all_short_vf) f.write('VF CLOSENESS count: %d\n' % all_short_vf_closeness) f.write('Non vf count: %d\n' % all_short_nonvf) f.write('Non vf CLOSENESS count: %d\n' % all_short_nonvf_closeness) if all_short_vf + all_short_nonvf > 0: f.write('VF perc: %f\n' % (all_short_vf / float(all_short_vf + all_short_nonvf))) if all_short_vf_closeness + all_short_nonvf_closeness > 0: f.write( 'VF CLOSENESS perc: %f\n' % (all_short_vf_closeness / float(all_short_vf_closeness + all_short_nonvf_closeness))) f.write('Mean VF prob: %f\n' % np.mean(short_results)) f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(short_results3)) f.write('Mean VF2 prob: %f\n' % np.mean(short_results2)) f.write('=-----------------\n') f.write('VF count: %d\n' % all_long_vf) f.write('VF CLOSENESS count: %d\n' % all_long_vf_closeness) f.write('Non vf count: %d\n' % all_long_nonvf) f.write('Non vf CLOSENESS count: %d\n' % all_long_nonvf_closeness) f.write('VF perc: %f\n' % (all_long_vf / float(all_long_vf + all_long_nonvf))) f.write('VF CLOSENESS perc: %f\n' % (all_long_vf_closeness / float(all_long_vf_closeness + all_long_nonvf_closeness))) f.write('Mean VF prob: %f\n' % np.mean(long_results)) f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(long_results3)) f.write('Mean VF2 prob: %f\n' % np.mean(long_results2))
def graphs(g, out, network_path): # g = g.Erdos_Renyi(n=g.vcount(), m=g.ecount(), directed=False, loops=False) # i.plot(g) degrees = g.degree() print 'Generate random graph' max_random_graph = g.Degree_Sequence(g.degree(mode=i.ALL, loops=False), method='vl') rand_degrees = max_random_graph.degree(mode=i.ALL, loops=False) fi_values = [] ratio_values = [] print 'Calculate coeff' edge_helper = [min(g.degree(x.source), g.degree(x.target)) for x in g.es] rand_edge_helper = [ min(max_random_graph.degree(x.source), max_random_graph.degree(x.target)) for x in max_random_graph.es ] progress = progressbar1.AnimatedProgressBar(end=max(degrees), width=15) for k in range(1, max(degrees) + 1): progress += 1 progress.show_progress() try: fi = rich_club_coeff(g, k, degrees=degrees, edge_helper=edge_helper) fi_ran = rich_club_coeff(max_random_graph, k, degrees=rand_degrees, edge_helper=rand_edge_helper) ratio = fi / float(fi_ran) except ZeroDivisionError: fi_values.append(0) ratio_values.append(0) continue fi_values.append(fi) ratio_values.append(ratio) fig, ax = plt.subplots() ax.axhline(y=1, xmin=1, xmax=len(ratio_values)) ax.plot(range(1, len(fi_values) + 1), fi_values, 'o') ax.set_xlabel('k') ax.set_ylabel('fi(k)') ax.set_title('Rich club coeff ratio') ax.set_yscale('log') ax.set_xscale('log') # ax.set_ylim(0.0, 2.0) # ax.set_xlim([1, 10000]) ax.yaxis.grid(True, which='major') # plt.gca().invert_yaxis() fname = '%s_rich_club_simple_coeff.pdf' % out print fname pdf_file = PdfPages(fname) plt.savefig(pdf_file, format='pdf') pdf_file.close() plt.show() # i.plot(g.degree_distribution(), log="xy") fig, ax = plt.subplots() ax.axhline(y=1, xmin=1, xmax=len(ratio_values)) ax.plot(range(1, len(ratio_values) + 1), ratio_values, 'o') ax.set_xlabel('k') ax.set_ylabel('P_ran(k)') ax.set_title('Rich club coeff ratio') ax.set_xscale('log') ax.set_ylim(0.0, max(2.0, max(ratio_values))) # ax.set_xlim([1, 10000]) ax.yaxis.grid(True, which='major') fname = '%s_rich_club_ratio_coeff.pdf' % out print fname pdf_file = PdfPages(fname) plt.savefig(pdf_file, format='pdf') pdf_file.close() plt.show()
def filter(g, traceroutes): results = list() # remove traces with unknown nodes traceroutes = vft.trace_in_vertex_id(g, traceroutes) progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() if not vft.trace_exists(g, trace): print 'BUG?' continue for x in range(0, g.vcount()): g.vs[x]['traces'] = dict() trace = tuple(trace) s, t = trace[0], trace[-1] sh_len = g.shortest_paths(s, t, mode=i.ALL)[0][0] sh_len += 1 # igraph's hop count to node count all_routes = helpers.dfs_mark(g, s, t, sh_len + 1) # all_routes2 = helpers.dfs_simple(g, s, t, sh_len + 1, ()) # if set(all_routes) - set(all_routes2) != set(all_routes2) - set(all_routes): # print 'AJAJAJ' # print all_routes # print '----------' # print all_routes2 sh_routes = [x for x in all_routes if len(x) == sh_len] all_vf_routes = [x for x in all_routes if vft.is_valley_free(g, x)] prediction_set = set(sh_routes) | set(all_vf_routes) result = [ trace, len(trace), sh_len, len(sh_routes), trace in sh_routes, len(all_vf_routes), trace in all_vf_routes, len(all_routes), trace in all_routes, len(prediction_set), trace in prediction_set, vft.is_valley_free(g, trace), # sh_routes, all_vf_routes, all_routes, vft.trace_to_string(g, trace) ] results.append(result) print >> sys.stderr, ( 'TRACE\tTRACE_LEN\tSH_LEN', '\t#SH_ROUTE\tOK', '\t#ALL_VF\tOK', '\t#ALL_ROUTE\tOK', '\t#PREDICTION_SET\tOK', '\tIS_VF', # '\tSH_ROUTES\tALL_VF_ROUTES\tALL_ROUTE', '\tTRACE_STR') for result in results: result = [str(r) for r in result] print >> sys.stderr, '\t'.join(result) return results
def trace_dir(g, meta, vf_g, arguments): N = g.vcount() vf_sum_lp_w = sum([e['LPweight'] for e in vf_g.es]) out_instead_core = 0 core_instead_out = 0 in_customer = 0 in_customer_but_use_up = 0 progress = progressbar1.DummyProgressBar(end=10, width=15) if arguments.progressbar: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) for m in meta: progress += 1 progress.show_progress() trace = m[helpers.TRACE] first_edge = [trace[0], trace[1]] first_hop_type = vft.edge_dir(g, first_edge, vfmode=vft.CLOSENESS) s_idx = vft.node_to_nodeid(g, trace[0]) t_idx = vft.node_to_nodeid(g, trace[-1]) sh_vf = vf_g.get_all_shortest_paths(s_idx + N, t_idx + N) if len(sh_vf) > 0: in_customer += 1 if first_hop_type != LinkDir.D: in_customer_but_use_up += 1 if first_hop_type == LinkDir.D: # 'remove' all downward path: for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT): down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True) vf_g.es[down_edge]['LPweight_old'] = vf_g.es[down_edge][ 'LPweight'] vf_g.es[down_edge]['LPweight'] = vf_sum_lp_w + 1 lp_sh = vft.get_shortest_vf_route(g, s_idx, t_idx, mode='lp', vf_g=vf_g, vfmode=vft.CLOSENESS) pretty_plotter.pretty_plot(g, trace, lp_sh, []) first_new_edge = [lp_sh[0], lp_sh[1]] first_new_hop_type = vft.edge_dir(g, first_new_edge, vfmode=vft.CLOSENESS) if first_hop_type != first_new_hop_type: out_instead_core += 1 print '' print "Original trace: %s" % trace print 'Original trace dir: %s' % vft.trace_to_string(g, trace) print 'Original closeness: %s' % [ g.vs.find(x)['closeness'] for x in trace ] print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh] print 'LP dir: %s' % vft.trace_to_string(g, lp_sh) print 'LP closeness: %s' % [ g.vs.find(x)['closeness'] for x in lp_sh ] # raw_input() for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT): down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True) vf_g.es[down_edge]['LPweight'] = vf_g.es[down_edge][ 'LPweight_old'] elif first_hop_type == LinkDir.U: lp_sh = vft.get_shortest_vf_route(g, s_idx, t_idx, mode='lp', vf_g=vf_g, vfmode=vft.CLOSENESS) lp_dir = vft.trace_to_string(g, lp_sh) # if lp_dir[1:].startswith('U') or lp_dir[1:].startswith('D'): # continue first_new_edge = [lp_sh[0], lp_sh[1]] first_new_hop_type = vft.edge_dir(g, first_new_edge, vfmode=vft.CLOSENESS) if first_hop_type != first_new_hop_type: core_instead_out += 1 print '' print "Original trace: %s" % trace print 'Original trace dir: %s' % vft.trace_to_string(g, trace) print 'Original closeness: %s' % [ g.vs.find(x)['closeness'] for x in trace ] print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh] print 'LP dir: %s' % vft.trace_to_string(g, lp_sh) print 'LP closeness: %s' % [ g.vs.find(x)['closeness'] for x in lp_sh ] pretty_plotter.pretty_plot(g, trace, lp_sh, []) # raw_input() logger.info('Core instead down: %d' % core_instead_out) logger.info('Down instead core: %d' % out_instead_core) logger.info('In customer cone: %d' % in_customer) logger.info('In customer but use UP: %d' % in_customer_but_use_up)
def purify(g, meta_original, out, count=1000, try_per_race=1, show_progress=False): empty = 0 # remove traces with already calculated random paths logger.warn('[r]ONLY NOT FILLED PATHS[/]') meta_filled = [ x for x in meta_original if helpers.RANDOM_NONVF_WALK_RUN_COUNT not in x ] logger.info('All trace count: %d' % len(meta_filled)) tr_count = min(len(meta_filled), count) meta_random = random.sample(meta_filled, tr_count) logger.info('Chosen subset count: %d' % len(meta_random)) real_vf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 1] real_nonvf_degree = [ x for x in meta_random if x[helpers.IS_VF_DEGREE] == 0 ] assert len(real_nonvf_degree) == tr_count - len(real_vf_degree) real_vf_prelabeled = [ x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 1 ] real_nonvf_prelabeled = [ x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 0 ] assert len(real_nonvf_prelabeled) == tr_count - len(real_vf_prelabeled) real_vf_closeness = [ x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 1 ] real_nonvf_closeness = [ x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 0 ] assert len(real_nonvf_closeness) == tr_count - len(real_vf_closeness) logger.info('Real vf degree: %f[%d]' % ((len(real_vf_degree) / float(tr_count), len(real_vf_degree)))) logger.info( 'Real nonvf degree: %f[%d]' % ((len(real_nonvf_degree) / float(tr_count), len(real_nonvf_degree)))) logger.info( 'Real vf prelabeled: %f[%d]' % ((len(real_vf_prelabeled) / float(tr_count), len(real_vf_prelabeled)))) logger.info('Real nonvf prelabeled: %f[%d]' % ((len(real_nonvf_prelabeled) / float(tr_count), len(real_nonvf_prelabeled)))) logger.info( 'Real vf closeness: %f[%d]' % ((len(real_vf_closeness) / float(tr_count), len(real_vf_closeness)))) logger.info('Real nonvf closeness: %f[%d]' % ((len(real_nonvf_closeness) / float(tr_count), len(real_nonvf_closeness)))) progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(meta_random), width=15) for trace_meta in meta_random: progress += 1 progress.show_progress() trace = vft.trace_in_vertex_id(g, [ trace_meta[helpers.TRACE], ]) if len(trace) != 1: logger.error('PROBLEM') logger.error('%s' % trace_meta) continue trace = trace[0] random_nonvf_walk_closeness_route_count = 0 random_nonvf_walk_closeness_route_len = [] random_nonvf_walk_degree_route_count = 0 random_nonvf_walk_degree_route_len = [] random_nonvf_walk_prelabeled_route_count = 0 random_nonvf_walk_prelabeled_route_len = [] random_nonvf_walk_lp_soft_closeness_route_count = 0 random_nonvf_walk_lp_soft_degree_route_count = 0 random_nonvf_walk_lp_soft_prelabeled_route_count = 0 random_nonvf_walk_lp_hard_closeness_route_count = 0 random_nonvf_walk_lp_hard_degree_route_count = 0 random_nonvf_walk_lp_hard_prelabeled_route_count = 0 s, t = trace[0], trace[-1] for counter in xrange(0, try_per_race): isvf, random_path = helpers.random_nonvf_route( g, s, t, len(trace), vfmode=vft.CLOSENESS) assert len(random_path) > 0 if isvf: random_nonvf_walk_closeness_route_count += 1 lp_soft = vft.is_local_preferenced(g, trace, first_edge=True, vfmode=vft.CLOSENESS) lp_hard = vft.is_local_preferenced(g, trace, first_edge=False, vfmode=vft.CLOSENESS) if lp_soft: random_nonvf_walk_lp_soft_closeness_route_count += 1 if lp_hard: random_nonvf_walk_lp_hard_closeness_route_count += 1 random_nonvf_walk_closeness_route_len.append(len(random_path)) isvf, random_path = helpers.random_nonvf_route(g, s, t, len(trace), vfmode=vft.DEGREE) assert len(random_path) > 0 if isvf: random_nonvf_walk_degree_route_count += 1 lp_soft = vft.is_local_preferenced(g, trace, first_edge=True, vfmode=vft.DEGREE) lp_hard = vft.is_local_preferenced(g, trace, first_edge=False, vfmode=vft.DEGREE) if lp_soft: random_nonvf_walk_lp_soft_degree_route_count += 1 if lp_hard: random_nonvf_walk_lp_hard_degree_route_count += 1 random_nonvf_walk_degree_route_len.append(len(random_path)) isvf, random_path = helpers.random_nonvf_route( g, s, t, len(trace), vfmode=vft.PRELABELED) assert len(random_path) > 0 if isvf: random_nonvf_walk_prelabeled_route_count += 1 lp_soft = vft.is_local_preferenced(g, trace, first_edge=True, vfmode=vft.PRELABELED) lp_hard = vft.is_local_preferenced(g, trace, first_edge=False, vfmode=vft.PRELABELED) if lp_soft: random_nonvf_walk_lp_soft_prelabeled_route_count += 1 if lp_hard: random_nonvf_walk_lp_hard_prelabeled_route_count += 1 random_nonvf_walk_prelabeled_route_len.append(len(random_path)) # sanity check # if random_path[0] != s or random_path[-1] != t: # logger.error('ALERT') if len(random_path) != len(set(random_path)): logger.error('LENGTH ERROR') extra_meta = { helpers.RANDOM_NONVF_WALK_RUN_COUNT: try_per_race, helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE: random_nonvf_walk_closeness_route_count, helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE_LEN: random_nonvf_walk_closeness_route_len, helpers.RANDOM_NONVF_WALK_VF_DEGREE_ROUTE: random_nonvf_walk_degree_route_count, helpers.RANDOM_NONVF_WALK_VF_DEGREE_ROUTE_LEN: random_nonvf_walk_degree_route_len, helpers.RANDOM_NONVF_WALK_VF_PRELABELED_ROUTE: random_nonvf_walk_prelabeled_route_count, helpers.RANDOM_NONVF_WALK_VF_PRELABELED_ROUTE_LEN: random_nonvf_walk_prelabeled_route_len, helpers.RANDOM_NONVF_WALK_LP_SOFT_DEGREE_ROUTE: random_nonvf_walk_lp_soft_degree_route_count, helpers.RANDOM_NONVF_WALK_LP_SOFT_CLOSENESS_ROUTE: random_nonvf_walk_lp_soft_closeness_route_count, helpers.RANDOM_NONVF_WALK_LP_SOFT_PRELABELED_ROUTE: random_nonvf_walk_lp_soft_prelabeled_route_count, helpers.RANDOM_NONVF_WALK_LP_HARD_DEGREE_ROUTE: random_nonvf_walk_lp_hard_degree_route_count, helpers.RANDOM_NONVF_WALK_LP_HARD_CLOSENESS_ROUTE: random_nonvf_walk_lp_hard_closeness_route_count, helpers.RANDOM_NONVF_WALK_LP_HARD_PRELABELED_ROUTE: random_nonvf_walk_lp_hard_prelabeled_route_count } trace_meta.update(extra_meta) ## save modified meta # all meta_* get only references from meta_original helpers.save_to_json(out, meta_original) # calculate results real_vf = [x[helpers.IS_VF_CLOSENESS] for x in meta_random] real_vf_ratio = np.mean(real_vf) random_nonvf_walk_vf_ratio_per_element = [ x[helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE] / x[helpers.RANDOM_NONVF_WALK_RUN_COUNT] for x in meta_random ] random_nonvf_walk_vf_ratio = np.mean( random_nonvf_walk_vf_ratio_per_element) # print results logger.info('') logger.info('Empty: %d' % empty) logger.info('Tested trace count: %d' % len(meta_random)) logger.info('VF ratio in tested traces: %f' % real_vf_ratio) logger.info('VF ratio in random walks: %f' % random_nonvf_walk_vf_ratio)
logger.info('Generate activity matrix for all user in all time slot') activity_matrix = functional_mat['TS'][0, res][:, :, :] activity_matrix[np.isnan(activity_matrix)] = 0 activity_tmp = np.copy(activity_matrix) activity_matrix[activity_tmp > ts_limit] = 1 activity_matrix[activity_tmp <= ts_limit] = 0 del activity_tmp voxel_coords = structural_mat['centroids'][0, res] meta = [] vf_ratio = [] progress = progressbar1.AnimatedProgressBar(end=len(user_range) * len(time_range), width=15) measurement_id = helpers.id_generator(32) web_log = { 'measurement_id': measurement_id, 'msg_type': MSG_MEASUREMENT, 'resolution': res, 'user_range': user_range, 'adjacency_threshold': adjacency_threshold, 'time_range': time_range, 'corr_limit': corr_limit, 'max_stretch': MAX_STRETCH, 'ts_limit': ts_limit } logger.log(misc.WEB, json.dumps(web_log))
def purify(g, meta_original, out, count=1000, try_per_race=1, show_progress=False, with_lp=True): empty = 0 # remove traces with already calculated random paths logger.warn('[r]ONLY NOT FILLED PATHS[/]') meta_filled = [ x for x in meta_original if helpers.RANDOM_WALK_RUN_COUNT not in x ] # Filter if interested only in routes of stretch 1 # meta_filled = [x for x in meta_original # if x[helpers.TRACE_LEN]-x[helpers.SH_LEN] == 1] ## traces with a maximum stretch # logger.warn('[r]!!!ONLY WITH STRETCH[/]') # meta = [x for x in meta if x[helpers.STRETCH] > -1] # # shorter meta records # logger.warn('[r]!!!ONLY SHORT TRACES[/]') # meta = [x for x in meta if len(x[helpers.TRACE]) < 5] # meta_map = {tuple(x[helpers.TRACE]): x for x in meta_filled} logger.info('All trace count: %d' % len(meta_filled)) tr_count = min(len(meta_filled), count) meta_random = random.sample(meta_filled, tr_count) logger.info('Chosen subset count: %d' % len(meta_random)) # real_vf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 1] # real_nonvf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 0] # assert len(real_nonvf_degree) == tr_count - len(real_vf_degree) # real_vf_prelabeled = [x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 1] # real_nonvf_prelabeled = [x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 0] # assert len(real_nonvf_prelabeled) == tr_count - len(real_vf_prelabeled) # real_vf_closeness = [x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 1] # real_nonvf_closeness = [x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 0] # assert len(real_nonvf_closeness) == tr_count - len(real_vf_closeness) # logger.info('Real vf degree: %f[%d]' % ((len(real_vf_degree) / float(tr_count), # len(real_vf_degree)))) # logger.info('Real nonvf degree: %f[%d]' % ((len(real_nonvf_degree) / float(tr_count), # len(real_nonvf_degree)))) # logger.info('Real vf prelabeled: %f[%d]' % ((len(real_vf_prelabeled) / float(tr_count), # len(real_vf_prelabeled)))) # logger.info('Real nonvf prelabeled: %f[%d]' % ((len(real_nonvf_prelabeled) / float(tr_count), # len(real_nonvf_prelabeled)))) # logger.info('Real vf closeness: %f[%d]' % ((len(real_vf_closeness)/float(tr_count), len(real_vf_closeness)))) # logger.info('Real nonvf closeness: %f[%d]' % ((len(real_nonvf_closeness)/float(tr_count), len(real_nonvf_closeness)))) # traceroutes = [x[helpers.TRACE] for x in meta_random] # traceroutes = vft.trace_in_vertex_id(g, traceroutes) try: meta_random[0][helpers.TRACE] except Exception: meta_random = [{helpers.TRACE: x} for x in meta_random] progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(meta_random), width=15) stretch_list = [] max_stretch = max( [x[helpers.TRACE_LEN] - x[helpers.SH_LEN] for x in meta_random]) for stretch in range(0, max_stretch + 1): metas = [ x for x in meta_random if x[helpers.TRACE_LEN] - x[helpers.SH_LEN] == stretch ] stretch_list.extend(list(repeat(stretch, len(metas)))) # print(stretch_list) lenghts = random.shuffle(stretch_list) strx_array = [] for idx, trace_meta in enumerate(meta_random): progress += 1 progress.show_progress() # print(trace_meta[helpers.TRACE]) shl = trace_meta[helpers.SH_LEN] trace = vft.trace_in_vertex_id(g, [ trace_meta[helpers.TRACE], ]) if len(trace) != 1: print 'PROBLEM' print trace_meta continue trace = trace[0] # print(trace) random_walk_closeness_route_vf = 0 random_walk_closeness_route_lp_soft = 0 random_walk_closeness_route_lp_hard = 0 random_walk_degree_route_vf = 0 random_walk_degree_route_lp_soft = 0 random_walk_degree_route_lp_hard = 0 random_walk_prelabeled_route_vf = 0 random_walk_prelabeled_route_lp_soft = 0 random_walk_prelabeled_route_lp_hard = 0 s, t = trace[0], trace[-1] for counter in xrange(0, try_per_race): # random_path = helpers.random_route_walk(g, s, t, len(trace)) # Modified random_path = helpers.random_route_walk( g, s, t, shl + stretch_list[idx]) # Modified if len(random_path) == 0: empty += 1 if vft.is_valley_free(g, random_path, vfmode=vft.CLOSENESS): random_walk_closeness_route_vf += 1 if (len(random_path) == shl + 1): strx_array.append(1) if with_lp: lp_soft = vft.is_local_preferenced(g, random_path, first_edge=True, vfmode=vft.CLOSENESS) lp_hard = vft.is_local_preferenced(g, random_path, first_edge=False, vfmode=vft.CLOSENESS) if lp_soft: random_walk_closeness_route_lp_soft += 1 if lp_hard: random_walk_closeness_route_lp_hard += 1 else: if (len(random_path) == shl + 1): strx_array.append(0) # if vft.is_valley_free(g, random_path, vfmode=vft.DEGREE): # random_walk_degree_route_vf += 1 # if with_lp: # lp_soft = vft.is_local_preferenced(g, random_path, # first_edge=True, # vfmode=vft.DEGREE) # lp_hard = vft.is_local_preferenced(g, random_path, # first_edge=False, # vfmode=vft.DEGREE) # if lp_soft: # random_walk_degree_route_lp_soft += 1 # if lp_hard: # random_walk_degree_route_lp_hard += 1 # if vft.is_valley_free(g, random_path, vfmode=vft.PRELABELED): # random_walk_prelabeled_route_vf += 1 # if with_lp: # lp_soft = vft.is_local_preferenced(g, random_path, # first_edge=True, # vfmode=vft.PRELABELED) # lp_hard = vft.is_local_preferenced(g, random_path, # first_edge=False, # vfmode=vft.PRELABELED) # if lp_soft: # random_walk_prelabeled_route_lp_soft += 1 # if lp_hard: # random_walk_prelabeled_route_lp_hard += 1 # sanity check # if random_path[0] != s or random_path[-1] != t: # logger.error('ALERT') if len(random_path) != len(set(random_path)): logger.error('LENGTH ERROR') extra_meta = { helpers.RANDOM_WALK_RUN_COUNT: try_per_race, helpers.RANDOM_WALK_VF_CLOSENESS_ROUTE: random_walk_closeness_route_vf, helpers.RANDOM_WALK_VF_DEGREE_ROUTE: random_walk_degree_route_vf, helpers.RANDOM_WALK_VF_PRELABELED_ROUTE: random_walk_prelabeled_route_vf, } if with_lp: extra_meta.update({ helpers.RANDOM_WALK_LP_SOFT_CLOSENESS_ROUTE: random_walk_closeness_route_lp_soft, helpers.RANDOM_WALK_LP_HARD_CLOSENESS_ROUTE: random_walk_closeness_route_lp_hard, helpers.RANDOM_WALK_LP_SOFT_DEGREE_ROUTE: random_walk_degree_route_lp_soft, helpers.RANDOM_WALK_LP_HARD_DEGREE_ROUTE: random_walk_degree_route_lp_hard, helpers.RANDOM_WALK_LP_SOFT_PRELABELED_ROUTE: random_walk_prelabeled_route_lp_soft, helpers.RANDOM_WALK_LP_HARD_PRELABELED_ROUTE: random_walk_prelabeled_route_lp_hard }) trace_meta.update(extra_meta) ## save modified meta # all meta_* get only references from meta_original helpers.save_to_json(out, meta_random) # meta_mod = [x for x in meta_map.itervalues()] # helpers.save_to_json(out, meta_mod) # calculate results # real_vf = [x[helpers.IS_VF_CLOSENESS] for x in meta_random] # real_vf_ratio = np.mean(real_vf) random_walk_vf_ratio_per_element = [ x[helpers.RANDOM_WALK_VF_CLOSENESS_ROUTE] / x[helpers.RANDOM_WALK_RUN_COUNT] for x in meta_random ] random_walk_vf_ratio = np.mean(random_walk_vf_ratio_per_element) # print results logger.info('') logger.info('Empty: %d' % empty) logger.info('Tested trace count: %d' % len(meta_random)) # logger.info('VF ratio in tested traces: %f' % real_vf_ratio) logger.info('VF ratio in random walks: %f' % random_walk_vf_ratio) logger.info('VF ratio in random walks for path stretch 1: %f' % np.mean(strx_array))
def purify(g, meta, top_nodes, show_progress): progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) logger.info('Purify') triplet_count_closeness = 0 top_nodes_triplet_closeness = 0 triplet_count_prelabeled = 0 top_nodes_triplet_prelabeled = 0 triplet_count_degree = 0 top_nodes_triplet_degree = 0 for row in meta: progress += 1 progress.show_progress() trace = row[helpers.TRACE] trace = vft.trace_in_vertex_id(g, [trace, ])[0] if row[helpers.IS_VF_CLOSENESS] == 0: logger.debug('Closeness valley') vt = vft.get_valley_triplets(g, trace, vft.CLOSENESS) triplet_count_closeness += len(vt) for trip in vt: if any([x in top_nodes for x in trip]): top_nodes_triplet_closeness += 1 if row[helpers.IS_VF_PRELABELED] == 0: logger.debug('Prelabeled valley') vt = vft.get_valley_triplets(g, trace, vft.PRELABELED) triplet_count_prelabeled += len(vt) for trip in vt: if any([x in top_nodes for x in trip]): top_nodes_triplet_prelabeled += 1 if row[helpers.IS_VF_DEGREE] == 0: logger.debug('Degree valley') vt = vft.get_valley_triplets(g, trace, vft.DEGREE) triplet_count_degree += len(vt) for trip in vt: if any([x in top_nodes for x in trip]): top_nodes_triplet_degree += 1 print try: ratio_closeness = top_nodes_triplet_closeness / float(triplet_count_closeness) except ZeroDivisionError: ratio_closeness = 0.0 try: ratio_prelabeled = top_nodes_triplet_prelabeled / float(triplet_count_prelabeled) except ZeroDivisionError: ratio_prelabeled = 0.0 try: ratio_degree = top_nodes_triplet_degree / float(triplet_count_degree) except ZeroDivisionError: ratio_degree = 0.0 logger.info('CLOSENESS: %6.3f[%d/%d]' % (ratio_closeness, top_nodes_triplet_closeness, triplet_count_closeness)) logger.info('PRELABELED: %6.3f[%d/%d]' % (ratio_prelabeled, top_nodes_triplet_prelabeled, triplet_count_prelabeled)) logger.info('DEGREE: %6.3f[%d/%d]' % (ratio_degree, top_nodes_triplet_degree, triplet_count_degree))
def main(): parser = argparse.ArgumentParser( description=('Syntetic route generator'), formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('network') parser.add_argument('meta') parser.add_argument('all_trace_out', metavar='all-trace-out') parser.add_argument('syntetic_out', metavar='syntetic-out') parser.add_argument('--trace-count', '-tc', type=int, dest='trace_count', default=5000) parser.add_argument('--random-sample', dest='random_sample', action='store_true') parser.add_argument('--closeness-error', '-ce', type=float, dest='closeness_error', default=0.0) parser.add_argument('--core-limit-percentile', '-cl', type=int, dest='core_limit', default=0) parser.add_argument('--toggle-node-error-mode', action='store_true') arguments = parser.parse_args() arguments.verbose = min(len(helpers.LEVELS) - 1, arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) show_progress = arguments.progressbar g = helpers.load_network(arguments.network) g = g.simplify() meta = helpers.load_from_json(arguments.meta) if arguments.random_sample: random.shuffle(meta) meta = meta[:arguments.trace_count] N = g.vcount() cl = sorted([x['closeness'] for x in g.vs], reverse=True) logger.info('Min closeness: %s' % np.min(cl)) logger.info('Max closeness: %s' % np.max(cl)) logger.info('Mean closenss: %s' % np.mean(cl)) logger.info('10%% closeness: %s' % np.percentile(cl, 10)) logger.info('90%% closeness: %s' % np.percentile(cl, 90)) logger.info('Core limit: [r]%d%%[/]' % arguments.core_limit) change_probability = 100 * arguments.closeness_error logger.info('Change probability: [r]%6.2f%%[/]' % change_probability) core_limit = np.percentile(cl, arguments.core_limit) logger.info('Core limit in closeness: [bb]%f[/]' % core_limit) if arguments.toggle_node_error_mode: logger.info('[r]Node error mode[/]') msg = ( "If given node's closeness >= core_limit then the new ", "closeness in this node is ", #"rand(closeness_error ... old closeness)" "OLD_CLOSENSS * +/- closeness_error%") logger.info(''.join(msg)) logger.info('Minimum node closeness: [g]%f[/]' % arguments.closeness_error) for n in g.vs: if n['closeness'] < core_limit: continue # sign = -1 if random.uniform(-1, 1) < 0 else 1 # n['closeness'] = n['closeness'] * (1 + sign * arguments.closeness_error) new_closeness = random.uniform(arguments.closeness_error, n['closeness']) n['closeness'] = new_closeness g_labeled = vft.label_graph_edges(g, vfmode=vft.CLOSENESS) peer_edge_count = len([x for x in g_labeled.es if x['dir'] == LinkDir.P]) logger.info('Peer edge count: %d' % peer_edge_count) changed_edges = [] if not arguments.toggle_node_error_mode: msg = ("If the closeness values of the endpoints in given edge is ", "larger than the core_limit and ", "random(0,1) < closeness_error then change the direction ", "for this edge") logger.info(''.join(msg)) changed_u = changed_d = 0 changed_edges = [] changed_edgess = [] for edge in g_labeled.es: s, t = edge.source, edge.target s_cl = g_labeled.vs[s]['closeness'] t_cl = g_labeled.vs[t]['closeness'] if (s_cl < core_limit or t_cl < core_limit): continue if random.uniform(0, 1) > arguments.closeness_error: continue # if abs(s_cl - t_cl) / min(s_cl, t_cl) > 0.02: continue new_edge_dir = LinkDir.U if random.uniform(0, 1) > 0.5 else LinkDir.D if new_edge_dir != edge['dir']: if edge['dir'] == LinkDir.U: changed_u += 1 else: changed_d += 1 edge['dir'] = new_edge_dir changed_edges.append(edge) changed_edgess.append((edge.source, edge.target)) # if edge['dir'] == LinkDir.U: # changed_u += 1 # changed_edgess.append((edge.source, edge.target)) # edge['dir'] = LinkDir.D # changed_edges.append(edge) # elif edge['dir'] == LinkDir.D: # changed_d += 1 # changed_edgess.append((edge.source, edge.target)) # edge['dir'] = LinkDir.U # changed_edges.append(edge) logger.info('E count: %d' % g_labeled.ecount()) logger.info('Changed U: %d' % changed_u) logger.info('Changed D: %d' % changed_d) logger.info('Changed: %d' % (changed_d + changed_u)) changed_e = [(g_labeled.vs[e.source]['name'], g_labeled.vs[e.target]['name']) for e in changed_edges] changed_e = changed_e + [(x[1], x[0]) for x in changed_e] changed_e = set(changed_e) vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS, labeled_graph=g_labeled) # e_colors = [] # for e in vf_g_closeness.es: # if e.source < N and e.target < N: col = 'grey' # elif e.source < N and e.target >= N: col = 'blue' # elif e.source >= N and e.target >= N: col = 'red' # else: col = 'cyan' # e_colors.append(col) # igraph.plot(vf_g_closeness, "/tmp/closeness.pdf", # vertex_label=vf_g_closeness.vs['name'], # vertex_size=0.2, # edge_color=e_colors) pairs = [(g.vs.find(x[helpers.TRACE][0]).index, g.vs.find(x[helpers.TRACE][-1]).index, tuple(x[helpers.TRACE])) for x in meta] # pairs = list(set(pairs)) # random.shuffle(pairs) # visited = set() # pairs2 = [] # for x in pairs: # k = (x[0], x[1]) # if k in visited: continue # visited.add(k) # visited.add((k[1], k[0])) # pairs2.append(x) # pairs = pairs2 traces = [x[2] for x in pairs] stretches = [] syntetic_traces = [] sh_traces = [] base_traces = [] original_traces = [] bad = 0 progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15) for s, t, trace_original in pairs: progress += 1 progress.show_progress() trace_original_idx = [g.vs.find(x).index for x in trace_original] logger.debug('Original trace: %s -- %s -- %s', [g.vs[x]['name'] for x in trace_original_idx], vft.trace_to_string(g, trace_original_idx, vft.CLOSENESS), [g.vs[x]['closeness'] for x in trace_original_idx]) sh_routes = g.get_all_shortest_paths(s, t) sh_len = len(sh_routes[0]) sh_routes_named = [[g.vs[y]['name'] for y in x] for x in sh_routes] sh_trace_name = random.choice(sh_routes_named) base_trace_name = random.choice(sh_routes_named) candidates = vf_g_closeness.get_all_shortest_paths(s + N, t + N) candidates = [vft.vf_route_converter(x, N) for x in candidates] # candidates = [] if len(candidates) == 0: candidates = vft.get_shortest_vf_route(g_labeled, s, t, mode='vf', vf_g=vf_g_closeness, _all=True, vfmode=vft.CLOSENESS) if len(candidates) == 0: s_name, t_name = g.vs[s]['name'], g.vs[t]['name'] logger.debug("!!!No syntetic route from %s to %s" % (s_name, t_name)) continue logger.debug('Candidates from %s to %s:' % (g.vs[s]['name'], g.vs[t]['name'])) for c in candidates: logger.debug('%s -- %s -- %s' % ([g.vs[x]['name'] for x in c], vft.trace_to_string(g_labeled, c, vft.PRELABELED), [g.vs[x]['closeness'] for x in c])) chosen_one = random.choice(candidates) chosen_one_name = [g.vs[x]['name'] for x in chosen_one] # print chosen_one # print trace_original # pretty_plotter.pretty_plot(g, trace_original_idx, # chosen_one, changed_edgess, # spec_color=(0, 0, 0, 155)) hop_stretch = len(chosen_one) - sh_len stretches.append(hop_stretch) trace_original_e = zip(trace_original, trace_original[1:]) chosen_one_e = zip(chosen_one_name, chosen_one_name[1:]) trace_affected = any([x in changed_e for x in trace_original_e]) chosen_affected = any([x in changed_e for x in chosen_one_e]) logger.debug('Trace affected: %s' % trace_affected) logger.debug('Chosen affected: %s' % chosen_affected) # if hop_stretch > 2: # logger.debug('Base: %s' % trace_to_string(g_labeled, base_trace_name)) # logger.debug('SH: %s' % trace_to_string(g_labeled, sh_trace_name)) # logger.debug('Trace: %s' % trace_to_string(g_labeled, trace_original)) # logger.debug('Syntetic: %s' % trace_to_string(g_labeled, chosen_one_name)) if trace_affected or chosen_affected or hop_stretch > 2: # pretty_plotter.pretty_plot_all(g, traces, # chosen_one, changed_edgess, # spec_color=(0, 0, 0, 255)) bad += 1 syntetic_traces.append(chosen_one_name) sh_traces.append(sh_trace_name) base_traces.append(base_trace_name) original_traces.append(trace_original) logger.debug('From %s to %s chosen one %s' % (g.vs[s]['name'], g.vs[t]['name'], chosen_one_name)) result = zip(base_traces, sh_traces, original_traces, syntetic_traces) helpers.save_to_json(arguments.all_trace_out, result) helpers.save_to_json(arguments.syntetic_out, syntetic_traces) print 'Bad: %d' % bad c = collections.Counter(stretches) trace_count = len(syntetic_traces) logger.info('Stretch dist:') for k in c: logger.info('\t%d: %5.2f%%[%d]' % (k, 100 * c[k] / float(trace_count), c[k])) logger.info('Valid route count: %d' % trace_count) logger.info('Route count parameter: %d' % arguments.trace_count) logger.info('Generated valid pair count: %d' % len(pairs))
def purify(g, meta, out, count=1000): results = list() results2 = list() results3 = list() all_vf = 0 all_nonvf = 0 all_vf_closeness = 0 all_nonvf_closeness = 0 short_results = list() short_results2 = list() short_results3 = list() all_short_vf = 0 all_short_nonvf = 0 all_short_vf_closeness = 0 all_short_nonvf_closeness = 0 long_results = list() long_results2 = list() long_results3 = list() all_long_vf = 0 all_long_nonvf = 0 all_long_vf_closeness = 0 all_long_nonvf_closeness = 0 # remove traces with already calculated all_path logger.warn('[r]ONLY NOT FILLED PATHS[/]') meta = [x for x in meta if not helpers.ALL_PATH_COUNT in x] # traces with a maximum stretch logger.warn('[r]!!!ONLY WITH LOW STRETCH[/]') meta = [x for x in meta if x[helpers.STRETCH] < 4] # shorter meta records logger.warn('[r]!!!ONLY SHORT TRACES[/]') meta = [x for x in meta if len(x[helpers.TRACE]) < 5] meta_map = {tuple(x[helpers.TRACE]): x for x in meta} # traceroutes = [x for x in meta if x[TRACE_LEN] == x[SH_LEN]] logger.info('All trace count: %d' % len(meta)) tr_count = min(len(meta), count) meta = random.sample(meta, tr_count) logger.info('Chosen trace count: %d' % len(meta)) real_vf = [x for x in meta if x[helpers.IS_VF] == 1] real_nonvf = [x for x in meta if x[helpers.IS_VF] == 0] real_vf_closeness = [x for x in meta if x[helpers.IS_VF_CLOSENESS] == 1] real_nonvf_closeness = [x for x in meta if x[helpers.IS_VF_CLOSENESS] == 0] logger.info('Real vf: %f[%d]' % ((len(real_vf)/float(len(meta)), len(real_vf)))) logger.info('Real nonvf: %f[%d]' % ((len(real_nonvf)/float(len(meta)), len(real_nonvf)))) logger.info('Real vf closeness: %f[%d]' % ((len(real_vf_closeness)/float(len(meta)), len(real_vf_closeness)))) logger.info('Real nonvf closeness: %f[%d]' % ((len(real_nonvf_closeness)/float(len(meta)), len(real_nonvf_closeness)))) logger.info('Remove unknown traces. Trace count before: %d' % len(meta)) traceroutes = [x[helpers.TRACE] for x in meta] traceroutes, ignored = vft.trace_clean(g, traceroutes) logger.info('Traceroutes after: %d. Ignored: %d' % (len(traceroutes), ignored)) traceroutes = vft.trace_in_vertex_id(g, traceroutes) progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() for x in range(0, g.vcount()): g.vs[x]['traces'] = dict() s, t = trace[0], trace[-1] sh_path = g.get_all_shortest_paths(s, t, mode=i.OUT) all_path = helpers.dfs_mark(copy.deepcopy(g), s, t, len(trace)) # if len(sh_path) != len(all_path): # print len(sh_path) # print len(all_path) # print s, t # sanity check for x in all_path: if x[0] != s or x[-1] != t: logger.error('ALERT') if len(set([tuple(x) for x in all_path])) != len(all_path): logger.error('LENGTH ALERT') logger.error('%s' % len(all_path)) logger.error('%s' % len(set([tuple(x) for x in all_path]))) logger.error('%s' % sorted(all_path)) long_path = [x for x in all_path if len(x) == len(trace)] short_path = [x for x in all_path if len(x) < len(trace)] named_trace = [g.vs[x]['name'] for x in trace] extra_meta = { helpers.ALL_PATH_COUNT: len(all_path), helpers.SAME_LONG_PATH_COUNT: len(long_path), helpers.SHORTER_PATH_COUNT: len(short_path) } meta_map[tuple(named_trace)].update(extra_meta) vf_count = sum([1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in all_path]) nonvf = len(all_path) - vf_count vf_closeness_count = sum([1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in all_path]) nonvf_closeness = len(all_path) - vf_closeness_count tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in short_path] short_vf_count = sum(tmp) short_nonvf = len(tmp) - short_vf_count tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in short_path] short_vf_closeness_count = sum(tmp) short_nonvf_closeness = len(tmp) - short_vf_closeness_count tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in long_path] long_vf_count = sum(tmp) long_nonvf = len(tmp) - long_vf_count tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in long_path] long_vf_closeness_count = sum(tmp) long_nonvf_closeness = len(tmp) - long_vf_closeness_count extra_meta = { helpers.ALL_PATH_VF_COUNT: vf_closeness_count, helpers.SAME_LONG_PATH_VF_COUNT: long_vf_closeness_count, helpers.SHORTER_PATH_VF_COUNT: short_vf_closeness_count } meta_map[tuple(named_trace)].update(extra_meta) all_vf += vf_count all_nonvf += nonvf all_vf_closeness += vf_closeness_count all_nonvf_closeness += nonvf_closeness all_long_vf += long_vf_count all_long_nonvf += long_nonvf all_long_vf_closeness += long_vf_closeness_count all_long_nonvf_closeness += long_nonvf_closeness all_short_vf += short_vf_count all_short_nonvf += short_nonvf all_short_vf_closeness += short_vf_closeness_count all_short_nonvf_closeness += short_nonvf_closeness results.append(vf_count / float(len(all_path))) results3.append(vf_closeness_count / float(len(all_path))) if len(all_path) > 1: results2.append(vf_count / float(len(all_path))) long_results.append(long_vf_count / float(len(long_path))) long_results3.append(long_vf_closeness_count / float(len(long_path))) if len(long_path) > 1: long_results2.append(long_vf_count / float(len(long_path))) if len(short_path) > 0: short_results.append(short_vf_count / float(len(short_path))) short_results3.append(short_vf_closeness_count / float(len(short_path))) else: pass # short_results.append(0) # short_results3.append(0) if len(short_path) > 1: short_results2.append(short_vf_count / float(len(short_path))) # save mofified meta meta_mod = [x for x in meta_map.itervalues()] helpers.save_to_json(out, meta_mod) # print results print 'ALL' print 'VF count: %d' % all_vf print 'VF CLOSENESS count: %d' % all_vf_closeness print 'Non vf count: %d' % all_nonvf print 'Non vf CLOSENESS count: %d' % all_nonvf_closeness print 'VF perc: %f' % (all_vf/float(all_vf + all_nonvf)) print 'VF CLOSENESS perc: %f' % (all_vf_closeness/float(all_vf_closeness + all_nonvf_closeness)) print 'Mean VF prob: %f' % np.mean(results) print 'Mean VF CLOSENESS prob: %f' % np.mean(results3) print 'Mean VF2 prob: %f' % np.mean(results2) print '==========' print 'SHORT' print 'VF count: %d' % all_short_vf print 'VF CLOSENESS count: %d' % all_short_vf_closeness print 'Non vf count: %d' % all_short_nonvf print 'Non vf CLOSENESS count: %d' % all_short_nonvf_closeness if all_short_vf + all_short_nonvf > 0: print 'VF perc: %f' % (all_short_vf/float(all_short_vf + all_short_nonvf)) if all_short_vf_closeness + all_short_nonvf_closeness > 0: print 'VF CLOSENESS perc: %f' % (all_short_vf_closeness/float(all_short_vf_closeness + all_short_nonvf_closeness)) print 'Mean VF prob: %f' % np.mean(short_results) print 'Mean VF CLOSENESS prob: %f' % np.mean(short_results3) print 'Mean VF2 prob: %f' % np.mean(short_results2) print '=-----------------' print 'LONG' print 'VF count: %d' % all_long_vf print 'VF CLOSENESS count: %d' % all_long_vf_closeness print 'Non vf count: %d' % all_long_nonvf print 'Non vf CLOSENESS count: %d' % all_long_nonvf_closeness print 'VF perc: %f' % (all_long_vf/float(all_long_vf + all_long_nonvf)) print 'VF CLOSENESS perc: %f' % (all_long_vf_closeness/float(all_long_vf_closeness + all_long_nonvf_closeness)) print 'Mean VF prob: %f' % np.mean(long_results) print 'Mean VF CLOSENESS prob: %f' % np.mean(long_results3) print 'Mean VF2 prob: %f' % np.mean(long_results2)
def main(): parser = argparse.ArgumentParser( description=('SANDBOX mode. ', 'Write something ', 'useful here'), formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--progressbar', action='store_true') parser.add_argument('--verbose', '-v', action='count', default=0) parser.add_argument('--edge-drop', dest='edge_drop', type=float, default=0.0) parser.add_argument('--closeness-limit', dest='closeness_limit', type=float, default=0.0) parser.add_argument('network') parser.add_argument('traceroutes') arguments = parser.parse_args() show_progress = arguments.progressbar arguments.verbose = min(len(helpers.LEVELS), arguments.verbose) logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose]) g = helpers.load_network(arguments.network) traceroutes = helpers.load_from_json(arguments.traceroutes) logger.info('ecount: %d' % g.ecount()) logger.info('vcount: %d' % g.vcount()) logger.info('trace count: %d' % len(traceroutes)) g_dummy = g.copy() progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) closeness_list = [] for x in g_dummy.vs: progress += 1 progress.show_progress() closeness_list.append((x.index, g_dummy.closeness(x))) end = int(arguments.closeness_limit * g_dummy.vcount()) logger.debug('Top node count: %d' % end) top_nodes = sorted(closeness_list, key=lambda x: x[1], reverse=True)[:end] top_nodes_index = [x[0] for x in top_nodes] top_nodes_name = [g_dummy.vs[x[0]]['name'] for x in top_nodes] top_edges = [ e for e in g_dummy.es if e.source in top_nodes_index and e.target in top_nodes_index ] logger.debug('Top edge count: %d' % len(top_edges)) random.shuffle(top_edges) edge_drop = top_edges[:int(len(top_edges) * arguments.edge_drop)] logger.debug('Dropped edge count: %d' % len(edge_drop)) # edges = [x.index for x in g_dummy.es] # random.shuffle(edges) # edge_drop = edges[:int(g.ecount() * arguments.edge_drop)] g_dummy.delete_edges(edge_drop) traceroutes = traceroutes[:10000] all_edges = [] for trace in traceroutes: edges = zip(trace, trace[1:]) edges = [tuple(sorted(e)) for e in edges] all_edges.extend(edges) all_edges = list(set(all_edges)) top_edges = [ e for e in all_edges if e[0] in top_nodes_name and e[1] in top_nodes_name ] logger.info('TOP edge count in real traceroutes: %d' % len(top_edges)) found_top_edges = [] increments = [] for trace in traceroutes: edges = zip(trace, trace[1:]) edges = [tuple(sorted(e)) for e in edges] top_edges = [ x for x in edges if x[0] in top_nodes_name and x[1] in top_nodes_name ] found_top_edges.extend(top_edges) found_top_edges = list(set(found_top_edges)) increments.append(len(found_top_edges)) logger.info('Found top edge count: %d' % len(found_top_edges)) dummy_sh_traceroutes_meta = [] original_sh_traceroutes_meta = [] stretches = [] progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() s, t = trace[0], trace[-1] # logger.debug('Get shortest paths from {s} to {t}'.format(s=s, t=t)) sh_dummy = random.choice(g_dummy.get_shortest_paths(s, t)) sh_original = random.choice(g.get_shortest_paths(s, t)) stretch = len(sh_dummy) - len(sh_original) dummy_sh_traceroutes_meta.append((sh_dummy, stretch)) original_sh_traceroutes_meta.append((sh_original, 0)) stretches.append(stretch) # logger.debug('Stretch: %d' % stretch) # logger.debug('SH DUMMY: %s' % [g_dummy.vs[x]['name'] for x in sh_dummy]) # logger.debug('SH ORIG: %s' % [g.vs[x]['name'] for x in sh_original]) dummy_sh_meta = [(x[0], x[1], vft.is_valley_free(g_dummy, x[0], vft.CLOSENESS)) for x in dummy_sh_traceroutes_meta] dummy_sh_len_hist = collections.Counter( [len(x[0]) for x in dummy_sh_traceroutes_meta]) original_sh_len_hist = collections.Counter( [len(x[0]) for x in original_sh_traceroutes_meta]) original_len_hist = collections.Counter([len(x) for x in traceroutes]) stretches = [x for x in stretches if x >= 0] stretch_hist = collections.Counter(stretches) import matplotlib.pyplot as plt print print[(x, 100 * y / float(len(traceroutes)), y) for x, y in stretch_hist.iteritems()] plt.plot([x for x in stretch_hist.iterkeys()], [x for x in stretch_hist.itervalues()], 'g^') plt.ylabel('some numbers') # plt.show() logger.info('Dummy VF stat') max_stretch = max(dummy_sh_meta, key=lambda x: x[1])[1] for stretch in range(0, max_stretch + 1): stretched_traces = [x for x in dummy_sh_meta if x[1] == stretch] count = len(stretched_traces) vf_count = len([x for x in stretched_traces if x[2]]) vf_perc = vf_count / float(count) nonvf_count = count - vf_count nonvf_perc = nonvf_count / float(count) logger.info( '{stretch} -- {vf_perc}[{vf_count}]\t{nonvf_perc}[{nonvf_count}]'. format(stretch=stretch, vf_perc=vf_perc, vf_count=vf_count, nonvf_perc=nonvf_perc, nonvf_count=nonvf_count)) import matplotlib.pyplot as plt plt.plot(increments, 'g^') plt.ylabel('some numbers') plt.show()
def purify(g, meta, flags, try_per_trace, show_progress=False): # generate valley-free graph if flags[FLAG_PRELABELED]: logger.info('Generate VF_G_PRE') vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED) else: logger.info('Skip prelabeled graph') if flags[FLAG_DEGREE]: logger.info('Generate VF_G_DEGREE') vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE) else: logger.info('Skip degree graph') if flags[FLAG_CLOSENESS]: logger.info('Generate VF_G_CLOSENESS') vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS) else: logger.info('Skip closeness graph') # Randomize stretch dispersion stretches = [x[helpers.HOP_STRETCH] for x in meta] # a veletlen stretchek az eredeti stretchek veletlen, nem # visszateveses mintavetelezese. Mindez annyiszor, ahany # veletlen utat akarunk generalni minden valos trace vegpontja # kozott. stretch_list = [random.sample(stretches, len(stretches)) for x in xrange(0, try_per_trace)] # A kovetkezo ciklusban minden meta sorhoz rogton kiszamolunk # annyi random utat, amennyit parameterben megadtunk. Ehhez at kell # alakitani a stretch lista strukturat, hogy minden elem egy meta sorhoz # tartalmazza a random stretch ertekeket # # Pelda: elozo lepesnel kijott ez: [ [1,2,3,4], [2,4,1,3], [3,1,4,2] ] # vagyis elso lepesben a metaban tarolt tracek rendre 1,2,3,4 stretchet # kell felvegyenek, a masodikban 2,4,1,3 stb. A ciklusban viszont a meta # elso elemehez rogton ki akarjuk szamolni a veletlen stretchekhez tartozo # random utakat, vagyis [ [1,2,3], [2,4,1], [3,1,4], [4,3,2] ] formaban # van szukseg az ertekekre stretch_list = zip(*stretch_list) progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) for idx, record in enumerate(meta): progress += 1 progress.show_progress() trace = vft.trace_in_vertex_id(g, [record[helpers.TRACE], ]) if len(trace) != 1: print 'PROBLEM' print record continue trace = trace[0] if len(trace) == 1: continue sh_len = record[helpers.SH_LEN] s, t = trace[0], trace[-1] is_vf_prelabeled_l = [] is_lp_prelabeled_hard_l = [] is_lp_prelabeled_soft_l = [] is_vf_degree_l = [] is_lp_degree_hard_l = [] is_lp_degree_soft_l = [] is_vf_closeness_l = [] is_lp_closeness_hard_l = [] is_lp_closeness_soft_l = [] stretch_dist = stretch_list[idx] real_stretch_dist = [] for current_stretch in stretch_dist: random_length = sh_len + current_stretch random_path = helpers.random_route_walk(g, s, t, random_length) real_stretch_dist.append(len(random_path) - sh_len) if len(random_path) == 0: empty += 1 if flags[FLAG_PRELABELED]: (is_vf_prelabeled, is_lp_prelabeled_soft, is_lp_prelabeled_hard) = vf_attributes(g,random_path, vft.PRELABELED, flags[FLAG_LP_SOFT], flags[FLAG_LP_HARD], vf_g_pre) is_vf_prelabeled_l.append(is_vf_prelabeled) is_lp_prelabeled_soft_l.append(is_lp_prelabeled_soft) is_lp_prelabeled_hard_l.append(is_lp_prelabeled_hard) if flags[FLAG_DEGREE]: (is_vf_degree, is_lp_degree_soft, is_lp_degree_hard) = vf_attributes(g, random_path, vft.DEGREE, flags[FLAG_LP_SOFT], flags[FLAG_LP_HARD], vf_g_degree) is_vf_degree_l.append(is_vf_degree) is_lp_degree_soft_l.append(is_lp_degree_soft) is_lp_degree_hard_l.append(is_lp_degree_hard) if flags[FLAG_CLOSENESS]: (is_vf_closeness, is_lp_closeness_soft, is_lp_closeness_hard) = vf_attributes(g, random_path, vft.CLOSENESS, flags[FLAG_LP_SOFT], flags[FLAG_LP_HARD], vf_g_closeness) is_vf_closeness_l.append(is_vf_closeness) is_lp_closeness_soft_l.append(is_lp_closeness_soft) is_lp_closeness_hard_l.append(is_lp_closeness_hard) result = { helpers.RANDOM_GULYAS_WALK_ROUTES_RQ_STRETCH: stretch_dist, helpers.RANDOM_GULYAS_WALK_ROUTES_STRETCH: real_stretch_dist, helpers.RANDOM_GULYAS_WALK_ROUTES_VF_PRELABELED: is_vf_prelabeled_l, helpers.RANDOM_GULYAS_WALK_ROUTES_VF_DEGREE: is_vf_degree_l, helpers.RANDOM_GULYAS_WALK_ROUTES_VF_CLOSENESS: is_vf_closeness_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_PRELABELED: is_lp_prelabeled_soft_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_PRELABELED: is_lp_prelabeled_hard_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_DEGREE: is_lp_degree_soft_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_DEGREE: is_lp_degree_hard_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_CLOSENESS: is_lp_closeness_soft_l, helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_CLOSENESS: is_lp_closeness_hard_l, } record.update(result)
def generate_db(g, meta, locations, f_name): print 'Traceroute count %d' % len(meta) meta = [x for x in meta if x[TRACE_LEN] - x[SH_LEN] > 0] print 'Stretched traceroute count %d' % len(meta) weights = { x['name']: { 'sh': 0, 'real': 0, 'vfreal': 0, 'randomsh': 0 } for x in g.vs } random_paths = [] if locations is None: rands = numpy.random.randint(0, 10000, len(weights) * 2) loc = zip(rands[::2], rands[1::2]) locations = { n: { 'x': loc[idx][0], 'y': loc[idx][1] } for idx, n in enumerate(weights.iterkeys()) } # check locations delkeys = [] for x in weights: try: locations[x] except KeyError: delkeys.append(x) print 'Get closeness' progress = progressbar1.AnimatedProgressBar(end=len(weights), width=15) for cntr, x in enumerate(weights): progress += 1 progress.show_progress() weights[x]['closeness'] = g.closeness(x, mode=igraph.OUT) print 'Trace calculation' progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15) for cntr, m in enumerate(meta): progress += 1 progress.show_progress() trace = m[TRACE] # for x in trace: # if x in delkeys: continue for node in trace: weights[node]['real'] += 1 if m[IS_VF_CLOSENESS]: weights[node]['vfreal'] += 1 sh_paths = g.get_all_shortest_paths(trace[0], trace[-1]) sh_paths = [[g.vs[x]['name'] for x in p] for p in sh_paths] random_path = random.choice(sh_paths) random_paths.append(random_path) for node in random_path: weights[node]['randomsh'] += 1 for sh_path in sh_paths: for node in sh_path: weights[node]['sh'] += 1 print 'Delete nodes because has no location: %s' % delkeys g.delete_vertices([g.vs.find(x).index for x in delkeys]) print 'Nodes remained in graph: %d' % g.vcount() with open('%s.csv' % f_name, 'w') as f: f.write('NODE;CLOSENESS;X;Y;SH;REAL;VFREAL;RANDOMSH\n') for node, w in weights.iteritems(): struct = '{node};{closeness};{locx};{locy};{sh};{real};{vfreal};{randomsh}\n' f.write( struct.format(node=node, sh=w['sh'], real=w['real'], vfreal=w['vfreal'], randomsh=w['randomsh'], closeness=w['closeness'], locx=float(locations[node]['x']), locy=float(locations[node]['y']))) randomsh_hops = [y for x in random_paths for y in zip(x, x[1:])] randomsh_edges = (dict(source=s, target=t) for s, t in randomsh_hops) randomsh_graph = igraph.Graph.DictList(edges=randomsh_edges, vertices=None) randomsh_graph.simplify() randomsh_graph.save('%srandomsh.gml' % f_name) trace_hops = [y for x in meta for y in zip(x[TRACE], x[TRACE][1:])] trace_edges = (dict(source=s, target=t) for s, t in trace_hops) trace_graph = igraph.Graph.DictList(edges=trace_edges, vertices=None) trace_graph.simplify() trace_graph.save('%strace.gml' % f_name) g.save('%scalculation_graph.gml' % f_name)
def purify(g, traceroutes, flags, show_progress=False): results = list() # remove traces with unknown nodes traceroutes = vft.trace_in_vertex_id(g, traceroutes) # generate valley-free graph if flags[FLAG_PRELABELED]: logger.info('Generate VF_G_PRE') vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED) else: logger.info('Skip prelabeled graph') if flags[FLAG_DEGREE]: logger.info('Generate VF_G_DEGREE') vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE) else: logger.info('Skip degree graph') if flags[FLAG_CLOSENESS]: logger.info('Generate VF_G_CLOSENESS') vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS) else: logger.info('Skip closeness graph') progress = progressbar1.DummyProgressBar(end=10, width=15) if show_progress: progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15) for trace in traceroutes: progress += 1 progress.show_progress() logger.debug('Current trace: %s' % ([g.vs[x]['name'] for x in trace])) if len(trace) == 1: continue s, t = trace[0], trace[-1] is_vf_prelabeled = -1 is_lp_prelabeled_hard = -1 is_lp_prelabeled_soft = -1 is_vf_degree = -1 is_lp_degree_hard = -1 is_lp_degree_soft = -1 is_vf_closeness = -1 is_lp_closeness_hard = -1 is_lp_closeness_soft = -1 trace_len = len(trace) sh_len = g.shortest_paths(s, t, mode=i.OUT)[0][0] sh_len += 1 # convert hop count to node Counter if flags[FLAG_PRELABELED]: is_vf_prelabeled = vft.is_valley_free(g, trace, vft.PRELABELED) is_vf_prelabeled = int(is_vf_prelabeled) if is_vf_prelabeled: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_pre, first_edge=True, vfmode=vft.PRELABELED) is_lp_prelabeled_soft = 1 if lp_soft else 0 else: is_lp_prelabeled_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_pre, first_edge=False, vfmode=vft.PRELABELED) is_lp_prelabeled_hard = 1 if lp_hard else 0 else: is_lp_prelabeled_hard = -1 if flags[FLAG_DEGREE]: is_vf_degree = vft.is_valley_free(g, trace, vft.DEGREE) is_vf_degree = int(is_vf_degree) if is_vf_degree: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_degree, first_edge=True, vfmode=vft.DEGREE) is_lp_degree_soft = 1 if lp_soft else 0 else: is_lp_degree_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_degree, first_edge=False, vfmode=vft.DEGREE) is_lp_degree_hard = 1 if lp_hard else 0 else: is_lp_degree_hard = -1 if flags[FLAG_CLOSENESS]: is_vf_closeness = vft.is_valley_free(g, trace, vft.CLOSENESS) is_vf_closeness = int(is_vf_closeness) if is_vf_closeness: if flags[FLAG_LP_SOFT]: lp_soft = vft.is_local_preferenced(g, trace, vf_g=vf_g_closeness, first_edge=True, vfmode=vft.CLOSENESS) is_lp_closeness_soft = 1 if lp_soft else 0 else: is_lp_closeness_soft = -1 if flags[FLAG_LP_HARD]: lp_hard = vft.is_local_preferenced(g, trace, vf_g=vf_g_closeness, first_edge=False, vfmode=vft.CLOSENESS) is_lp_closeness_hard = 1 if lp_hard else 0 else: is_lp_closeness_hard = -1 if False: sh_vf_len = vft.get_shortest_vf_route(g, s, t, mode='vf', vf_g=vf_g_pre, _all=True, vfmode=vft.PRELABELED) # ugy tunik, mintha nem mindig lenne pontos? fentartassal kezelendo # ez az ertek azert is kerult bele, hogy ellenorizzuk in_vf_prediction = 1 if sh_vf_len and trace in sh_vf_len else 0 else: sh_vf_len = -1 in_vf_prediction = -1 sh_vf_len = len(sh_vf_len[0]) if isinstance(sh_vf_len, list) else -1 percentage_stretch = trace_len / float(sh_len) named_trace = [g.vs[_id]["name"] for _id in trace] result = { helpers.TRACE: named_trace, helpers.TRACE_LEN: trace_len, helpers.SH_LEN: sh_len, helpers.SH_VF_LEN: sh_vf_len, helpers.IS_VF_PRELABELED: is_vf_prelabeled, helpers.IS_VF_DEGREE: is_vf_degree, helpers.IS_VF_CLOSENESS: is_vf_closeness, helpers.HOP_STRETCH: trace_len - sh_len, helpers.PERC_STRETCH: percentage_stretch, helpers.IN_VF_PRED: in_vf_prediction, helpers.IS_LP_SOFT_PRELABELED: is_lp_prelabeled_soft, helpers.IS_LP_HARD_PRELABELED: is_lp_prelabeled_hard, helpers.IS_LP_SOFT_DEGREE: is_lp_degree_soft, helpers.IS_LP_HARD_DEGREE: is_lp_degree_hard, helpers.IS_LP_SOFT_CLOSENESS: is_lp_closeness_soft, helpers.IS_LP_HARD_CLOSENESS: is_lp_closeness_hard, } results.append(result) # print >> sys.stderr, ('TRACE\tTRACE_LEN\tSH_LEN\tSH_VF_LEN\tIS_VF', # '\tSTRETCH\tIN_VF_PREDICTION\tIS_LP_F\tIS_LP_ALL') # for result in results: # result = [str(r) for r in result] # print >> sys.stderr, '\t'.join(result) # statistic = statistics.purify(g, results, # 'nc+ec+tc+rt+vf+vf_closeness+pred+lp_soft_prelabeled+lp_hard_prelabeled+lp_soft_degree+lp_hard_degree+lp_soft_closeness+lp_hard_closeness'.split('+')) # statistics.stat_printer(statistic) return results