def watts_converter(network, show_progress=False):
logger.info('Get giant component')
g = network.clusters().giant()
logger.info('Calculate closeness')
closenesses = []
progress = progressbar1.DummyProgressBar(end=g.vcount(), width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15)
for x in g.vs:
progress += 1
progress.show_progress()
closenesses.append(g.closeness(x, mode=igraph.ALL))
g.vs['closeness'] = closenesses
logger.info('Add edges')
new_edge_list = []
watts_graph = g.copy()
watts_graph.delete_edges(watts_graph.es)
watts_graph.to_directed()
progress = progressbar1.DummyProgressBar(end=g.ecount(), width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=g.ecount(), width=15)
for e in g.es:
progress += 1
progress.show_progress()
s, t = e.source, e.target
if g.vs[s]['closeness'] > g.vs[t]['closeness']:
t, s = s, t
new_edge_list.append((s, t))
watts_graph.add_edges(new_edge_list)
for e in watts_graph.es:
e['provider'] = False
logger.info('Get provider edges')
progress = progressbar1.DummyProgressBar(end=g.vcount(), width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=g.vcount(), width=15)
for n in g.vs:
progress += 1
progress.show_progress()
provider_eid = get_provider_edge_highest_closeness(g, watts_graph, n)
if provider_eid is None: continue
watts_graph.es[provider_eid]['provider'] = True
return watts_graph
def main():
parser = argparse.ArgumentParser(
description='Save closeness values for all node',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('network')
parser.add_argument('out')
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS), arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
g = helpers.load_network(arguments.network)
logger.info('Graph loaded from: %s' % arguments.network)
logger.info('Graph vertex count: %d' % g.vcount())
if 'closeness' not in g.vs:
logger.info('Calculate closeness values')
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=g.vcount(),
width=15)
for n in g.vs:
progress += 1
progress.show_progress()
closeness = g.closeness(n, mode=igraph.OUT)
n['closeness'] = closeness
g.save(arguments.out)
def purify(g, sample_size, show_progress):
nodes = range(0, g.vcount())
samples = [random.sample(nodes, 4) for idx in range(0, sample_size)]
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(samples), width=15)
logger.info('Avg distance calculation')
avg_len = g.average_path_length()
logger.info('Avg len: %d' % avg_len)
hbcs = []
for quadruplet in samples:
progress += 1
progress.show_progress()
hbc = hyperbolicity(g, quadruplet)
hbcs.append(hbc)
avg_hbc = sum(hbcs) / float(len(hbcs))
logger.info('Avg hbc num: %f' % avg_hbc)
logger.info('Avg hbc: %f' % (avg_hbc / float(avg_len)))
def filter(g,
traceroutes,
filters=['sh', 'loop', 'ex', 'vf', 'lp'],
first_edge=True):
logger.info('Traceroutes: %d', len(traceroutes))
# remove empty traces
traceroutes = [x for x in traceroutes if len(x) > 0]
logger.info('Non empty traceroutes: %d', (len(traceroutes)))
traceroutes = [x for x in traceroutes if len(x) > 1]
logger.info('Larger than one hop traceroutes: %d', (len(traceroutes)))
# remove traces with unknown nodes
traceroutes, _ = vft.trace_clean(g, traceroutes)
logger.info('Ignored: %d', _)
traceroutes = vft.trace_in_vertex_id(g, traceroutes)
logger.info('Trace count: %d', len(traceroutes))
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15)
good_traceroutes = traceroutes[:]
if 'sh' in filters:
logger.debug('Remove short traces')
good_traceroutes = [x for x in good_traceroutes if len(x) >= 3]
logger.debug('Remained: %d', len(good_traceroutes))
if 'loop' in filters:
logger.debug('Remove traces with loops')
good_traceroutes = [
x for x in good_traceroutes if len(set(x)) == len(x)
]
logger.debug('Remained: %d' % len(good_traceroutes))
if 'ex' in filters:
logger.debug('Remove non existent traces')
good_traceroutes = [
x for x in good_traceroutes if vft.trace_exists(g, x)
]
logger.debug('Remained: %d', len(good_traceroutes))
if 'vf' in filters:
logger.debug('Remove non vf traces')
good_traceroutes = [
x for x in good_traceroutes if vft.is_valley_free(g, x)
]
logger.debug('Remained: %d' % len(good_traceroutes))
if 'lp' in filters:
logger.debug('Remove non lp traces')
good_traceroutes = [
x for x in good_traceroutes
if vft.is_local_preferenced(g, x, first_edge=first_edge)
]
logger.debug('Remained: %d' % len(good_traceroutes))
# convert back node ids to node names
good_traceroutes = [[g.vs[id]["name"] for id in trace]
for trace in good_traceroutes]
logger.debug(len(good_traceroutes))
return good_traceroutes
def upwalker_counter(g, meta, vf_g, arguments):
N = g.vcount()
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
# trace_up_map = {}
# random_up_map = {}
trace_up_counter = []
random_up_counter = []
for m in meta:
progress += 1
progress.show_progress()
trace = m[helpers.TRACE]
trace_dir = vft.trace_to_string(g, trace)
trace_up_count = trace_dir.count('U')
trace_up_counter.append(trace_up_count)
# trace_up_map[trace_up_count] = trace_up_map.get(trace_up_count, 0) +
# 1
s, t = trace[0], trace[-1]
s_idx, t_idx = vft.node_to_nodeid(g, s), vft.node_to_nodeid(g, t)
random_vf_route = helpers.random_route_walk(vf_g,
s_idx,
t_idx + N,
len(trace),
named=False,
weight_field='VFweight')
random_vf_route = vft.vf_route_converter(random_vf_route, N)
random_vf_dir = vft.trace_to_string(g, random_vf_route)
random_vf_count = random_vf_dir.count('U')
random_up_counter.append(random_vf_count)
# random_up_map[random_vf_count] = random_up_map.get(random_vf_count,
# 0) + 1
real_counter = collections.Counter(trace_up_counter)
real_up = ' '.join(
['%s: %s' % (k, real_counter[k]) for k in sorted(list(real_counter))])
random_counter = collections.Counter(random_up_counter)
random_up = ' '.join([
'%s: %s' % (k, random_counter[k]) for k in sorted(list(random_counter))
])
logger.info('')
logger.info('Real trace UP counter: %s' % real_up)
logger.info('Random vf trace up counter: %s' % random_up)
helpers.save_to_json(arguments.out, {
'REAL': dict(real_counter),
'RANDOM': dict(random_counter)
})
keys = sorted(set(list(real_counter) + list(random_counter)))
logger.info('IDX;REAL;RANDOM')
for k in keys:
logger.info('%s;%d;%d' % (k, real_counter[k], random_counter[k]))
def ba_gen(arguments):
g = helpers.load_network(arguments.input_graph)
if arguments.use_degree_seq:
method = 'Degree Seq.'
generated_graph = _degree_graph(g)
else:
method = 'Barabasi-Albert'
generated_graph = _ba_graph(g)
logger.info('Used method: [b]%s[/]' % method)
logger.debug('Graph created. Fill up closeness values')
N = generated_graph.vcount()
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=N, width=15)
for vs in generated_graph.vs:
progress += 1
progress.show_progress()
vs['name'] = 'BA%s' % vs.index
vs['closeness'] = generated_graph.closeness(vs, mode=igraph.OUT)
logger.info('Save to %s' % arguments.ba_output)
generated_graph.save(arguments.ba_output)
def main():
parser = argparse.ArgumentParser(
description='SANDBOX mode. Write something useful here',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('network')
parser.add_argument('--sample-size',
type=int,
default=1000,
dest='sample_size')
parser.add_argument('--node-drop',
type=float,
default=.1,
dest='node_drop')
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS), arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
g = helpers.load_network(arguments.network)
g = g.components().giant()
logger.info('Graph loaded from: %s' % arguments.network)
logger.info('Graph vertex count: %d' % g.vcount())
end = int(g.vcount() * arguments.node_drop)
logger.info('Remaining node count: %d' % end)
try:
nodes = [(x.index, x['closeness']) for x in g.vs]
except KeyError:
logger.info('Calculate closeness values')
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=g.vcount(),
width=15)
for n in g.vs:
progress += 1
progress.show_progress()
closeness = g.closeness(n)
n['closeness'] = closeness
g.save('with_closeness.gml')
nodes = [(x.index, x['closeness']) for x in g.vs]
nodes = sorted(nodes, reverse=True, key=lambda x: x[1])
top_nodes = [x[0] for x in nodes[:end]]
delete_nodes = [x.index for x in g.vs if x.index not in top_nodes]
g.delete_vertices(delete_nodes)
logger.info('Left nodes: %d' % g.vcount())
purify(g, arguments.sample_size, arguments.progressbar)
def ba_generator(ba_graph, sh_paths, stretch, vf_g, progressbar=False):
vf_count = 0
trace_count = 0
lp_count = 0
progress = progressbar1.DummyProgressBar(end=10, width=15)
if progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(sh_paths),
width=15)
for (s, t), shl in sh_paths:
progress += 1
progress.show_progress()
logger.debug('SH from %s to %s is %d' % (s, t, shl))
random_route = helpers.random_route_walk(ba_graph, s, t, shl + stretch)
logger.debug('Random route: %s' % random_route)
real_stretch = len(random_route) - shl
if real_stretch != stretch:
continue
trace_count += 1
is_vf = vft.is_valley_free(ba_graph,
random_route,
vfmode=vft.CLOSENESS)
logger.debug(
'Trace edge dir: %s' %
vft.trace_to_string(ba_graph, random_route, vfmode=vft.CLOSENESS))
logger.debug('Is VF: %s' % is_vf)
if is_vf:
is_lp = vft.is_local_preferenced(ba_graph,
random_route,
first_edge=True,
vfmode=vft.CLOSENESS,
vf_g=vf_g)
else:
is_lp = 0
logger.debug('Is LP: %s' % is_lp)
vf_count += int(is_vf)
lp_count += int(is_lp)
logger.info('Stretch %d' % stretch)
logger.info('Trace count: %d' % trace_count)
logger.info('VF count: %d' % vf_count)
logger.info('LP count: %d' % lp_count)
return (stretch, trace_count, vf_count, lp_count)
def upwalker_deepness(g, meta, vf_g, arguments):
N = g.vcount()
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
graph_max = max([x['closeness'] for x in g.vs])
with open(arguments.out, 'w') as f:
f.write(
'TRACE_MIN\tTRACE_AVG\tTRACE_MEDIAN\tTRACE_MAX\tSH_MIN\tSH_AVG\tSH_MEDIAN\tSH_MAX\tTRACE_MAX\tGRAPH_MAX\tSH_MAX\tSTRETCH\tLENGTH\n'
)
for m in meta:
progress += 1
progress.show_progress()
trace = m[helpers.TRACE]
s, t = trace[0], trace[-1]
sh_trace = g.get_all_shortest_paths(s, t)
sh_trace = random.choice(sh_trace)
sh_closeness = [g.vs[x]['closeness'] for x in sh_trace]
trace_closeness = [g.vs.find(x)['closeness'] for x in trace]
stretch = m[helpers.HOP_STRETCH] if m[
helpers.HOP_STRETCH] else len(trace) - len(sh_trace)
trace_max = max(trace_closeness)
sh_max = max(sh_trace)
with open('{}_trace{}'.format(arguments.out, len(trace)), 'a') as fo:
for x in trace_closeness:
fo.write('{} '.format(x / float(graph_max)))
fo.write('\n')
with open('{}_sh{}'.format(arguments.out, len(sh_trace)), 'a') as fo:
for x in sh_closeness:
fo.write('{} '.format(x / float(graph_max)))
fo.write('\n')
with open(arguments.out, 'a') as f:
f.write(
"{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\t{}\n".format(
min(trace_closeness), np.average(trace_closeness),
np.median(trace_closeness), np.max(trace_closeness),
min(sh_closeness), np.average(sh_closeness),
np.median(sh_closeness), np.max(sh_closeness), trace_max,
graph_max, sh_max, stretch, len(trace)))
def main():
parser = argparse.ArgumentParser(description='SANDBOX mode. Write something useful here', formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('network')
parser.add_argument('meta')
parser.add_argument('--top-node-ratio',
type=float, default=.1, dest='top_node_ratio')
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS), arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
g = helpers.load_network(arguments.network)
meta = helpers.load_from_json(arguments.meta)
logger.info('Graph loaded from: %s' % arguments.network)
logger.info('Graph vertex count: %d' % g.vcount())
end = int(g.vcount() * arguments.top_node_ratio)
logger.info('Top node count: %d' % end)
try:
nodes = [(x.index, x['closeness']) for x in g.vs]
except KeyError:
logger.info('Calculate closeness values')
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=g.vcount(),
width=15)
for n in g.vs:
progress += 1
progress.show_progress()
closeness = g.closeness(n)
n['closeness'] = closeness
g.save('%s_with_closeness.gml' % arguments.network)
nodes = [(x.index, x['closeness']) for x in g.vs]
nodes = sorted(nodes, reverse=True, key=lambda x: x[1])
top_nodes = set([x[0] for x in nodes[:end]])
purify(g, meta, top_nodes, arguments.progressbar)
def watts_trace_gen(g, traceroutes, show_progress=False):
watts_traceroutes = []
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
s, t = trace[0], trace[-1]
logger.debug('Get route from {s} to {t}'.format(s=s, t=t))
logger.debug('Original trace[[g]%d[/]]: %s' % (len(trace), trace))
try:
watts_trace = get_watts_route(g, s, t)
watts_trace = [g.vs[x]['name'] for x in watts_trace]
watts_traceroutes.append(watts_trace)
logger.debug('Watts trace[[p]%d[/]]: %s' % (len(watts_trace),
watts_trace))
except RuntimeError:
logger.warning('[b]No watts route[/]')
return watts_traceroutes
def purify(g, out, count=1000, show_progress=False):
logger.info('Started')
nodes = range(0, g.vcount())
endpoints = [random.sample(nodes, 2) for idx in range(0, count)]
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(
end=len(endpoints), width=15)
traces = []
for endpoint in endpoints:
progress += 1
progress.show_progress()
src, dst = endpoint
trace = g.get_shortest_paths(src, dst)[0]
if len(trace) > 0:
traces.append([g.vs[x]['name'] for x in trace])
logger.info('Len last trace: %d' % len(trace))
helpers.save_to_json(out, traces)
def main():
parser = argparse.ArgumentParser(
description='Pretty plot',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('network')
parser.add_argument('meta')
# parser.add_argument('output')
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS), arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
g = helpers.load_network(arguments.network)
meta = helpers.load_from_json(arguments.meta)
edges = [(e.source, e.target) for e in g.es]
rand_edges = random.sample(edges, 40)
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
for m in meta[:15]:
progress += 1
progress.show_progress()
trace = m[helpers.TRACE]
if len(trace) < 3: continue
s, t = trace[0], trace[-1]
sh = g.get_shortest_paths(s, t)[0]
pretty_plot(g, trace, sh, rand_edges)
def purify(labeled_g, out, network_path, extra_hop=0):
vs = [x.index for x in labeled_g.vs]
## Jus like in R
# print '================'
# for x in orig_vs:
# shp = labeled_g.get_all_shortest_paths(x, orig_vs, mode=i.ALL)
# res = []
# mes = 0
# for p in shp:
# mes += 1
# # print [labeled_g.vs[u]['name'] for u in p]
# vf_indicator = 1 if vft.is_valley_free(labeled_g, p) else 0
# # if vf_indicator == 0:
# # print [labeled_g.vs[u]['name'] for u in [p[0], p[-1]]]
# # print [labeled_g.degree(u) for u in p]
# # print vft.trace_to_string(labeled_g, p)
# # print vf_indicator == 1
# res.append(vf_indicator)
# # raw_input()
# # print mes
# print np.mean(res)
# raw_input()
# print '///////////////////////////'
pairs = random_pairs(vs, NODE_PAIRS)
print 'Random pairs: %d' % len(pairs)
probed_pairs = 0
all_vf = 0
all_nonvf = 0
all_vf_closeness = 0
all_nonvf_closeness = 0
results = []
results2 = []
results3 = []
short_results = list()
short_results2 = list()
short_results3 = list()
all_short_vf = 0
all_short_nonvf = 0
all_short_vf_closeness = 0
all_short_nonvf_closeness = 0
long_results = list()
long_results2 = list()
long_results3 = list()
all_long_vf = 0
all_long_nonvf = 0
all_long_vf_closeness = 0
all_long_nonvf_closeness = 0
results_closeness = []
results3_closeness = []
progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15)
for s, t in pairs:
progress += 1
progress.show_progress()
for x in range(0, labeled_g.vcount()):
labeled_g.vs[x]['traces'] = dict()
# all_path = labeled_g.get_all_shortest_paths(s, t, mode=i.ALL)
sh_len = labeled_g.shortest_paths(s, t, mode=i.ALL)[0][0]
sh_len += 1 # convert to hop count
all_path = helpers.dfs_mark(copy.deepcopy(labeled_g), s, t,
sh_len + extra_hop)
if all_path is None or len(all_path) < 1:
print 'No path between %s %s' % (s, t)
continue
probed_pairs += 1
vf_indicator = [
1 if vft.is_valley_free(labeled_g, x) else 0 for x in all_path
]
vf_closeness_indicator = [
1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS)
else 0 for x in all_path
]
vf_count = sum(vf_indicator)
vf_closeness_count = sum(vf_closeness_indicator)
nonvf = len(all_path) - vf_count
nonvf_closeness = len(all_path) - vf_closeness_count
all_vf += vf_count
all_nonvf += nonvf
all_vf_closeness += vf_closeness_count
all_nonvf_closeness += nonvf_closeness
long_path = [x for x in all_path if len(x) == sh_len + extra_hop]
short_path = [x for x in all_path if len(x) < sh_len + extra_hop]
tmp = [
1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_PRELABELED)
else 0 for x in all_path if len(x) < sh_len + extra_hop
]
short_vf_count = sum(tmp)
short_nonvf = len(tmp) - short_vf_count
tmp = [
1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS)
else 0 for x in all_path if len(x) < sh_len + extra_hop
]
short_vf_closeness_count = sum(tmp)
short_nonvf_closeness = len(tmp) - short_vf_closeness_count
tmp = [
1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_PRELABELED)
else 0 for x in all_path if len(x) >= sh_len + extra_hop
]
long_vf_count = sum(tmp)
long_nonvf = len(tmp) - long_vf_count
tmp = [
1 if vft.is_valley_free(labeled_g, x, vfmode=vft.ORDER_CLOSENESS)
else 0 for x in all_path if len(x) >= sh_len + extra_hop
]
long_vf_closeness_count = sum(tmp)
long_nonvf_closeness = len(tmp) - long_vf_closeness_count
if len(all_path) > 0:
results.append(vf_count / float(len(all_path)))
results_closeness.append(vf_closeness_count / float(len(all_path)))
else:
results.append(0)
results_closeness.append(0)
results3.append([vf_count, nonvf])
results3_closeness.append([vf_closeness_count, nonvf_closeness])
if len(all_path) > 1: results2.append(vf_count / float(len(all_path)))
all_long_vf += long_vf_count
all_long_nonvf += long_nonvf
all_long_vf_closeness += long_vf_closeness_count
all_long_nonvf_closeness += long_nonvf_closeness
all_short_vf += short_vf_count
all_short_nonvf += short_nonvf
all_short_vf_closeness += short_vf_closeness_count
all_short_nonvf_closeness += short_nonvf_closeness
if len(long_path) > 0:
long_results.append(long_vf_count / float(len(long_path)))
long_results3.append(long_vf_closeness_count /
float(len(long_path)))
else:
long_results.append(0)
long_results3.append(0)
if len(long_path) > 1:
long_results2.append(long_vf_count / float(len(long_path)))
if len(short_path) > 0:
short_results.append(short_vf_count / float(len(short_path)))
short_results3.append(short_vf_closeness_count /
float(len(short_path)))
else:
short_results.append(0)
short_results3.append(0)
print
with open(out, 'w') as f:
f.write('%s\n' % network_path)
f.write('Probed pairs: %d\n' % probed_pairs)
f.write('VF count: %d\n' % all_vf)
f.write('Non vf count: %d\n' % all_nonvf)
f.write('VF perc: %f\n' % (all_vf / float(all_vf + all_nonvf)))
f.write('Mean VF prob: %f\n' % np.mean(results))
f.write('Mean VF2 prob: %f\n' % np.mean(results2))
f.write('\n')
f.write('VF CLOSENESS count: %d\n' % all_vf_closeness)
f.write('Non vf CLOSENESS count: %d\n' % all_nonvf_closeness)
f.write(
'VF CLOSENESS perc: %f\n' %
(all_vf_closeness / float(all_vf_closeness + all_nonvf_closeness)))
f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(results_closeness))
f.write('\n')
f.write('==========\n')
f.write('VF count: %d\n' % all_short_vf)
f.write('VF CLOSENESS count: %d\n' % all_short_vf_closeness)
f.write('Non vf count: %d\n' % all_short_nonvf)
f.write('Non vf CLOSENESS count: %d\n' % all_short_nonvf_closeness)
if all_short_vf + all_short_nonvf > 0:
f.write('VF perc: %f\n' %
(all_short_vf / float(all_short_vf + all_short_nonvf)))
if all_short_vf_closeness + all_short_nonvf_closeness > 0:
f.write(
'VF CLOSENESS perc: %f\n' %
(all_short_vf_closeness /
float(all_short_vf_closeness + all_short_nonvf_closeness)))
f.write('Mean VF prob: %f\n' % np.mean(short_results))
f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(short_results3))
f.write('Mean VF2 prob: %f\n' % np.mean(short_results2))
f.write('=-----------------\n')
f.write('VF count: %d\n' % all_long_vf)
f.write('VF CLOSENESS count: %d\n' % all_long_vf_closeness)
f.write('Non vf count: %d\n' % all_long_nonvf)
f.write('Non vf CLOSENESS count: %d\n' % all_long_nonvf_closeness)
f.write('VF perc: %f\n' %
(all_long_vf / float(all_long_vf + all_long_nonvf)))
f.write('VF CLOSENESS perc: %f\n' %
(all_long_vf_closeness /
float(all_long_vf_closeness + all_long_nonvf_closeness)))
f.write('Mean VF prob: %f\n' % np.mean(long_results))
f.write('Mean VF CLOSENESS prob: %f\n' % np.mean(long_results3))
f.write('Mean VF2 prob: %f\n' % np.mean(long_results2))
def graphs(g, out, network_path):
# g = g.Erdos_Renyi(n=g.vcount(), m=g.ecount(), directed=False, loops=False)
# i.plot(g)
degrees = g.degree()
print 'Generate random graph'
max_random_graph = g.Degree_Sequence(g.degree(mode=i.ALL, loops=False),
method='vl')
rand_degrees = max_random_graph.degree(mode=i.ALL, loops=False)
fi_values = []
ratio_values = []
print 'Calculate coeff'
edge_helper = [min(g.degree(x.source), g.degree(x.target)) for x in g.es]
rand_edge_helper = [
min(max_random_graph.degree(x.source),
max_random_graph.degree(x.target)) for x in max_random_graph.es
]
progress = progressbar1.AnimatedProgressBar(end=max(degrees), width=15)
for k in range(1, max(degrees) + 1):
progress += 1
progress.show_progress()
try:
fi = rich_club_coeff(g,
k,
degrees=degrees,
edge_helper=edge_helper)
fi_ran = rich_club_coeff(max_random_graph,
k,
degrees=rand_degrees,
edge_helper=rand_edge_helper)
ratio = fi / float(fi_ran)
except ZeroDivisionError:
fi_values.append(0)
ratio_values.append(0)
continue
fi_values.append(fi)
ratio_values.append(ratio)
fig, ax = plt.subplots()
ax.axhline(y=1, xmin=1, xmax=len(ratio_values))
ax.plot(range(1, len(fi_values) + 1), fi_values, 'o')
ax.set_xlabel('k')
ax.set_ylabel('fi(k)')
ax.set_title('Rich club coeff ratio')
ax.set_yscale('log')
ax.set_xscale('log')
# ax.set_ylim(0.0, 2.0)
# ax.set_xlim([1, 10000])
ax.yaxis.grid(True, which='major')
# plt.gca().invert_yaxis()
fname = '%s_rich_club_simple_coeff.pdf' % out
print fname
pdf_file = PdfPages(fname)
plt.savefig(pdf_file, format='pdf')
pdf_file.close()
plt.show()
# i.plot(g.degree_distribution(), log="xy")
fig, ax = plt.subplots()
ax.axhline(y=1, xmin=1, xmax=len(ratio_values))
ax.plot(range(1, len(ratio_values) + 1), ratio_values, 'o')
ax.set_xlabel('k')
ax.set_ylabel('P_ran(k)')
ax.set_title('Rich club coeff ratio')
ax.set_xscale('log')
ax.set_ylim(0.0, max(2.0, max(ratio_values)))
# ax.set_xlim([1, 10000])
ax.yaxis.grid(True, which='major')
fname = '%s_rich_club_ratio_coeff.pdf' % out
print fname
pdf_file = PdfPages(fname)
plt.savefig(pdf_file, format='pdf')
pdf_file.close()
plt.show()
def filter(g, traceroutes):
results = list()
# remove traces with unknown nodes
traceroutes = vft.trace_in_vertex_id(g, traceroutes)
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
if not vft.trace_exists(g, trace):
print 'BUG?'
continue
for x in range(0, g.vcount()):
g.vs[x]['traces'] = dict()
trace = tuple(trace)
s, t = trace[0], trace[-1]
sh_len = g.shortest_paths(s, t, mode=i.ALL)[0][0]
sh_len += 1 # igraph's hop count to node count
all_routes = helpers.dfs_mark(g, s, t, sh_len + 1)
# all_routes2 = helpers.dfs_simple(g, s, t, sh_len + 1, ())
# if set(all_routes) - set(all_routes2) != set(all_routes2) - set(all_routes):
# print 'AJAJAJ'
# print all_routes
# print '----------'
# print all_routes2
sh_routes = [x for x in all_routes if len(x) == sh_len]
all_vf_routes = [x for x in all_routes if vft.is_valley_free(g, x)]
prediction_set = set(sh_routes) | set(all_vf_routes)
result = [
trace,
len(trace),
sh_len,
len(sh_routes),
trace in sh_routes,
len(all_vf_routes),
trace in all_vf_routes,
len(all_routes),
trace in all_routes,
len(prediction_set),
trace in prediction_set,
vft.is_valley_free(g, trace),
# sh_routes, all_vf_routes, all_routes,
vft.trace_to_string(g, trace)
]
results.append(result)
print >> sys.stderr, (
'TRACE\tTRACE_LEN\tSH_LEN',
'\t#SH_ROUTE\tOK',
'\t#ALL_VF\tOK',
'\t#ALL_ROUTE\tOK',
'\t#PREDICTION_SET\tOK',
'\tIS_VF',
# '\tSH_ROUTES\tALL_VF_ROUTES\tALL_ROUTE',
'\tTRACE_STR')
for result in results:
result = [str(r) for r in result]
print >> sys.stderr, '\t'.join(result)
return results
def trace_dir(g, meta, vf_g, arguments):
N = g.vcount()
vf_sum_lp_w = sum([e['LPweight'] for e in vf_g.es])
out_instead_core = 0
core_instead_out = 0
in_customer = 0
in_customer_but_use_up = 0
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
for m in meta:
progress += 1
progress.show_progress()
trace = m[helpers.TRACE]
first_edge = [trace[0], trace[1]]
first_hop_type = vft.edge_dir(g, first_edge, vfmode=vft.CLOSENESS)
s_idx = vft.node_to_nodeid(g, trace[0])
t_idx = vft.node_to_nodeid(g, trace[-1])
sh_vf = vf_g.get_all_shortest_paths(s_idx + N, t_idx + N)
if len(sh_vf) > 0:
in_customer += 1
if first_hop_type != LinkDir.D:
in_customer_but_use_up += 1
if first_hop_type == LinkDir.D:
# 'remove' all downward path:
for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT):
down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True)
vf_g.es[down_edge]['LPweight_old'] = vf_g.es[down_edge][
'LPweight']
vf_g.es[down_edge]['LPweight'] = vf_sum_lp_w + 1
lp_sh = vft.get_shortest_vf_route(g,
s_idx,
t_idx,
mode='lp',
vf_g=vf_g,
vfmode=vft.CLOSENESS)
pretty_plotter.pretty_plot(g, trace, lp_sh, [])
first_new_edge = [lp_sh[0], lp_sh[1]]
first_new_hop_type = vft.edge_dir(g,
first_new_edge,
vfmode=vft.CLOSENESS)
if first_hop_type != first_new_hop_type:
out_instead_core += 1
print ''
print "Original trace: %s" % trace
print 'Original trace dir: %s' % vft.trace_to_string(g, trace)
print 'Original closeness: %s' % [
g.vs.find(x)['closeness'] for x in trace
]
print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh]
print 'LP dir: %s' % vft.trace_to_string(g, lp_sh)
print 'LP closeness: %s' % [
g.vs.find(x)['closeness'] for x in lp_sh
]
# raw_input()
for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT):
down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True)
vf_g.es[down_edge]['LPweight'] = vf_g.es[down_edge][
'LPweight_old']
elif first_hop_type == LinkDir.U:
lp_sh = vft.get_shortest_vf_route(g,
s_idx,
t_idx,
mode='lp',
vf_g=vf_g,
vfmode=vft.CLOSENESS)
lp_dir = vft.trace_to_string(g, lp_sh)
# if lp_dir[1:].startswith('U') or lp_dir[1:].startswith('D'):
# continue
first_new_edge = [lp_sh[0], lp_sh[1]]
first_new_hop_type = vft.edge_dir(g,
first_new_edge,
vfmode=vft.CLOSENESS)
if first_hop_type != first_new_hop_type:
core_instead_out += 1
print ''
print "Original trace: %s" % trace
print 'Original trace dir: %s' % vft.trace_to_string(g, trace)
print 'Original closeness: %s' % [
g.vs.find(x)['closeness'] for x in trace
]
print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh]
print 'LP dir: %s' % vft.trace_to_string(g, lp_sh)
print 'LP closeness: %s' % [
g.vs.find(x)['closeness'] for x in lp_sh
]
pretty_plotter.pretty_plot(g, trace, lp_sh, [])
# raw_input()
logger.info('Core instead down: %d' % core_instead_out)
logger.info('Down instead core: %d' % out_instead_core)
logger.info('In customer cone: %d' % in_customer)
logger.info('In customer but use UP: %d' % in_customer_but_use_up)
def purify(g,
meta_original,
out,
count=1000,
try_per_race=1,
show_progress=False):
empty = 0
# remove traces with already calculated random paths
logger.warn('[r]ONLY NOT FILLED PATHS[/]')
meta_filled = [
x for x in meta_original
if helpers.RANDOM_NONVF_WALK_RUN_COUNT not in x
]
logger.info('All trace count: %d' % len(meta_filled))
tr_count = min(len(meta_filled), count)
meta_random = random.sample(meta_filled, tr_count)
logger.info('Chosen subset count: %d' % len(meta_random))
real_vf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 1]
real_nonvf_degree = [
x for x in meta_random if x[helpers.IS_VF_DEGREE] == 0
]
assert len(real_nonvf_degree) == tr_count - len(real_vf_degree)
real_vf_prelabeled = [
x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 1
]
real_nonvf_prelabeled = [
x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 0
]
assert len(real_nonvf_prelabeled) == tr_count - len(real_vf_prelabeled)
real_vf_closeness = [
x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 1
]
real_nonvf_closeness = [
x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 0
]
assert len(real_nonvf_closeness) == tr_count - len(real_vf_closeness)
logger.info('Real vf degree: %f[%d]' %
((len(real_vf_degree) / float(tr_count), len(real_vf_degree))))
logger.info(
'Real nonvf degree: %f[%d]' %
((len(real_nonvf_degree) / float(tr_count), len(real_nonvf_degree))))
logger.info(
'Real vf prelabeled: %f[%d]' %
((len(real_vf_prelabeled) / float(tr_count), len(real_vf_prelabeled))))
logger.info('Real nonvf prelabeled: %f[%d]' %
((len(real_nonvf_prelabeled) / float(tr_count),
len(real_nonvf_prelabeled))))
logger.info(
'Real vf closeness: %f[%d]' %
((len(real_vf_closeness) / float(tr_count), len(real_vf_closeness))))
logger.info('Real nonvf closeness: %f[%d]' %
((len(real_nonvf_closeness) / float(tr_count),
len(real_nonvf_closeness))))
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(meta_random),
width=15)
for trace_meta in meta_random:
progress += 1
progress.show_progress()
trace = vft.trace_in_vertex_id(g, [
trace_meta[helpers.TRACE],
])
if len(trace) != 1:
logger.error('PROBLEM')
logger.error('%s' % trace_meta)
continue
trace = trace[0]
random_nonvf_walk_closeness_route_count = 0
random_nonvf_walk_closeness_route_len = []
random_nonvf_walk_degree_route_count = 0
random_nonvf_walk_degree_route_len = []
random_nonvf_walk_prelabeled_route_count = 0
random_nonvf_walk_prelabeled_route_len = []
random_nonvf_walk_lp_soft_closeness_route_count = 0
random_nonvf_walk_lp_soft_degree_route_count = 0
random_nonvf_walk_lp_soft_prelabeled_route_count = 0
random_nonvf_walk_lp_hard_closeness_route_count = 0
random_nonvf_walk_lp_hard_degree_route_count = 0
random_nonvf_walk_lp_hard_prelabeled_route_count = 0
s, t = trace[0], trace[-1]
for counter in xrange(0, try_per_race):
isvf, random_path = helpers.random_nonvf_route(
g, s, t, len(trace), vfmode=vft.CLOSENESS)
assert len(random_path) > 0
if isvf:
random_nonvf_walk_closeness_route_count += 1
lp_soft = vft.is_local_preferenced(g,
trace,
first_edge=True,
vfmode=vft.CLOSENESS)
lp_hard = vft.is_local_preferenced(g,
trace,
first_edge=False,
vfmode=vft.CLOSENESS)
if lp_soft:
random_nonvf_walk_lp_soft_closeness_route_count += 1
if lp_hard:
random_nonvf_walk_lp_hard_closeness_route_count += 1
random_nonvf_walk_closeness_route_len.append(len(random_path))
isvf, random_path = helpers.random_nonvf_route(g,
s,
t,
len(trace),
vfmode=vft.DEGREE)
assert len(random_path) > 0
if isvf:
random_nonvf_walk_degree_route_count += 1
lp_soft = vft.is_local_preferenced(g,
trace,
first_edge=True,
vfmode=vft.DEGREE)
lp_hard = vft.is_local_preferenced(g,
trace,
first_edge=False,
vfmode=vft.DEGREE)
if lp_soft:
random_nonvf_walk_lp_soft_degree_route_count += 1
if lp_hard:
random_nonvf_walk_lp_hard_degree_route_count += 1
random_nonvf_walk_degree_route_len.append(len(random_path))
isvf, random_path = helpers.random_nonvf_route(
g, s, t, len(trace), vfmode=vft.PRELABELED)
assert len(random_path) > 0
if isvf:
random_nonvf_walk_prelabeled_route_count += 1
lp_soft = vft.is_local_preferenced(g,
trace,
first_edge=True,
vfmode=vft.PRELABELED)
lp_hard = vft.is_local_preferenced(g,
trace,
first_edge=False,
vfmode=vft.PRELABELED)
if lp_soft:
random_nonvf_walk_lp_soft_prelabeled_route_count += 1
if lp_hard:
random_nonvf_walk_lp_hard_prelabeled_route_count += 1
random_nonvf_walk_prelabeled_route_len.append(len(random_path))
# sanity check
# if random_path[0] != s or random_path[-1] != t:
# logger.error('ALERT')
if len(random_path) != len(set(random_path)):
logger.error('LENGTH ERROR')
extra_meta = {
helpers.RANDOM_NONVF_WALK_RUN_COUNT:
try_per_race,
helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE:
random_nonvf_walk_closeness_route_count,
helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE_LEN:
random_nonvf_walk_closeness_route_len,
helpers.RANDOM_NONVF_WALK_VF_DEGREE_ROUTE:
random_nonvf_walk_degree_route_count,
helpers.RANDOM_NONVF_WALK_VF_DEGREE_ROUTE_LEN:
random_nonvf_walk_degree_route_len,
helpers.RANDOM_NONVF_WALK_VF_PRELABELED_ROUTE:
random_nonvf_walk_prelabeled_route_count,
helpers.RANDOM_NONVF_WALK_VF_PRELABELED_ROUTE_LEN:
random_nonvf_walk_prelabeled_route_len,
helpers.RANDOM_NONVF_WALK_LP_SOFT_DEGREE_ROUTE:
random_nonvf_walk_lp_soft_degree_route_count,
helpers.RANDOM_NONVF_WALK_LP_SOFT_CLOSENESS_ROUTE:
random_nonvf_walk_lp_soft_closeness_route_count,
helpers.RANDOM_NONVF_WALK_LP_SOFT_PRELABELED_ROUTE:
random_nonvf_walk_lp_soft_prelabeled_route_count,
helpers.RANDOM_NONVF_WALK_LP_HARD_DEGREE_ROUTE:
random_nonvf_walk_lp_hard_degree_route_count,
helpers.RANDOM_NONVF_WALK_LP_HARD_CLOSENESS_ROUTE:
random_nonvf_walk_lp_hard_closeness_route_count,
helpers.RANDOM_NONVF_WALK_LP_HARD_PRELABELED_ROUTE:
random_nonvf_walk_lp_hard_prelabeled_route_count
}
trace_meta.update(extra_meta)
## save modified meta
# all meta_* get only references from meta_original
helpers.save_to_json(out, meta_original)
# calculate results
real_vf = [x[helpers.IS_VF_CLOSENESS] for x in meta_random]
real_vf_ratio = np.mean(real_vf)
random_nonvf_walk_vf_ratio_per_element = [
x[helpers.RANDOM_NONVF_WALK_VF_CLOSENESS_ROUTE] /
x[helpers.RANDOM_NONVF_WALK_RUN_COUNT] for x in meta_random
]
random_nonvf_walk_vf_ratio = np.mean(
random_nonvf_walk_vf_ratio_per_element)
# print results
logger.info('')
logger.info('Empty: %d' % empty)
logger.info('Tested trace count: %d' % len(meta_random))
logger.info('VF ratio in tested traces: %f' % real_vf_ratio)
logger.info('VF ratio in random walks: %f' % random_nonvf_walk_vf_ratio)
logger.info('Generate activity matrix for all user in all time slot')
activity_matrix = functional_mat['TS'][0, res][:, :, :]
activity_matrix[np.isnan(activity_matrix)] = 0
activity_tmp = np.copy(activity_matrix)
activity_matrix[activity_tmp > ts_limit] = 1
activity_matrix[activity_tmp <= ts_limit] = 0
del activity_tmp
voxel_coords = structural_mat['centroids'][0, res]
meta = []
vf_ratio = []
progress = progressbar1.AnimatedProgressBar(end=len(user_range) *
len(time_range),
width=15)
measurement_id = helpers.id_generator(32)
web_log = {
'measurement_id': measurement_id,
'msg_type': MSG_MEASUREMENT,
'resolution': res,
'user_range': user_range,
'adjacency_threshold': adjacency_threshold,
'time_range': time_range,
'corr_limit': corr_limit,
'max_stretch': MAX_STRETCH,
'ts_limit': ts_limit
}
logger.log(misc.WEB, json.dumps(web_log))
def purify(g,
meta_original,
out,
count=1000,
try_per_race=1,
show_progress=False,
with_lp=True):
empty = 0
# remove traces with already calculated random paths
logger.warn('[r]ONLY NOT FILLED PATHS[/]')
meta_filled = [
x for x in meta_original if helpers.RANDOM_WALK_RUN_COUNT not in x
]
# Filter if interested only in routes of stretch 1
# meta_filled = [x for x in meta_original
# if x[helpers.TRACE_LEN]-x[helpers.SH_LEN] == 1]
## traces with a maximum stretch
# logger.warn('[r]!!!ONLY WITH STRETCH[/]')
# meta = [x for x in meta if x[helpers.STRETCH] > -1]
# # shorter meta records
# logger.warn('[r]!!!ONLY SHORT TRACES[/]')
# meta = [x for x in meta if len(x[helpers.TRACE]) < 5]
# meta_map = {tuple(x[helpers.TRACE]): x for x in meta_filled}
logger.info('All trace count: %d' % len(meta_filled))
tr_count = min(len(meta_filled), count)
meta_random = random.sample(meta_filled, tr_count)
logger.info('Chosen subset count: %d' % len(meta_random))
# real_vf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 1]
# real_nonvf_degree = [x for x in meta_random if x[helpers.IS_VF_DEGREE] == 0]
# assert len(real_nonvf_degree) == tr_count - len(real_vf_degree)
# real_vf_prelabeled = [x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 1]
# real_nonvf_prelabeled = [x for x in meta_random if x[helpers.IS_VF_PRELABELED] == 0]
# assert len(real_nonvf_prelabeled) == tr_count - len(real_vf_prelabeled)
# real_vf_closeness = [x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 1]
# real_nonvf_closeness = [x for x in meta_random if x[helpers.IS_VF_CLOSENESS] == 0]
# assert len(real_nonvf_closeness) == tr_count - len(real_vf_closeness)
# logger.info('Real vf degree: %f[%d]' % ((len(real_vf_degree) / float(tr_count),
# len(real_vf_degree))))
# logger.info('Real nonvf degree: %f[%d]' % ((len(real_nonvf_degree) / float(tr_count),
# len(real_nonvf_degree))))
# logger.info('Real vf prelabeled: %f[%d]' % ((len(real_vf_prelabeled) / float(tr_count),
# len(real_vf_prelabeled))))
# logger.info('Real nonvf prelabeled: %f[%d]' % ((len(real_nonvf_prelabeled) / float(tr_count),
# len(real_nonvf_prelabeled))))
# logger.info('Real vf closeness: %f[%d]' % ((len(real_vf_closeness)/float(tr_count), len(real_vf_closeness))))
# logger.info('Real nonvf closeness: %f[%d]' % ((len(real_nonvf_closeness)/float(tr_count), len(real_nonvf_closeness))))
# traceroutes = [x[helpers.TRACE] for x in meta_random]
# traceroutes = vft.trace_in_vertex_id(g, traceroutes)
try:
meta_random[0][helpers.TRACE]
except Exception:
meta_random = [{helpers.TRACE: x} for x in meta_random]
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(meta_random),
width=15)
stretch_list = []
max_stretch = max(
[x[helpers.TRACE_LEN] - x[helpers.SH_LEN] for x in meta_random])
for stretch in range(0, max_stretch + 1):
metas = [
x for x in meta_random
if x[helpers.TRACE_LEN] - x[helpers.SH_LEN] == stretch
]
stretch_list.extend(list(repeat(stretch, len(metas))))
# print(stretch_list)
lenghts = random.shuffle(stretch_list)
strx_array = []
for idx, trace_meta in enumerate(meta_random):
progress += 1
progress.show_progress()
# print(trace_meta[helpers.TRACE])
shl = trace_meta[helpers.SH_LEN]
trace = vft.trace_in_vertex_id(g, [
trace_meta[helpers.TRACE],
])
if len(trace) != 1:
print 'PROBLEM'
print trace_meta
continue
trace = trace[0]
# print(trace)
random_walk_closeness_route_vf = 0
random_walk_closeness_route_lp_soft = 0
random_walk_closeness_route_lp_hard = 0
random_walk_degree_route_vf = 0
random_walk_degree_route_lp_soft = 0
random_walk_degree_route_lp_hard = 0
random_walk_prelabeled_route_vf = 0
random_walk_prelabeled_route_lp_soft = 0
random_walk_prelabeled_route_lp_hard = 0
s, t = trace[0], trace[-1]
for counter in xrange(0, try_per_race):
# random_path = helpers.random_route_walk(g, s, t, len(trace)) # Modified
random_path = helpers.random_route_walk(
g, s, t, shl + stretch_list[idx]) # Modified
if len(random_path) == 0:
empty += 1
if vft.is_valley_free(g, random_path, vfmode=vft.CLOSENESS):
random_walk_closeness_route_vf += 1
if (len(random_path) == shl + 1):
strx_array.append(1)
if with_lp:
lp_soft = vft.is_local_preferenced(g,
random_path,
first_edge=True,
vfmode=vft.CLOSENESS)
lp_hard = vft.is_local_preferenced(g,
random_path,
first_edge=False,
vfmode=vft.CLOSENESS)
if lp_soft:
random_walk_closeness_route_lp_soft += 1
if lp_hard:
random_walk_closeness_route_lp_hard += 1
else:
if (len(random_path) == shl + 1):
strx_array.append(0)
# if vft.is_valley_free(g, random_path, vfmode=vft.DEGREE):
# random_walk_degree_route_vf += 1
# if with_lp:
# lp_soft = vft.is_local_preferenced(g, random_path,
# first_edge=True,
# vfmode=vft.DEGREE)
# lp_hard = vft.is_local_preferenced(g, random_path,
# first_edge=False,
# vfmode=vft.DEGREE)
# if lp_soft:
# random_walk_degree_route_lp_soft += 1
# if lp_hard:
# random_walk_degree_route_lp_hard += 1
# if vft.is_valley_free(g, random_path, vfmode=vft.PRELABELED):
# random_walk_prelabeled_route_vf += 1
# if with_lp:
# lp_soft = vft.is_local_preferenced(g, random_path,
# first_edge=True,
# vfmode=vft.PRELABELED)
# lp_hard = vft.is_local_preferenced(g, random_path,
# first_edge=False,
# vfmode=vft.PRELABELED)
# if lp_soft:
# random_walk_prelabeled_route_lp_soft += 1
# if lp_hard:
# random_walk_prelabeled_route_lp_hard += 1
# sanity check
# if random_path[0] != s or random_path[-1] != t:
# logger.error('ALERT')
if len(random_path) != len(set(random_path)):
logger.error('LENGTH ERROR')
extra_meta = {
helpers.RANDOM_WALK_RUN_COUNT:
try_per_race,
helpers.RANDOM_WALK_VF_CLOSENESS_ROUTE:
random_walk_closeness_route_vf,
helpers.RANDOM_WALK_VF_DEGREE_ROUTE:
random_walk_degree_route_vf,
helpers.RANDOM_WALK_VF_PRELABELED_ROUTE:
random_walk_prelabeled_route_vf,
}
if with_lp:
extra_meta.update({
helpers.RANDOM_WALK_LP_SOFT_CLOSENESS_ROUTE:
random_walk_closeness_route_lp_soft,
helpers.RANDOM_WALK_LP_HARD_CLOSENESS_ROUTE:
random_walk_closeness_route_lp_hard,
helpers.RANDOM_WALK_LP_SOFT_DEGREE_ROUTE:
random_walk_degree_route_lp_soft,
helpers.RANDOM_WALK_LP_HARD_DEGREE_ROUTE:
random_walk_degree_route_lp_hard,
helpers.RANDOM_WALK_LP_SOFT_PRELABELED_ROUTE:
random_walk_prelabeled_route_lp_soft,
helpers.RANDOM_WALK_LP_HARD_PRELABELED_ROUTE:
random_walk_prelabeled_route_lp_hard
})
trace_meta.update(extra_meta)
## save modified meta
# all meta_* get only references from meta_original
helpers.save_to_json(out, meta_random)
# meta_mod = [x for x in meta_map.itervalues()]
# helpers.save_to_json(out, meta_mod)
# calculate results
# real_vf = [x[helpers.IS_VF_CLOSENESS] for x in meta_random]
# real_vf_ratio = np.mean(real_vf)
random_walk_vf_ratio_per_element = [
x[helpers.RANDOM_WALK_VF_CLOSENESS_ROUTE] /
x[helpers.RANDOM_WALK_RUN_COUNT] for x in meta_random
]
random_walk_vf_ratio = np.mean(random_walk_vf_ratio_per_element)
# print results
logger.info('')
logger.info('Empty: %d' % empty)
logger.info('Tested trace count: %d' % len(meta_random))
# logger.info('VF ratio in tested traces: %f' % real_vf_ratio)
logger.info('VF ratio in random walks: %f' % random_walk_vf_ratio)
logger.info('VF ratio in random walks for path stretch 1: %f' %
np.mean(strx_array))
def purify(g, meta, top_nodes, show_progress):
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(meta),
width=15)
logger.info('Purify')
triplet_count_closeness = 0
top_nodes_triplet_closeness = 0
triplet_count_prelabeled = 0
top_nodes_triplet_prelabeled = 0
triplet_count_degree = 0
top_nodes_triplet_degree = 0
for row in meta:
progress += 1
progress.show_progress()
trace = row[helpers.TRACE]
trace = vft.trace_in_vertex_id(g, [trace, ])[0]
if row[helpers.IS_VF_CLOSENESS] == 0:
logger.debug('Closeness valley')
vt = vft.get_valley_triplets(g, trace, vft.CLOSENESS)
triplet_count_closeness += len(vt)
for trip in vt:
if any([x in top_nodes for x in trip]):
top_nodes_triplet_closeness += 1
if row[helpers.IS_VF_PRELABELED] == 0:
logger.debug('Prelabeled valley')
vt = vft.get_valley_triplets(g, trace, vft.PRELABELED)
triplet_count_prelabeled += len(vt)
for trip in vt:
if any([x in top_nodes for x in trip]):
top_nodes_triplet_prelabeled += 1
if row[helpers.IS_VF_DEGREE] == 0:
logger.debug('Degree valley')
vt = vft.get_valley_triplets(g, trace, vft.DEGREE)
triplet_count_degree += len(vt)
for trip in vt:
if any([x in top_nodes for x in trip]):
top_nodes_triplet_degree += 1
print
try:
ratio_closeness = top_nodes_triplet_closeness / float(triplet_count_closeness)
except ZeroDivisionError:
ratio_closeness = 0.0
try:
ratio_prelabeled = top_nodes_triplet_prelabeled / float(triplet_count_prelabeled)
except ZeroDivisionError:
ratio_prelabeled = 0.0
try:
ratio_degree = top_nodes_triplet_degree / float(triplet_count_degree)
except ZeroDivisionError:
ratio_degree = 0.0
logger.info('CLOSENESS: %6.3f[%d/%d]' % (ratio_closeness, top_nodes_triplet_closeness, triplet_count_closeness))
logger.info('PRELABELED: %6.3f[%d/%d]' % (ratio_prelabeled, top_nodes_triplet_prelabeled, triplet_count_prelabeled))
logger.info('DEGREE: %6.3f[%d/%d]' % (ratio_degree, top_nodes_triplet_degree, triplet_count_degree))
def main():
parser = argparse.ArgumentParser(
description=('Syntetic route generator'),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('network')
parser.add_argument('meta')
parser.add_argument('all_trace_out', metavar='all-trace-out')
parser.add_argument('syntetic_out', metavar='syntetic-out')
parser.add_argument('--trace-count',
'-tc',
type=int,
dest='trace_count',
default=5000)
parser.add_argument('--random-sample',
dest='random_sample',
action='store_true')
parser.add_argument('--closeness-error',
'-ce',
type=float,
dest='closeness_error',
default=0.0)
parser.add_argument('--core-limit-percentile',
'-cl',
type=int,
dest='core_limit',
default=0)
parser.add_argument('--toggle-node-error-mode', action='store_true')
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS) - 1, arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
show_progress = arguments.progressbar
g = helpers.load_network(arguments.network)
g = g.simplify()
meta = helpers.load_from_json(arguments.meta)
if arguments.random_sample:
random.shuffle(meta)
meta = meta[:arguments.trace_count]
N = g.vcount()
cl = sorted([x['closeness'] for x in g.vs], reverse=True)
logger.info('Min closeness: %s' % np.min(cl))
logger.info('Max closeness: %s' % np.max(cl))
logger.info('Mean closenss: %s' % np.mean(cl))
logger.info('10%% closeness: %s' % np.percentile(cl, 10))
logger.info('90%% closeness: %s' % np.percentile(cl, 90))
logger.info('Core limit: [r]%d%%[/]' % arguments.core_limit)
change_probability = 100 * arguments.closeness_error
logger.info('Change probability: [r]%6.2f%%[/]' % change_probability)
core_limit = np.percentile(cl, arguments.core_limit)
logger.info('Core limit in closeness: [bb]%f[/]' % core_limit)
if arguments.toggle_node_error_mode:
logger.info('[r]Node error mode[/]')
msg = (
"If given node's closeness >= core_limit then the new ",
"closeness in this node is ",
#"rand(closeness_error ... old closeness)"
"OLD_CLOSENSS * +/- closeness_error%")
logger.info(''.join(msg))
logger.info('Minimum node closeness: [g]%f[/]' %
arguments.closeness_error)
for n in g.vs:
if n['closeness'] < core_limit:
continue
# sign = -1 if random.uniform(-1, 1) < 0 else 1
# n['closeness'] = n['closeness'] * (1 + sign * arguments.closeness_error)
new_closeness = random.uniform(arguments.closeness_error,
n['closeness'])
n['closeness'] = new_closeness
g_labeled = vft.label_graph_edges(g, vfmode=vft.CLOSENESS)
peer_edge_count = len([x for x in g_labeled.es if x['dir'] == LinkDir.P])
logger.info('Peer edge count: %d' % peer_edge_count)
changed_edges = []
if not arguments.toggle_node_error_mode:
msg = ("If the closeness values of the endpoints in given edge is ",
"larger than the core_limit and ",
"random(0,1) < closeness_error then change the direction ",
"for this edge")
logger.info(''.join(msg))
changed_u = changed_d = 0
changed_edges = []
changed_edgess = []
for edge in g_labeled.es:
s, t = edge.source, edge.target
s_cl = g_labeled.vs[s]['closeness']
t_cl = g_labeled.vs[t]['closeness']
if (s_cl < core_limit or t_cl < core_limit): continue
if random.uniform(0, 1) > arguments.closeness_error: continue
# if abs(s_cl - t_cl) / min(s_cl, t_cl) > 0.02: continue
new_edge_dir = LinkDir.U if random.uniform(0,
1) > 0.5 else LinkDir.D
if new_edge_dir != edge['dir']:
if edge['dir'] == LinkDir.U:
changed_u += 1
else:
changed_d += 1
edge['dir'] = new_edge_dir
changed_edges.append(edge)
changed_edgess.append((edge.source, edge.target))
# if edge['dir'] == LinkDir.U:
# changed_u += 1
# changed_edgess.append((edge.source, edge.target))
# edge['dir'] = LinkDir.D
# changed_edges.append(edge)
# elif edge['dir'] == LinkDir.D:
# changed_d += 1
# changed_edgess.append((edge.source, edge.target))
# edge['dir'] = LinkDir.U
# changed_edges.append(edge)
logger.info('E count: %d' % g_labeled.ecount())
logger.info('Changed U: %d' % changed_u)
logger.info('Changed D: %d' % changed_d)
logger.info('Changed: %d' % (changed_d + changed_u))
changed_e = [(g_labeled.vs[e.source]['name'],
g_labeled.vs[e.target]['name']) for e in changed_edges]
changed_e = changed_e + [(x[1], x[0]) for x in changed_e]
changed_e = set(changed_e)
vf_g_closeness = vft.convert_to_vf(g,
vfmode=vft.CLOSENESS,
labeled_graph=g_labeled)
# e_colors = []
# for e in vf_g_closeness.es:
# if e.source < N and e.target < N: col = 'grey'
# elif e.source < N and e.target >= N: col = 'blue'
# elif e.source >= N and e.target >= N: col = 'red'
# else: col = 'cyan'
# e_colors.append(col)
# igraph.plot(vf_g_closeness, "/tmp/closeness.pdf",
# vertex_label=vf_g_closeness.vs['name'],
# vertex_size=0.2,
# edge_color=e_colors)
pairs = [(g.vs.find(x[helpers.TRACE][0]).index,
g.vs.find(x[helpers.TRACE][-1]).index, tuple(x[helpers.TRACE]))
for x in meta]
# pairs = list(set(pairs))
# random.shuffle(pairs)
# visited = set()
# pairs2 = []
# for x in pairs:
# k = (x[0], x[1])
# if k in visited: continue
# visited.add(k)
# visited.add((k[1], k[0]))
# pairs2.append(x)
# pairs = pairs2
traces = [x[2] for x in pairs]
stretches = []
syntetic_traces = []
sh_traces = []
base_traces = []
original_traces = []
bad = 0
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15)
for s, t, trace_original in pairs:
progress += 1
progress.show_progress()
trace_original_idx = [g.vs.find(x).index for x in trace_original]
logger.debug('Original trace: %s -- %s -- %s',
[g.vs[x]['name'] for x in trace_original_idx],
vft.trace_to_string(g, trace_original_idx, vft.CLOSENESS),
[g.vs[x]['closeness'] for x in trace_original_idx])
sh_routes = g.get_all_shortest_paths(s, t)
sh_len = len(sh_routes[0])
sh_routes_named = [[g.vs[y]['name'] for y in x] for x in sh_routes]
sh_trace_name = random.choice(sh_routes_named)
base_trace_name = random.choice(sh_routes_named)
candidates = vf_g_closeness.get_all_shortest_paths(s + N, t + N)
candidates = [vft.vf_route_converter(x, N) for x in candidates]
# candidates = []
if len(candidates) == 0:
candidates = vft.get_shortest_vf_route(g_labeled,
s,
t,
mode='vf',
vf_g=vf_g_closeness,
_all=True,
vfmode=vft.CLOSENESS)
if len(candidates) == 0:
s_name, t_name = g.vs[s]['name'], g.vs[t]['name']
logger.debug("!!!No syntetic route from %s to %s" %
(s_name, t_name))
continue
logger.debug('Candidates from %s to %s:' %
(g.vs[s]['name'], g.vs[t]['name']))
for c in candidates:
logger.debug('%s -- %s -- %s' %
([g.vs[x]['name'] for x in c],
vft.trace_to_string(g_labeled, c, vft.PRELABELED),
[g.vs[x]['closeness'] for x in c]))
chosen_one = random.choice(candidates)
chosen_one_name = [g.vs[x]['name'] for x in chosen_one]
# print chosen_one
# print trace_original
# pretty_plotter.pretty_plot(g, trace_original_idx,
# chosen_one, changed_edgess,
# spec_color=(0, 0, 0, 155))
hop_stretch = len(chosen_one) - sh_len
stretches.append(hop_stretch)
trace_original_e = zip(trace_original, trace_original[1:])
chosen_one_e = zip(chosen_one_name, chosen_one_name[1:])
trace_affected = any([x in changed_e for x in trace_original_e])
chosen_affected = any([x in changed_e for x in chosen_one_e])
logger.debug('Trace affected: %s' % trace_affected)
logger.debug('Chosen affected: %s' % chosen_affected)
# if hop_stretch > 2:
# logger.debug('Base: %s' % trace_to_string(g_labeled, base_trace_name))
# logger.debug('SH: %s' % trace_to_string(g_labeled, sh_trace_name))
# logger.debug('Trace: %s' % trace_to_string(g_labeled, trace_original))
# logger.debug('Syntetic: %s' % trace_to_string(g_labeled, chosen_one_name))
if trace_affected or chosen_affected or hop_stretch > 2:
# pretty_plotter.pretty_plot_all(g, traces,
# chosen_one, changed_edgess,
# spec_color=(0, 0, 0, 255))
bad += 1
syntetic_traces.append(chosen_one_name)
sh_traces.append(sh_trace_name)
base_traces.append(base_trace_name)
original_traces.append(trace_original)
logger.debug('From %s to %s chosen one %s' %
(g.vs[s]['name'], g.vs[t]['name'], chosen_one_name))
result = zip(base_traces, sh_traces, original_traces, syntetic_traces)
helpers.save_to_json(arguments.all_trace_out, result)
helpers.save_to_json(arguments.syntetic_out, syntetic_traces)
print 'Bad: %d' % bad
c = collections.Counter(stretches)
trace_count = len(syntetic_traces)
logger.info('Stretch dist:')
for k in c:
logger.info('\t%d: %5.2f%%[%d]' %
(k, 100 * c[k] / float(trace_count), c[k]))
logger.info('Valid route count: %d' % trace_count)
logger.info('Route count parameter: %d' % arguments.trace_count)
logger.info('Generated valid pair count: %d' % len(pairs))
def purify(g, meta, out, count=1000):
results = list()
results2 = list()
results3 = list()
all_vf = 0
all_nonvf = 0
all_vf_closeness = 0
all_nonvf_closeness = 0
short_results = list()
short_results2 = list()
short_results3 = list()
all_short_vf = 0
all_short_nonvf = 0
all_short_vf_closeness = 0
all_short_nonvf_closeness = 0
long_results = list()
long_results2 = list()
long_results3 = list()
all_long_vf = 0
all_long_nonvf = 0
all_long_vf_closeness = 0
all_long_nonvf_closeness = 0
# remove traces with already calculated all_path
logger.warn('[r]ONLY NOT FILLED PATHS[/]')
meta = [x for x in meta if not helpers.ALL_PATH_COUNT in x]
# traces with a maximum stretch
logger.warn('[r]!!!ONLY WITH LOW STRETCH[/]')
meta = [x for x in meta if x[helpers.STRETCH] < 4]
# shorter meta records
logger.warn('[r]!!!ONLY SHORT TRACES[/]')
meta = [x for x in meta if len(x[helpers.TRACE]) < 5]
meta_map = {tuple(x[helpers.TRACE]): x for x in meta}
# traceroutes = [x for x in meta if x[TRACE_LEN] == x[SH_LEN]]
logger.info('All trace count: %d' % len(meta))
tr_count = min(len(meta), count)
meta = random.sample(meta, tr_count)
logger.info('Chosen trace count: %d' % len(meta))
real_vf = [x for x in meta if x[helpers.IS_VF] == 1]
real_nonvf = [x for x in meta if x[helpers.IS_VF] == 0]
real_vf_closeness = [x for x in meta if x[helpers.IS_VF_CLOSENESS] == 1]
real_nonvf_closeness = [x for x in meta if x[helpers.IS_VF_CLOSENESS] == 0]
logger.info('Real vf: %f[%d]' % ((len(real_vf)/float(len(meta)), len(real_vf))))
logger.info('Real nonvf: %f[%d]' % ((len(real_nonvf)/float(len(meta)), len(real_nonvf))))
logger.info('Real vf closeness: %f[%d]' % ((len(real_vf_closeness)/float(len(meta)), len(real_vf_closeness))))
logger.info('Real nonvf closeness: %f[%d]' % ((len(real_nonvf_closeness)/float(len(meta)), len(real_nonvf_closeness))))
logger.info('Remove unknown traces. Trace count before: %d' % len(meta))
traceroutes = [x[helpers.TRACE] for x in meta]
traceroutes, ignored = vft.trace_clean(g, traceroutes)
logger.info('Traceroutes after: %d. Ignored: %d' % (len(traceroutes), ignored))
traceroutes = vft.trace_in_vertex_id(g, traceroutes)
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes), width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
for x in range(0, g.vcount()):
g.vs[x]['traces'] = dict()
s, t = trace[0], trace[-1]
sh_path = g.get_all_shortest_paths(s, t, mode=i.OUT)
all_path = helpers.dfs_mark(copy.deepcopy(g), s, t, len(trace))
# if len(sh_path) != len(all_path):
# print len(sh_path)
# print len(all_path)
# print s, t
# sanity check
for x in all_path:
if x[0] != s or x[-1] != t:
logger.error('ALERT')
if len(set([tuple(x) for x in all_path])) != len(all_path):
logger.error('LENGTH ALERT')
logger.error('%s' % len(all_path))
logger.error('%s' % len(set([tuple(x) for x in all_path])))
logger.error('%s' % sorted(all_path))
long_path = [x for x in all_path if len(x) == len(trace)]
short_path = [x for x in all_path if len(x) < len(trace)]
named_trace = [g.vs[x]['name'] for x in trace]
extra_meta = {
helpers.ALL_PATH_COUNT: len(all_path),
helpers.SAME_LONG_PATH_COUNT: len(long_path),
helpers.SHORTER_PATH_COUNT: len(short_path)
}
meta_map[tuple(named_trace)].update(extra_meta)
vf_count = sum([1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in all_path])
nonvf = len(all_path) - vf_count
vf_closeness_count = sum([1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in all_path])
nonvf_closeness = len(all_path) - vf_closeness_count
tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in short_path]
short_vf_count = sum(tmp)
short_nonvf = len(tmp) - short_vf_count
tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in short_path]
short_vf_closeness_count = sum(tmp)
short_nonvf_closeness = len(tmp) - short_vf_closeness_count
tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.PRELABELED) else 0 for x in long_path]
long_vf_count = sum(tmp)
long_nonvf = len(tmp) - long_vf_count
tmp = [1 if vft.is_valley_free(g, x, vfmode=vft.CLOSENESS) else 0 for x in long_path]
long_vf_closeness_count = sum(tmp)
long_nonvf_closeness = len(tmp) - long_vf_closeness_count
extra_meta = {
helpers.ALL_PATH_VF_COUNT: vf_closeness_count,
helpers.SAME_LONG_PATH_VF_COUNT: long_vf_closeness_count,
helpers.SHORTER_PATH_VF_COUNT: short_vf_closeness_count
}
meta_map[tuple(named_trace)].update(extra_meta)
all_vf += vf_count
all_nonvf += nonvf
all_vf_closeness += vf_closeness_count
all_nonvf_closeness += nonvf_closeness
all_long_vf += long_vf_count
all_long_nonvf += long_nonvf
all_long_vf_closeness += long_vf_closeness_count
all_long_nonvf_closeness += long_nonvf_closeness
all_short_vf += short_vf_count
all_short_nonvf += short_nonvf
all_short_vf_closeness += short_vf_closeness_count
all_short_nonvf_closeness += short_nonvf_closeness
results.append(vf_count / float(len(all_path)))
results3.append(vf_closeness_count / float(len(all_path)))
if len(all_path) > 1: results2.append(vf_count / float(len(all_path)))
long_results.append(long_vf_count / float(len(long_path)))
long_results3.append(long_vf_closeness_count / float(len(long_path)))
if len(long_path) > 1: long_results2.append(long_vf_count / float(len(long_path)))
if len(short_path) > 0:
short_results.append(short_vf_count / float(len(short_path)))
short_results3.append(short_vf_closeness_count / float(len(short_path)))
else:
pass
# short_results.append(0)
# short_results3.append(0)
if len(short_path) > 1: short_results2.append(short_vf_count / float(len(short_path)))
# save mofified meta
meta_mod = [x for x in meta_map.itervalues()]
helpers.save_to_json(out, meta_mod)
# print results
print 'ALL'
print 'VF count: %d' % all_vf
print 'VF CLOSENESS count: %d' % all_vf_closeness
print 'Non vf count: %d' % all_nonvf
print 'Non vf CLOSENESS count: %d' % all_nonvf_closeness
print 'VF perc: %f' % (all_vf/float(all_vf + all_nonvf))
print 'VF CLOSENESS perc: %f' % (all_vf_closeness/float(all_vf_closeness + all_nonvf_closeness))
print 'Mean VF prob: %f' % np.mean(results)
print 'Mean VF CLOSENESS prob: %f' % np.mean(results3)
print 'Mean VF2 prob: %f' % np.mean(results2)
print '=========='
print 'SHORT'
print 'VF count: %d' % all_short_vf
print 'VF CLOSENESS count: %d' % all_short_vf_closeness
print 'Non vf count: %d' % all_short_nonvf
print 'Non vf CLOSENESS count: %d' % all_short_nonvf_closeness
if all_short_vf + all_short_nonvf > 0:
print 'VF perc: %f' % (all_short_vf/float(all_short_vf + all_short_nonvf))
if all_short_vf_closeness + all_short_nonvf_closeness > 0:
print 'VF CLOSENESS perc: %f' % (all_short_vf_closeness/float(all_short_vf_closeness + all_short_nonvf_closeness))
print 'Mean VF prob: %f' % np.mean(short_results)
print 'Mean VF CLOSENESS prob: %f' % np.mean(short_results3)
print 'Mean VF2 prob: %f' % np.mean(short_results2)
print '=-----------------'
print 'LONG'
print 'VF count: %d' % all_long_vf
print 'VF CLOSENESS count: %d' % all_long_vf_closeness
print 'Non vf count: %d' % all_long_nonvf
print 'Non vf CLOSENESS count: %d' % all_long_nonvf_closeness
print 'VF perc: %f' % (all_long_vf/float(all_long_vf + all_long_nonvf))
print 'VF CLOSENESS perc: %f' % (all_long_vf_closeness/float(all_long_vf_closeness + all_long_nonvf_closeness))
print 'Mean VF prob: %f' % np.mean(long_results)
print 'Mean VF CLOSENESS prob: %f' % np.mean(long_results3)
print 'Mean VF2 prob: %f' % np.mean(long_results2)
def main():
parser = argparse.ArgumentParser(
description=('SANDBOX mode. ', 'Write something ', 'useful here'),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('--edge-drop',
dest='edge_drop',
type=float,
default=0.0)
parser.add_argument('--closeness-limit',
dest='closeness_limit',
type=float,
default=0.0)
parser.add_argument('network')
parser.add_argument('traceroutes')
arguments = parser.parse_args()
show_progress = arguments.progressbar
arguments.verbose = min(len(helpers.LEVELS), arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
g = helpers.load_network(arguments.network)
traceroutes = helpers.load_from_json(arguments.traceroutes)
logger.info('ecount: %d' % g.ecount())
logger.info('vcount: %d' % g.vcount())
logger.info('trace count: %d' % len(traceroutes))
g_dummy = g.copy()
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes),
width=15)
closeness_list = []
for x in g_dummy.vs:
progress += 1
progress.show_progress()
closeness_list.append((x.index, g_dummy.closeness(x)))
end = int(arguments.closeness_limit * g_dummy.vcount())
logger.debug('Top node count: %d' % end)
top_nodes = sorted(closeness_list, key=lambda x: x[1], reverse=True)[:end]
top_nodes_index = [x[0] for x in top_nodes]
top_nodes_name = [g_dummy.vs[x[0]]['name'] for x in top_nodes]
top_edges = [
e for e in g_dummy.es
if e.source in top_nodes_index and e.target in top_nodes_index
]
logger.debug('Top edge count: %d' % len(top_edges))
random.shuffle(top_edges)
edge_drop = top_edges[:int(len(top_edges) * arguments.edge_drop)]
logger.debug('Dropped edge count: %d' % len(edge_drop))
# edges = [x.index for x in g_dummy.es]
# random.shuffle(edges)
# edge_drop = edges[:int(g.ecount() * arguments.edge_drop)]
g_dummy.delete_edges(edge_drop)
traceroutes = traceroutes[:10000]
all_edges = []
for trace in traceroutes:
edges = zip(trace, trace[1:])
edges = [tuple(sorted(e)) for e in edges]
all_edges.extend(edges)
all_edges = list(set(all_edges))
top_edges = [
e for e in all_edges
if e[0] in top_nodes_name and e[1] in top_nodes_name
]
logger.info('TOP edge count in real traceroutes: %d' % len(top_edges))
found_top_edges = []
increments = []
for trace in traceroutes:
edges = zip(trace, trace[1:])
edges = [tuple(sorted(e)) for e in edges]
top_edges = [
x for x in edges
if x[0] in top_nodes_name and x[1] in top_nodes_name
]
found_top_edges.extend(top_edges)
found_top_edges = list(set(found_top_edges))
increments.append(len(found_top_edges))
logger.info('Found top edge count: %d' % len(found_top_edges))
dummy_sh_traceroutes_meta = []
original_sh_traceroutes_meta = []
stretches = []
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes),
width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
s, t = trace[0], trace[-1]
# logger.debug('Get shortest paths from {s} to {t}'.format(s=s, t=t))
sh_dummy = random.choice(g_dummy.get_shortest_paths(s, t))
sh_original = random.choice(g.get_shortest_paths(s, t))
stretch = len(sh_dummy) - len(sh_original)
dummy_sh_traceroutes_meta.append((sh_dummy, stretch))
original_sh_traceroutes_meta.append((sh_original, 0))
stretches.append(stretch)
# logger.debug('Stretch: %d' % stretch)
# logger.debug('SH DUMMY: %s' % [g_dummy.vs[x]['name'] for x in sh_dummy])
# logger.debug('SH ORIG: %s' % [g.vs[x]['name'] for x in sh_original])
dummy_sh_meta = [(x[0], x[1],
vft.is_valley_free(g_dummy, x[0], vft.CLOSENESS))
for x in dummy_sh_traceroutes_meta]
dummy_sh_len_hist = collections.Counter(
[len(x[0]) for x in dummy_sh_traceroutes_meta])
original_sh_len_hist = collections.Counter(
[len(x[0]) for x in original_sh_traceroutes_meta])
original_len_hist = collections.Counter([len(x) for x in traceroutes])
stretches = [x for x in stretches if x >= 0]
stretch_hist = collections.Counter(stretches)
import matplotlib.pyplot as plt
print
print[(x, 100 * y / float(len(traceroutes)), y)
for x, y in stretch_hist.iteritems()]
plt.plot([x for x in stretch_hist.iterkeys()],
[x for x in stretch_hist.itervalues()], 'g^')
plt.ylabel('some numbers')
# plt.show()
logger.info('Dummy VF stat')
max_stretch = max(dummy_sh_meta, key=lambda x: x[1])[1]
for stretch in range(0, max_stretch + 1):
stretched_traces = [x for x in dummy_sh_meta if x[1] == stretch]
count = len(stretched_traces)
vf_count = len([x for x in stretched_traces if x[2]])
vf_perc = vf_count / float(count)
nonvf_count = count - vf_count
nonvf_perc = nonvf_count / float(count)
logger.info(
'{stretch} -- {vf_perc}[{vf_count}]\t{nonvf_perc}[{nonvf_count}]'.
format(stretch=stretch,
vf_perc=vf_perc,
vf_count=vf_count,
nonvf_perc=nonvf_perc,
nonvf_count=nonvf_count))
import matplotlib.pyplot as plt
plt.plot(increments, 'g^')
plt.ylabel('some numbers')
plt.show()
def purify(g, meta, flags, try_per_trace, show_progress=False):
# generate valley-free graph
if flags[FLAG_PRELABELED]:
logger.info('Generate VF_G_PRE')
vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED)
else:
logger.info('Skip prelabeled graph')
if flags[FLAG_DEGREE]:
logger.info('Generate VF_G_DEGREE')
vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE)
else:
logger.info('Skip degree graph')
if flags[FLAG_CLOSENESS]:
logger.info('Generate VF_G_CLOSENESS')
vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS)
else:
logger.info('Skip closeness graph')
# Randomize stretch dispersion
stretches = [x[helpers.HOP_STRETCH] for x in meta]
# a veletlen stretchek az eredeti stretchek veletlen, nem
# visszateveses mintavetelezese. Mindez annyiszor, ahany
# veletlen utat akarunk generalni minden valos trace vegpontja
# kozott.
stretch_list = [random.sample(stretches, len(stretches))
for x in xrange(0, try_per_trace)]
# A kovetkezo ciklusban minden meta sorhoz rogton kiszamolunk
# annyi random utat, amennyit parameterben megadtunk. Ehhez at kell
# alakitani a stretch lista strukturat, hogy minden elem egy meta sorhoz
# tartalmazza a random stretch ertekeket
#
# Pelda: elozo lepesnel kijott ez: [ [1,2,3,4], [2,4,1,3], [3,1,4,2] ]
# vagyis elso lepesben a metaban tarolt tracek rendre 1,2,3,4 stretchet
# kell felvegyenek, a masodikban 2,4,1,3 stb. A ciklusban viszont a meta
# elso elemehez rogton ki akarjuk szamolni a veletlen stretchekhez tartozo
# random utakat, vagyis [ [1,2,3], [2,4,1], [3,1,4], [4,3,2] ] formaban
# van szukseg az ertekekre
stretch_list = zip(*stretch_list)
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
for idx, record in enumerate(meta):
progress += 1
progress.show_progress()
trace = vft.trace_in_vertex_id(g, [record[helpers.TRACE], ])
if len(trace) != 1:
print 'PROBLEM'
print record
continue
trace = trace[0]
if len(trace) == 1: continue
sh_len = record[helpers.SH_LEN]
s, t = trace[0], trace[-1]
is_vf_prelabeled_l = []
is_lp_prelabeled_hard_l = []
is_lp_prelabeled_soft_l = []
is_vf_degree_l = []
is_lp_degree_hard_l = []
is_lp_degree_soft_l = []
is_vf_closeness_l = []
is_lp_closeness_hard_l = []
is_lp_closeness_soft_l = []
stretch_dist = stretch_list[idx]
real_stretch_dist = []
for current_stretch in stretch_dist:
random_length = sh_len + current_stretch
random_path = helpers.random_route_walk(g, s, t, random_length)
real_stretch_dist.append(len(random_path) - sh_len)
if len(random_path) == 0:
empty += 1
if flags[FLAG_PRELABELED]:
(is_vf_prelabeled,
is_lp_prelabeled_soft,
is_lp_prelabeled_hard) = vf_attributes(g,random_path,
vft.PRELABELED,
flags[FLAG_LP_SOFT],
flags[FLAG_LP_HARD],
vf_g_pre)
is_vf_prelabeled_l.append(is_vf_prelabeled)
is_lp_prelabeled_soft_l.append(is_lp_prelabeled_soft)
is_lp_prelabeled_hard_l.append(is_lp_prelabeled_hard)
if flags[FLAG_DEGREE]:
(is_vf_degree,
is_lp_degree_soft,
is_lp_degree_hard) = vf_attributes(g, random_path,
vft.DEGREE,
flags[FLAG_LP_SOFT],
flags[FLAG_LP_HARD],
vf_g_degree)
is_vf_degree_l.append(is_vf_degree)
is_lp_degree_soft_l.append(is_lp_degree_soft)
is_lp_degree_hard_l.append(is_lp_degree_hard)
if flags[FLAG_CLOSENESS]:
(is_vf_closeness,
is_lp_closeness_soft,
is_lp_closeness_hard) = vf_attributes(g, random_path,
vft.CLOSENESS,
flags[FLAG_LP_SOFT],
flags[FLAG_LP_HARD],
vf_g_closeness)
is_vf_closeness_l.append(is_vf_closeness)
is_lp_closeness_soft_l.append(is_lp_closeness_soft)
is_lp_closeness_hard_l.append(is_lp_closeness_hard)
result = {
helpers.RANDOM_GULYAS_WALK_ROUTES_RQ_STRETCH: stretch_dist,
helpers.RANDOM_GULYAS_WALK_ROUTES_STRETCH: real_stretch_dist,
helpers.RANDOM_GULYAS_WALK_ROUTES_VF_PRELABELED: is_vf_prelabeled_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_VF_DEGREE: is_vf_degree_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_VF_CLOSENESS: is_vf_closeness_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_PRELABELED: is_lp_prelabeled_soft_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_PRELABELED: is_lp_prelabeled_hard_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_DEGREE: is_lp_degree_soft_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_DEGREE: is_lp_degree_hard_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_SOFT_CLOSENESS: is_lp_closeness_soft_l,
helpers.RANDOM_GULYAS_WALK_ROUTES_LP_HARD_CLOSENESS: is_lp_closeness_hard_l,
}
record.update(result)
def generate_db(g, meta, locations, f_name):
print 'Traceroute count %d' % len(meta)
meta = [x for x in meta if x[TRACE_LEN] - x[SH_LEN] > 0]
print 'Stretched traceroute count %d' % len(meta)
weights = {
x['name']: {
'sh': 0,
'real': 0,
'vfreal': 0,
'randomsh': 0
}
for x in g.vs
}
random_paths = []
if locations is None:
rands = numpy.random.randint(0, 10000, len(weights) * 2)
loc = zip(rands[::2], rands[1::2])
locations = {
n: {
'x': loc[idx][0],
'y': loc[idx][1]
}
for idx, n in enumerate(weights.iterkeys())
}
# check locations
delkeys = []
for x in weights:
try:
locations[x]
except KeyError:
delkeys.append(x)
print 'Get closeness'
progress = progressbar1.AnimatedProgressBar(end=len(weights), width=15)
for cntr, x in enumerate(weights):
progress += 1
progress.show_progress()
weights[x]['closeness'] = g.closeness(x, mode=igraph.OUT)
print 'Trace calculation'
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
for cntr, m in enumerate(meta):
progress += 1
progress.show_progress()
trace = m[TRACE]
# for x in trace:
# if x in delkeys: continue
for node in trace:
weights[node]['real'] += 1
if m[IS_VF_CLOSENESS]:
weights[node]['vfreal'] += 1
sh_paths = g.get_all_shortest_paths(trace[0], trace[-1])
sh_paths = [[g.vs[x]['name'] for x in p] for p in sh_paths]
random_path = random.choice(sh_paths)
random_paths.append(random_path)
for node in random_path:
weights[node]['randomsh'] += 1
for sh_path in sh_paths:
for node in sh_path:
weights[node]['sh'] += 1
print 'Delete nodes because has no location: %s' % delkeys
g.delete_vertices([g.vs.find(x).index for x in delkeys])
print 'Nodes remained in graph: %d' % g.vcount()
with open('%s.csv' % f_name, 'w') as f:
f.write('NODE;CLOSENESS;X;Y;SH;REAL;VFREAL;RANDOMSH\n')
for node, w in weights.iteritems():
struct = '{node};{closeness};{locx};{locy};{sh};{real};{vfreal};{randomsh}\n'
f.write(
struct.format(node=node,
sh=w['sh'],
real=w['real'],
vfreal=w['vfreal'],
randomsh=w['randomsh'],
closeness=w['closeness'],
locx=float(locations[node]['x']),
locy=float(locations[node]['y'])))
randomsh_hops = [y for x in random_paths for y in zip(x, x[1:])]
randomsh_edges = (dict(source=s, target=t) for s, t in randomsh_hops)
randomsh_graph = igraph.Graph.DictList(edges=randomsh_edges, vertices=None)
randomsh_graph.simplify()
randomsh_graph.save('%srandomsh.gml' % f_name)
trace_hops = [y for x in meta for y in zip(x[TRACE], x[TRACE][1:])]
trace_edges = (dict(source=s, target=t) for s, t in trace_hops)
trace_graph = igraph.Graph.DictList(edges=trace_edges, vertices=None)
trace_graph.simplify()
trace_graph.save('%strace.gml' % f_name)
g.save('%scalculation_graph.gml' % f_name)
def purify(g, traceroutes, flags, show_progress=False):
results = list()
# remove traces with unknown nodes
traceroutes = vft.trace_in_vertex_id(g, traceroutes)
# generate valley-free graph
if flags[FLAG_PRELABELED]:
logger.info('Generate VF_G_PRE')
vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED)
else:
logger.info('Skip prelabeled graph')
if flags[FLAG_DEGREE]:
logger.info('Generate VF_G_DEGREE')
vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE)
else:
logger.info('Skip degree graph')
if flags[FLAG_CLOSENESS]:
logger.info('Generate VF_G_CLOSENESS')
vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS)
else:
logger.info('Skip closeness graph')
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes),
width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
logger.debug('Current trace: %s' % ([g.vs[x]['name'] for x in trace]))
if len(trace) == 1: continue
s, t = trace[0], trace[-1]
is_vf_prelabeled = -1
is_lp_prelabeled_hard = -1
is_lp_prelabeled_soft = -1
is_vf_degree = -1
is_lp_degree_hard = -1
is_lp_degree_soft = -1
is_vf_closeness = -1
is_lp_closeness_hard = -1
is_lp_closeness_soft = -1
trace_len = len(trace)
sh_len = g.shortest_paths(s, t, mode=i.OUT)[0][0]
sh_len += 1 # convert hop count to node Counter
if flags[FLAG_PRELABELED]:
is_vf_prelabeled = vft.is_valley_free(g, trace, vft.PRELABELED)
is_vf_prelabeled = int(is_vf_prelabeled)
if is_vf_prelabeled:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_pre,
first_edge=True,
vfmode=vft.PRELABELED)
is_lp_prelabeled_soft = 1 if lp_soft else 0
else:
is_lp_prelabeled_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_pre,
first_edge=False,
vfmode=vft.PRELABELED)
is_lp_prelabeled_hard = 1 if lp_hard else 0
else:
is_lp_prelabeled_hard = -1
if flags[FLAG_DEGREE]:
is_vf_degree = vft.is_valley_free(g, trace, vft.DEGREE)
is_vf_degree = int(is_vf_degree)
if is_vf_degree:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_degree,
first_edge=True,
vfmode=vft.DEGREE)
is_lp_degree_soft = 1 if lp_soft else 0
else:
is_lp_degree_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_degree,
first_edge=False,
vfmode=vft.DEGREE)
is_lp_degree_hard = 1 if lp_hard else 0
else:
is_lp_degree_hard = -1
if flags[FLAG_CLOSENESS]:
is_vf_closeness = vft.is_valley_free(g, trace, vft.CLOSENESS)
is_vf_closeness = int(is_vf_closeness)
if is_vf_closeness:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_closeness,
first_edge=True,
vfmode=vft.CLOSENESS)
is_lp_closeness_soft = 1 if lp_soft else 0
else:
is_lp_closeness_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_closeness,
first_edge=False,
vfmode=vft.CLOSENESS)
is_lp_closeness_hard = 1 if lp_hard else 0
else:
is_lp_closeness_hard = -1
if False:
sh_vf_len = vft.get_shortest_vf_route(g,
s,
t,
mode='vf',
vf_g=vf_g_pre,
_all=True,
vfmode=vft.PRELABELED)
# ugy tunik, mintha nem mindig lenne pontos? fentartassal kezelendo
# ez az ertek azert is kerult bele, hogy ellenorizzuk
in_vf_prediction = 1 if sh_vf_len and trace in sh_vf_len else 0
else:
sh_vf_len = -1
in_vf_prediction = -1
sh_vf_len = len(sh_vf_len[0]) if isinstance(sh_vf_len, list) else -1
percentage_stretch = trace_len / float(sh_len)
named_trace = [g.vs[_id]["name"] for _id in trace]
result = {
helpers.TRACE: named_trace,
helpers.TRACE_LEN: trace_len,
helpers.SH_LEN: sh_len,
helpers.SH_VF_LEN: sh_vf_len,
helpers.IS_VF_PRELABELED: is_vf_prelabeled,
helpers.IS_VF_DEGREE: is_vf_degree,
helpers.IS_VF_CLOSENESS: is_vf_closeness,
helpers.HOP_STRETCH: trace_len - sh_len,
helpers.PERC_STRETCH: percentage_stretch,
helpers.IN_VF_PRED: in_vf_prediction,
helpers.IS_LP_SOFT_PRELABELED: is_lp_prelabeled_soft,
helpers.IS_LP_HARD_PRELABELED: is_lp_prelabeled_hard,
helpers.IS_LP_SOFT_DEGREE: is_lp_degree_soft,
helpers.IS_LP_HARD_DEGREE: is_lp_degree_hard,
helpers.IS_LP_SOFT_CLOSENESS: is_lp_closeness_soft,
helpers.IS_LP_HARD_CLOSENESS: is_lp_closeness_hard,
}
results.append(result)
# print >> sys.stderr, ('TRACE\tTRACE_LEN\tSH_LEN\tSH_VF_LEN\tIS_VF',
# '\tSTRETCH\tIN_VF_PREDICTION\tIS_LP_F\tIS_LP_ALL')
# for result in results:
# result = [str(r) for r in result]
# print >> sys.stderr, '\t'.join(result)
# statistic = statistics.purify(g, results,
# 'nc+ec+tc+rt+vf+vf_closeness+pred+lp_soft_prelabeled+lp_hard_prelabeled+lp_soft_degree+lp_hard_degree+lp_soft_closeness+lp_hard_closeness'.split('+'))
# statistics.stat_printer(statistic)
return results
