def test_shortest_vf_route(self):
s, t = 0, 1 # shortest vf: N0 - N5 - N1
shortest_vf = [0, 5, 1]
sh = vft.get_shortest_vf_route(self.sample_graph, s, t)
self.assertEqual(sh, shortest_vf)
s, t = 0, 2
shortest_vf = [0, 5, 6, 2]
sh = vft.get_shortest_vf_route(self.sample_graph, s, t)
self.assertEqual(sh, shortest_vf)
def test_all_shortest_vf_route(self):
self.sample_graph.add_vertex('N8')
self.sample_graph.add_edges([['N8', 'N5'], ['N6', 'N8']])
self.prelabeled[8] = self.prelabeled[4]
self.sample_graph.delete_edges([
self.sample_graph.get_eid(5, 6),
])
s, t = 0, 2
shortest_vf1 = [0, 5, 4, 6, 2]
shortest_vf2 = [0, 5, 8, 6, 2]
sh = vft.get_shortest_vf_route(self.sample_graph, s, t, _all=True)
self.assertEqual(len(sh), 2)
self.assertIn(shortest_vf1, sh)
self.assertIn(shortest_vf2, sh)
def test_lp_choose_peer_if_no_customer_cone(self):
self.sample_graph.add_vertices(['N8', 'N9'])
self.prelabeled[8] = self.prelabeled[7]
self.prelabeled[9] = self.prelabeled[7] * 2
self.closenesses[8] = self.closenesses[7]
self.closenesses[9] = self.closenesses[7] * 2
self.sample_graph.add_edges([['N9', 'N7'], ['N8', 'N7'], ['N6', 'N8'],
['N9', 'N3']])
s = self.sample_graph.vs.find('N7').index
t = self.sample_graph.vs.find('N3').index
sh = vft.get_shortest_vf_route(self.sample_graph,
s,
t,
_all=True,
mode='lp')
self.assertEqual(len(sh), 1)
self.assertEqual(sh[0], [7, 8, 6, 3])
def test_all_shortest_lp_route(self):
self.sample_graph.add_vertices(['N8', 'N9'])
self.sample_graph.add_edges([['N8', 'N5'], ['N2', 'N8'], ['N9', 'N5'],
['N2', 'N9']])
self.prelabeled[8] = 0.5
self.prelabeled[9] = 0.5
s, t = 1, 2
shortest_lp = [1, 5, 8, 2]
shortest_lp2 = [1, 5, 9, 2]
sh = vft.get_shortest_vf_route(self.sample_graph,
s,
t,
_all=True,
mode='lp')
self.assertEqual(len(sh), 2)
self.assertIn(shortest_lp, sh)
self.assertIn(shortest_lp2, sh)
def trace_dir(g, meta, vf_g, arguments):
N = g.vcount()
vf_sum_lp_w = sum([e['LPweight'] for e in vf_g.es])
out_instead_core = 0
core_instead_out = 0
in_customer = 0
in_customer_but_use_up = 0
progress = progressbar1.DummyProgressBar(end=10, width=15)
if arguments.progressbar:
progress = progressbar1.AnimatedProgressBar(end=len(meta), width=15)
for m in meta:
progress += 1
progress.show_progress()
trace = m[helpers.TRACE]
first_edge = [trace[0], trace[1]]
first_hop_type = vft.edge_dir(g, first_edge, vfmode=vft.CLOSENESS)
s_idx = vft.node_to_nodeid(g, trace[0])
t_idx = vft.node_to_nodeid(g, trace[-1])
sh_vf = vf_g.get_all_shortest_paths(s_idx + N, t_idx + N)
if len(sh_vf) > 0:
in_customer += 1
if first_hop_type != LinkDir.D:
in_customer_but_use_up += 1
if first_hop_type == LinkDir.D:
# 'remove' all downward path:
for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT):
down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True)
vf_g.es[down_edge]['LPweight_old'] = vf_g.es[down_edge][
'LPweight']
vf_g.es[down_edge]['LPweight'] = vf_sum_lp_w + 1
lp_sh = vft.get_shortest_vf_route(g,
s_idx,
t_idx,
mode='lp',
vf_g=vf_g,
vfmode=vft.CLOSENESS)
pretty_plotter.pretty_plot(g, trace, lp_sh, [])
first_new_edge = [lp_sh[0], lp_sh[1]]
first_new_hop_type = vft.edge_dir(g,
first_new_edge,
vfmode=vft.CLOSENESS)
if first_hop_type != first_new_hop_type:
out_instead_core += 1
print ''
print "Original trace: %s" % trace
print 'Original trace dir: %s' % vft.trace_to_string(g, trace)
print 'Original closeness: %s' % [
g.vs.find(x)['closeness'] for x in trace
]
print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh]
print 'LP dir: %s' % vft.trace_to_string(g, lp_sh)
print 'LP closeness: %s' % [
g.vs.find(x)['closeness'] for x in lp_sh
]
# raw_input()
for neighbor in vf_g.neighbors(s_idx + N, mode=igraph.OUT):
down_edge = vf_g.get_eid(s_idx + N, neighbor, directed=True)
vf_g.es[down_edge]['LPweight'] = vf_g.es[down_edge][
'LPweight_old']
elif first_hop_type == LinkDir.U:
lp_sh = vft.get_shortest_vf_route(g,
s_idx,
t_idx,
mode='lp',
vf_g=vf_g,
vfmode=vft.CLOSENESS)
lp_dir = vft.trace_to_string(g, lp_sh)
# if lp_dir[1:].startswith('U') or lp_dir[1:].startswith('D'):
# continue
first_new_edge = [lp_sh[0], lp_sh[1]]
first_new_hop_type = vft.edge_dir(g,
first_new_edge,
vfmode=vft.CLOSENESS)
if first_hop_type != first_new_hop_type:
core_instead_out += 1
print ''
print "Original trace: %s" % trace
print 'Original trace dir: %s' % vft.trace_to_string(g, trace)
print 'Original closeness: %s' % [
g.vs.find(x)['closeness'] for x in trace
]
print 'LP trace: %s' % [g.vs[x]['name'] for x in lp_sh]
print 'LP dir: %s' % vft.trace_to_string(g, lp_sh)
print 'LP closeness: %s' % [
g.vs.find(x)['closeness'] for x in lp_sh
]
pretty_plotter.pretty_plot(g, trace, lp_sh, [])
# raw_input()
logger.info('Core instead down: %d' % core_instead_out)
logger.info('Down instead core: %d' % out_instead_core)
logger.info('In customer cone: %d' % in_customer)
logger.info('In customer but use UP: %d' % in_customer_but_use_up)
def purify(g, traceroutes, flags, show_progress=False):
results = list()
# remove traces with unknown nodes
traceroutes = vft.trace_in_vertex_id(g, traceroutes)
# generate valley-free graph
if flags[FLAG_PRELABELED]:
logger.info('Generate VF_G_PRE')
vf_g_pre = vft.convert_to_vf(g, vfmode=vft.PRELABELED)
else:
logger.info('Skip prelabeled graph')
if flags[FLAG_DEGREE]:
logger.info('Generate VF_G_DEGREE')
vf_g_degree = vft.convert_to_vf(g, vfmode=vft.DEGREE)
else:
logger.info('Skip degree graph')
if flags[FLAG_CLOSENESS]:
logger.info('Generate VF_G_CLOSENESS')
vf_g_closeness = vft.convert_to_vf(g, vfmode=vft.CLOSENESS)
else:
logger.info('Skip closeness graph')
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(traceroutes),
width=15)
for trace in traceroutes:
progress += 1
progress.show_progress()
logger.debug('Current trace: %s' % ([g.vs[x]['name'] for x in trace]))
if len(trace) == 1: continue
s, t = trace[0], trace[-1]
is_vf_prelabeled = -1
is_lp_prelabeled_hard = -1
is_lp_prelabeled_soft = -1
is_vf_degree = -1
is_lp_degree_hard = -1
is_lp_degree_soft = -1
is_vf_closeness = -1
is_lp_closeness_hard = -1
is_lp_closeness_soft = -1
trace_len = len(trace)
sh_len = g.shortest_paths(s, t, mode=i.OUT)[0][0]
sh_len += 1 # convert hop count to node Counter
if flags[FLAG_PRELABELED]:
is_vf_prelabeled = vft.is_valley_free(g, trace, vft.PRELABELED)
is_vf_prelabeled = int(is_vf_prelabeled)
if is_vf_prelabeled:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_pre,
first_edge=True,
vfmode=vft.PRELABELED)
is_lp_prelabeled_soft = 1 if lp_soft else 0
else:
is_lp_prelabeled_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_pre,
first_edge=False,
vfmode=vft.PRELABELED)
is_lp_prelabeled_hard = 1 if lp_hard else 0
else:
is_lp_prelabeled_hard = -1
if flags[FLAG_DEGREE]:
is_vf_degree = vft.is_valley_free(g, trace, vft.DEGREE)
is_vf_degree = int(is_vf_degree)
if is_vf_degree:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_degree,
first_edge=True,
vfmode=vft.DEGREE)
is_lp_degree_soft = 1 if lp_soft else 0
else:
is_lp_degree_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_degree,
first_edge=False,
vfmode=vft.DEGREE)
is_lp_degree_hard = 1 if lp_hard else 0
else:
is_lp_degree_hard = -1
if flags[FLAG_CLOSENESS]:
is_vf_closeness = vft.is_valley_free(g, trace, vft.CLOSENESS)
is_vf_closeness = int(is_vf_closeness)
if is_vf_closeness:
if flags[FLAG_LP_SOFT]:
lp_soft = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_closeness,
first_edge=True,
vfmode=vft.CLOSENESS)
is_lp_closeness_soft = 1 if lp_soft else 0
else:
is_lp_closeness_soft = -1
if flags[FLAG_LP_HARD]:
lp_hard = vft.is_local_preferenced(g,
trace,
vf_g=vf_g_closeness,
first_edge=False,
vfmode=vft.CLOSENESS)
is_lp_closeness_hard = 1 if lp_hard else 0
else:
is_lp_closeness_hard = -1
if False:
sh_vf_len = vft.get_shortest_vf_route(g,
s,
t,
mode='vf',
vf_g=vf_g_pre,
_all=True,
vfmode=vft.PRELABELED)
# ugy tunik, mintha nem mindig lenne pontos? fentartassal kezelendo
# ez az ertek azert is kerult bele, hogy ellenorizzuk
in_vf_prediction = 1 if sh_vf_len and trace in sh_vf_len else 0
else:
sh_vf_len = -1
in_vf_prediction = -1
sh_vf_len = len(sh_vf_len[0]) if isinstance(sh_vf_len, list) else -1
percentage_stretch = trace_len / float(sh_len)
named_trace = [g.vs[_id]["name"] for _id in trace]
result = {
helpers.TRACE: named_trace,
helpers.TRACE_LEN: trace_len,
helpers.SH_LEN: sh_len,
helpers.SH_VF_LEN: sh_vf_len,
helpers.IS_VF_PRELABELED: is_vf_prelabeled,
helpers.IS_VF_DEGREE: is_vf_degree,
helpers.IS_VF_CLOSENESS: is_vf_closeness,
helpers.HOP_STRETCH: trace_len - sh_len,
helpers.PERC_STRETCH: percentage_stretch,
helpers.IN_VF_PRED: in_vf_prediction,
helpers.IS_LP_SOFT_PRELABELED: is_lp_prelabeled_soft,
helpers.IS_LP_HARD_PRELABELED: is_lp_prelabeled_hard,
helpers.IS_LP_SOFT_DEGREE: is_lp_degree_soft,
helpers.IS_LP_HARD_DEGREE: is_lp_degree_hard,
helpers.IS_LP_SOFT_CLOSENESS: is_lp_closeness_soft,
helpers.IS_LP_HARD_CLOSENESS: is_lp_closeness_hard,
}
results.append(result)
# print >> sys.stderr, ('TRACE\tTRACE_LEN\tSH_LEN\tSH_VF_LEN\tIS_VF',
# '\tSTRETCH\tIN_VF_PREDICTION\tIS_LP_F\tIS_LP_ALL')
# for result in results:
# result = [str(r) for r in result]
# print >> sys.stderr, '\t'.join(result)
# statistic = statistics.purify(g, results,
# 'nc+ec+tc+rt+vf+vf_closeness+pred+lp_soft_prelabeled+lp_hard_prelabeled+lp_soft_degree+lp_hard_degree+lp_soft_closeness+lp_hard_closeness'.split('+'))
# statistics.stat_printer(statistic)
return results
def main():
parser = argparse.ArgumentParser(
description=('Syntetic route generator'),
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--progressbar', action='store_true')
parser.add_argument('--verbose', '-v', action='count', default=0)
parser.add_argument('network')
parser.add_argument('meta')
parser.add_argument('all_trace_out', metavar='all-trace-out')
parser.add_argument('syntetic_out', metavar='syntetic-out')
parser.add_argument('--trace-count',
'-tc',
type=int,
dest='trace_count',
default=5000)
parser.add_argument('--random-sample',
dest='random_sample',
action='store_true')
parser.add_argument('--closeness-error',
'-ce',
type=float,
dest='closeness_error',
default=0.0)
parser.add_argument('--core-limit-percentile',
'-cl',
type=int,
dest='core_limit',
default=0)
parser.add_argument('--toggle-node-error-mode', action='store_true')
arguments = parser.parse_args()
arguments.verbose = min(len(helpers.LEVELS) - 1, arguments.verbose)
logging.getLogger('compnet').setLevel(helpers.LEVELS[arguments.verbose])
show_progress = arguments.progressbar
g = helpers.load_network(arguments.network)
g = g.simplify()
meta = helpers.load_from_json(arguments.meta)
if arguments.random_sample:
random.shuffle(meta)
meta = meta[:arguments.trace_count]
N = g.vcount()
cl = sorted([x['closeness'] for x in g.vs], reverse=True)
logger.info('Min closeness: %s' % np.min(cl))
logger.info('Max closeness: %s' % np.max(cl))
logger.info('Mean closenss: %s' % np.mean(cl))
logger.info('10%% closeness: %s' % np.percentile(cl, 10))
logger.info('90%% closeness: %s' % np.percentile(cl, 90))
logger.info('Core limit: [r]%d%%[/]' % arguments.core_limit)
change_probability = 100 * arguments.closeness_error
logger.info('Change probability: [r]%6.2f%%[/]' % change_probability)
core_limit = np.percentile(cl, arguments.core_limit)
logger.info('Core limit in closeness: [bb]%f[/]' % core_limit)
if arguments.toggle_node_error_mode:
logger.info('[r]Node error mode[/]')
msg = (
"If given node's closeness >= core_limit then the new ",
"closeness in this node is ",
#"rand(closeness_error ... old closeness)"
"OLD_CLOSENSS * +/- closeness_error%")
logger.info(''.join(msg))
logger.info('Minimum node closeness: [g]%f[/]' %
arguments.closeness_error)
for n in g.vs:
if n['closeness'] < core_limit:
continue
# sign = -1 if random.uniform(-1, 1) < 0 else 1
# n['closeness'] = n['closeness'] * (1 + sign * arguments.closeness_error)
new_closeness = random.uniform(arguments.closeness_error,
n['closeness'])
n['closeness'] = new_closeness
g_labeled = vft.label_graph_edges(g, vfmode=vft.CLOSENESS)
peer_edge_count = len([x for x in g_labeled.es if x['dir'] == LinkDir.P])
logger.info('Peer edge count: %d' % peer_edge_count)
changed_edges = []
if not arguments.toggle_node_error_mode:
msg = ("If the closeness values of the endpoints in given edge is ",
"larger than the core_limit and ",
"random(0,1) < closeness_error then change the direction ",
"for this edge")
logger.info(''.join(msg))
changed_u = changed_d = 0
changed_edges = []
changed_edgess = []
for edge in g_labeled.es:
s, t = edge.source, edge.target
s_cl = g_labeled.vs[s]['closeness']
t_cl = g_labeled.vs[t]['closeness']
if (s_cl < core_limit or t_cl < core_limit): continue
if random.uniform(0, 1) > arguments.closeness_error: continue
# if abs(s_cl - t_cl) / min(s_cl, t_cl) > 0.02: continue
new_edge_dir = LinkDir.U if random.uniform(0,
1) > 0.5 else LinkDir.D
if new_edge_dir != edge['dir']:
if edge['dir'] == LinkDir.U:
changed_u += 1
else:
changed_d += 1
edge['dir'] = new_edge_dir
changed_edges.append(edge)
changed_edgess.append((edge.source, edge.target))
# if edge['dir'] == LinkDir.U:
# changed_u += 1
# changed_edgess.append((edge.source, edge.target))
# edge['dir'] = LinkDir.D
# changed_edges.append(edge)
# elif edge['dir'] == LinkDir.D:
# changed_d += 1
# changed_edgess.append((edge.source, edge.target))
# edge['dir'] = LinkDir.U
# changed_edges.append(edge)
logger.info('E count: %d' % g_labeled.ecount())
logger.info('Changed U: %d' % changed_u)
logger.info('Changed D: %d' % changed_d)
logger.info('Changed: %d' % (changed_d + changed_u))
changed_e = [(g_labeled.vs[e.source]['name'],
g_labeled.vs[e.target]['name']) for e in changed_edges]
changed_e = changed_e + [(x[1], x[0]) for x in changed_e]
changed_e = set(changed_e)
vf_g_closeness = vft.convert_to_vf(g,
vfmode=vft.CLOSENESS,
labeled_graph=g_labeled)
# e_colors = []
# for e in vf_g_closeness.es:
# if e.source < N and e.target < N: col = 'grey'
# elif e.source < N and e.target >= N: col = 'blue'
# elif e.source >= N and e.target >= N: col = 'red'
# else: col = 'cyan'
# e_colors.append(col)
# igraph.plot(vf_g_closeness, "/tmp/closeness.pdf",
# vertex_label=vf_g_closeness.vs['name'],
# vertex_size=0.2,
# edge_color=e_colors)
pairs = [(g.vs.find(x[helpers.TRACE][0]).index,
g.vs.find(x[helpers.TRACE][-1]).index, tuple(x[helpers.TRACE]))
for x in meta]
# pairs = list(set(pairs))
# random.shuffle(pairs)
# visited = set()
# pairs2 = []
# for x in pairs:
# k = (x[0], x[1])
# if k in visited: continue
# visited.add(k)
# visited.add((k[1], k[0]))
# pairs2.append(x)
# pairs = pairs2
traces = [x[2] for x in pairs]
stretches = []
syntetic_traces = []
sh_traces = []
base_traces = []
original_traces = []
bad = 0
progress = progressbar1.DummyProgressBar(end=10, width=15)
if show_progress:
progress = progressbar1.AnimatedProgressBar(end=len(pairs), width=15)
for s, t, trace_original in pairs:
progress += 1
progress.show_progress()
trace_original_idx = [g.vs.find(x).index for x in trace_original]
logger.debug('Original trace: %s -- %s -- %s',
[g.vs[x]['name'] for x in trace_original_idx],
vft.trace_to_string(g, trace_original_idx, vft.CLOSENESS),
[g.vs[x]['closeness'] for x in trace_original_idx])
sh_routes = g.get_all_shortest_paths(s, t)
sh_len = len(sh_routes[0])
sh_routes_named = [[g.vs[y]['name'] for y in x] for x in sh_routes]
sh_trace_name = random.choice(sh_routes_named)
base_trace_name = random.choice(sh_routes_named)
candidates = vf_g_closeness.get_all_shortest_paths(s + N, t + N)
candidates = [vft.vf_route_converter(x, N) for x in candidates]
# candidates = []
if len(candidates) == 0:
candidates = vft.get_shortest_vf_route(g_labeled,
s,
t,
mode='vf',
vf_g=vf_g_closeness,
_all=True,
vfmode=vft.CLOSENESS)
if len(candidates) == 0:
s_name, t_name = g.vs[s]['name'], g.vs[t]['name']
logger.debug("!!!No syntetic route from %s to %s" %
(s_name, t_name))
continue
logger.debug('Candidates from %s to %s:' %
(g.vs[s]['name'], g.vs[t]['name']))
for c in candidates:
logger.debug('%s -- %s -- %s' %
([g.vs[x]['name'] for x in c],
vft.trace_to_string(g_labeled, c, vft.PRELABELED),
[g.vs[x]['closeness'] for x in c]))
chosen_one = random.choice(candidates)
chosen_one_name = [g.vs[x]['name'] for x in chosen_one]
# print chosen_one
# print trace_original
# pretty_plotter.pretty_plot(g, trace_original_idx,
# chosen_one, changed_edgess,
# spec_color=(0, 0, 0, 155))
hop_stretch = len(chosen_one) - sh_len
stretches.append(hop_stretch)
trace_original_e = zip(trace_original, trace_original[1:])
chosen_one_e = zip(chosen_one_name, chosen_one_name[1:])
trace_affected = any([x in changed_e for x in trace_original_e])
chosen_affected = any([x in changed_e for x in chosen_one_e])
logger.debug('Trace affected: %s' % trace_affected)
logger.debug('Chosen affected: %s' % chosen_affected)
# if hop_stretch > 2:
# logger.debug('Base: %s' % trace_to_string(g_labeled, base_trace_name))
# logger.debug('SH: %s' % trace_to_string(g_labeled, sh_trace_name))
# logger.debug('Trace: %s' % trace_to_string(g_labeled, trace_original))
# logger.debug('Syntetic: %s' % trace_to_string(g_labeled, chosen_one_name))
if trace_affected or chosen_affected or hop_stretch > 2:
# pretty_plotter.pretty_plot_all(g, traces,
# chosen_one, changed_edgess,
# spec_color=(0, 0, 0, 255))
bad += 1
syntetic_traces.append(chosen_one_name)
sh_traces.append(sh_trace_name)
base_traces.append(base_trace_name)
original_traces.append(trace_original)
logger.debug('From %s to %s chosen one %s' %
(g.vs[s]['name'], g.vs[t]['name'], chosen_one_name))
result = zip(base_traces, sh_traces, original_traces, syntetic_traces)
helpers.save_to_json(arguments.all_trace_out, result)
helpers.save_to_json(arguments.syntetic_out, syntetic_traces)
print 'Bad: %d' % bad
c = collections.Counter(stretches)
trace_count = len(syntetic_traces)
logger.info('Stretch dist:')
for k in c:
logger.info('\t%d: %5.2f%%[%d]' %
(k, 100 * c[k] / float(trace_count), c[k]))
logger.info('Valid route count: %d' % trace_count)
logger.info('Route count parameter: %d' % arguments.trace_count)
logger.info('Generated valid pair count: %d' % len(pairs))