def check_valley_free(g, path, log=None): r = connection.Redis() if r.get(dbkeys.AS.status('peering_data')) != "True": sys.stderr.write("Attempted to check valley-free property, " "but have no peering data.\n") return (0, 1) # IF we have no peering data. It's irrelevant. if len(path) == 0: return (0, 1) went_down = False errct = 0.0 hopct = 0.0 asn_path = [g.node[hop]['asn'] for hop in path if g.node[hop]['asn'] != "N/A"] for as1, as2 in pairwise(asn_path): hopct += 1 if as1 == as2: continue relationship = r().hget(dbkeys.AS.relationship(as1), as2) if not relationship: # Let's try the other side. relationship = r().hget(dbkeys.AS.relationship(as2), as1) if not relationship: if log: log.write("No relationship for %s <-> %s\n" % (as1, as2)) errct += 1 continue else: # We need to swap -1 and 1 for this side # since it's the opposite perspective. # If we swap a 2 to a -2, who cares. relationship *= -1 if relationship == 1: went_down = True elif relationship == -1: # AS1 is a customer of AS2 if went_down: raise ValleyFreeError() return (errct, hopct)
def identify_ixps(self, as_path): """Identify the IXP and MetaIXPs that occur along a given AS path :as_path: An iterable of AS numbers representing a path :returns: A tuple of the form (ixps, metaixps), which are sets """ if as_path is None: return ([], []) path_ixps = set() for pair in pairwise(as_path.split()): if pair in self.ixps: path_ixps |= self.ixps[pair] path_metaixps = set([self.lookup_metaixp(ixp) for ixp in path_ixps]) return (path_ixps, path_metaixps)
def thread_shortest_path(graphpath, sp_key, type_key, node_key, path_key, used_key): global thread_graph log_out = redirect_output() log_out.write("Spawned with sp_key = {0}; type_key = {1}; " "node_key={2}; path_key={3}\n" .format(sp_key, type_key, used_key, path_key)) log_out.flush() r = connection.Redis() log_out.write("Reading graph...") log_out.flush() thread_graph = nx.read_graphml(graphpath) log_out.write(" Complete\n") log_out.flush() target = r.spop(sp_key) while target: used_nodes = set() used_paths = set() log_out.write("Obtaining shortest paths for %s... " % target) log_out.flush() paths = nx.single_source_dijkstra_path( thread_graph, target, weight='latency') for path_target, path in paths.iteritems(): if r.hget(type_key, path_target) in ('relay', 'client', 'dest'): # We only care for relays, clients and destinations try: errors, total = check_valley_free( thread_graph, path, log=log_out) # if total > 0: # log_out.write("Path from %{0} to %s was valley-free. " # " %0.0f/%0.0f (%0.2f) missing links in calc\n" # % (target, path_target, # errors, total, errors / total)) except ValleyFreeError: # If it's not valley free, we rebuild a new path log_out.write("Path from %s to %s is not valley-free. " "Building a new one... " % (target, path_target)) try: path, time_taken = valley_free_path( thread_graph, target, path_target) log_out.write(" finished in %s seconds. New Path: %s " "[%0.0f/%0.0f (%0.2f%%) of links had no " "information]\n" % (Color.wrap(time_taken, Color.OKBLUE), str(path), path.errct, len(path) - 1, float(path.errct) / float(len(path) - 1))) except ValleyFreeError: log_out.write( Color.warn( "Couldn't produce valley-free path from %s to %s\n" % (target, path_target))) raise else: # Now store the links and nodes from this # path so that we keep them. used_nodes.update(path) hops = list() for hop in pairwise(path): hops.append(hop) #Add each hop to our set of paths used_paths.add(hop) r.sadd(path_key, *used_paths) r.sadd(used_key, *used_nodes) log_out.write("Done\n") log_out.flush() target = r.spop(sp_key) log_out.write("Exiting ") log_out.flush() log_out.close() sys.exit(0)
def collapse_degree_two(self, protected=[]): log.info("Cleaning up collapse dbkeys...") r = connection.Redis() p = r.pipeline() for key in r.keys("graph:collapsed:*"): p.delete(key) write_failed(p.execute()) pass_ctr = 0 collapsable = True ignoreable = set() clogout = open('collapse.log', 'w') while collapsable: pass_ctr += 1 sys.stderr.write("\n") collapsable = False degree2nodes = filter( lambda val: (len(val[1]) == 2 and val[0] not in ignoreable), self.iteritems()) counter = 0 n = 0 deferred = 0 collapsed = set() timer = ProgressTimer(len(degree2nodes)) for node, connections in degree2nodes: if n % 50 == 0 or n == timer.total - 1: timer.tick(50) sys.stderr.write( "{0}Pass {1}: {2} {3}".format( Color.NEWL, pass_ctr, Color.wrapformat( "[{0} processed, {1} collapsed, {2} deferred]", Color.HEADER, n, counter, deferred ), Color.wrapformat( "[eta: {0}]", Color.OKGREEN, timer.eta() )) ) n += 1 asns = [r.get(dbkeys.POP.asn(x)) for x in connections | set([node])] countries = [r.smembers(dbkeys.POP.countries(x)) for x in connections | set([node])] same_asn = reduce(lambda x, y: x if x == y else False, asns) same_country = True for x, y in pairwise(countries): if x & y != x: same_country = False if (same_asn is False or same_country is False or node in protected): ignoreable.update(connections | set([node])) continue if len(collapsed & (connections | set([node]))) != 0: deferred += 1 continue collapsed.update(connections | set([node])) side1 = connections.pop() side2 = connections.pop() connections.update(set([side1, side2])) try: #side1_delay = median(get_delays(dbkeys.Link.interlink(node, side1))) side1_delays = decile_transform( [float(delay) for edge in r.smembers(dbkeys.Link.interlink(node, side1)) for delay in r.smembers(dbkeys.delay_key(*eval(edge)))]) except: side1_delays = eval(r.get("graph:collapsed:%s" % (dbkeys.Link.interlink(node, side1)))) try: #side2_delay = median(get_delays(dbkeys.Link.interlink(node, side2))) side2_delays = decile_transform( [float(delay) for edge in r.smembers(dbkeys.Link.interlink(node, side2)) for delay in r.smembers(dbkeys.delay_key(*eval(edge)))]) except: side2_delays = eval(r.get("graph:collapsed:%s" % (dbkeys.Link.interlink(node, side2)))) combined_delays = [s1 + s2 for s1 in side1_delays for s2 in side2_delays] r.set('graph:collapsed:%s' % (dbkeys.Link.interlink(*list(connections))), decile_transform(combined_delays)) clogout.write("Collapsed %s <-> %s <-> %s\n" % (side1, node, side2)) collapsable = True del self[node] self[side1].add(side2) self[side2].add(side1) self[side1].remove(node) self[side2].remove(node) counter += 1 clogout.close()