def handle_same_pop_link(link):
""" Handle links which should belong to the same PoP
a. Neither has a PoP assigned
- Assign both the same pop and set links:intra
b. One side has a PoP assigned
- Assign the other one the existing PoP and add links:intra
c. Both sides have a PoP assigned
- add to delayed_job:popjoins
"""
r = connection.Redis()
ip1, ip2 = link.split(":")[2:]
with r.pipeline() as pipe:
try:
pipe.watch(dbkeys.ip_key(ip1))
pipe.watch(dbkeys.ip_key(ip2))
pop1 = dbkeys.get_pop(ip1, pipe=pipe)
pop2 = dbkeys.get_pop(ip2, pipe=pipe)
pipe.multi()
if pop1 is None and pop2 is None:
pop1 = dbkeys.setpopnumber(dbkeys.mutex_popnum(), ip1, pipe=pipe)
pipe.hset(dbkeys.ip_key(ip2), 'pop', pop1)
pipe.sadd(dbkeys.POP.members(pop1), ip2)
store_link(r, (ip1, ip2), pop1, pipe=pipe)
elif pop1 is not None and pop2 is not None:
if not r.sismember('delayed_job:popjoins:known', (pop1, pop2)):
pipe.lpush("delayed_job:popjoins", (pop1, pop2))
pipe.sadd('delayed_job:popjoins:known', (pop1, pop2))
else:
if pop1 is None:
knownpop = pop2
pipe.hset(dbkeys.ip_key(ip1), 'pop', knownpop)
pipe.sadd(dbkeys.POP.members(knownpop), ip1)
else:
knownpop = pop1
pipe.hset(dbkeys.ip_key(ip2), 'pop', knownpop)
pipe.sadd(dbkeys.POP.members(knownpop), ip2)
store_link(r, (ip1, ip2), knownpop, pipe=pipe)
pipe.execute()
return True
except redis.WatchError:
return False
finally:
pipe.reset()
def handle_cross_pop_link(link):
""" Handle a situation where the two IPs on either end of a
link should be in different PoPs.
- Neither has a PoP assigned
- Assign two new PoPs, and create links:inter
- One side has a PoP assigned
- Assign 1 new PoP and create links:inter
- Both sides have a PoP assigned
- Add it to the links:inter
"""
r = connection.Redis()
ip1, ip2 = link.split(":")[2:]
with r.pipeline() as pipe:
try:
pipe.watch(dbkeys.ip_key(ip1))
pipe.watch(dbkeys.ip_key(ip2))
pop1 = dbkeys.get_pop(ip1, pipe=pipe)
pop2 = dbkeys.get_pop(ip2, pipe=pipe)
pipe.multi()
if pop1 is None and pop2 is None:
pop1 = dbkeys.setpopnumber(dbkeys.mutex_popnum(), ip1, pipe=pipe)
pop2 = dbkeys.setpopnumber(dbkeys.mutex_popnum(), ip2, pipe=pipe)
elif pop1 is not None and pop2 is not None:
pass
else:
if pop1 is None:
pop1 = dbkeys.setpopnumber(dbkeys.mutex_popnum(), ip1, pipe=pipe)
else:
pop2 = dbkeys.setpopnumber(dbkeys.mutex_popnum(), ip2, pipe=pipe)
store_link(r, (ip1, ip2), pop1, pop2, pipe=pipe)
pipe.execute()
return True
except redis.WatchError:
return False
finally:
pipe.reset()
def add_alexa_destinations(vertex_list, linklist, count):
"""
Add potential destination endpoints based on the top 10000 destinations
"""
r = connection.Redis()
aslookup = preprocess.MaxMindGeoIPReader.Instance()
attached = 0
failed = 0
pops = set()
with pkg_resources.resource_stream(
'inettopology_popmap.resources',
'alexa_top_dests.txt') as destlist:
for line in destlist:
if line[0] == '#':
continue
ip, url, matched_ip, matched_bits = line.split()
db_ip_pop = dbkeys.get_pop(matched_ip)
if db_ip_pop is None:
log.debug("Couldn't attach {0} with ip {1}. No matching IP found"
.format(url, matched_ip))
failed += 1
continue
nodeid = "dest_{0}".format(ip.replace('.', '_'))
if nodeid in vertex_list:
continue # Don't add the same url twice
countries = r.smembers(dbkeys.POP.countries(db_ip_pop))
if len(countries) == 1:
country = countries.pop()
else:
country = aslookup.lookup_country_codes(matched_ip)[0]
pops.add(db_ip_pop)
vertex_list.add_vertex(nodeid,
nodeid=nodeid,
nodetype="dest",
url=url,
ip=ip,
asn=r.get(dbkeys.POP.asn(db_ip_pop)),
country=country)
linkkey = dbkeys.Link.intralink(db_ip_pop)
linkdelays = [
delay
for edge in r.smembers(linkkey)
for delay in r.smembers(dbkeys.delay_key(*eval(edge)))]
try:
latency = util.decile_transform(linkdelays)
except util.EmptyListError:
latency = [5 for x in xrange(10)]
linklist.append(
EdgeLink(nodeid,
db_ip_pop,
{'latency': latency,
'med_latency': latency[len(latency)/2]}))
attached += 1
if attached % 10 == 0:
log.info("Attached {0} destinations. Couldn't attach {1}"
.format(attached, failed))
if attached >= count:
break
return (attached, len(pops))
def assert_pops_ok(r, *ips):
for ip in ips:
pop = dbkeys.get_pop(ip)
if pop is not None and not r.sismember('poplist', pop):
raise NoPopExistsError(
"ip %s has pop of %s, which doesn't exist" % (ip, pop))
