def run(self):
while FINISH_EVENT.isSet():
try:
rpc(MASTER_SERVER, 'heartbeat', ())
except Exception, e:
pass
time.sleep(BEAT_PERIOD)
def exposed_gather_browser_stats(self, type):
# write to etc/hosts.xia
check_output('echo "%s %s" > %s' % (BROWSER_ADDR, rpc(MASTER_SERVER, 'get_neighbor_webserver', (type,)), ETC_HOSTS))
print check_output('cat %s' % ETC_HOSTS)
while True:
try:
check_output('ps -e | grep proxy.py')
break
except:
if type == 'tunneling':
Popen(PROXY_CMD, shell=True)
else:
proxy_dag = rpc(MASTER_SERVER, 'exposed_get_proxy_address', (type,))
print 'proxy going to start'
print proxy_dag
Popen('%s "%s"' % (PROXY_CMD, proxy_dag), shell=True)
time.sleep(1)
time.sleep(3)
print 'running browser cmd'
p = Popen(BROWSER_CMD,shell=True)
p.wait()
time.sleep(3)
while True:
(out, rc) = check_both(BROWSER_CMD, check=False)
if rc == 1:
raise Execption("BROWSER_CMD failed")
if rc == 0:
break
time.sleep(1)
out = out[0].split('\n')[-2].split(' ')[-1]
rpc(MASTER_SERVER, 'browser_stats', (out,type))
check_output('killall proxy.py')
def exposed_get_gateway_xhost(self, type):
host = self.exposed_get_gateway_host(type, self._host)
try:
xhost = "RE AD:%s HID:%s" % (rpc(host, "get_ad", ()), rpc(host, "get_hid", ()))
return xhost
except:
self.exposed_error("get_xhost: %s" % type, host)
def exposed_gather_stats(self):
printtime('<<<<GATHER STATS>>>>')
neighbors = rpc(MASTER_SERVER, 'get_backbone', ())
out = multi_ping(neighbors)
latency = out[0][0]
my_backbone = out[0][1]
hops = traceroute(my_backbone)
rpc(MASTER_SERVER, 'stats', (my_backbone, latency, hops))
return ['Sent stats: (%s, %s, %s)' % (my_backbone, latency, hops), my_backbone]
def run(self):
for backbone in BACKBONES:
while True:
try:
rpc("localhost", "hard_restart", (backbone,))
break
except Exception, e:
printtime("%s" % e)
time.sleep(1)
def run(self):
while FINISH_EVENT.isSet():
try:
rpc('localhost', 'run_commands', ())
break
except Exception, e:
printtime('%s' % e)
#rpc(MASTER_SERVER, 'error', ('command broke?', my_name))
time.sleep(1)
def exposed_get_gateway_xhost(self, type):
host = self.exposed_get_gateway_host(type,self._host)
try:
xhost = 'RE AD:%s HID:%s' % \
(rpc(host, 'get_ad', ()), rpc(host, 'get_hid', ()))
return xhost
except:
self.exposed_error('get_xhost: %s' % type, host)
def run(self):
while self.goOnEvent.isSet():
try:
myHID = rpc('localhost', 'get_hid', ())
neighbors = rpc('localhost', 'get_neighbors', ())
rpc(MASTER_SERVER, 'heartbeat', (myHID, neighbors))
except Exception, e:
pass
time.sleep(BEAT_PERIOD)
def run(self):
for backbone in BACKBONES:
while True:
try:
rpc('localhost', 'hard_restart', (backbone, ))
break;
except Exception, e:
printtime('%s' % e)
time.sleep(1)
def exposed_get_neighbor_xhost(self):
printtime("<<<< GET NEIGHBORS >>>>")
if self._host in CLIENTS:
neighbor = [client for client in CLIENTS if client != self._host][0]
while True:
try:
return "RE AD:%s HID:%s" % (rpc(neighbor, "get_ad", ()), rpc(neighbor, "get_hid", ()))
except:
pass
return None
def run(self):
for t in NODES:
for node in t:
while True:
try:
rpc('localhost', 'hard_restart', (node, ))
break;
except Exception, e:
printtime('%s' % e)
time.sleep(1)
def exposed_run_webserver(self, type):
myAD = self.exposed_get_ad()
myHID = self.exposed_get_hid()
neighbor = rpc(MASTER_SERVER, 'get_neighbor_host', ())
my_gateway = rpc(MASTER_SERVER, 'get_gateway_host', (type, my_name))
their_gateway = rpc(MASTER_SERVER, 'get_gateway_host', (type, neighbor))
g4ID = rpc(my_gateway, 'get_fourid', ())
gAD = rpc(their_gateway, 'get_ad', ())
print myAD, myHID, neighbor, my_gateway, their_gateway, g4ID, gAD
Popen('%s AD:%s HID:%s IP:%s AD:%s' % (WEBSERVER_CMD, myAD, myHID, g4ID, gAD),shell=True)
time.sleep(3)
def exposed_get_neighbor_xhost(self):
printtime("<<<< GET NEIGHBORS >>>>")
if self._host in CLIENTS:
neighbor = [client for client in CLIENTS if client != self._host][0]
while True:
try:
return 'RE AD:%s HID:%s' % \
(rpc(neighbor, 'get_ad', ()), rpc(neighbor, 'get_hid', ()))
except:
pass
return None
def exposed_gather_xstats(self):
printtime('<<<<GATHER XSTATS>>>>')
neighbor = rpc(MASTER_SERVER, 'get_neighbor_xhost', ())
printtime('neighbor: %s' % neighbor)
xlatency = xping(neighbor)
printtime('xlatency: %s' % xlatency)
time.sleep(2)
xhops = xtraceroute(neighbor)
printtime('xhops: %s' % xhops)
rpc(MASTER_SERVER, 'xstats', (xlatency, xhops))
return 'Sent xstats: (%s, %s, %s)' % (neighbor, xlatency, xhops)
def exposed_get_fourid_neighbor_xhost(self, type, src):
printtime("<<<< GET 4ID NEIGHBORS >>>>")
if type == '6RD': gA = self.exposed_get_sixrd_A(); gB = self.exposed_get_sixrd_B()
if type == '4ID': gA = self.exposed_get_fourid_GA(); gB = self.exposed_get_fourid_GB()
if type == 'SDN': gA = self.exposed_get_sdn_A(); gB = self.exposed_get_sdn_B()
print "GETTING FROM: %s" % self.exposed_get_fourid_CA()
s = 'AD:%s HID:%s' % (rpc(self.exposed_get_fourid_CA(), 'get_ad', ()),
rpc(self.exposed_get_fourid_CA(), 'get_hid', ()))
print "GETTING FROM: %s" % self.exposed_get_fourid_CB()
d = 'AD:%s HID:%s' % (rpc(self.exposed_get_fourid_CB(), 'get_ad', ()),
rpc(self.exposed_get_fourid_CB(), 'get_hid', ()))
if self._host in CLIENTS:
src = not src
if src:
print "GETTING FROM: %s" % gB
ad = rpc(gB, 'get_ad', ())
print "GETTING FROM: %s" % gA
fourid = rpc(gA, 'get_fourid', ())
return 'RE AD:%s IP:%s %s' % (ad, fourid, s)
else:
print "GETTING FROM: %s" % gA
ad = rpc(gA, 'get_ad', ())
print "GETTING FROM: %s" % gB
fourid = rpc(gB, 'get_fourid', ())
return 'RE AD:%s IP:%s %s' % (ad, fourid, d)
def run(self):
should_run = True
while should_run:
should_run = False
self.print_write('launching...')
self.hard_stop()
try:
self.ssh_run(START_CMD, waitForCompletion=False)
except Exception, e:
if self.finishEvent.isSet():
self.print_write('NW.run Exception: %s' % e)
try:
rpc('localhost', 'error', ('Startup', self.host))
except:
pass
def run(self):
while FINISH_EVENT.isSet():
try:
printtime('<<<<DISCOVERY>>>>')
nodes_to_check = rpc(MASTER_SERVER, 'get_nodes_to_check', ())
for node in nodes_to_check:
if check_neighbor(node) == "": continue
cmd = 'ab http://%s:%d/test | grep "Transfer rate"' % (node, HTTP_PORT)
out = check_output(cmd)[0]
printtime(out)
stats = (node, int(float(out.split(':')[1].split('[')[0].strip())))
rpc(MASTER_SERVER, 'discovery_stats', (stats,))
except Exception, e:
print e
pass
time.sleep(DISCOVERY_PERIOD)
def exposed_get_proxy_address(self, type):
printtime("<<<< GET PROXY ADDR >>>>")
me = CLIENTS[0]
neighbor = BACKBONES[0]
while True:
try:
gad = "AD:%s" % rpc(self.exposed_get_gateway_host(type, neighbor), "get_ad", ())
ad = "AD:%s" % rpc(self._host, "get_ad", ())
hid = "HID:%s" % rpc(self._host, "get_hid", ())
fourid = "IP:%s" % rpc(self.exposed_get_gateway_host(type, self._host), "get_fourid", ())
s = "RE %s %s %s %s SID:%s" % (gad, fourid, ad, hid, PROXY_SID)
return s
except:
time.sleep(1)
print "Trying to get proxy addres..."
pass
def exposed_run_commands(self):
printtime('requesting commands!')
commands = rpc(MASTER_SERVER, 'get_commands', ())
printtime('commands received!')
printtime('commands: %s' % commands)
for command in commands:
printtime(command)
exec(command)
def exposed_get_proxy_address(self, type):
printtime("<<<< GET PROXY ADDR >>>>")
me = CLIENTS[0]
neighbor = BACKBONES[0]
while True:
try:
gad = 'AD:%s' % rpc( \
self.exposed_get_gateway_host(type,neighbor), 'get_ad', ())
ad = 'AD:%s' % rpc(self._host, 'get_ad', ())
hid = 'HID:%s' % rpc(self._host, 'get_hid', ())
fourid = 'IP:%s' % rpc( \
self.exposed_get_gateway_host(type,self._host), 'get_fourid', ())
s = 'RE %s %s %s %s SID:%s' % \
(gad, fourid, ad, hid, PROXY_SID)
return s
except:
time.sleep(1)
print "Trying to get proxy addres..."
pass
def exposed_wait_for_neighbors(self, neighbors, msg):
while True:
for neighbor in neighbors:
try:
printtime('waiting on: %s' % neighbor)
out = rpc(neighbor, 'get_hid', ())
return out
except:
printtime(msg)
time.sleep(1)
def exposed_get_neighbor_webserver(self, type):
printtime("<<<< GET WEBSERVER >>>>")
if type == "tunneling" and self._host in CLIENTS:
neighbor = [client for client in CLIENTS if client != self._host][0]
elif type in ["6RD", "4ID", "SDN"]:
neighbor = BACKBONES[0]
while True:
try:
ad = "AD:%s" % rpc(neighbor, "get_ad", ())
hid = "HID:%s" % rpc(neighbor, "get_hid", ())
if type == "tunneling":
fourid = "IP:%s" % BLANK_FOURID
s = "RE %s %s %s %s SID:%s" % (ad, fourid, ad, hid, WEBSERVER_SID)
else:
gad = "AD:%s" % rpc(self.exposed_get_gateway_host(type, self._host), "get_ad", ())
fourid = "IP:%s" % rpc(self.exposed_get_gateway_host(type, neighbor), "get_fourid", ())
s = "RE %s %s %s %s SID:%s" % (gad, fourid, ad, hid, WEBSERVER_SID)
return s
except:
pass
def run(self):
should_run = True
while should_run:
should_run = False
self.print_write('launching...')
self.hard_stop()
try:
if self.fourid_setup:
self.ssh_run(FOURID_SETUP_CMD, waitForCompletion=False)
else:
self.ssh_run(START_CMD, waitForCompletion=False)
except Exception, e:
if self.finishEvent.isSet():
self.print_write('NW.run Exception: %s' % e)
try:
rpc('localhost', 'error', ('Startup', self.host))
except:
pass
if self.finishEvent.isSet() and self.host in BACKBONES:
should_run = True
self.goOnEvent.set()
def run(self):
should_run = True
while should_run:
should_run = False
self.print_write("launching...")
self.hard_stop()
try:
if self.fourid_setup:
self.ssh_run(FOURID_SETUP_CMD, waitForCompletion=False)
else:
self.ssh_run(START_CMD, waitForCompletion=False)
except Exception, e:
if self.finishEvent.isSet():
self.print_write("NW.run Exception: %s" % e)
try:
rpc("localhost", "error", ("Startup", self.host))
except:
pass
if self.finishEvent.isSet() and self.host in BACKBONES:
should_run = True
self.goOnEvent.set()
def exposed_get_neighbor_webserver(self, type):
printtime("<<<< GET WEBSERVER >>>>")
if type == "tunneling" and self._host in CLIENTS:
neighbor = [client for client in CLIENTS if client != self._host][0]
elif type in ['6RD','4ID','SDN']:
neighbor = BACKBONES[0]
while True:
try:
ad = 'AD:%s' % rpc(neighbor, 'get_ad', ())
hid = 'HID:%s' % rpc(neighbor, 'get_hid', ())
if type == "tunneling":
fourid = 'IP:%s' % BLANK_FOURID
s = 'RE %s %s %s %s SID:%s' % \
(ad, fourid, ad, hid, WEBSERVER_SID)
else:
gad = 'AD:%s' % rpc( \
self.exposed_get_gateway_host(type,self._host), 'get_ad', ())
fourid = 'IP:%s' % rpc( \
self.exposed_get_gateway_host(type,neighbor), 'get_fourid', ())
s = 'RE %s %s %s %s SID:%s' % \
(gad, fourid, ad, hid, WEBSERVER_SID)
return s
except:
pass
class Runner(threading.Thread):
def run(self):
for backbone in BACKBONES:
while True:
try:
rpc('localhost', 'hard_restart', (backbone, ))
break;
except Exception, e:
printtime('%s' % e)
time.sleep(1)
while True:
try:
rpc('localhost', 'new_exp', ())
break
except Exception, e:
printtime('%s' % e)
time.sleep(1)
def orchestrate_pairwise_ping(self):
PAIRWISE_PING_LOCK.acquire()
cur_exp = tuple(CLIENTS)
if cur_exp in PAIRWISE_PING:
PAIRWISE_PING_LOCK.release()
return PAIRWISE_PING[cur_exp]
min_ping = 5000
min_pair = None
for node in BACKBONES[1:]:
while True:
try:
out = rpc(node, 'get_ping', (CLIENTS[1:],))[0]
if float(out[0]) == -1: continue
if float(out[0]) < min_ping:
min_ping = float(out[0])
min_pair = (node, out[1])
break; # try next node
except Exception, e:
time.sleep(1)
def exposed_update_table(self, new_table):
#ForwardingTable = new_table
#if not new_table:
# return
s = open(HTTPD_CONF_TOP).read()
if rpc(MASTER_SERVER, 'should_cache', ()):
s += open(HTTPD_CONF_CACHE).read()
for k in new_table:
v = new_table[k]
s += '<Proxy balancer://%d>\n' % (k)
for t in v:
s += 'BalancerMember http://%s:%d/ loadfactor=1\n' % (t[0], HTTP_PORT)
s += 'ProxySet lbmethod=bytraffic\n'
s += '</Proxy>\n'
s += 'ProxyPass /%d/ balancer://%d\n' % (k, k)
f = open(HTTPD_TEMP, "w")
f.write(s)
f.close()
start_httpd(HTTPD_TEMP)
def exposed_get_fourid_neighbor_xhost(self, type, src):
printtime("<<<< GET 4ID NEIGHBORS >>>>")
if type == "6RD":
gA = self.exposed_get_sixrd_A()
gB = self.exposed_get_sixrd_B()
if type == "4ID":
gA = self.exposed_get_fourid_GA()
gB = self.exposed_get_fourid_GB()
if type == "SDN":
gA = self.exposed_get_sdn_A()
gB = self.exposed_get_sdn_B()
print "GETTING FROM: %s" % self.exposed_get_fourid_CA()
s = "AD:%s HID:%s" % (
rpc(self.exposed_get_fourid_CA(), "get_ad", ()),
rpc(self.exposed_get_fourid_CA(), "get_hid", ()),
)
print "GETTING FROM: %s" % self.exposed_get_fourid_CB()
d = "AD:%s HID:%s" % (
rpc(self.exposed_get_fourid_CB(), "get_ad", ()),
rpc(self.exposed_get_fourid_CB(), "get_hid", ()),
)
if self._host in CLIENTS:
src = not src
if src:
print "GETTING FROM: %s" % gB
ad = rpc(gB, "get_ad", ())
print "GETTING FROM: %s" % gA
fourid = rpc(gA, "get_fourid", ())
return "RE AD:%s IP:%s %s" % (ad, fourid, s)
else:
print "GETTING FROM: %s" % gA
ad = rpc(gA, "get_ad", ())
print "GETTING FROM: %s" % gB
fourid = rpc(gB, "get_fourid", ())
return "RE AD:%s IP:%s %s" % (ad, fourid, d)
def orchestrate_pairwise_ping(self):
PAIRWISE_PING_LOCK.acquire()
cur_exp = tuple(CLIENTS)
if cur_exp in PAIRWISE_PING:
PAIRWISE_PING_LOCK.release()
return PAIRWISE_PING[cur_exp]
min_ping = 5000
min_pair = None
for node in BACKBONES[1:]:
while True:
try:
out = rpc(node, "get_ping", (CLIENTS[1:],))[0]
if float(out[0]) == -1:
continue
if float(out[0]) < min_ping:
min_ping = float(out[0])
min_pair = (node, out[1])
break
# try next node
except Exception, e:
time.sleep(1)
def exposed_soft_restart(self, neighbor):
xianetcmd = rpc(MASTER_SERVER, 'get_xianet', (neighbor, ))
check_output(KILL_XIANET)
printtime('running %s' % xianetcmd)
exec(xianetcmd)
return xianetcmd
def exposed_new_exp(self, host=None):
global CLIENTS, NUMEXP, NEW_EXP_TIMER, PRINT_VERB, NEW_EXP_LOCK, FOURID_G_INDEX, TYPE
if SINGLE_EXPERIMENT and NUMEXP == 2:
return
if NEW_EXP_LOCK.locked():
return
NEW_EXP_LOCK.acquire()
print "<<< lock aquired >>>"
# some host errored that's not currently in the experiment
if host and host not in CLIENTS and host not in BACKBONES:
NEW_EXP_LOCK.release()
print "<<< nope: lock released >>>"
return
try:
cur_exp = tuple(CLIENTS)
if len(STATSD[cur_exp]) != 4 and not FOURID_EXPERIMENT:
printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp)
if len(STATSD[cur_exp]) != 3 and FOURID_EXPERIMENT:
printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp)
except:
pass
# pick new test clients
while True:
if not FOURID_EXPERIMENT:
while True:
new_clients = sample(NAMES, 2)
new_clients = [NAME_LOOKUP[client] for client in new_clients]
if new_clients[0] not in CLIENTS and new_clients[1] not in CLIENTS:
break
if SAME_TEST_NODES:
new_clients = [NAME_LOOKUP["planetlab1.tsuniv.edu"], NAME_LOOKUP["planetlab5.cs.cornell.edu"]]
new_clients = sorted(new_clients)
else:
while True:
while True:
new_clients = [NAME_LOOKUP[choice(NAMES)]]
if new_clients[0] not in CLIENTS:
break
if SAME_TEST_NODES:
new_clients = [NAME_LOOKUP["DC"]]
print new_clients[0]
IP_LOOKUP[socket.gethostbyname(new_clients[0])] = new_clients[0]
self.exposed_hard_restart(new_clients[0], setup=True)
i = 0
while new_clients[0] in NODE_WATCHERS and i < 30:
try:
rpc(new_clients[0], "get_hello", ())
break
except:
time.sleep(1)
i += 1
if new_clients[0] not in NODE_WATCHERS or i >= 30: # client crashed
continue
i = 0
while i < 5: # only try five times since some nodes have ping issue and we don't want to lock
try:
print "Trying Ping"
test_clients = [NAME_LOOKUP[c] for c in sample(NAMES, 50)]
if new_clients[0] in test_clients:
continue
pings = rpc(new_clients[0], "get_ping", (test_clients,))
print pings
if float(pings[3][0]) < FOURID_MAX_PING_TIME:
break
except Exception, e:
print "Ping didn't work"
print e
# print pings
time.sleep(1)
i += 1
if i < 5: # if we successfully got our topo
break
else: # take down client
print "need to try new ping client"
NODE_WATCHERS[new_clients[0]].clearGoOn()
while new_clients[0] in NODE_WATCHERS:
print "waiting for node to go down"
time.sleep(1)
print "Done Ping"
ping_hosts = [p[1] for p in pings]
new_clients += ping_hosts[:4]
if SAME_TEST_NODES:
new_clients = [new_clients[0]] + ["NYC", "Atlanta", "Cleveland", "Houston"]
print new_clients
new_clients = [new_clients[0]] + [NAME_LOOKUP[c] for c in new_clients[1:]]
print "building topo"
self.build_topo(new_clients)
print "done topo"
NODE_WATCHERS[new_clients[0]].clearGoOn()
while new_clients[0] in NODE_WATCHERS:
print "waiting for node to go down"
time.sleep(1)
# Make sure we haven't done this experiment before
if tuple(new_clients) not in STATSD:
self.init_statsd(new_clients)
break
def run(self):
global REQ, ST, F, E, DE_FLOPPER, DE_FLOP_COUNT, SORTED_E, NODERMAP, G
while FINISHED_EVENT.isSet():
# if not REQ:
# time.sleep(1)
# continue
nodeMap = {}
NODERMAP = {}
i = 1
for type in xrange(3):
for j,key in enumerate(HOSTNAME_LOOKUP):
if key in NODES[type]:
nodeMap[key] = i
NODERMAP[i] = key
i += 1
NODERMAP[0] = 'dummy'
SORTED_E = {}
# if len(DSTATSD) == 0:
# LA = 'planetlab1.cs.ucla.edu'
# SF = 'planetslug4.cse.ucsc.edu'
# SE = 'planetlab01.cs.washington.edu'
# DE = 'planetlab2.cs.colorado.edu'
# HO = 'ricepl-1.cs.rice.edu'
# DSTATSD[DE] = {}
# DSTATSD[DE][SF] = 1500
# DSTATSD[DE][SE] = 1500
# # DSTATSD[HO] = {}
# # DSTATSD[HO][SF] = 1500
# # DSTATSD[HO][SE] = 1500
# DSTATSD[SF] = {}
# DSTATSD[SF][LA] = 1000
# DSTATSD[SF][HO] = 2000
# DSTATSD[SE] = {}
# DSTATSD[SE][LA] = 1000
for node in NODES[SOURCES]: #SOURCES
link = (0, nodeMap[node], MAX_LINK)
try:
SORTED_E[0] += [link]
except:
SORTED_E[0] = [link]
for key, value in DSTATSD.iteritems():
for k, v in value.iteritems():
link = (nodeMap[key], nodeMap[k], v)
ken = KILLED_EDGE_NAME
print ken
if ken and key == ken[0] and k == ken[1]:
# print 'KEN HERE'
# SORTED_E[nodeMap[key]] += [(link[0], link[1], 0)]
continue
try:
SORTED_E[nodeMap[key]] += [link]
except:
SORTED_E[nodeMap[key]] = [link]
print "<<<< HERE WE ARE>>"
print SORTED_E
G = []
gs = open('streams').read().split('\n')[:-1]
for g in gs:
l = g.split('\t')
br = l[2].translate(string.maketrans("","",), '{}[]kbps').strip().split(',')
br = [eval(b) for b in br]
T = l[3].translate(string.maketrans("","",), '{}[]').strip().split(',')
T = [(eval(t.split('=')[0].split('_')[1]), eval(t.split('=')[1])) for t in T]
G.append([eval(l[1]), br, T])
print G
print SORTED_E
if SORTED_E:
try:
if DECISION_RUNNING:
REQ, ST, F, E, avg_d, lpt = DecisionEngine(G, SORTED_E, False)
print "RAN DECISION ENGINE"
else:
not_REQ, ST, F, E, avg_d, lpt = DecisionEngine(G, SORTED_E, False)
except Exception, e:
print e
print REQ
print ST
print F
print E
print '<<<<<<REQ!!'
print REQ
req2 = copy.deepcopy(REQ)
for k,v in req2.iteritems():
for k2,v2 in v.iteritems():
v[k2] = [(NODERMAP[i[0]], i[1]) for i in v2]
print req2
for node,i in nodeMap.iteritems():
if i in req2 and req2[i]:
print node
try:
rpc(node, 'update_table', (req2[i],))
except:
pass
# DE_FLOPPER -= 1
# if DE_FLOPPER == 0:
# rpc('localhost', 'shut_off_DE', ())
# print 'CONTROLCRASHER (on): ' + str(oavg)
# elif DE_FLOPPER == -3:
# print 'CONTROLCRASHER (off): ' + str(oavg)
# DE_FLOP_COUNT += 1
# print 'DE_FLOP COUNT: ' + str(DE_FLOP_COUNT)
# DE_FLOPPER = 3
# rpc('localhost', 'turn_on_DE', ())
time.sleep(DECISION_PERIOD)
#!/usr/bin/python
from plcommon import rpc
RPC_PORT = 43278
rpc("localhost", "shut_off_DE", ())
def check_neighbor(neighbor):
try:
printtime('checking on: %s' % neighbor)
return rpc(neighbor, 'check_httpd', ())
except:
return ""
def exposed_new_exp(self, host=None):
global CLIENTS, NUMEXP, NEW_EXP_TIMER, PRINT_VERB, NEW_EXP_LOCK, FOURID_G_INDEX, TYPE
if SINGLE_EXPERIMENT and NUMEXP == 2:
return
if NEW_EXP_LOCK.locked():
return
NEW_EXP_LOCK.acquire()
print "<<< lock aquired >>>"
# some host errored that's not currently in the experiment
if host and host not in CLIENTS and host not in BACKBONES:
NEW_EXP_LOCK.release()
print "<<< nope: lock released >>>"
return
try:
cur_exp = tuple(CLIENTS)
if len(STATSD[cur_exp]) != 4 and not FOURID_EXPERIMENT:
printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp)
if len(STATSD[cur_exp]) != 3 and FOURID_EXPERIMENT:
printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp)
except:
pass
# pick new test clients
while True:
if not FOURID_EXPERIMENT:
while True:
new_clients = sample(NAMES, 2)
new_clients = [NAME_LOOKUP[client] for client in new_clients]
if new_clients[0] not in CLIENTS and new_clients[1] not in CLIENTS:
break
if SAME_TEST_NODES:
new_clients = [NAME_LOOKUP['planetlab1.tsuniv.edu'],NAME_LOOKUP['planetlab5.cs.cornell.edu']]
new_clients = sorted(new_clients)
else:
while True:
while True:
new_clients = [NAME_LOOKUP[choice(NAMES)]]
if new_clients[0] not in CLIENTS:
break
if SAME_TEST_NODES:
new_clients = [NAME_LOOKUP['DC']]
print new_clients[0]
IP_LOOKUP[socket.gethostbyname(new_clients[0])] = new_clients[0]
self.exposed_hard_restart(new_clients[0], setup=True)
i = 0
while new_clients[0] in NODE_WATCHERS and i < 30:
try:
rpc(new_clients[0],'get_hello',())
break
except:
time.sleep(1)
i += 1
if new_clients[0] not in NODE_WATCHERS or i >= 30: # client crashed
continue
i = 0
while i < 5: # only try five times since some nodes have ping issue and we don't want to lock
try:
print "Trying Ping"
test_clients = [NAME_LOOKUP[c] for c in sample(NAMES, 50)]
if new_clients[0] in test_clients: continue
pings = rpc(new_clients[0], 'get_ping', (test_clients, ))
print pings
if float(pings[3][0]) < FOURID_MAX_PING_TIME:
break
except Exception, e:
print "Ping didn't work"
print e
#print pings
time.sleep(1)
i += 1
if i < 5: # if we successfully got our topo
break
else: # take down client
print 'need to try new ping client'
NODE_WATCHERS[new_clients[0]].clearGoOn()
while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down"; time.sleep(1)
print "Done Ping"
ping_hosts = [p[1] for p in pings]
new_clients += ping_hosts[:4]
if SAME_TEST_NODES:
new_clients = [new_clients[0]]+['NYC','Atlanta','Cleveland', 'Houston']
print new_clients
new_clients = [new_clients[0]]+[NAME_LOOKUP[c] for c in new_clients[1:]]
print "building topo"
self.build_topo(new_clients)
print "done topo"
NODE_WATCHERS[new_clients[0]].clearGoOn()
while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down"; time.sleep(1)
# Make sure we haven't done this experiment before
if tuple(new_clients) not in STATSD:
self.init_statsd(new_clients)
break
def exposed_gather_fourid_stats(self, type):
printtime('<<<<GATHER FOURIDSTATS>>>>')
# [CA, 6RDA, GA, SDNA, CB, 6RDB, GB, SDNB]
fidn = rpc(MASTER_SERVER, 'get_fourid_nodes', ())
print my_name, fidn
if my_name not in fidn:
return
elif my_name in [fidn[0], fidn[4]]: #CA, CB
printtime('<<<<<NODE STARTING 4IDSTAT>>>>')
i = fidn.index(my_name)
gpair = 'AG' if i == 0 else 'BG'
npair = 'AB' if i == 0 else 'BA'
neighbor = fidn[4] if i == 0 else fidn[0]
#for type in ['SDN', '4ID', '6RD']:
printtime('<<<<NODE STARTING %s STATS>>>>' % type)
gateway = rpc(MASTER_SERVER,'get_gateway_host',(type,my_name))
# DST = rpc(MASTER_SERVER,'get_gateway_xhost',(type,))
# xlatency = xping(DST)
# xhops = xtraceroute(DST)
# DST_h = rpc(MASTER_SERVER,'get_gateway_host',(type,my_name))
latency = multi_ping([gateway])[0][0]
hops = traceroute(gateway, gateway=True)
# rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, gpair), gateway, xlatency, xhops))
rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, gpair), gateway, latency, hops))
SRC = rpc(MASTER_SERVER,'get_fourid_neighbor_xhost',(type,True))
DST = rpc(MASTER_SERVER,'get_fourid_neighbor_xhost',(type,False))
xlatency = xping(DST, src=SRC)
xhops = -1
rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, npair), neighbor, xlatency, xhops))
if my_name == fidn[4]: #CB
self.exposed_gather_browser_stats(type)
elif my_name == fidn[0]: #CA
self.exposed_run_webserver(type)
while not rpc(MASTER_SERVER, 'check_done_type', (type,)):
print "Waiting for browser test to complete..."
time.sleep(1)
elif my_name in [fidn[1], fidn[2], fidn[3]]: # 6RDA, GA, SDNA
indices = [i for i,x in enumerate(fidn) if x == my_name]
for i in indices:
t = ['6RD', '4ID', 'SDN'][i-1]
if t == type:
partner_node = fidn[i+4]
latency = multi_ping([partner_node])[0][0]
hops = traceroute(partner_node)
rpc(MASTER_SERVER, 'fidstats', ('%s-GG' % type, partner_node, latency, hops))
