def run(self): while FINISH_EVENT.isSet(): try: rpc(MASTER_SERVER, 'heartbeat', ()) except Exception, e: pass time.sleep(BEAT_PERIOD)
def exposed_gather_browser_stats(self, type): # write to etc/hosts.xia check_output('echo "%s %s" > %s' % (BROWSER_ADDR, rpc(MASTER_SERVER, 'get_neighbor_webserver', (type,)), ETC_HOSTS)) print check_output('cat %s' % ETC_HOSTS) while True: try: check_output('ps -e | grep proxy.py') break except: if type == 'tunneling': Popen(PROXY_CMD, shell=True) else: proxy_dag = rpc(MASTER_SERVER, 'exposed_get_proxy_address', (type,)) print 'proxy going to start' print proxy_dag Popen('%s "%s"' % (PROXY_CMD, proxy_dag), shell=True) time.sleep(1) time.sleep(3) print 'running browser cmd' p = Popen(BROWSER_CMD,shell=True) p.wait() time.sleep(3) while True: (out, rc) = check_both(BROWSER_CMD, check=False) if rc == 1: raise Execption("BROWSER_CMD failed") if rc == 0: break time.sleep(1) out = out[0].split('\n')[-2].split(' ')[-1] rpc(MASTER_SERVER, 'browser_stats', (out,type)) check_output('killall proxy.py')
def exposed_get_gateway_xhost(self, type): host = self.exposed_get_gateway_host(type, self._host) try: xhost = "RE AD:%s HID:%s" % (rpc(host, "get_ad", ()), rpc(host, "get_hid", ())) return xhost except: self.exposed_error("get_xhost: %s" % type, host)
def exposed_gather_stats(self): printtime('<<<<GATHER STATS>>>>') neighbors = rpc(MASTER_SERVER, 'get_backbone', ()) out = multi_ping(neighbors) latency = out[0][0] my_backbone = out[0][1] hops = traceroute(my_backbone) rpc(MASTER_SERVER, 'stats', (my_backbone, latency, hops)) return ['Sent stats: (%s, %s, %s)' % (my_backbone, latency, hops), my_backbone]
def run(self): for backbone in BACKBONES: while True: try: rpc("localhost", "hard_restart", (backbone,)) break except Exception, e: printtime("%s" % e) time.sleep(1)
def run(self): while FINISH_EVENT.isSet(): try: rpc('localhost', 'run_commands', ()) break except Exception, e: printtime('%s' % e) #rpc(MASTER_SERVER, 'error', ('command broke?', my_name)) time.sleep(1)
def exposed_get_gateway_xhost(self, type): host = self.exposed_get_gateway_host(type,self._host) try: xhost = 'RE AD:%s HID:%s' % \ (rpc(host, 'get_ad', ()), rpc(host, 'get_hid', ())) return xhost except: self.exposed_error('get_xhost: %s' % type, host)
def run(self): while self.goOnEvent.isSet(): try: myHID = rpc('localhost', 'get_hid', ()) neighbors = rpc('localhost', 'get_neighbors', ()) rpc(MASTER_SERVER, 'heartbeat', (myHID, neighbors)) except Exception, e: pass time.sleep(BEAT_PERIOD)
def run(self): for backbone in BACKBONES: while True: try: rpc('localhost', 'hard_restart', (backbone, )) break; except Exception, e: printtime('%s' % e) time.sleep(1)
def exposed_get_neighbor_xhost(self): printtime("<<<< GET NEIGHBORS >>>>") if self._host in CLIENTS: neighbor = [client for client in CLIENTS if client != self._host][0] while True: try: return "RE AD:%s HID:%s" % (rpc(neighbor, "get_ad", ()), rpc(neighbor, "get_hid", ())) except: pass return None
def run(self): for t in NODES: for node in t: while True: try: rpc('localhost', 'hard_restart', (node, )) break; except Exception, e: printtime('%s' % e) time.sleep(1)
def exposed_run_webserver(self, type): myAD = self.exposed_get_ad() myHID = self.exposed_get_hid() neighbor = rpc(MASTER_SERVER, 'get_neighbor_host', ()) my_gateway = rpc(MASTER_SERVER, 'get_gateway_host', (type, my_name)) their_gateway = rpc(MASTER_SERVER, 'get_gateway_host', (type, neighbor)) g4ID = rpc(my_gateway, 'get_fourid', ()) gAD = rpc(their_gateway, 'get_ad', ()) print myAD, myHID, neighbor, my_gateway, their_gateway, g4ID, gAD Popen('%s AD:%s HID:%s IP:%s AD:%s' % (WEBSERVER_CMD, myAD, myHID, g4ID, gAD),shell=True) time.sleep(3)
def exposed_get_neighbor_xhost(self): printtime("<<<< GET NEIGHBORS >>>>") if self._host in CLIENTS: neighbor = [client for client in CLIENTS if client != self._host][0] while True: try: return 'RE AD:%s HID:%s' % \ (rpc(neighbor, 'get_ad', ()), rpc(neighbor, 'get_hid', ())) except: pass return None
def exposed_gather_xstats(self): printtime('<<<<GATHER XSTATS>>>>') neighbor = rpc(MASTER_SERVER, 'get_neighbor_xhost', ()) printtime('neighbor: %s' % neighbor) xlatency = xping(neighbor) printtime('xlatency: %s' % xlatency) time.sleep(2) xhops = xtraceroute(neighbor) printtime('xhops: %s' % xhops) rpc(MASTER_SERVER, 'xstats', (xlatency, xhops)) return 'Sent xstats: (%s, %s, %s)' % (neighbor, xlatency, xhops)
def exposed_get_fourid_neighbor_xhost(self, type, src): printtime("<<<< GET 4ID NEIGHBORS >>>>") if type == '6RD': gA = self.exposed_get_sixrd_A(); gB = self.exposed_get_sixrd_B() if type == '4ID': gA = self.exposed_get_fourid_GA(); gB = self.exposed_get_fourid_GB() if type == 'SDN': gA = self.exposed_get_sdn_A(); gB = self.exposed_get_sdn_B() print "GETTING FROM: %s" % self.exposed_get_fourid_CA() s = 'AD:%s HID:%s' % (rpc(self.exposed_get_fourid_CA(), 'get_ad', ()), rpc(self.exposed_get_fourid_CA(), 'get_hid', ())) print "GETTING FROM: %s" % self.exposed_get_fourid_CB() d = 'AD:%s HID:%s' % (rpc(self.exposed_get_fourid_CB(), 'get_ad', ()), rpc(self.exposed_get_fourid_CB(), 'get_hid', ())) if self._host in CLIENTS: src = not src if src: print "GETTING FROM: %s" % gB ad = rpc(gB, 'get_ad', ()) print "GETTING FROM: %s" % gA fourid = rpc(gA, 'get_fourid', ()) return 'RE AD:%s IP:%s %s' % (ad, fourid, s) else: print "GETTING FROM: %s" % gA ad = rpc(gA, 'get_ad', ()) print "GETTING FROM: %s" % gB fourid = rpc(gB, 'get_fourid', ()) return 'RE AD:%s IP:%s %s' % (ad, fourid, d)
def run(self): should_run = True while should_run: should_run = False self.print_write('launching...') self.hard_stop() try: self.ssh_run(START_CMD, waitForCompletion=False) except Exception, e: if self.finishEvent.isSet(): self.print_write('NW.run Exception: %s' % e) try: rpc('localhost', 'error', ('Startup', self.host)) except: pass
def run(self): while FINISH_EVENT.isSet(): try: printtime('<<<<DISCOVERY>>>>') nodes_to_check = rpc(MASTER_SERVER, 'get_nodes_to_check', ()) for node in nodes_to_check: if check_neighbor(node) == "": continue cmd = 'ab http://%s:%d/test | grep "Transfer rate"' % (node, HTTP_PORT) out = check_output(cmd)[0] printtime(out) stats = (node, int(float(out.split(':')[1].split('[')[0].strip()))) rpc(MASTER_SERVER, 'discovery_stats', (stats,)) except Exception, e: print e pass time.sleep(DISCOVERY_PERIOD)
def exposed_get_proxy_address(self, type): printtime("<<<< GET PROXY ADDR >>>>") me = CLIENTS[0] neighbor = BACKBONES[0] while True: try: gad = "AD:%s" % rpc(self.exposed_get_gateway_host(type, neighbor), "get_ad", ()) ad = "AD:%s" % rpc(self._host, "get_ad", ()) hid = "HID:%s" % rpc(self._host, "get_hid", ()) fourid = "IP:%s" % rpc(self.exposed_get_gateway_host(type, self._host), "get_fourid", ()) s = "RE %s %s %s %s SID:%s" % (gad, fourid, ad, hid, PROXY_SID) return s except: time.sleep(1) print "Trying to get proxy addres..." pass
def exposed_run_commands(self): printtime('requesting commands!') commands = rpc(MASTER_SERVER, 'get_commands', ()) printtime('commands received!') printtime('commands: %s' % commands) for command in commands: printtime(command) exec(command)
def exposed_get_proxy_address(self, type): printtime("<<<< GET PROXY ADDR >>>>") me = CLIENTS[0] neighbor = BACKBONES[0] while True: try: gad = 'AD:%s' % rpc( \ self.exposed_get_gateway_host(type,neighbor), 'get_ad', ()) ad = 'AD:%s' % rpc(self._host, 'get_ad', ()) hid = 'HID:%s' % rpc(self._host, 'get_hid', ()) fourid = 'IP:%s' % rpc( \ self.exposed_get_gateway_host(type,self._host), 'get_fourid', ()) s = 'RE %s %s %s %s SID:%s' % \ (gad, fourid, ad, hid, PROXY_SID) return s except: time.sleep(1) print "Trying to get proxy addres..." pass
def exposed_wait_for_neighbors(self, neighbors, msg): while True: for neighbor in neighbors: try: printtime('waiting on: %s' % neighbor) out = rpc(neighbor, 'get_hid', ()) return out except: printtime(msg) time.sleep(1)
def exposed_get_neighbor_webserver(self, type): printtime("<<<< GET WEBSERVER >>>>") if type == "tunneling" and self._host in CLIENTS: neighbor = [client for client in CLIENTS if client != self._host][0] elif type in ["6RD", "4ID", "SDN"]: neighbor = BACKBONES[0] while True: try: ad = "AD:%s" % rpc(neighbor, "get_ad", ()) hid = "HID:%s" % rpc(neighbor, "get_hid", ()) if type == "tunneling": fourid = "IP:%s" % BLANK_FOURID s = "RE %s %s %s %s SID:%s" % (ad, fourid, ad, hid, WEBSERVER_SID) else: gad = "AD:%s" % rpc(self.exposed_get_gateway_host(type, self._host), "get_ad", ()) fourid = "IP:%s" % rpc(self.exposed_get_gateway_host(type, neighbor), "get_fourid", ()) s = "RE %s %s %s %s SID:%s" % (gad, fourid, ad, hid, WEBSERVER_SID) return s except: pass
def run(self): should_run = True while should_run: should_run = False self.print_write('launching...') self.hard_stop() try: if self.fourid_setup: self.ssh_run(FOURID_SETUP_CMD, waitForCompletion=False) else: self.ssh_run(START_CMD, waitForCompletion=False) except Exception, e: if self.finishEvent.isSet(): self.print_write('NW.run Exception: %s' % e) try: rpc('localhost', 'error', ('Startup', self.host)) except: pass if self.finishEvent.isSet() and self.host in BACKBONES: should_run = True self.goOnEvent.set()
def run(self): should_run = True while should_run: should_run = False self.print_write("launching...") self.hard_stop() try: if self.fourid_setup: self.ssh_run(FOURID_SETUP_CMD, waitForCompletion=False) else: self.ssh_run(START_CMD, waitForCompletion=False) except Exception, e: if self.finishEvent.isSet(): self.print_write("NW.run Exception: %s" % e) try: rpc("localhost", "error", ("Startup", self.host)) except: pass if self.finishEvent.isSet() and self.host in BACKBONES: should_run = True self.goOnEvent.set()
def exposed_get_neighbor_webserver(self, type): printtime("<<<< GET WEBSERVER >>>>") if type == "tunneling" and self._host in CLIENTS: neighbor = [client for client in CLIENTS if client != self._host][0] elif type in ['6RD','4ID','SDN']: neighbor = BACKBONES[0] while True: try: ad = 'AD:%s' % rpc(neighbor, 'get_ad', ()) hid = 'HID:%s' % rpc(neighbor, 'get_hid', ()) if type == "tunneling": fourid = 'IP:%s' % BLANK_FOURID s = 'RE %s %s %s %s SID:%s' % \ (ad, fourid, ad, hid, WEBSERVER_SID) else: gad = 'AD:%s' % rpc( \ self.exposed_get_gateway_host(type,self._host), 'get_ad', ()) fourid = 'IP:%s' % rpc( \ self.exposed_get_gateway_host(type,neighbor), 'get_fourid', ()) s = 'RE %s %s %s %s SID:%s' % \ (gad, fourid, ad, hid, WEBSERVER_SID) return s except: pass
class Runner(threading.Thread): def run(self): for backbone in BACKBONES: while True: try: rpc('localhost', 'hard_restart', (backbone, )) break; except Exception, e: printtime('%s' % e) time.sleep(1) while True: try: rpc('localhost', 'new_exp', ()) break except Exception, e: printtime('%s' % e) time.sleep(1)
def orchestrate_pairwise_ping(self): PAIRWISE_PING_LOCK.acquire() cur_exp = tuple(CLIENTS) if cur_exp in PAIRWISE_PING: PAIRWISE_PING_LOCK.release() return PAIRWISE_PING[cur_exp] min_ping = 5000 min_pair = None for node in BACKBONES[1:]: while True: try: out = rpc(node, 'get_ping', (CLIENTS[1:],))[0] if float(out[0]) == -1: continue if float(out[0]) < min_ping: min_ping = float(out[0]) min_pair = (node, out[1]) break; # try next node except Exception, e: time.sleep(1)
def exposed_update_table(self, new_table): #ForwardingTable = new_table #if not new_table: # return s = open(HTTPD_CONF_TOP).read() if rpc(MASTER_SERVER, 'should_cache', ()): s += open(HTTPD_CONF_CACHE).read() for k in new_table: v = new_table[k] s += '<Proxy balancer://%d>\n' % (k) for t in v: s += 'BalancerMember http://%s:%d/ loadfactor=1\n' % (t[0], HTTP_PORT) s += 'ProxySet lbmethod=bytraffic\n' s += '</Proxy>\n' s += 'ProxyPass /%d/ balancer://%d\n' % (k, k) f = open(HTTPD_TEMP, "w") f.write(s) f.close() start_httpd(HTTPD_TEMP)
def exposed_get_fourid_neighbor_xhost(self, type, src): printtime("<<<< GET 4ID NEIGHBORS >>>>") if type == "6RD": gA = self.exposed_get_sixrd_A() gB = self.exposed_get_sixrd_B() if type == "4ID": gA = self.exposed_get_fourid_GA() gB = self.exposed_get_fourid_GB() if type == "SDN": gA = self.exposed_get_sdn_A() gB = self.exposed_get_sdn_B() print "GETTING FROM: %s" % self.exposed_get_fourid_CA() s = "AD:%s HID:%s" % ( rpc(self.exposed_get_fourid_CA(), "get_ad", ()), rpc(self.exposed_get_fourid_CA(), "get_hid", ()), ) print "GETTING FROM: %s" % self.exposed_get_fourid_CB() d = "AD:%s HID:%s" % ( rpc(self.exposed_get_fourid_CB(), "get_ad", ()), rpc(self.exposed_get_fourid_CB(), "get_hid", ()), ) if self._host in CLIENTS: src = not src if src: print "GETTING FROM: %s" % gB ad = rpc(gB, "get_ad", ()) print "GETTING FROM: %s" % gA fourid = rpc(gA, "get_fourid", ()) return "RE AD:%s IP:%s %s" % (ad, fourid, s) else: print "GETTING FROM: %s" % gA ad = rpc(gA, "get_ad", ()) print "GETTING FROM: %s" % gB fourid = rpc(gB, "get_fourid", ()) return "RE AD:%s IP:%s %s" % (ad, fourid, d)
def orchestrate_pairwise_ping(self): PAIRWISE_PING_LOCK.acquire() cur_exp = tuple(CLIENTS) if cur_exp in PAIRWISE_PING: PAIRWISE_PING_LOCK.release() return PAIRWISE_PING[cur_exp] min_ping = 5000 min_pair = None for node in BACKBONES[1:]: while True: try: out = rpc(node, "get_ping", (CLIENTS[1:],))[0] if float(out[0]) == -1: continue if float(out[0]) < min_ping: min_ping = float(out[0]) min_pair = (node, out[1]) break # try next node except Exception, e: time.sleep(1)
def exposed_soft_restart(self, neighbor): xianetcmd = rpc(MASTER_SERVER, 'get_xianet', (neighbor, )) check_output(KILL_XIANET) printtime('running %s' % xianetcmd) exec(xianetcmd) return xianetcmd
def exposed_new_exp(self, host=None): global CLIENTS, NUMEXP, NEW_EXP_TIMER, PRINT_VERB, NEW_EXP_LOCK, FOURID_G_INDEX, TYPE if SINGLE_EXPERIMENT and NUMEXP == 2: return if NEW_EXP_LOCK.locked(): return NEW_EXP_LOCK.acquire() print "<<< lock aquired >>>" # some host errored that's not currently in the experiment if host and host not in CLIENTS and host not in BACKBONES: NEW_EXP_LOCK.release() print "<<< nope: lock released >>>" return try: cur_exp = tuple(CLIENTS) if len(STATSD[cur_exp]) != 4 and not FOURID_EXPERIMENT: printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp) if len(STATSD[cur_exp]) != 3 and FOURID_EXPERIMENT: printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp) except: pass # pick new test clients while True: if not FOURID_EXPERIMENT: while True: new_clients = sample(NAMES, 2) new_clients = [NAME_LOOKUP[client] for client in new_clients] if new_clients[0] not in CLIENTS and new_clients[1] not in CLIENTS: break if SAME_TEST_NODES: new_clients = [NAME_LOOKUP["planetlab1.tsuniv.edu"], NAME_LOOKUP["planetlab5.cs.cornell.edu"]] new_clients = sorted(new_clients) else: while True: while True: new_clients = [NAME_LOOKUP[choice(NAMES)]] if new_clients[0] not in CLIENTS: break if SAME_TEST_NODES: new_clients = [NAME_LOOKUP["DC"]] print new_clients[0] IP_LOOKUP[socket.gethostbyname(new_clients[0])] = new_clients[0] self.exposed_hard_restart(new_clients[0], setup=True) i = 0 while new_clients[0] in NODE_WATCHERS and i < 30: try: rpc(new_clients[0], "get_hello", ()) break except: time.sleep(1) i += 1 if new_clients[0] not in NODE_WATCHERS or i >= 30: # client crashed continue i = 0 while i < 5: # only try five times since some nodes have ping issue and we don't want to lock try: print "Trying Ping" test_clients = [NAME_LOOKUP[c] for c in sample(NAMES, 50)] if new_clients[0] in test_clients: continue pings = rpc(new_clients[0], "get_ping", (test_clients,)) print pings if float(pings[3][0]) < FOURID_MAX_PING_TIME: break except Exception, e: print "Ping didn't work" print e # print pings time.sleep(1) i += 1 if i < 5: # if we successfully got our topo break else: # take down client print "need to try new ping client" NODE_WATCHERS[new_clients[0]].clearGoOn() while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down" time.sleep(1) print "Done Ping" ping_hosts = [p[1] for p in pings] new_clients += ping_hosts[:4] if SAME_TEST_NODES: new_clients = [new_clients[0]] + ["NYC", "Atlanta", "Cleveland", "Houston"] print new_clients new_clients = [new_clients[0]] + [NAME_LOOKUP[c] for c in new_clients[1:]] print "building topo" self.build_topo(new_clients) print "done topo" NODE_WATCHERS[new_clients[0]].clearGoOn() while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down" time.sleep(1) # Make sure we haven't done this experiment before if tuple(new_clients) not in STATSD: self.init_statsd(new_clients) break
def run(self): global REQ, ST, F, E, DE_FLOPPER, DE_FLOP_COUNT, SORTED_E, NODERMAP, G while FINISHED_EVENT.isSet(): # if not REQ: # time.sleep(1) # continue nodeMap = {} NODERMAP = {} i = 1 for type in xrange(3): for j,key in enumerate(HOSTNAME_LOOKUP): if key in NODES[type]: nodeMap[key] = i NODERMAP[i] = key i += 1 NODERMAP[0] = 'dummy' SORTED_E = {} # if len(DSTATSD) == 0: # LA = 'planetlab1.cs.ucla.edu' # SF = 'planetslug4.cse.ucsc.edu' # SE = 'planetlab01.cs.washington.edu' # DE = 'planetlab2.cs.colorado.edu' # HO = 'ricepl-1.cs.rice.edu' # DSTATSD[DE] = {} # DSTATSD[DE][SF] = 1500 # DSTATSD[DE][SE] = 1500 # # DSTATSD[HO] = {} # # DSTATSD[HO][SF] = 1500 # # DSTATSD[HO][SE] = 1500 # DSTATSD[SF] = {} # DSTATSD[SF][LA] = 1000 # DSTATSD[SF][HO] = 2000 # DSTATSD[SE] = {} # DSTATSD[SE][LA] = 1000 for node in NODES[SOURCES]: #SOURCES link = (0, nodeMap[node], MAX_LINK) try: SORTED_E[0] += [link] except: SORTED_E[0] = [link] for key, value in DSTATSD.iteritems(): for k, v in value.iteritems(): link = (nodeMap[key], nodeMap[k], v) ken = KILLED_EDGE_NAME print ken if ken and key == ken[0] and k == ken[1]: # print 'KEN HERE' # SORTED_E[nodeMap[key]] += [(link[0], link[1], 0)] continue try: SORTED_E[nodeMap[key]] += [link] except: SORTED_E[nodeMap[key]] = [link] print "<<<< HERE WE ARE>>" print SORTED_E G = [] gs = open('streams').read().split('\n')[:-1] for g in gs: l = g.split('\t') br = l[2].translate(string.maketrans("","",), '{}[]kbps').strip().split(',') br = [eval(b) for b in br] T = l[3].translate(string.maketrans("","",), '{}[]').strip().split(',') T = [(eval(t.split('=')[0].split('_')[1]), eval(t.split('=')[1])) for t in T] G.append([eval(l[1]), br, T]) print G print SORTED_E if SORTED_E: try: if DECISION_RUNNING: REQ, ST, F, E, avg_d, lpt = DecisionEngine(G, SORTED_E, False) print "RAN DECISION ENGINE" else: not_REQ, ST, F, E, avg_d, lpt = DecisionEngine(G, SORTED_E, False) except Exception, e: print e print REQ print ST print F print E print '<<<<<<REQ!!' print REQ req2 = copy.deepcopy(REQ) for k,v in req2.iteritems(): for k2,v2 in v.iteritems(): v[k2] = [(NODERMAP[i[0]], i[1]) for i in v2] print req2 for node,i in nodeMap.iteritems(): if i in req2 and req2[i]: print node try: rpc(node, 'update_table', (req2[i],)) except: pass # DE_FLOPPER -= 1 # if DE_FLOPPER == 0: # rpc('localhost', 'shut_off_DE', ()) # print 'CONTROLCRASHER (on): ' + str(oavg) # elif DE_FLOPPER == -3: # print 'CONTROLCRASHER (off): ' + str(oavg) # DE_FLOP_COUNT += 1 # print 'DE_FLOP COUNT: ' + str(DE_FLOP_COUNT) # DE_FLOPPER = 3 # rpc('localhost', 'turn_on_DE', ()) time.sleep(DECISION_PERIOD)
#!/usr/bin/python from plcommon import rpc RPC_PORT = 43278 rpc("localhost", "shut_off_DE", ())
def check_neighbor(neighbor): try: printtime('checking on: %s' % neighbor) return rpc(neighbor, 'check_httpd', ()) except: return ""
def exposed_new_exp(self, host=None): global CLIENTS, NUMEXP, NEW_EXP_TIMER, PRINT_VERB, NEW_EXP_LOCK, FOURID_G_INDEX, TYPE if SINGLE_EXPERIMENT and NUMEXP == 2: return if NEW_EXP_LOCK.locked(): return NEW_EXP_LOCK.acquire() print "<<< lock aquired >>>" # some host errored that's not currently in the experiment if host and host not in CLIENTS and host not in BACKBONES: NEW_EXP_LOCK.release() print "<<< nope: lock released >>>" return try: cur_exp = tuple(CLIENTS) if len(STATSD[cur_exp]) != 4 and not FOURID_EXPERIMENT: printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp) if len(STATSD[cur_exp]) != 3 and FOURID_EXPERIMENT: printtime("<<<< TIMEOUT!! (%s) >>>>" % cur_exp) except: pass # pick new test clients while True: if not FOURID_EXPERIMENT: while True: new_clients = sample(NAMES, 2) new_clients = [NAME_LOOKUP[client] for client in new_clients] if new_clients[0] not in CLIENTS and new_clients[1] not in CLIENTS: break if SAME_TEST_NODES: new_clients = [NAME_LOOKUP['planetlab1.tsuniv.edu'],NAME_LOOKUP['planetlab5.cs.cornell.edu']] new_clients = sorted(new_clients) else: while True: while True: new_clients = [NAME_LOOKUP[choice(NAMES)]] if new_clients[0] not in CLIENTS: break if SAME_TEST_NODES: new_clients = [NAME_LOOKUP['DC']] print new_clients[0] IP_LOOKUP[socket.gethostbyname(new_clients[0])] = new_clients[0] self.exposed_hard_restart(new_clients[0], setup=True) i = 0 while new_clients[0] in NODE_WATCHERS and i < 30: try: rpc(new_clients[0],'get_hello',()) break except: time.sleep(1) i += 1 if new_clients[0] not in NODE_WATCHERS or i >= 30: # client crashed continue i = 0 while i < 5: # only try five times since some nodes have ping issue and we don't want to lock try: print "Trying Ping" test_clients = [NAME_LOOKUP[c] for c in sample(NAMES, 50)] if new_clients[0] in test_clients: continue pings = rpc(new_clients[0], 'get_ping', (test_clients, )) print pings if float(pings[3][0]) < FOURID_MAX_PING_TIME: break except Exception, e: print "Ping didn't work" print e #print pings time.sleep(1) i += 1 if i < 5: # if we successfully got our topo break else: # take down client print 'need to try new ping client' NODE_WATCHERS[new_clients[0]].clearGoOn() while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down"; time.sleep(1) print "Done Ping" ping_hosts = [p[1] for p in pings] new_clients += ping_hosts[:4] if SAME_TEST_NODES: new_clients = [new_clients[0]]+['NYC','Atlanta','Cleveland', 'Houston'] print new_clients new_clients = [new_clients[0]]+[NAME_LOOKUP[c] for c in new_clients[1:]] print "building topo" self.build_topo(new_clients) print "done topo" NODE_WATCHERS[new_clients[0]].clearGoOn() while new_clients[0] in NODE_WATCHERS: print "waiting for node to go down"; time.sleep(1) # Make sure we haven't done this experiment before if tuple(new_clients) not in STATSD: self.init_statsd(new_clients) break
def exposed_gather_fourid_stats(self, type): printtime('<<<<GATHER FOURIDSTATS>>>>') # [CA, 6RDA, GA, SDNA, CB, 6RDB, GB, SDNB] fidn = rpc(MASTER_SERVER, 'get_fourid_nodes', ()) print my_name, fidn if my_name not in fidn: return elif my_name in [fidn[0], fidn[4]]: #CA, CB printtime('<<<<<NODE STARTING 4IDSTAT>>>>') i = fidn.index(my_name) gpair = 'AG' if i == 0 else 'BG' npair = 'AB' if i == 0 else 'BA' neighbor = fidn[4] if i == 0 else fidn[0] #for type in ['SDN', '4ID', '6RD']: printtime('<<<<NODE STARTING %s STATS>>>>' % type) gateway = rpc(MASTER_SERVER,'get_gateway_host',(type,my_name)) # DST = rpc(MASTER_SERVER,'get_gateway_xhost',(type,)) # xlatency = xping(DST) # xhops = xtraceroute(DST) # DST_h = rpc(MASTER_SERVER,'get_gateway_host',(type,my_name)) latency = multi_ping([gateway])[0][0] hops = traceroute(gateway, gateway=True) # rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, gpair), gateway, xlatency, xhops)) rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, gpair), gateway, latency, hops)) SRC = rpc(MASTER_SERVER,'get_fourid_neighbor_xhost',(type,True)) DST = rpc(MASTER_SERVER,'get_fourid_neighbor_xhost',(type,False)) xlatency = xping(DST, src=SRC) xhops = -1 rpc(MASTER_SERVER, 'fidstats', ('%s-%s' % (type, npair), neighbor, xlatency, xhops)) if my_name == fidn[4]: #CB self.exposed_gather_browser_stats(type) elif my_name == fidn[0]: #CA self.exposed_run_webserver(type) while not rpc(MASTER_SERVER, 'check_done_type', (type,)): print "Waiting for browser test to complete..." time.sleep(1) elif my_name in [fidn[1], fidn[2], fidn[3]]: # 6RDA, GA, SDNA indices = [i for i,x in enumerate(fidn) if x == my_name] for i in indices: t = ['6RD', '4ID', 'SDN'][i-1] if t == type: partner_node = fidn[i+4] latency = multi_ping([partner_node])[0][0] hops = traceroute(partner_node) rpc(MASTER_SERVER, 'fidstats', ('%s-GG' % type, partner_node, latency, hops))