def _stop_capture(self, cap): if not cap: return True ts = time.time() #Kill tcpdump shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) try: shell.run(["/bin/bash", "-i", "-c", config.capture_kill_cmd], allow_error=True) except Exception as e: pass cap.wait_for_result() if len(self.last_cap) > 0: os.system( "scp -q -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no %s@%s:/root/capture.dmp %s\n" % (config.vm_ssh_key, config.vm_user, mv.vm2ip( self.tc[0]), self.last_cap)) self.log.write('[timer] Stop Capture: %f sec.\n' % (time.time() - ts)) return True
def _start_monitor(self): monitor = None ts = time.time() cmd = config.monitor_cmd.format(port=str(config.monitor_com_port), proxy=mv.vm2ip(self.tc[0]), proxyport=str(config.proxy_com_port)) self.log.write("Monitor CMD: " + cmd + "\n") shell = spur.SshShell(hostname=mv.vm2ip(self.mon[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) #Start Proxy monitor = shell.spawn(["/bin/bash", "-i", "-c", cmd], store_pid=True, allow_error=True) if not monitor.is_running(): res = monitor.wait_for_result() self.log.write("Monitor Failed to Start: " + res.output + res.stderr_output) return None else: self.log.write("Started monitor on " + str(mv.vm2ip(self.mon[0])) + "...\n") #Wait for proxy to come up if (self._waitListening(mv.vm2ip(self.mon[0]), config.monitor_com_port, 240, False) == False): self.log.write("Monitor Failed to start after 240 seconds!\n") print "Monitor Failed to Start after 240 seconds!" return None self.log.write('[timer] Start monitor: %f sec.\n' % (time.time() - ts)) self.monitor_running = True return monitor
def _start_proxy(self): proxy = None ts = time.time() cmd = config.proxy_cmd.format(port=str(config.proxy_com_port)) self.log.write("Proxy CMD: " + cmd + "\n") shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) #Setup NetEM try: ret = shell.run(["/bin/bash", "-i", "-c", config.limit_cmd]) self.log.write("Setting up netem:\n" + ret.output) except Exception as e: print "Setting up netem failed: " + str(e) self.log.write("Setting up netem failed: " + str(e) + "\n") return None #Start Proxy proxy = shell.spawn(["/bin/bash", "-i", "-c", cmd], store_pid=True, allow_error=True) if not proxy.is_running(): res = proxy.wait_for_result() self.log.write("Proxy Failed to Start: " + res.output + res.stderr_output) return None else: self.log.write("Started proxy on " + str(mv.vm2ip(self.tc[0])) + "...\n") #Wait for proxy to come up if (self._waitListening(mv.vm2ip(self.tc[0]), config.proxy_com_port, 240, False) == False): self.log.write("Proxy Failed to start after 240 seconds!\n") print "Proxy Failed to Start after 240 seconds!" return None self.log.write('[timer] Start proxy: %f sec.\n' % (time.time() - ts)) self.proxy_running = True return proxy
def _cleanup(self): ts = time.time() shell = spur.SshShell(hostname=mv.vm2ip(self.mininet[0]), username=config.mininet_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) res = shell.run(["/bin/bash", "-i", "-c", config.mininet_cleanup_cmd]) if config.enable_stat: self.log.write('[timer] Clean up mininet: %f sec.\n' % (time.time() - ts)) return True
def _stop_controllers(self): if self.controllers_running is False: return True ts = time.time() for c in self.controllers: shell = spur.SshShell( hostname=mv.vm2ip(c), username=config.controller_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) try: self.log.write("Stopping controller (" + mv.vm2ip(c) + ")...\n") res = shell.run([ "/bin/bash", "-i", "-c", "~/monitors/control.sh {0} {1}".format( config.controller_type, "stop") ], allow_error=True) try: self.controller_stat_dict = ProcMonStat.extract_stat( res.output) except Exception as e: import traceback self.log.write( 'Could not extract controller procmon stat: ' + str(e) + '.\n') print(traceback.format_exc()) self.log.write(res.output) except Exception as e: print e self.log.write("Exception: " + str(e) + "\n") self.log.flush() return False if config.enable_stat: self.log.write('[timer] Stop controllers: %f sec.\n' % (time.time() - ts)) self.controllers_running = False return True
def _start_controllers(self): ts = time.time() for c in self.controllers: shell = spur.SshShell( hostname=mv.vm2ip(c), username=config.controller_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) res = shell.run([ "/bin/bash", "-i", "-c", "~/monitors/control.sh {0} {1}".format(config.controller_type, "start") ]) self.log.write("Starting Controller (" + mv.vm2ip(c) + ")...\n" + res.output) self.log.flush() for c in self.controllers: if (self._waitListening(mv.vm2ip(c), config.controller_port, 60) == False): self.log.write( "Controller %s failed to start after %d seconds" % (mv.vm2ip(c), 60)) self.log.flush() return False else: # Controller starts. res = shell.run([ "/bin/bash", "-i", "-c", "~/monitors/control.sh {0} {1}".format( config.controller_type, "mon") ]) self.log.write("Starting resource monitor for controller (" + mv.vm2ip(c) + ")...\n" + res.output) self.log.flush() if config.enable_stat: self.log.write('[timer] Start all controllers: %f sec.\n' % (time.time() - ts)) self.controllers_running = True return True
def startVms(self): for c in self.controllers: mv.startvm(c) for m in self.mininet: mv.startvm(m) for c in self.controllers: if (self._waitListening(mv.vm2ip(c), 22, 240, True) == False): print "Error: Controller VM %d not started!" % (c) return False else: os.system( "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n" % (config.vm_ssh_key, monitor_tools_path, config.controller_user, mv.vm2ip(c))) shell = spur.SshShell( hostname=mv.vm2ip(c), username=config.controller_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) proc = shell.run([ "/bin/bash", "-i", "-c", "cd monitors && make clean && make" ]) if proc.return_code is not 0: print "Error: Make failed!" return False for m in self.mininet: if (self._waitListening(mv.vm2ip(m), 22, 240, True) == False): print "Error: Mininet %d not started!" % (c) return False else: if config.mininet_replace_scripts: os.system( "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s/* %s@%s:~\n" % (config.vm_ssh_key, mininet_config_path, config.mininet_user, mv.vm2ip(m))) os.system( "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n" % (config.vm_ssh_key, monitor_tools_path, config.mininet_user, mv.vm2ip(m))) shell = spur.SshShell( hostname=mv.vm2ip(m), username=config.mininet_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) proc = shell.run([ "/bin/bash", "-i", "-c", "cd monitors && make clean && make" ]) if proc.return_code is not 0: print "Error: Make failed!" return False return True
def _cleanup(self): ts = time.time() #Kill Proxy shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) try: ret = shell.run(["/bin/bash", "-i", "-c", config.proxy_kill_cmd]) except Exception as e: return False #Kill Monitor shell = spur.SshShell(hostname=mv.vm2ip(self.mon[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) try: ret = shell.run(["/bin/bash", "-i", "-c", config.monitor_kill_cmd]) except Exception as e: return False self.log.write('[timer] Clean up: %f sec.\n' % (time.time() - ts)) return True
def _query_proxy_done(self): last = 0 cmd = "*,*,{0},0,0,*,ACTIVE,*".format(config.protocol) for t in self.tc: res = self._proxy_communicate((mv.vm2ip(t), config.proxy_com_port), cmd, wait_for_response=True) if type(res) is bool and res is False: return False if type(res) is not str: return False last = float(res) if last < 10: return False if time.time() - last >= config.test_max_idle: return True
def _send_proxy_strategy(self, strategy): strat = "" # Default strategy if strategy == None: strategy = [ "*,*,TCP,0,0,*,CLEAR,*", "{0},{1},{2},0,0,*,CLEAR,*".format(config.target_client_ip, config.target_server_ip, config.protocol) ] ts = time.time() for l in strategy: if type(l) is dict: if 'action' not in l: return False strat = l['action'] if 'time' in l and l['time'] > 0.01: strat = dict(l) strat['time'] = 0 tmr = threading.Timer(l['time'], self._send_proxy_strategy, [[strat], proxyaddrs]) tmr.start() self.timers.append(tmr) continue elif type(l) is str: strat = l else: return False self.log.write("Strategy CMD: " + strat + "\n") self.log.flush() for t in self.tc: res = self._proxy_communicate( (mv.vm2ip(t), config.proxy_com_port), strat) if (res == False): self.log.write("Failed to Send Command\n") self.log.flush() return False self.log.write('[timer] Send strategy: %f sec.\n' % (time.time() - ts)) return True
def _call_test(self, test_script, cmd, proxyaddrs): # Build Config and default command cfg = { 'controllers': proxyaddrs, 'topo_discovery': config.topo_discovery_delay, 'fail_mode': config.mininet_fail_mode } if cmd == None: cmd = [{'cmd': 'basic'}] ts = time.time() res = None exec_res = None m = self.mininet[0] shell = spur.SshShell(hostname=mv.vm2ip(m), username=config.mininet_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) self.log.write("Starting Test: " + test_script + "\n") self.log.write("Test Config: %s\n" % cfg) for c in cmd: self.log.write("Test Command: %s\n" % c) self.log.flush() try: proc = shell.spawn(["/bin/bash", "-i", "-c", test_script]) proc.stdin_write("%s\n" % repr(cfg)) for c in cmd: proc.stdin_write("%s\n" % repr(c)) exec_res = proc.wait_for_result() res = eval(exec_res.output) except Exception as e: print e self.log.write("Exception: " + str(e) + "\n") self.log.flush() return [None, None] if config.enable_stat: self.log.write('[timer] Do all tests: %f sec.\n' % (time.time() - ts)) return [res, exec_res.stderr_output]
def _query_proxy_conn_info(self): length = 0 total_data = 0 cmd = "{0},{1},{2},0,0,*,TIME,*".format(config.target_client_ip, config.target_server_ip, config.protocol) for t in self.tc: res = self._proxy_communicate((mv.vm2ip(t), config.proxy_com_port), cmd, wait_for_response=True) if type(res) is bool and res is False: return (False, 0, "System Error") if type(res) is not str: return (False, 0, "System Error") lns = res.split() if len(lns) != 2: return (False, 0, "System Error") length = float(lns[0]) total_data = int(lns[1]) if length < 0 or length > 200: return False, length, total_data return True, length, total_data
def _start_capture(self): self.last_cap = "" if not self.do_capture: return None #Generate Capture Name time_str = time.strftime(config.captures_time_str) fname = config.captures_loc.format(tm=time_str, exe=self.instance) self.last_cap = fname #Generate capture command cmd = config.capture_cmd #Start SSH Shell shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) #Start Capture cap = None try: cap = shell.spawn(["/bin/bash", "-i", "-c", cmd], store_pid=True, allow_error=True) if not cap.is_running(): res = proxy.wait_for_result() self.log.write("Capture Failed to Start: " + res.stderr_output) return None except Exception as e: print e self.log.write("Exception: " + str(e) + "\n") self.log.flush() return None return cap
def _call_test(self): ts = time.time() #Start servers for s in self.servers: if mv.vmHasSSH(s): shell = spur.SshShell( hostname=mv.vm2ip(s), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) try: ret = shell.run( ["/bin/bash", "-i", "-c", config.server_start_cmd]) except Exception as e: print "Failed to start server" self.log.write("Failed to start server\n") return False, 0 if mv.vmCanPing(s) and self._waitListening(mv.vm2ip(s), 80, 240, False) == False: print "Failed to start server" self.log.write("Failed to start server\n") return False, 0 self.log.write("Servers Started...\n") time.sleep(0.5) #Start background traffic shell = spur.SshShell(hostname=mv.vm2ip(self.clients[1]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) background = shell.spawn([ "/bin/bash", "-i", "-c", config.background_client_cmd.format(tm=str(config.max_time)) ], store_pid=True, allow_error=True) if not background.is_running(): ret = background.wait_for_result() print "Background traffic command failed: %s %s" % ( ret.output, ret.stderr_output) self.log.write("Background traffic command failed: %s %s\n" % (ret.output, ret.stderr_output)) return False, 0 bts = time.time() #Start main traffic shell = spur.SshShell(hostname=mv.vm2ip(self.clients[0]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) mts = time.time() ret = None speed = 0 bspeed = 0 main = shell.spawn([ "/bin/bash", "-i", "-c", config.main_client_cmd.format(tm=str(config.max_time)) ], store_pid=True, allow_error=True) if not main.is_running(): ret = main.wait_for_result() print "Main Traffic Command failed: %s %s" % (ret.output, ret.stderr_output) self.log.write("Main Traffic Command Failed: %s %s\n" % (ret.output, ret.stderr_output)) try: background.send_signal(2) except Exception as e: pass return False, 0 #Wait to finish self.last_termination_idle = False while background.is_running() or main.is_running(): if background.is_running(): bspeed = time.time() - bts if main.is_running(): speed = time.time() - mts if self._query_proxy_done(): self.last_termination_idle = True try: background.send_signal(2) except Exception as e: pass try: main.send_signal(2) except Exception as e: pass break time.sleep(1) #Check Main Return code self.return_code_error = False ret = main.wait_for_result() if ret.return_code is not 0: self.log.write("Main Traffic Command Failed! Return Code: %d\n" % (ret.return_code)) print "Main Traffic Command Failed! Return Code: %d" % ( ret.return_code) speed = config.max_time if ret.return_code is 56: self.return_code_error = True self.log.write("Main Traffic command output: \n" + ret.stderr_output) #Check Background Return code ret = background.wait_for_result() if ret.return_code is not 0: self.log.write( "Background Traffic Command Failed! Return Code: %d\n" % (ret.return_code)) print "Background Traffic Command Failed! Return Code: %d" % ( ret.return_code) bspeed = config.max_time if ret.return_code is 56: self.return_code_error = True self.log.write("Background Traffic command output: \n" + ret.stderr_output) return True, speed
def startVms(self): for c in self.clients: mv.startvm(c) for s in self.servers: mv.startvm(s) for t in self.tc: mv.startvm(t) for m in self.mon: mv.startvm(m) for c in self.clients: if (self._waitListening(mv.vm2ip(c), 22, 240, True) == False): print "Error: client VM %d not started!" % (c) return False for s in self.servers: if not mv.vmCanPing(s): time.sleep(30) else: if (self._waitListening(mv.vm2ip(s), 22 if mv.vmHasSSH(s) else 80, 240, True) == False): print "Error: server VM %d not started!" % (s) return False if hasattr(config, 'background_server_config') and len( config.background_server_config) > 0 and mv.vmHasSSH( self.servers[1]): shell = spur.SshShell( hostname=mv.vm2ip(self.servers[1]), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) proc = shell.run( ["/bin/bash", "-i", "-c", config.background_server_config]) if proc.return_code is not 0: print "Error: Background server config failed!" return False for t in self.tc: if (self._waitListening(mv.vm2ip(t), 22, 240, True) == False): print "Error: Traffic Shaping VM %d not started!" % (t) return False else: if config.vm_replace_data: os.system( "scp -r -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n" % (config.vm_ssh_key, proxy_path, config.vm_user, mv.vm2ip(t))) shell = spur.SshShell( hostname=mv.vm2ip(t), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) proc = shell.run([ "/bin/bash", "-i", "-c", "cd proxy && make clean && make" ]) if proc.return_code is not 0: print "Error: Make failed!" return False for m in self.mon: if (self._waitListening(mv.vm2ip(m), 22, 240, True) == False): print "Error: Monitor VM %d not started!" % (t) return False else: if config.vm_replace_data: os.system( "scp -r -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n" % (config.vm_ssh_key, monitor_path, config.vm_user, mv.vm2ip(m))) shell = spur.SshShell( hostname=mv.vm2ip(m), username=config.vm_user, missing_host_key=spur.ssh.MissingHostKey.accept, private_key_file=config.vm_ssh_key) proc = shell.run([ "/bin/bash", "-i", "-c", "cd monitor && make clean && make" ]) if proc.return_code is not 0: print "Error: Make failed!" return False return True
def doTest(self, strategy): """ :return [True | False, str, int]: First boolean value indicates pass or fail, followed by an explanation, and an int indicating the suggestion of doing rebaseline. """ if 'topo' not in strategy: return (False, "System Failure", 0) test_script = strategy['topo'] self.msg_types = [] if hasattr(self, 'switch_stat_dict'): del self.switch_stat_dict if hasattr(self, 'controller_stat_dict'): del self.controller_stat_dict result = [True, "Success!", 0] self.log.write('#' * 30 + "Starting Test " + str(self.testnum) + '#' * 30 + '\n') self.log.write(str(datetime.today()) + "\n") # Create Address/Port strings controlleraddrs = list() proxyaddrs = list() for c in self.controllers: controlleraddrs.append( mv.vm2ip(c) + ":" + str(config.controller_port)) proxyaddrs.append((config.proxy_addr, config.proxy_base_port + c)) # Start Controllers if self._start_controllers() == False: return (False, "System Failure", 0) # Start Proxy proxy = self._start_proxy(controlleraddrs, proxyaddrs) if proxy is None: self._stop_controllers() return (False, "System Failure", 0) # Send Proxy Strategy if self._send_proxy_strategy(strategy['switch'], proxyaddrs) == False: self._stop_controllers() proxy.terminate() return (False, "System Failure", 0) # Initialize Controller strategies, if any if 'controller' in strategy: if self._handle_controller_actions(strategy['controller'], proxy) == False: self._stop_controllers() proxy.terminate() return (False, "System Failure", 0) # VeriFlow veriflow = None if config.veriflow_enabled == True: vf_port = config.veriflow_base_port + self.controllers[0] veriflow = self._start_veriflow(test_script, proxyaddrs, vf_port) proxyaddrs = list() proxyaddrs.append( (config.proxy_addr, vf_port, confg.proxy_base_port + 1)) if veriflow is None: self._stop_controllers() proxy.terminate() return (False, "System Failure", 0) # Do Test test_std_err = None res, test_std_err = self._call_test(test_script, strategy['host'], proxyaddrs) if res is None: self._stop_controllers() if veriflow is not None: veriflow.terminate() proxy.terminate() return (False, "System Failure", 0) # Evaluate Results if isinstance(res, (dict)) and "results" in res: res_list = res["results"] for r in res_list: if r == False: result[0] = False result[1] = "Network Tests" # Check Rules if "rules" in res: if self._check_rule_dump(res["rules"]) == False: result[0] = False result[1] = "Network State" # Save stat dictionary for switch if 'stat' in res: self.switch_stat_dict = res['stat'] else: result[0] = False result[1] = "System Failure" # Check Message Types if self._get_msg_types( ("localhost", config.proxy_com_port + self.mininet[0])) == False: if veriflow is not None: veriflow.terminate() proxy.terminate() self._stop_controllers() return (False, "System Failure", 0) if self._check_for_error_msgs(): result[0] = False result[1] = "Error Message" # Check and Stop VeriFlow if config.veriflow_enabled: if self._stop_veriflow(veriflow) == False: proxy.terminate() self._stop_controllers() return (False, "System Failure", 0) res = self._check_veriflow_results() if result[0] == True: result = res # Stop Proxy if self._stop_proxy(proxy) == False: self._stop_controllers() return (False, "System Failure", 0) # Stop Controllers self._stop_controllers() # Cleanup Any Mininet Remnants self._cleanup() # result = [result[0], result[1], 0] # Check Performance sw_perf = '' if self.creating_baseline else self._validate_stat( 'Switch', result, self.switch_stat, 'switch_stat_dict') ctl_perf = 'Controller stat: ' + repr( self.controller_stat_dict ) if self.creating_baseline else self._validate_stat( 'Controller', result, self.controller_stat, 'controller_stat_dict') # Log self.log.flush() self.log.write("*****************\n") self.log.write("********* Test Script output ********\n") if test_std_err: self.log.write(test_std_err) self.log.write("*****************\n") self.log.write("Switch Performance: %s\n" % (sw_perf if len(sw_perf) else "Expected")) self.log.write("*****************\n") self.log.write("Controller Performance: %s\n" % (ctl_perf if len(ctl_perf) else "Expected")) self.log.write("*****************\n") self.log.write("Rebaseline Suggestion: %d\n" % result[2]) self.log.write("*****************\n") self.log.write("Veriflow Flips: %s\n" % (str(self.veriflow_flips))) self.log.write("*****************\n") if result[0] == False and result[1] == "VeriFlow": self.log.write("VeriFlow Output:\n") for f in self.veriflow_output: self.log.write(f + "\n") self.log.write("*****************\n") self.log.write("Rule State:\n") for r in self.rule_state: self.log.write(r + "\n") self.log.write("Num: %d\n" % (len(self.rule_state))) self.log.write("*****************\n") self.log.write("Message Types Seen: %s\n" % (str(self.msg_types))) self.log.write("*****************\n") self.log.write("Test Result: " + str(result[0]) + " , Reason:" + str(result[1]) + "\n") self.log.write(str(datetime.today()) + "\n") self.log.write("##############################Ending Test " + str(self.testnum) + "###################################\n") self.log.flush() self.testnum += 1 return result