Esempio n. 1
0
    def _stop_capture(self, cap):
        if not cap:
            return True

        ts = time.time()

        #Kill tcpdump
        shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        try:
            shell.run(["/bin/bash", "-i", "-c", config.capture_kill_cmd],
                      allow_error=True)
        except Exception as e:
            pass
        cap.wait_for_result()

        if len(self.last_cap) > 0:
            os.system(
                "scp -q -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no %s@%s:/root/capture.dmp %s\n"
                % (config.vm_ssh_key, config.vm_user, mv.vm2ip(
                    self.tc[0]), self.last_cap))

        self.log.write('[timer] Stop Capture: %f sec.\n' % (time.time() - ts))
        return True
Esempio n. 2
0
    def _start_monitor(self):
        monitor = None
        ts = time.time()
        cmd = config.monitor_cmd.format(port=str(config.monitor_com_port),
                                        proxy=mv.vm2ip(self.tc[0]),
                                        proxyport=str(config.proxy_com_port))
        self.log.write("Monitor CMD: " + cmd + "\n")

        shell = spur.SshShell(hostname=mv.vm2ip(self.mon[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        #Start Proxy
        monitor = shell.spawn(["/bin/bash", "-i", "-c", cmd],
                              store_pid=True,
                              allow_error=True)
        if not monitor.is_running():
            res = monitor.wait_for_result()
            self.log.write("Monitor Failed to Start: " + res.output +
                           res.stderr_output)
            return None
        else:
            self.log.write("Started monitor on " + str(mv.vm2ip(self.mon[0])) +
                           "...\n")

        #Wait for proxy to come up
        if (self._waitListening(mv.vm2ip(self.mon[0]), config.monitor_com_port,
                                240, False) == False):
            self.log.write("Monitor Failed to start after 240 seconds!\n")
            print "Monitor Failed to Start after 240 seconds!"
            return None

        self.log.write('[timer] Start monitor: %f sec.\n' % (time.time() - ts))
        self.monitor_running = True
        return monitor
Esempio n. 3
0
    def _start_proxy(self):
        proxy = None
        ts = time.time()
        cmd = config.proxy_cmd.format(port=str(config.proxy_com_port))
        self.log.write("Proxy CMD: " + cmd + "\n")

        shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)

        #Setup NetEM
        try:
            ret = shell.run(["/bin/bash", "-i", "-c", config.limit_cmd])
            self.log.write("Setting up netem:\n" + ret.output)
        except Exception as e:
            print "Setting up netem failed: " + str(e)
            self.log.write("Setting up netem failed: " + str(e) + "\n")
            return None

        #Start Proxy
        proxy = shell.spawn(["/bin/bash", "-i", "-c", cmd],
                            store_pid=True,
                            allow_error=True)
        if not proxy.is_running():
            res = proxy.wait_for_result()
            self.log.write("Proxy Failed to Start: " + res.output +
                           res.stderr_output)
            return None
        else:
            self.log.write("Started proxy on " + str(mv.vm2ip(self.tc[0])) +
                           "...\n")

        #Wait for proxy to come up
        if (self._waitListening(mv.vm2ip(self.tc[0]), config.proxy_com_port,
                                240, False) == False):
            self.log.write("Proxy Failed to start after 240 seconds!\n")
            print "Proxy Failed to Start after 240 seconds!"
            return None
        self.log.write('[timer] Start proxy: %f sec.\n' % (time.time() - ts))
        self.proxy_running = True
        return proxy
Esempio n. 4
0
 def _cleanup(self):
     ts = time.time()
     shell = spur.SshShell(hostname=mv.vm2ip(self.mininet[0]),
                           username=config.mininet_user,
                           missing_host_key=spur.ssh.MissingHostKey.accept,
                           private_key_file=config.vm_ssh_key)
     res = shell.run(["/bin/bash", "-i", "-c", config.mininet_cleanup_cmd])
     if config.enable_stat:
         self.log.write('[timer] Clean up mininet: %f sec.\n' %
                        (time.time() - ts))
     return True
Esempio n. 5
0
 def _stop_controllers(self):
     if self.controllers_running is False:
         return True
     ts = time.time()
     for c in self.controllers:
         shell = spur.SshShell(
             hostname=mv.vm2ip(c),
             username=config.controller_user,
             missing_host_key=spur.ssh.MissingHostKey.accept,
             private_key_file=config.vm_ssh_key)
         try:
             self.log.write("Stopping controller (" + mv.vm2ip(c) +
                            ")...\n")
             res = shell.run([
                 "/bin/bash", "-i", "-c",
                 "~/monitors/control.sh {0} {1}".format(
                     config.controller_type, "stop")
             ],
                             allow_error=True)
             try:
                 self.controller_stat_dict = ProcMonStat.extract_stat(
                     res.output)
             except Exception as e:
                 import traceback
                 self.log.write(
                     'Could not extract controller procmon stat: ' +
                     str(e) + '.\n')
                 print(traceback.format_exc())
                 self.log.write(res.output)
         except Exception as e:
             print e
             self.log.write("Exception: " + str(e) + "\n")
             self.log.flush()
             return False
     if config.enable_stat:
         self.log.write('[timer] Stop controllers: %f sec.\n' %
                        (time.time() - ts))
     self.controllers_running = False
     return True
Esempio n. 6
0
 def _start_controllers(self):
     ts = time.time()
     for c in self.controllers:
         shell = spur.SshShell(
             hostname=mv.vm2ip(c),
             username=config.controller_user,
             missing_host_key=spur.ssh.MissingHostKey.accept,
             private_key_file=config.vm_ssh_key)
         res = shell.run([
             "/bin/bash", "-i", "-c",
             "~/monitors/control.sh {0} {1}".format(config.controller_type,
                                                    "start")
         ])
         self.log.write("Starting Controller (" + mv.vm2ip(c) + ")...\n" +
                        res.output)
         self.log.flush()
     for c in self.controllers:
         if (self._waitListening(mv.vm2ip(c), config.controller_port,
                                 60) == False):
             self.log.write(
                 "Controller %s failed to start after %d seconds" %
                 (mv.vm2ip(c), 60))
             self.log.flush()
             return False
         else:
             # Controller starts.
             res = shell.run([
                 "/bin/bash", "-i", "-c",
                 "~/monitors/control.sh {0} {1}".format(
                     config.controller_type, "mon")
             ])
             self.log.write("Starting resource monitor for controller (" +
                            mv.vm2ip(c) + ")...\n" + res.output)
             self.log.flush()
     if config.enable_stat:
         self.log.write('[timer] Start all controllers: %f sec.\n' %
                        (time.time() - ts))
     self.controllers_running = True
     return True
Esempio n. 7
0
 def startVms(self):
     for c in self.controllers:
         mv.startvm(c)
     for m in self.mininet:
         mv.startvm(m)
     for c in self.controllers:
         if (self._waitListening(mv.vm2ip(c), 22, 240, True) == False):
             print "Error: Controller VM %d not started!" % (c)
             return False
         else:
             os.system(
                 "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n"
                 % (config.vm_ssh_key, monitor_tools_path,
                    config.controller_user, mv.vm2ip(c)))
             shell = spur.SshShell(
                 hostname=mv.vm2ip(c),
                 username=config.controller_user,
                 missing_host_key=spur.ssh.MissingHostKey.accept,
                 private_key_file=config.vm_ssh_key)
             proc = shell.run([
                 "/bin/bash", "-i", "-c",
                 "cd monitors && make clean && make"
             ])
             if proc.return_code is not 0:
                 print "Error: Make failed!"
                 return False
     for m in self.mininet:
         if (self._waitListening(mv.vm2ip(m), 22, 240, True) == False):
             print "Error: Mininet %d not started!" % (c)
             return False
         else:
             if config.mininet_replace_scripts:
                 os.system(
                     "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s/* %s@%s:~\n"
                     % (config.vm_ssh_key, mininet_config_path,
                        config.mininet_user, mv.vm2ip(m)))
                 os.system(
                     "scp -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n"
                     % (config.vm_ssh_key, monitor_tools_path,
                        config.mininet_user, mv.vm2ip(m)))
                 shell = spur.SshShell(
                     hostname=mv.vm2ip(m),
                     username=config.mininet_user,
                     missing_host_key=spur.ssh.MissingHostKey.accept,
                     private_key_file=config.vm_ssh_key)
                 proc = shell.run([
                     "/bin/bash", "-i", "-c",
                     "cd monitors && make clean && make"
                 ])
                 if proc.return_code is not 0:
                     print "Error: Make failed!"
                     return False
     return True
Esempio n. 8
0
    def _cleanup(self):
        ts = time.time()

        #Kill Proxy
        shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        try:
            ret = shell.run(["/bin/bash", "-i", "-c", config.proxy_kill_cmd])
        except Exception as e:
            return False

        #Kill Monitor
        shell = spur.SshShell(hostname=mv.vm2ip(self.mon[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        try:
            ret = shell.run(["/bin/bash", "-i", "-c", config.monitor_kill_cmd])
        except Exception as e:
            return False
        self.log.write('[timer] Clean up: %f sec.\n' % (time.time() - ts))
        return True
Esempio n. 9
0
    def _query_proxy_done(self):
        last = 0
        cmd = "*,*,{0},0,0,*,ACTIVE,*".format(config.protocol)
        for t in self.tc:
            res = self._proxy_communicate((mv.vm2ip(t), config.proxy_com_port),
                                          cmd,
                                          wait_for_response=True)
            if type(res) is bool and res is False:
                return False
            if type(res) is not str:
                return False
            last = float(res)

        if last < 10:
            return False

        if time.time() - last >= config.test_max_idle:
            return True
Esempio n. 10
0
    def _send_proxy_strategy(self, strategy):
        strat = ""
        # Default strategy
        if strategy == None:
            strategy = [
                "*,*,TCP,0,0,*,CLEAR,*",
                "{0},{1},{2},0,0,*,CLEAR,*".format(config.target_client_ip,
                                                   config.target_server_ip,
                                                   config.protocol)
            ]
        ts = time.time()

        for l in strategy:
            if type(l) is dict:
                if 'action' not in l:
                    return False
                strat = l['action']
                if 'time' in l and l['time'] > 0.01:
                    strat = dict(l)
                    strat['time'] = 0
                    tmr = threading.Timer(l['time'], self._send_proxy_strategy,
                                          [[strat], proxyaddrs])
                    tmr.start()
                    self.timers.append(tmr)
                    continue
            elif type(l) is str:
                strat = l
            else:
                return False
            self.log.write("Strategy CMD: " + strat + "\n")
            self.log.flush()
            for t in self.tc:
                res = self._proxy_communicate(
                    (mv.vm2ip(t), config.proxy_com_port), strat)
                if (res == False):
                    self.log.write("Failed to Send Command\n")
                    self.log.flush()
                    return False
        self.log.write('[timer] Send strategy: %f sec.\n' % (time.time() - ts))
        return True
Esempio n. 11
0
    def _call_test(self, test_script, cmd, proxyaddrs):
        # Build Config and default command
        cfg = {
            'controllers': proxyaddrs,
            'topo_discovery': config.topo_discovery_delay,
            'fail_mode': config.mininet_fail_mode
        }
        if cmd == None:
            cmd = [{'cmd': 'basic'}]

        ts = time.time()
        res = None
        exec_res = None
        m = self.mininet[0]
        shell = spur.SshShell(hostname=mv.vm2ip(m),
                              username=config.mininet_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        self.log.write("Starting Test: " + test_script + "\n")
        self.log.write("Test Config: %s\n" % cfg)
        for c in cmd:
            self.log.write("Test Command: %s\n" % c)
        self.log.flush()
        try:
            proc = shell.spawn(["/bin/bash", "-i", "-c", test_script])
            proc.stdin_write("%s\n" % repr(cfg))
            for c in cmd:
                proc.stdin_write("%s\n" % repr(c))
            exec_res = proc.wait_for_result()
            res = eval(exec_res.output)
        except Exception as e:
            print e
            self.log.write("Exception: " + str(e) + "\n")
            self.log.flush()
            return [None, None]
        if config.enable_stat:
            self.log.write('[timer] Do all tests: %f sec.\n' %
                           (time.time() - ts))
        return [res, exec_res.stderr_output]
Esempio n. 12
0
    def _query_proxy_conn_info(self):
        length = 0
        total_data = 0
        cmd = "{0},{1},{2},0,0,*,TIME,*".format(config.target_client_ip,
                                                config.target_server_ip,
                                                config.protocol)
        for t in self.tc:
            res = self._proxy_communicate((mv.vm2ip(t), config.proxy_com_port),
                                          cmd,
                                          wait_for_response=True)
            if type(res) is bool and res is False:
                return (False, 0, "System Error")
            if type(res) is not str:
                return (False, 0, "System Error")
            lns = res.split()
            if len(lns) != 2:
                return (False, 0, "System Error")
            length = float(lns[0])
            total_data = int(lns[1])

        if length < 0 or length > 200:
            return False, length, total_data
        return True, length, total_data
Esempio n. 13
0
    def _start_capture(self):
        self.last_cap = ""
        if not self.do_capture:
            return None

        #Generate Capture Name
        time_str = time.strftime(config.captures_time_str)
        fname = config.captures_loc.format(tm=time_str, exe=self.instance)
        self.last_cap = fname

        #Generate capture command
        cmd = config.capture_cmd

        #Start SSH Shell
        shell = spur.SshShell(hostname=mv.vm2ip(self.tc[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)

        #Start Capture
        cap = None
        try:
            cap = shell.spawn(["/bin/bash", "-i", "-c", cmd],
                              store_pid=True,
                              allow_error=True)
            if not cap.is_running():
                res = proxy.wait_for_result()
                self.log.write("Capture Failed to Start: " + res.stderr_output)
                return None
        except Exception as e:
            print e
            self.log.write("Exception: " + str(e) + "\n")
            self.log.flush()
            return None

        return cap
Esempio n. 14
0
    def _call_test(self):
        ts = time.time()

        #Start servers
        for s in self.servers:
            if mv.vmHasSSH(s):
                shell = spur.SshShell(
                    hostname=mv.vm2ip(s),
                    username=config.vm_user,
                    missing_host_key=spur.ssh.MissingHostKey.accept,
                    private_key_file=config.vm_ssh_key)
                try:
                    ret = shell.run(
                        ["/bin/bash", "-i", "-c", config.server_start_cmd])
                except Exception as e:
                    print "Failed to start server"
                    self.log.write("Failed to start server\n")
                    return False, 0
            if mv.vmCanPing(s) and self._waitListening(mv.vm2ip(s), 80, 240,
                                                       False) == False:
                print "Failed to start server"
                self.log.write("Failed to start server\n")
                return False, 0
        self.log.write("Servers Started...\n")

        time.sleep(0.5)

        #Start background traffic
        shell = spur.SshShell(hostname=mv.vm2ip(self.clients[1]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        background = shell.spawn([
            "/bin/bash", "-i", "-c",
            config.background_client_cmd.format(tm=str(config.max_time))
        ],
                                 store_pid=True,
                                 allow_error=True)
        if not background.is_running():
            ret = background.wait_for_result()
            print "Background traffic command failed: %s %s" % (
                ret.output, ret.stderr_output)
            self.log.write("Background traffic command failed: %s %s\n" %
                           (ret.output, ret.stderr_output))
            return False, 0
        bts = time.time()

        #Start main traffic
        shell = spur.SshShell(hostname=mv.vm2ip(self.clients[0]),
                              username=config.vm_user,
                              missing_host_key=spur.ssh.MissingHostKey.accept,
                              private_key_file=config.vm_ssh_key)
        mts = time.time()
        ret = None
        speed = 0
        bspeed = 0
        main = shell.spawn([
            "/bin/bash", "-i", "-c",
            config.main_client_cmd.format(tm=str(config.max_time))
        ],
                           store_pid=True,
                           allow_error=True)
        if not main.is_running():
            ret = main.wait_for_result()
            print "Main Traffic Command failed: %s %s" % (ret.output,
                                                          ret.stderr_output)
            self.log.write("Main Traffic Command Failed: %s %s\n" %
                           (ret.output, ret.stderr_output))
            try:
                background.send_signal(2)
            except Exception as e:
                pass
            return False, 0

        #Wait to finish
        self.last_termination_idle = False
        while background.is_running() or main.is_running():
            if background.is_running():
                bspeed = time.time() - bts
            if main.is_running():
                speed = time.time() - mts
            if self._query_proxy_done():
                self.last_termination_idle = True
                try:
                    background.send_signal(2)
                except Exception as e:
                    pass
                try:
                    main.send_signal(2)
                except Exception as e:
                    pass
                break
            time.sleep(1)

        #Check Main Return code
        self.return_code_error = False
        ret = main.wait_for_result()
        if ret.return_code is not 0:
            self.log.write("Main Traffic Command Failed! Return Code: %d\n" %
                           (ret.return_code))
            print "Main Traffic Command Failed! Return Code: %d" % (
                ret.return_code)
            speed = config.max_time
        if ret.return_code is 56:
            self.return_code_error = True
        self.log.write("Main Traffic command output: \n" + ret.stderr_output)

        #Check Background Return code
        ret = background.wait_for_result()
        if ret.return_code is not 0:
            self.log.write(
                "Background Traffic Command Failed! Return Code: %d\n" %
                (ret.return_code))
            print "Background Traffic Command Failed! Return Code: %d" % (
                ret.return_code)
            bspeed = config.max_time
        if ret.return_code is 56:
            self.return_code_error = True
        self.log.write("Background Traffic command output: \n" +
                       ret.stderr_output)

        return True, speed
Esempio n. 15
0
    def startVms(self):
        for c in self.clients:
            mv.startvm(c)
        for s in self.servers:
            mv.startvm(s)
        for t in self.tc:
            mv.startvm(t)
        for m in self.mon:
            mv.startvm(m)
        for c in self.clients:
            if (self._waitListening(mv.vm2ip(c), 22, 240, True) == False):
                print "Error: client VM %d not started!" % (c)
                return False
        for s in self.servers:
            if not mv.vmCanPing(s):
                time.sleep(30)
            else:
                if (self._waitListening(mv.vm2ip(s),
                                        22 if mv.vmHasSSH(s) else 80, 240,
                                        True) == False):
                    print "Error: server VM %d not started!" % (s)
                    return False

        if hasattr(config, 'background_server_config') and len(
                config.background_server_config) > 0 and mv.vmHasSSH(
                    self.servers[1]):
            shell = spur.SshShell(
                hostname=mv.vm2ip(self.servers[1]),
                username=config.vm_user,
                missing_host_key=spur.ssh.MissingHostKey.accept,
                private_key_file=config.vm_ssh_key)
            proc = shell.run(
                ["/bin/bash", "-i", "-c", config.background_server_config])
            if proc.return_code is not 0:
                print "Error: Background server config failed!"
                return False

        for t in self.tc:
            if (self._waitListening(mv.vm2ip(t), 22, 240, True) == False):
                print "Error: Traffic Shaping VM %d not started!" % (t)
                return False
            else:
                if config.vm_replace_data:
                    os.system(
                        "scp -r -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n"
                        % (config.vm_ssh_key, proxy_path, config.vm_user,
                           mv.vm2ip(t)))
                    shell = spur.SshShell(
                        hostname=mv.vm2ip(t),
                        username=config.vm_user,
                        missing_host_key=spur.ssh.MissingHostKey.accept,
                        private_key_file=config.vm_ssh_key)
                    proc = shell.run([
                        "/bin/bash", "-i", "-c",
                        "cd proxy && make clean && make"
                    ])
                    if proc.return_code is not 0:
                        print "Error: Make failed!"
                        return False
        for m in self.mon:
            if (self._waitListening(mv.vm2ip(m), 22, 240, True) == False):
                print "Error: Monitor VM %d not started!" % (t)
                return False
            else:
                if config.vm_replace_data:
                    os.system(
                        "scp -r -p -i %s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -r %s %s@%s:~\n"
                        % (config.vm_ssh_key, monitor_path, config.vm_user,
                           mv.vm2ip(m)))
                    shell = spur.SshShell(
                        hostname=mv.vm2ip(m),
                        username=config.vm_user,
                        missing_host_key=spur.ssh.MissingHostKey.accept,
                        private_key_file=config.vm_ssh_key)
                    proc = shell.run([
                        "/bin/bash", "-i", "-c",
                        "cd monitor && make clean && make"
                    ])
                    if proc.return_code is not 0:
                        print "Error: Make failed!"
                        return False

        return True
Esempio n. 16
0
    def doTest(self, strategy):
        """
        :return [True | False, str, int]: First boolean value indicates pass or fail,
        	followed by an explanation, and an int indicating the suggestion of doing rebaseline.
        """
        if 'topo' not in strategy:
            return (False, "System Failure", 0)
        test_script = strategy['topo']
        self.msg_types = []
        if hasattr(self, 'switch_stat_dict'):
            del self.switch_stat_dict
        if hasattr(self, 'controller_stat_dict'):
            del self.controller_stat_dict
        result = [True, "Success!", 0]
        self.log.write('#' * 30 + "Starting Test " + str(self.testnum) +
                       '#' * 30 + '\n')
        self.log.write(str(datetime.today()) + "\n")
        # Create Address/Port strings
        controlleraddrs = list()
        proxyaddrs = list()
        for c in self.controllers:
            controlleraddrs.append(
                mv.vm2ip(c) + ":" + str(config.controller_port))
            proxyaddrs.append((config.proxy_addr, config.proxy_base_port + c))

        # Start Controllers
        if self._start_controllers() == False:
            return (False, "System Failure", 0)

        # Start Proxy
        proxy = self._start_proxy(controlleraddrs, proxyaddrs)
        if proxy is None:
            self._stop_controllers()
            return (False, "System Failure", 0)

        # Send Proxy Strategy
        if self._send_proxy_strategy(strategy['switch'], proxyaddrs) == False:
            self._stop_controllers()
            proxy.terminate()
            return (False, "System Failure", 0)

        # Initialize Controller strategies, if any
        if 'controller' in strategy:
            if self._handle_controller_actions(strategy['controller'],
                                               proxy) == False:
                self._stop_controllers()
                proxy.terminate()
                return (False, "System Failure", 0)

        # VeriFlow
        veriflow = None
        if config.veriflow_enabled == True:
            vf_port = config.veriflow_base_port + self.controllers[0]
            veriflow = self._start_veriflow(test_script, proxyaddrs, vf_port)
            proxyaddrs = list()
            proxyaddrs.append(
                (config.proxy_addr, vf_port, confg.proxy_base_port + 1))
            if veriflow is None:
                self._stop_controllers()
                proxy.terminate()
                return (False, "System Failure", 0)

        # Do Test
        test_std_err = None
        res, test_std_err = self._call_test(test_script, strategy['host'],
                                            proxyaddrs)
        if res is None:
            self._stop_controllers()
            if veriflow is not None:
                veriflow.terminate()
            proxy.terminate()
            return (False, "System Failure", 0)

        # Evaluate Results
        if isinstance(res, (dict)) and "results" in res:
            res_list = res["results"]
            for r in res_list:
                if r == False:
                    result[0] = False
                    result[1] = "Network Tests"

            # Check Rules
            if "rules" in res:
                if self._check_rule_dump(res["rules"]) == False:
                    result[0] = False
                    result[1] = "Network State"

            # Save stat dictionary for switch
            if 'stat' in res:
                self.switch_stat_dict = res['stat']
        else:
            result[0] = False
            result[1] = "System Failure"

        # Check Message Types
        if self._get_msg_types(
            ("localhost", config.proxy_com_port + self.mininet[0])) == False:
            if veriflow is not None:
                veriflow.terminate()
            proxy.terminate()
            self._stop_controllers()
            return (False, "System Failure", 0)
        if self._check_for_error_msgs():
            result[0] = False
            result[1] = "Error Message"

        # Check and Stop VeriFlow
        if config.veriflow_enabled:
            if self._stop_veriflow(veriflow) == False:
                proxy.terminate()
                self._stop_controllers()
                return (False, "System Failure", 0)
            res = self._check_veriflow_results()
            if result[0] == True:
                result = res

        # Stop Proxy
        if self._stop_proxy(proxy) == False:
            self._stop_controllers()
            return (False, "System Failure", 0)

        # Stop Controllers
        self._stop_controllers()

        # Cleanup Any Mininet Remnants
        self._cleanup()

        # result = [result[0], result[1], 0]

        # Check Performance
        sw_perf = '' if self.creating_baseline else self._validate_stat(
            'Switch', result, self.switch_stat, 'switch_stat_dict')
        ctl_perf = 'Controller stat: ' + repr(
            self.controller_stat_dict
        ) if self.creating_baseline else self._validate_stat(
            'Controller', result, self.controller_stat, 'controller_stat_dict')

        # Log
        self.log.flush()
        self.log.write("*****************\n")
        self.log.write("********* Test Script output ********\n")
        if test_std_err:
            self.log.write(test_std_err)
        self.log.write("*****************\n")
        self.log.write("Switch Performance: %s\n" %
                       (sw_perf if len(sw_perf) else "Expected"))
        self.log.write("*****************\n")
        self.log.write("Controller Performance: %s\n" %
                       (ctl_perf if len(ctl_perf) else "Expected"))
        self.log.write("*****************\n")
        self.log.write("Rebaseline Suggestion: %d\n" % result[2])
        self.log.write("*****************\n")
        self.log.write("Veriflow Flips: %s\n" % (str(self.veriflow_flips)))
        self.log.write("*****************\n")
        if result[0] == False and result[1] == "VeriFlow":
            self.log.write("VeriFlow Output:\n")
            for f in self.veriflow_output:
                self.log.write(f + "\n")
            self.log.write("*****************\n")
        self.log.write("Rule State:\n")
        for r in self.rule_state:
            self.log.write(r + "\n")
        self.log.write("Num: %d\n" % (len(self.rule_state)))
        self.log.write("*****************\n")
        self.log.write("Message Types Seen: %s\n" % (str(self.msg_types)))
        self.log.write("*****************\n")
        self.log.write("Test Result: " + str(result[0]) + " , Reason:" +
                       str(result[1]) + "\n")
        self.log.write(str(datetime.today()) + "\n")
        self.log.write("##############################Ending Test " +
                       str(self.testnum) +
                       "###################################\n")
        self.log.flush()
        self.testnum += 1
        return result