def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info("ONOS HA test: Restart all ONOS nodes - " + "initialization") # set global variables # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() main.Cluster.numCtrls = 1 try: from tests.HA.dependencies.HA import HA main.HA = HA() from tests.HA.HAswapNodes.dependencies.Server import Server main.Server = Server() # load some variables from the params file cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] stepResult = main.testSetUp.envSetup() except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) main.testSetUp.ONOSSetUp(main.Cluster, cellName=cellName, removeLog=True, extraApply=[ main.HA.setServerForCluster, main.HA.scalingMetadata, main.HA.startingMininet, main.HA.copyBackupConfig, main.HA.setMetadataUrl ], extraClean=main.HA.cleanUpOnosService, installMax=True) main.HA.initialSetUp(serviceClean=True)
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ import imp import pexpect import time import json main.log.info("ONOS HA test: Stop a minority of ONOS nodes - " + "initialization") # set global variables # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() # load some variables from the params file cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] main.numCtrls = int(main.params['num_controllers']) stepResult = main.testSetUp.envSetup() except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) main.testSetUp.ONOSSetUp(main.Mininet1, main.Cluster, cellName=cellName, removeLog=True, extraApply=[ main.HA.startingMininet, main.HA.customizeOnosGenPartitions ], extraClean=main.HA.cleanUpGenPartition) main.HA.initialSetUp()
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info("ONOS HA test: Restart a minority of ONOS nodes - " + "initialization") # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] stepResult = main.testSetUp.envSetup() except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) main.HA.generateGraph("HAkillNodes") main.testSetUp.ONOSSetUp(main.Mininet1, main.Cluster, cellName=cellName, removeLog=True, extraApply=[ main.HA.startingMininet, main.HA.customizeOnosGenPartitions, main.HA.copyBackupConfig, main.ONOSbench.preventAutoRespawn ], extraClean=main.HA.cleanUpGenPartition) main.HA.initialSetUp(serviceClean=True)
def CASE1( self, main ): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info( "ONOS HA Sanity test - initialization" ) # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error( "ONOSSetup not found. exiting the test" ) main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() cellName = main.params[ 'ENV' ][ 'cellName' ] main.apps = main.params[ 'ENV' ][ 'appString' ] stepResult = main.testSetUp.envSetup() except Exception as e: main.testSetUp.envSetupException( e ) main.testSetUp.evnSetupConclusion( stepResult ) main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True, extraApply=main.HA.startingMininet ) main.HA.initialSetUp()
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info("ONOS HA test: Stop a minority of ONOS nodes - " + "initialization") # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] stepResult = main.testSetUp.envSetup(includeCaseDesc=False) except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) applyFuncs = [main.HA.copyBackupConfig] applyArgs = [None] try: if main.params['topology']['topoFile']: main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' ) else: applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) except (KeyError, IndexError): applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) main.testSetUp.ONOSSetUp(main.Cluster, cellName=cellName, removeLog=True, extraApply=applyFuncs, applyArgs=applyArgs, extraClean=main.HA.cleanUpGenPartition, includeCaseDesc=False) main.HA.initialSetUp(serviceClean=True) main.step('Set logging levels') logging = True try: logs = main.params.get('ONOS_Logging', False) if logs: for namespace, level in logs.items(): for ctrl in main.Cluster.active(): ctrl.CLI.logSet(level, namespace) except AttributeError: logging = False utilities.assert_equals(expect=True, actual=logging, onpass="******", onfail="Failed to set log levels")
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info("ONOS Single node cluster restart " + "HA test - initialization") # set global variables # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() # load some variables from the params file cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] main.numCtrls = int(main.params['num_controllers']) stepResult = main.testSetUp.envSetup(includeCaseDesc=False) except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) cellApps = str(main.params["ENV"]["appString"]) cellNAme = str(main.params["ENV"]["appString"]) applyFuncs = [main.testSetUp.createApplyCell] applyArgs = [[ main.Cluster, True, cellName, cellApps, "", True, main.Cluster.runningNodes[0].ipAddress ]] try: if main.params['topology']['topoFile']: main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' ) else: applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) except (KeyError, IndexError): applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) main.Cluster.setRunningNode(int(main.params['num_controllers'])) ip = main.Cluster.getIps(allNode=True) main.testSetUp.ONOSSetUp(main.Cluster, cellName="SingleHA", extraApply=applyFuncs, applyArgs=applyArgs, includeCaseDesc=False) main.HA.initialSetUp() main.step('Set logging levels') logging = True try: logs = main.params.get('ONOS_Logging', False) if logs: for namespace, level in logs.items(): for ctrl in main.Cluster.active(): ctrl.CLI.logSet(level, namespace) except AttributeError: logging = False utilities.assert_equals(expect=True, actual=logging, onpass="******", onfail="Failed to set log levels")
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ import re main.log.info("ONOS HA test: Restart all ONOS nodes - " + "initialization") # set global variables # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() main.Cluster.numCtrls = 1 try: from tests.HA.dependencies.HA import HA main.HA = HA() # load some variables from the params file cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] stepResult = main.testSetUp.envSetup(includeCaseDesc=False) except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) main.scaling = main.params['scaling'].split(",") main.log.debug(main.scaling) scale = main.scaling.pop(0) main.log.debug(scale) main.Cluster.setRunningNode(int(re.search("\d+", scale).group(0))) applyFuncs = [main.HA.removeKarafConsoleLogging] applyArgs = [None] try: if main.params['topology']['topoFile']: main.log.info( 'Skipping start of Mininet in this case, make sure you start it elsewhere' ) else: applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) except (KeyError, IndexError): applyFuncs.append(main.HA.startingMininet) applyArgs.append(None) main.testSetUp.ONOSSetUp(main.Cluster, cellName=cellName, extraApply=applyFuncs, applyArgs=applyArgs, installMax=True, atomixClusterSize=3, includeCaseDesc=False) main.HA.initialSetUp() main.step('Set logging levels') logging = True try: logs = main.params.get('ONOS_Logging', False) if logs: for namespace, level in logs.items(): for ctrl in main.Cluster.active(): ctrl.CLI.logSet(level, namespace) except AttributeError: logging = False utilities.assert_equals(expect=True, actual=logging, onpass="******", onfail="Failed to set log levels")
def CASE300(self, main): """ Stop 1 ONOS node at a time and test connectivity """ from tests.USECASE.VPLS.dependencies import vpls from tests.HA.dependencies.HA import HA assert vpls, "vpls not defined" main.HA = HA() main.case("Stop one ONOS node at a time and test connectivity.") result = main.TRUE for i in range(0, main.Cluster.numCtrls): stri = str(i) ip_address = main.Cluster.active(i).ipAddress # Stop an ONOS node: i main.step("Stop ONOS node " + stri + ".") stopResult = main.ONOSbench.onosStop(ip_address) main.Cluster.runningNodes[i].active = False utilities.assert_equals( expect=main.TRUE, actual=stopResult, onpass="******", onfail="ONOS nodes NOT successfully stopped.") # Check connectivity connectivityResult = vpls.testConnectivityVpls(main, isNodeUp=False) result = result and connectivityResult # Restart ONOS node main.step("Restart ONOS node " + stri + " and checking status of restart.") startResult = main.ONOSbench.onosStart(ip_address) utilities.assert_equals( expect=main.TRUE, actual=startResult, onpass="******", onfail="ONOS nodes NOT successfully started.") result = result and startResult # Check if ONOS is up yet main.log.info("Checking if ONOS node " + stri + " is up.") upResult = main.ONOSbench.isup(ip_address) utilities.assert_equals( expect=main.TRUE, actual=upResult, onpass="******", onfail="ONOS did NOT successfully restart.") # Restart CLI main.log.info("Restarting ONOS node " + stri + "'s main.CLI.") cliResults = main.Cluster.active(0).CLI.startOnosCli(ip_address) main.Cluster.runningNodes[i].active = True utilities.assert_equals( expect=main.TRUE, actual=cliResults, onpass="******", onfail="ONOS CLI did NOT successfully restart.") # Run some basic checks to see if ONOS node truly has succesfully restarted: # Checking if all nodes appear with status READY using 'nodes' command main.step("Checking ONOS nodes.") nodeResults = utilities.retry(main.Cluster.nodesCheck, False, sleep=main.timeSleep, attempts=main.numAttempts) utilities.assert_equals(expect=True, actual=nodeResults, onpass="******", onfail="Nodes check NOT successful.") # All apps that are present are active main.log.info("Checking if apps are active.") compareAppsResult = vpls.compareApps(main) utilities.assert_equals( expect=main.TRUE, actual=compareAppsResult, onpass="******", onfail="Apps are NOT the same across all nodes.") result = result and compareAppsResult # Check connectivity connectivityResult = vpls.testConnectivityVpls(main) result = result and connectivityResult if not result: break utilities.assert_equals(expect=main.TRUE, actual=result, onpass="******", onfail="Connectivity is NOT as expected.")
def CASE2(self, main): """ Kill ONOS node, and measure the latency for INSTANCE_DEACTIVATED, MASTER_CHANGED, and role request ( tshark time ), then bring the node back up. """ import time import datetime import numpy from tests.HA.dependencies.HA import HA main.HA = HA() main.latencyData = { 'kill_to_deactivation': [], 'deactivation_to_role_request': [] } main.failCounter = 0 passingResult = True criticalError = False main.step("Gathering data starting with " + str(main.warmUp) + " warm ups and a sample size of " + str(main.sampleSize)) for iteration in range(0, main.sampleSize + main.warmUp): main.log.info("==========================================") main.log.info("================iteration:{}==============".format( str(iteration + 1))) ip_address = main.Cluster.active(0).ipAddress strNodeNumToKill = str(main.nodeNumToKill) main.log.info("Assigning mastership to ONOS node " + strNodeNumToKill) main.Cluster.active(0).CLI.deviceRole(main.switchID, ip_address) main.log.info("Sleeping for " + str(main.recoverySleep) + " seconds...") time.sleep(main.recoverySleep) mastershipCheck = main.Cluster.active(0).CLI.getMaster( main.switchID) == ip_address if not mastershipCheck: main.log.warn("Mastership is NOT as expected.") with open(main.tsharkResultPath, "w") as tshark: tshark.write("") main.log.info("Starting tshark capture.") main.ONOSbench.tsharkGrep(main.ofpRoleRequest, main.tsharkResultPath) time1 = time.time() * 1000.0 # Kill an ONOS node main.log.info("Killing ONOS node " + strNodeNumToKill + ".") killresult = main.ONOSbench.onosKill(ip_address) main.Cluster.runningNodes[main.nodeNumToKill].active = False # Stop an ONOS node main.log.info("Stopping ONOS node " + strNodeNumToKill + ".") stopresult = main.ONOSbench.onosStop(ip_address) killStopResult = stopresult == killresult and True if not killStopResult: main.log.error( "ONOS node was NOT successfully stopped and killed.") criticalError = True time.sleep(main.failoverSleep) # Stop tshark and get times main.log.info("Stopping tshark.") main.ONOSbench.tsharkStop() masterChangedLats = [] instanceDeactivatedLats = [] main.log.info("Obtaining latencies from 'events' output.") for CLInum in range(0, main.Cluster.numCtrls - 1): eventOutput = main.Cluster.active(CLInum).CLI.events( args='-a').split("\r\n") for line in reversed(eventOutput): timestamp = line[:23] if line[ 19] != '-' else line[:19] + '.000' timestamp = float( datetime.datetime.strptime( timestamp, "%Y-%m-%dT%H:%M:%S.%f").strftime('%s.%f')) * 1000.0 if timestamp - time1 >= 0: if "INSTANCE_DEACTIVATED" in line: instanceDeactivatedLats.append(timestamp - time1) elif "MASTER_CHANGED" in line: masterChangedLats.append(timestamp - time1) else: break if instanceDeactivatedLats and masterChangedLats: instanceDeactivatedLats.sort() masterChangedLats.sort() instanceDeactivated = instanceDeactivatedLats[0] masterChanged = masterChangedLats[0] eventLatCheck = True else: eventLatCheck = False main.log.warn( "Latencies were NOT obtained from 'events' successfully.") main.log.info("Obtain latency from tshark output.") tsharkLatCheck = True with open(main.tsharkResultPath, "r") as resultFile: resultText = resultFile.readline() main.log.info("Capture result: " + resultText) resultText = resultText.split() if len(resultText) > 1: roleRequestLat = int(float(resultText[1]) * 1000.0) - time1 resultFile.close() else: main.log.error("Tshark output file is NOT as expected.") tsharkLatCheck = False if not tsharkLatCheck: main.log.warn( "Latency was NOT obtained from tshark successfully.") validDataCheck = False if eventLatCheck and tsharkLatCheck: main.log.info("instanceDeactivated: " + str(instanceDeactivated)) main.log.info("masterChanged: " + str(masterChanged)) main.log.info("roleRequestLat: " + str(roleRequestLat)) if iteration >= main.warmUp: main.log.info("Verifying that the data are valid." ) # Don't record data during a warm-up validDataCheck = roleRequestLat >= 0 and \ instanceDeactivated >= 0 and \ masterChanged >= 0 if not validDataCheck: main.log.warn("Data are NOT valid.") if eventLatCheck and tsharkLatCheck and validDataCheck: main.log.info("Saving data...") if roleRequestLat >= instanceDeactivated: main.latencyData['kill_to_deactivation'].append( instanceDeactivated) main.latencyData[ 'deactivation_to_role_request'].append( roleRequestLat - instanceDeactivated) else: main.latencyData['kill_to_deactivation'].append( roleRequestLat) main.latencyData[ 'deactivation_to_role_request'].append(0) main.log.info( "kill_to_deactivation: " + str(main.latencyData['kill_to_deactivation'][-1])) main.log.info("deactivation_to_role_request: " + str( main.latencyData['deactivation_to_role_request'] [-1])) # Restart ONOS node main.log.info("Restart ONOS node " + strNodeNumToKill + " and checking status of restart.") startResult = main.ONOSbench.onosStart(ip_address) if not startResult: main.log.error("ONOS nodes NOT successfully started.") criticalError = True # Check if ONOS is up yet main.log.info("Checking if ONOS node " + strNodeNumToKill + " is up.") upResult = main.ONOSbench.isup(ip_address) if not upResult: main.log.error("ONOS did NOT successfully restart.") criticalError = True # Restart CLI main.log.info("Restarting ONOS node " + strNodeNumToKill + "'s main.CLI.") cliResult = main.Cluster.active( main.nodeNumToKill).CLI.startOnosCli(ip_address) main.Cluster.runningNodes[main.nodeNumToKill].active = True if not cliResult: main.log.error("ONOS CLI did NOT successfully restart.") criticalError = True main.log.info("Checking ONOS nodes.") nodeResults = utilities.retry(main.Cluster.nodesCheck, False, sleep=5, attempts=50) if not nodeResults: main.log.error("Nodes check NOT successful.") criticalError = True main.log.info("Sleeping for " + str(main.recoverySleep) + " seconds...") time.sleep(main.recoverySleep) if not ( mastershipCheck and eventLatCheck and tsharkLatCheck and validDataCheck ) and \ iteration >= main.warmUp: main.failCounter += 1 main.log.warn("Iteration failed. Failure count: " + str(main.failCounter)) if float(main.failCounter) / float( main.sampleSize) >= main.failPercent or criticalError: main.log.error( str(main.failPercent * 100) + "% or more of data is invalid, or a critical error has occurred." ) passingResult = False break utilities.assert_equals( expect=True, actual=passingResult, onpass="******" + str(main.Cluster.numCtrls) + " data gathering was successful.", onfail="Node scaling " + str(main.Cluster.numCtrls) + " data gathering FAILED. Stopping test.") if not passingResult: main.cleanAndExit()
def CASE1(self, main): """ CASE1 is to compile ONOS and push it to the test machines Startup sequence: cell <name> onos-verify-cell NOTE: temporary - onos-remove-raft-logs onos-uninstall start mininet git pull mvn clean install onos-package onos-install -f onos-wait-for-start start cli sessions start tcpdump """ main.log.info("ONOS Single node cluster restart " + "HA test - initialization") main.case("Setting up test environment") main.caseExplanation = "Setup the test environment including " +\ "installing ONOS, starting Mininet and ONOS" +\ "cli sessions." # set global variables # These are for csv plotting in jenkins main.HAlabels = [] main.HAdata = [] try: from tests.dependencies.ONOSSetup import ONOSSetup main.testSetUp = ONOSSetup() except ImportError: main.log.error("ONOSSetup not found. exiting the test") main.cleanAndExit() main.testSetUp.envSetupDescription() try: from tests.HA.dependencies.HA import HA main.HA = HA() # load some variables from the params file cellName = main.params['ENV']['cellName'] main.apps = main.params['ENV']['appString'] main.numCtrls = int(main.params['num_controllers']) stepResult = main.testSetUp.envSetup() except Exception as e: main.testSetUp.envSetupException(e) main.testSetUp.evnSetupConclusion(stepResult) main.Cluster.setRunningNode(int(main.params['num_controllers'])) ip = main.Cluster.getIps(allNode=True) main.testSetUp.ONOSSetUp( main.Cluster, cellName="SingleHA", removeLog=True, extraApply=[ main.testSetUp.createApplyCell, main.HA.startingMininet, main.testSetUp.createApplyCell ], applyArgs=[[main.Cluster, True, cellName, "", "", True, ip], None, [ main.Cluster, True, "SingleHA", "", "", True, main.Cluster.runningNodes[0].ipAddress ]]) main.HA.initialSetUp()