def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches, links ): """ Params: end1,end2: identify the end switches, ex.: 'leaf1', 'spine1' dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002' port1, port2: respective port of the end switches that connects to the link, ex.:'1' switches, links: number of expected switches and links after linkDown, ex.: '4', '6' Kill a link and verify ONOS can see the proper link change """ main.step( "Restore link between %s and %s" % (end1, end2) ) result = False count = 0 while True: count += 1 main.Mininet1.link( END1=end1, END2=end2, OPTION="up" ) main.Mininet1.link( END2=end1, END1=end2, OPTION="up" ) main.log.info( "Waiting %s seconds for link up to be discovered" % main.linkSleep ) time.sleep( main.linkSleep ) for i in range(0, main.numCtrls): onosIsUp = main.ONOSbench.isup( main.ONOSip[ i ] ) if onosIsUp == main.TRUE: main.CLIs[ i ].portstate( dpid=dpid1, port=port1 ) main.CLIs[ i ].portstate( dpid=dpid2, port=port2 ) time.sleep( main.linkSleep ) result = main.CLIs[ main.active ].checkStatus( numoswitch=switches, numolink=links ) if count > 5 or result: break utilities.assert_equals( expect=main.TRUE, actual=result, onpass="******", onfail="Failed to bring link up" )
def killLink(main, end1, end2, switches, links): """ end1,end2: identify the switches, ex.: 'leaf1', 'spine1' switches, links: number of expected switches and links after linkDown, ex.: '4', '6' Kill a link and verify ONOS can see the proper link change """ main.linkSleep = float(main.params['timers']['LinkDiscovery']) main.step("Kill link between %s and %s" % (end1, end2)) LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down") LinkDown = main.Mininet1.link(END2=end1, END1=end2, OPTION="down") main.log.info("Waiting %s seconds for link down to be discovered" % main.linkSleep) time.sleep(main.linkSleep) topology = utilities.retry(main.CLIs[main.active].checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links }, attempts=10, sleep=main.linkSleep) result = topology & LinkDown utilities.assert_equals(expect=main.TRUE, actual=result, onpass="******", onfail="Failed to turn off link?")
def restoreLink(main, end1, end2, dpid1, dpid2, port1, port2, switches, links): """ Params: end1,end2: identify the end switches, ex.: 'leaf1', 'spine1' dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002' port1, port2: respective port of the end switches that connects to the link, ex.:'1' switches, links: number of expected switches and links after linkDown, ex.: '4', '6' Kill a link and verify ONOS can see the proper link change """ main.step("Restore link between %s and %s" % (end1, end2)) result = False count = 0 while True: count += 0 main.Mininet1.link(END1=end1, END2=end2, OPTION="up") main.Mininet1.link(END2=end1, END1=end2, OPTION="up") main.log.info("Waiting %s seconds for link up to be discovered" % main.linkSleep) time.sleep(main.linkSleep) main.CLIs[main.active].portstate(dpid=dpid1, port=port1) main.CLIs[main.active].portstate(dpid=dpid2, port=port2) time.sleep(main.linkSleep) result = main.CLIs[main.active].checkStatus(numoswitch=switches, numolink=links) if count > 5 or result: break utilities.assert_equals( expect=main.TRUE, actual=result, onpass="******", onfail="Failed to bring link up" )
def killSwitch(main, switch, switches, links): """ Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6' Completely kill a switch and verify ONOS can see the proper change """ main.switchSleep = float(main.params['timers']['SwitchDiscovery']) main.step("Kill " + switch) main.log.info("Stopping" + switch) main.Mininet1.switch(SW=switch, OPTION="stop") # todo make this repeatable main.log.info("Waiting %s seconds for switch down to be discovered" % (main.switchSleep)) time.sleep(main.switchSleep) topology = utilities.retry(main.CLIs[main.active].checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links }, attempts=10, sleep=main.switchSleep) utilities.assert_equals(expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to kill switch?")
def initTest( main ): """ - Construct tests variables - GIT ( optional ) - Checkout ONOS master branch - Pull latest ONOS code - Building ONOS ( optional ) - Install ONOS package - Build ONOS package """ main.step( "Constructing test variables" ) # Test variables main.cellName = main.params[ 'ENV' ][ 'cellName' ] main.apps = main.params[ 'ENV' ][ 'cellApps' ] main.diff = main.params[ 'ENV' ][ 'diffApps' ] gitBranch = main.params[ 'GIT' ][ 'branch' ] main.path = os.path.dirname( main.testFile ) main.dependencyPath = main.path + "/../dependencies/" main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ] wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ] main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," ) main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] ) # main.ONOSport = main.params[ 'CTRL' ][ 'port' ] main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] ) main.cellData = { } # for creating cell file main.CLIs = [ ] main.ONOSip = [ ] main.RESTs = [ ] # Assigning ONOS cli handles to a list for i in range( 1, main.maxNodes + 1 ): main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) ) main.ONOSip.append( main.CLIs[ i - 1 ].ip_address ) # -- INIT SECTION, ONLY RUNS ONCE -- # main.startUp = imp.load_source( wrapperFile1, main.dependencyPath + wrapperFile1 + ".py" ) copyResult1 = main.ONOSbench.scp( main.Mininet1, main.dependencyPath + main.topology, main.Mininet1.home, direction="to" ) if main.CLIs: stepResult = main.TRUE else: main.log.error( "Did not properly created list of ONOS CLI handle" ) stepResult = main.FALSE utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******" + "test variables ", onfail="Failed to construct test variables" )
def pingAll(main, tag=""): main.log.report("Check full connectivity") main.step("Check full connectivity %s" % tag) pa = main.Mininet1.pingall() utilities.assert_equals( expect=main.TRUE, actual=pa, onpass="******", onfail="Full connectivity failed", ) main.ONOSbench.dumpFlows(main.ONOSip[main.active], main.logdir, "flowsOn" + tag) main.ONOSbench.dumpGroups(main.ONOSip[main.active], main.logdir, "groupsOn" + tag)
def pingAll(main, tag=""): main.log.report("Check full connectivity") main.step("Check full connectivity %s" % tag) pa = main.Mininet1.pingall() utilities.assert_equals(expect=main.TRUE, actual=pa, onpass="******", onfail="Full connectivity failed") main.ONOSbench.dumpFlows(main.ONOSip[main.active], main.logdir, "flowsOn" + tag) main.ONOSbench.dumpGroups(main.ONOSip[main.active], main.logdir, "groupsOn" + tag)
def addStaticOnosRoute(main, subnet, intf): """ Adds an ONOS static route with the use route-add command. """ main.step( "Add static route for subnet {0} towards router interface {1}". format(subnet, intf)) routeResult = main.Cluster.active(0).addStaticRoute(subnet, intf) utilities.assert_equals(expect=True, actual=(not routeResult), onpass="******", onfail="route-add command failed")
def startMininet(main, topology, args=""): main.step("Starting Mininet Topology") arg = "--onos %d %s" % (main.numCtrls, args) main.topology = topology topoResult = main.Mininet1.startNet(topoFile=main.dependencyPath + main.topology, args=arg) stepResult = topoResult utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to load topology" ) # Exit if topology did not load properly if not topoResult: main.cleanup() main.exit()
def checkDevices(main, switches, tag="", sleep=10): main.step("Check whether the switches count is equal to %s" % switches) if tag == "": tag = 'CASE%d' % main.CurrentTestCaseNumber result = utilities.retry(main.Cluster.active(0).CLI.checkStatus, main.FALSE, kwargs={'numoswitch': switches}, attempts=10, sleep=sleep) utilities.assert_equals(expect=main.TRUE, actual=result, onpass="******", onfail="Failed to boot up devices?")
def initTest(main): """ - Construct tests variables - GIT ( optional ) - Checkout ONOS master branch - Pull latest ONOS code - Building ONOS ( optional ) - Install ONOS package - Build ONOS package """ main.step("Constructing test variables") # Test variables main.cellName = main.params["ENV"]["cellName"] main.apps = main.params["ENV"]["cellApps"] main.diff = main.params["ENV"]["diffApps"] gitBranch = main.params["GIT"]["branch"] main.path = os.path.dirname(main.testFile) main.dependencyPath = main.path + "/../dependencies/" main.topology = main.params["DEPENDENCY"]["topology"] wrapperFile1 = main.params["DEPENDENCY"]["wrapper1"] main.scale = (main.params["SCALE"]["size"]).split(",") main.maxNodes = int(main.params["SCALE"]["max"]) # main.ONOSport = main.params[ 'CTRL' ][ 'port' ] main.startUpSleep = int(main.params["SLEEP"]["startup"]) main.cellData = {} # for creating cell file main.CLIs = [] main.ONOSip = [] # Assigning ONOS cli handles to a list for i in range(1, main.maxNodes + 1): main.CLIs.append(getattr(main, "ONOScli" + str(i))) main.ONOSip.append(main.CLIs[i - 1].ip_address) # -- INIT SECTION, ONLY RUNS ONCE -- # main.startUp = imp.load_source(wrapperFile1, main.dependencyPath + wrapperFile1 + ".py") copyResult1 = main.ONOSbench.scp( main.Mininet1, main.dependencyPath + main.topology, main.Mininet1.home, direction="to" ) if main.CLIs: stepResult = main.TRUE else: main.log.error("Did not properly created list of ONOS CLI handle") stepResult = main.FALSE utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******" + "test variables ", onfail="Failed to construct test variables", )
def startMininet(main, topology, args=""): main.step("Starting Mininet Topology") arg = "--onos %d %s" % (main.Cluster.numCtrls, args) main.topology = topology topoResult = main.Mininet1.startNet(topoFile=main.Mininet1.home + main.topology, args=arg) stepResult = topoResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to load topology") # Exit if topology did not load properly if not topoResult: main.cleanAndExit()
def installOnos(main, vlanCfg=True, skipPackage=False, cliSleep=10, parallel=True): """ - Set up cell - Create cell file - Set cell file - Verify cell file - Kill ONOS process - Uninstall ONOS cluster - Verify ONOS start up - Install ONOS cluster - Connect to cli """ # main.scale[ 0 ] determines the current number of ONOS controller if not main.apps: main.log.error("App list is empty") main.log.info("Cluster size: " + str(main.Cluster.numCtrls)) main.log.info("Cluster ips: " + ', '.join(main.Cluster.getIps())) main.dynamicHosts = ['in1', 'out1'] main.testSetUp.ONOSSetUp(main.Cluster, newCell=True, cellName=main.cellName, skipPack=skipPackage, useSSH=Testcaselib.useSSH, installParallel=parallel, includeCaseDesc=False) ready = utilities.retry(main.Cluster.active(0).CLI.summary, main.FALSE, sleep=cliSleep, attempts=10) if ready: ready = main.TRUE utilities.assert_equals(expect=main.TRUE, actual=ready, onpass="******", onfail="ONOS summary command failed") if not ready: main.log.error("ONOS startup failed!") main.cleanAndExit() for ctrl in main.Cluster.active(): ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting") ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline") ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl") ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def verifyNetworkHostIp(main, attempts=10, sleep=10): """ Verifies IP address assignment from the hosts """ main.step("Verify IP address assignment from hosts") ipResult = main.TRUE for hostName, ip in main.expectedHosts["network"].items(): ipResult = ipResult and utilities.retry(main.Network.verifyHostIp, main.FALSE, kwargs={ 'hostList': [hostName], 'prefix': ip }, attempts=attempts, sleep=sleep) utilities.assert_equals(expect=main.TRUE, actual=ipResult, onpass="******", onfail="Verify network host IP failed")
def pingAll(main, tag="", dumpflows=True): main.log.report("Check full connectivity") print main.pingChart for entry in main.pingChart.itervalues(): print entry hosts, expect = entry['hosts'], entry['expect'] expect = main.TRUE if expect else main.FALSE main.step("Connectivity for %s %s" % (str(hosts), tag)) pa = main.Mininet1.pingallHosts(hosts) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IP connectivity failed") if dumpflows: main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "flows", main.logdir, "flowsOn" + tag) main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "groups", main.logdir, "groupsOn" + tag)
def verifyOnosHostIp(main, attempts=10, sleep=10): """ Verifies host IP address assignment from ONOS """ main.step("Verify host IP address assignment in ONOS") ipResult = main.TRUE for hostName, ip in main.expectedHosts["onos"].items(): ipResult = ipResult and utilities.retry( main.Cluster.active(0).verifyHostIp, main.FALSE, kwargs={ 'hostList': [hostName], 'prefix': ip }, attempts=attempts, sleep=sleep) utilities.assert_equals(expect=main.TRUE, actual=ipResult, onpass="******", onfail="Verify ONOS host IP failed")
def recoverSwitch(main, switch, switches, links): """ Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6' Recover a switch and verify ONOS can see the proper change """ main.step("Recovering " + switch) main.log.info("Starting" + switch) main.Mininet1.switch(SW=switch, OPTION="start") main.log.info("Waiting %s seconds for switch up to be discovered" % (main.switchSleep)) time.sleep(main.switchSleep) topology = utilities.retry( main.CLIs[main.active].checkStatus, main.FALSE, kwargs={"numoswitch": switches, "numolink": links}, attempts=10, sleep=main.switchSleep, ) utilities.assert_equals( expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to recover switch?" )
def killSwitch(main, switch, switches, links): """ Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6' Completely kill a switch and verify ONOS can see the proper change """ main.switchSleep = float(main.params["timers"]["SwitchDiscovery"]) main.step("Kill " + switch) main.log.info("Stopping" + switch) main.Mininet1.switch(SW=switch, OPTION="stop") main.log.info("Waiting %s seconds for switch down to be discovered" % (main.switchSleep)) time.sleep(main.switchSleep) topology = utilities.retry( main.CLIs[main.active].checkStatus, main.FALSE, kwargs={"numoswitch": switches, "numolink": links}, attempts=10, sleep=main.switchSleep, ) utilities.assert_equals( expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to kill switch?" )
def recoverSwitch( main, switch, switches, links ): """ Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6' Recover a switch and verify ONOS can see the proper change """ # todo make this repeatable main.step( "Recovering " + switch ) main.log.info( "Starting" + switch ) main.Mininet1.switch( SW=switch, OPTION="start" ) main.log.info( "Waiting %s seconds for switch up to be discovered" % ( main.switchSleep) ) time.sleep( main.switchSleep ) topology = utilities.retry( main.CLIs[ main.active ].checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links }, attempts=10, sleep=main.switchSleep ) utilities.assert_equals( expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to recover switch?" )
def killOnos(main, nodes, switches, links, expNodes): """ Params: nodes, integer array with position of the ONOS nodes in the CLIs array switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6' Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change """ main.step("Killing ONOS instance") for i in nodes: killResult = main.ONOSbench.onosDie(main.CLIs[i].ip_address) utilities.assert_equals(expect=main.TRUE, actual=killResult, onpass="******", onfail="Error killing ONOS instance") if i == main.active: main.active = (i + 1) % main.numCtrls time.sleep(12) if len(nodes) < main.numCtrls: nodesToCheck = [] for x in range(0, main.numCtrls): if x not in nodes: nodesToCheck.append(x) nodeResults = utilities.retry(Testcaselib.nodesCheck, False, args=[nodesToCheck], attempts=5, sleep=10) utilities.assert_equals(expect=True, actual=nodeResults, onpass="******", onfail="Nodes check NOT successful") if not nodeResults: for i in nodes: cli = main.CLIs[i] main.log.debug("{} components not ACTIVE: \n{}".format( cli.name, cli.sendline("scr:list | grep -v ACTIVE"))) main.log.error("Failed to kill ONOS, stopping test") main.cleanup() main.exit() topology = utilities.retry(main.CLIs[main.active].checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links, 'numoctrl': expNodes }, attempts=10, sleep=12) utilities.assert_equals(expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to turn off ONOS Instance") else: main.active = -1
def killLink(main, end1, end2, switches, links): """ end1,end2: identify the switches, ex.: 'leaf1', 'spine1' switches, links: number of expected switches and links after linkDown, ex.: '4', '6' Kill a link and verify ONOS can see the proper link change """ main.linkSleep = float(main.params["timers"]["LinkDiscovery"]) main.step("Kill link between %s and %s" % (end1, end2)) LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down") main.log.info("Waiting %s seconds for link down to be discovered" % main.linkSleep) time.sleep(main.linkSleep) topology = utilities.retry( main.CLIs[main.active].checkStatus, main.FALSE, kwargs={"numoswitch": switches, "numolink": links}, attempts=10, sleep=main.linkSleep, ) result = topology & LinkDown utilities.assert_equals( expect=main.TRUE, actual=result, onpass="******", onfail="Failed to turn off link?" )
def startMininet(main, topology, args=""): copyResult = main.ONOSbench.scp(main.Mininet1, main.topoPath + main.topology, main.Mininet1.home, direction="to") if main.topologyLib: for lib in main.topologyLib.split(","): copyResult = copyResult and main.ONOSbench.scp( main.Mininet1, main.topoPath + lib, main.Mininet1.home, direction="to") if main.topologyConf: for conf in main.topologyConf.split(","): copyResult = copyResult and main.ONOSbench.scp( main.Mininet1, main.configPath + main.forConfig + conf, "~/", direction="to") stepResult = copyResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to copy topo files") main.step("Starting Mininet Topology") arg = "--onos-ip=%s %s" % (",".join( [ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args) main.topology = topology topoResult = main.Mininet1.startNet(topoFile=main.Mininet1.home + main.topology, args=arg) stepResult = topoResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to load topology") # Exit if topology did not load properly if not topoResult: main.cleanAndExit()
def connectToPhysicalNetwork(main, switchNames): main.step("Connecting to physical netowrk") topoResult = main.NetworkBench.connectToNet() stepResult = topoResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to load topology") # Exit if topology did not load properly if not topoResult: main.cleanAndExit() main.step("Assign switches to controllers.") assignResult = main.TRUE for name in switchNames: assignResult = assignResult & main.NetworkBench.assignSwController( sw=name, ip=main.Cluster.getIps(), port='6653') utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to assign switches to controllers")
def killOnos(main, nodes, switches, links, expNodes): """ Params: nodes, integer array with position of the ONOS nodes in the CLIs array switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6' Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change """ main.step("Killing ONOS instances with index(es): {}".format(nodes)) for i in nodes: killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[i].ipAddress) utilities.assert_equals(expect=main.TRUE, actual=killResult, onpass="******", onfail="Error killing ONOS instance") main.Cluster.runningNodes[i].active = False time.sleep(12) if len(nodes) < main.Cluster.numCtrls: nodeResults = utilities.retry(main.Cluster.nodesCheck, False, attempts=5, sleep=10) utilities.assert_equals(expect=True, actual=nodeResults, onpass="******", onfail="Nodes check NOT successful") if not nodeResults: for i in nodes: ctrl = main.Cluster.runningNodes[i] main.log.debug("{} components not ACTIVE: \n{}".format( ctrl.name, ctrl.CLI.sendline("scr:list | grep -v ACTIVE"))) main.log.error("Failed to kill ONOS, stopping test") main.cleanAndExit() topology = utilities.retry(main.Cluster.active(0).CLI.checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links, 'numoctrl': expNodes }, attempts=10, sleep=12) utilities.assert_equals(expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to turn off ONOS Instance")
def recoverOnos(main, nodes, switches, links, expNodes): """ Params: nodes, integer array with position of the ONOS nodes in the CLIs array switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6' Recover an ONOS instance and verify the ONOS cluster can see the proper change """ main.step("Recovering ONOS instance") [main.ONOSbench.onosStart(main.CLIs[i].ip_address) for i in nodes] for i in nodes: isUp = main.ONOSbench.isup(main.ONOSip[i]) utilities.assert_equals( expect=main.TRUE, actual=isUp, onpass="******", onfail="ONOS service did not start properly") for i in nodes: main.step("Checking if ONOS CLI is ready") main.CLIs[i].startCellCli() cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i], commandlineTimeout=60, onosStartTimeout=100) utilities.assert_equals(expect=main.TRUE, actual=cliResult, onpass="******", onfail="ONOS CLI is not ready") main.active = i if main.active == -1 else main.active main.step("Checking ONOS nodes") nodeResults = utilities.retry(Testcaselib.nodesCheck, False, args=[nodes], attempts=5, sleep=10) utilities.assert_equals(expect=True, actual=nodeResults, onpass="******", onfail="Nodes check NOT successful") if not nodeResults: for i in nodes: cli = main.CLIs[i] main.log.debug("{} components not ACTIVE: \n{}".format( cli.name, cli.sendline("scr:list | grep -v ACTIVE"))) main.log.error("Failed to start ONOS, stopping test") main.cleanup() main.exit() topology = utilities.retry(main.CLIs[main.active].checkStatus, main.FALSE, kwargs={ 'numoswitch': switches, 'numolink': links, 'numoctrl': expNodes }, attempts=10, sleep=12) utilities.assert_equals(expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to turn off ONOS Instance") for i in range(10): ready = True output = main.CLIs[main.active].summary() if not output: ready = False if ready: break time.sleep(10) utilities.assert_equals(expect=True, actual=ready, onpass="******", onfail="ONOS summary command failed") if not ready: main.log.error("ONOS startup failed!") main.cleanup() main.exit()
def installOnos(main, vlanCfg=True): """ - Set up cell - Create cell file - Set cell file - Verify cell file - Kill ONOS process - Uninstall ONOS cluster - Verify ONOS start up - Install ONOS cluster - Connect to cli """ # main.scale[ 0 ] determines the current number of ONOS controller if main.diff: main.apps = main.apps + "," + main.diff else: main.log.error("App list is empty") main.log.info("NODE COUNT = " + str(main.Cluster.numCtrls)) main.log.info(''.join(main.Cluster.getIps())) main.dynamicHosts = ['in1', 'out1'] main.testSetUp.createApplyCell(main.Cluster, newCell=True, cellName=main.cellName, Mininet=main.Mininet1, useSSH=Testcaselib.useSSH, ips=main.Cluster.getIps()) # kill off all onos processes main.log.info("Safety check, killing all ONOS processes" + " before initiating environment setup") for ctrl in main.Cluster.runningNodes: main.ONOSbench.onosDie(ctrl.ipAddress) main.testSetUp.buildOnos(main.Cluster) main.testSetUp.installOnos(main.Cluster, False) main.testSetUp.setupSsh(main.Cluster) main.testSetUp.checkOnosService(main.Cluster) cliResult = main.TRUE main.step("Checking if ONOS CLI is ready") for ctrl in main.Cluster.runningNodes: ctrl.CLI.startCellCli() cliResult = cliResult and ctrl.CLI.startOnosCli( ctrl.ipAddress, commandlineTimeout=60, onosStartTimeout=100) ctrl.active = True utilities.assert_equals(expect=main.TRUE, actual=cliResult, onpass="******", onfail="ONOS CLI is not ready") ready = utilities.retry(main.Cluster.active(0).CLI.summary, main.FALSE, sleep=10, attempts=10) if ready: ready = main.TRUE utilities.assert_equals(expect=main.TRUE, actual=ready, onpass="******", onfail="ONOS summary command failed") with open("%s/json/%s.json" % (main.dependencyPath, main.cfgName)) as cfg: main.Cluster.active(0).REST.setNetCfg(json.load(cfg)) with open("%s/json/%s.chart" % (main.dependencyPath, main.cfgName)) as chart: main.pingChart = json.load(chart) if not ready: main.log.error("ONOS startup failed!") main.cleanAndExit() for ctrl in main.Cluster.active(): ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting") ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline") ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl") ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def installOnos(main): """ - Set up cell - Create cell file - Set cell file - Verify cell file - Kill ONOS process - Uninstall ONOS cluster - Verify ONOS start up - Install ONOS cluster - Connect to cli """ # main.scale[ 0 ] determines the current number of ONOS controller apps = main.apps if main.diff: apps = main.apps + "," + main.diff else: main.log.error("App list is empty") main.case("Package and start ONOS using apps:" + apps) print "NODE COUNT = ", main.numCtrls print main.ONOSip tempOnosIp = [] for i in range(main.numCtrls): tempOnosIp.append(main.ONOSip[i]) onosUser = main.params["ENV"]["cellUser"] main.step("Create and Apply cell file") main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "temp", main.Mininet1.ip_address, apps, tempOnosIp, onosUser ) cellResult = main.ONOSbench.setCell("temp") verifyResult = main.ONOSbench.verifyCell() stepResult = cellResult and verifyResult utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******" + "environment", onfail="Failed to apply cell to environment ", ) # kill off all onos processes main.log.info("Safety check, killing all ONOS processes" + " before initiating environment setup") for i in range(main.maxNodes): main.ONOSbench.onosDie(main.ONOSip[i]) main.step("Create and Install ONOS package") main.ONOSbench.handle.sendline( "cp " + main.dependencyPath + "/" + main.cfgName + ".json ~/onos/tools/package/config/network-cfg.json" ) packageResult = main.ONOSbench.onosPackage() onosInstallResult = main.TRUE for i in range(main.numCtrls): onosInstallResult = onosInstallResult and main.ONOSbench.onosInstall(node=main.ONOSip[i]) stepResult = onosInstallResult utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to install ONOS package", ) main.step("Starting ONOS service") stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE for i in range(main.numCtrls): onosIsUp = onosIsUp and main.ONOSbench.isup(main.ONOSip[i]) if onosIsUp == main.TRUE: main.log.report("ONOS instance is up and ready") else: main.log.report("ONOS instance may not be up, stop and " + "start ONOS again ") for i in range(main.numCtrls): stopResult = stopResult and main.ONOSbench.onosStop(main.ONOSip[i]) for i in range(main.numCtrls): startResult = startResult and main.ONOSbench.onosStart(main.ONOSip[i]) stepResult = onosIsUp and stopResult and startResult utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******", onfail="ONOS service did not start properly", ) main.step("Checking if ONOS CLI is ready") for i in range(main.numCtrls): main.CLIs[i].startCellCli() cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i], commandlineTimeout=60, onosStartTimeout=100) utilities.assert_equals( expect=main.TRUE, actual=cliResult, onpass="******", onfail="ONOS CLI is not ready" ) main.active = 0 for i in range(10): ready = True output = main.CLIs[main.active].summary() if not output: ready = False if ready: break time.sleep(10) utilities.assert_equals( expect=True, actual=ready, onpass="******", onfail="ONOS summary command failed" ) if not ready: main.log.error("ONOS startup failed!") main.cleanup() main.exit() for i in range(main.numCtrls): main.CLIs[i].logSet("DEBUG", "org.onosproject.segmentrouting") main.CLIs[i].logSet("DEBUG", "org.onosproject.driver.pipeline") main.CLIs[i].logSet("DEBUG", "org.onosproject.store.group.impl") main.CLIs[i].logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def installOnos(main, vlanCfg=True): """ - Set up cell - Create cell file - Set cell file - Verify cell file - Kill ONOS process - Uninstall ONOS cluster - Verify ONOS start up - Install ONOS cluster - Connect to cli """ # main.scale[ 0 ] determines the current number of ONOS controller apps = main.apps if main.diff: apps = main.apps + "," + main.diff else: main.log.error("App list is empty") print "NODE COUNT = ", main.numCtrls print main.ONOSip tempOnosIp = [] main.dynamicHosts = ['in1', 'out1'] for i in range(main.numCtrls): tempOnosIp.append(main.ONOSip[i]) onosUser = main.params['ENV']['cellUser'] main.step("Create and Apply cell file") main.ONOSbench.createCellFile(main.ONOSbench.ip_address, "temp", main.Mininet1.ip_address, apps, tempOnosIp, onosUser, useSSH=Testcaselib.useSSH) cellResult = main.ONOSbench.setCell("temp") verifyResult = main.ONOSbench.verifyCell() stepResult = cellResult and verifyResult utilities.assert_equals( expect=main.TRUE, actual=stepResult, onpass="******" + \ "environment", onfail="Failed to apply cell to environment " ) # kill off all onos processes main.log.info("Safety check, killing all ONOS processes" + " before initiating environment setup") for i in range(main.maxNodes): main.ONOSbench.onosDie(main.ONOSip[i]) main.step("Create and Install ONOS package") packageResult = main.ONOSbench.buckBuild() onosInstallResult = main.TRUE for i in range(main.numCtrls): onosInstallResult = onosInstallResult and \ main.ONOSbench.onosInstall( node=main.ONOSip[ i ] ) stepResult = onosInstallResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to install ONOS package") if Testcaselib.useSSH: for i in range(main.numCtrls): onosInstallResult = onosInstallResult and \ main.ONOSbench.onosSecureSSH( node=main.ONOSip[ i ] ) stepResult = onosInstallResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to secure SSH") main.step("Starting ONOS service") stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE, for i in range(main.numCtrls): onosIsUp = onosIsUp and main.ONOSbench.isup(main.ONOSip[i]) if onosIsUp == main.TRUE: main.log.report("ONOS instance is up and ready") else: main.log.report("ONOS instance may not be up, stop and " + "start ONOS again ") for i in range(main.numCtrls): stopResult = stopResult and \ main.ONOSbench.onosStop( main.ONOSip[ i ] ) for i in range(main.numCtrls): startResult = startResult and \ main.ONOSbench.onosStart( main.ONOSip[ i ] ) stepResult = onosIsUp and stopResult and startResult utilities.assert_equals(expect=main.TRUE, actual=stepResult, onpass="******", onfail="ONOS service did not start properly") main.step("Checking if ONOS CLI is ready") for i in range(main.numCtrls): main.CLIs[i].startCellCli() cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i], commandlineTimeout=60, onosStartTimeout=100) utilities.assert_equals(expect=main.TRUE, actual=cliResult, onpass="******", onfail="ONOS CLI is not ready") main.active = 0 for i in range(10): ready = True output = main.CLIs[main.active].summary() if not output: ready = False if ready: break time.sleep(10) utilities.assert_equals(expect=True, actual=ready, onpass="******", onfail="ONOS summary command failed") with open("%s/json/%s.json" % (main.dependencyPath, main.cfgName)) as cfg: main.RESTs[main.active].setNetCfg(json.load(cfg)) with open("%s/json/%s.chart" % (main.dependencyPath, main.cfgName)) as chart: main.pingChart = json.load(chart) if not ready: main.log.error("ONOS startup failed!") main.cleanup() main.exit() for i in range(main.numCtrls): main.CLIs[i].logSet("DEBUG", "org.onosproject.segmentrouting") main.CLIs[i].logSet("DEBUG", "org.onosproject.driver.pipeline") main.CLIs[i].logSet("DEBUG", "org.onosproject.store.group.impl") main.CLIs[i].logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def compareTopos(self, Mininet, attempts=1, includeCaseDesc=True): """ Description: compares the links and hosts and switches of the onos to the mininet. Required: * Mininet - Mininet driver to use. * attempts - number of attempts to compare in case the result is different after a certain time. Returns: Returns main.TRUE if the results are matching else Returns main.FALSE """ if includeCaseDesc: main.case("Compare ONOS Topology view to Mininet topology") main.caseExplanation = "Compare topology elements between Mininet" +\ " and ONOS" main.log.info("Gathering topology information from Mininet") devicesResults = main.FALSE # Overall Boolean for device correctness linksResults = main.FALSE # Overall Boolean for link correctness hostsResults = main.FALSE # Overall Boolean for host correctness deviceFails = [] # Nodes where devices are incorrect linkFails = [] # Nodes where links are incorrect hostFails = [] # Nodes where hosts are incorrect mnSwitches = Mininet.getSwitches() mnLinks = Mininet.getLinks() mnHosts = Mininet.getHosts() main.step("Comparing Mininet topology to ONOS topology") while ( attempts >= 0 ) and\ ( not devicesResults or not linksResults or not hostsResults ): main.log.info("Sleeping {} seconds".format(2)) time.sleep(2) if not devicesResults: devices = self.getAll("devices", False) ports = self.getAll("ports", False) devicesResults = main.TRUE deviceFails = [] # Reset for each failed attempt if not linksResults: links = self.getAll("links", False) linksResults = main.TRUE linkFails = [] # Reset for each failed attempt if not hostsResults: hosts = self.getAll("hosts", False) hostsResults = main.TRUE hostFails = [] # Reset for each failed attempt # Check for matching topology on each node for controller in main.Cluster.getRunningPos(): controllerStr = str(controller + 1) # ONOS node number # Compare Devices currentDevicesResult = self.compareDevicePort( Mininet, controller, mnSwitches, devices, ports) if not currentDevicesResult: deviceFails.append(controllerStr) devicesResults = devicesResults and currentDevicesResult # Compare Links currentLinksResult = self.compareBase(links, controller, Mininet.compareLinks, [mnSwitches, mnLinks]) if not currentLinksResult: linkFails.append(controllerStr) linksResults = linksResults and currentLinksResult # Compare Hosts currentHostsResult = self.compareBase(hosts, controller, Mininet.compareHosts, mnHosts) if not currentHostsResult: hostFails.append(controllerStr) hostsResults = hostsResults and currentHostsResult # Decrement Attempts Remaining attempts -= 1 utilities.assert_equals( expect=[], actual=deviceFails, onpass="******", onfail="ONOS incorrectly discovered devices on nodes: " + str(deviceFails)) utilities.assert_equals( expect=[], actual=linkFails, onpass="******", onfail="ONOS incorrectly discovered links on nodes: " + str(linkFails)) utilities.assert_equals( expect=[], actual=hostFails, onpass="******", onfail="ONOS incorrectly discovered hosts on nodes: " + str(hostFails)) topoResults = hostsResults and linksResults and devicesResults utilities.assert_equals( expect=main.TRUE, actual=topoResults, onpass="******", onfail="ONOS incorrectly discovered the topology") return topoResults
def pingAll(main, tag="", dumpflows=True, acceptableFailed=0, basedOnIp=False, sleep=10, retryAttempts=1): ''' Verify connectivity between hosts according to the ping chart acceptableFailed: max number of acceptable failed pings. basedOnIp: if True, run ping or ping6 based on suffix of host names retryAttempts: the number of retry ping. Only works for IPv4 hosts. ''' main.log.report("Check host connectivity") main.log.debug("Ping chart: %s" % main.pingChart) if tag == "": tag = 'CASE%d' % main.CurrentTestCaseNumber for entry in main.pingChart.itervalues(): main.log.debug("Entry in ping chart: %s" % entry) expect = entry['expect'] if expect == "Unidirectional": # Verify ping from each src host to each dst host src = entry['src'] dst = entry['dst'] expect = main.TRUE main.step( "Verify unidirectional connectivity from %s to %s with tag %s" % (str(src), str(dst), tag)) if basedOnIp: if ("v4" in src[0]): pa = main.Network.pingallHostsUnidirectional( src, dst, acceptableFailed=acceptableFailed) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IPv4 connectivity failed") if ("v6" in src[0]): pa = main.Network.pingallHostsUnidirectional( src, dst, ipv6=True, acceptableFailed=acceptableFailed) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IPv6 connectivity failed") else: pa = main.Network.pingallHostsUnidirectional( src, dst, acceptableFailed=acceptableFailed) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IP connectivity failed") else: # Verify ping between each host pair hosts = entry['hosts'] try: expect = main.TRUE if str( expect).lower() == 'true' else main.FALSE except: expect = main.FALSE main.step("Verify full connectivity for %s with tag %s" % (str(hosts), tag)) if basedOnIp: if ("v4" in hosts[0]): pa = utilities.retry( main.Network.pingallHosts, main.FALSE if expect else main.TRUE, args=(hosts, ), attempts=retryAttempts, sleep=sleep) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IPv4 connectivity failed") if ("v6" in hosts[0]): pa = main.Network.pingIpv6Hosts( hosts, acceptableFailed=acceptableFailed) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IPv6 connectivity failed") else: pa = main.Network.pingallHosts(hosts) utilities.assert_equals( expect=expect, actual=pa, onpass="******", onfail="IP connectivity failed") if dumpflows: main.ONOSbench.dumpONOSCmd( main.Cluster.active(0).ipAddress, "flows", main.logdir, tag + "_FlowsOn") main.ONOSbench.dumpONOSCmd( main.Cluster.active(0).ipAddress, "groups", main.logdir, tag + "_GroupsOn")