def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
links ):
"""
Params:
end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
port1, port2: respective port of the end switches that connects to the link, ex.:'1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.step( "Restore link between %s and %s" % (end1, end2) )
result = False
count = 0
while True:
count += 1
main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
main.log.info(
"Waiting %s seconds for link up to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
for i in range(0, main.numCtrls):
onosIsUp = main.ONOSbench.isup( main.ONOSip[ i ] )
if onosIsUp == main.TRUE:
main.CLIs[ i ].portstate( dpid=dpid1, port=port1 )
main.CLIs[ i ].portstate( dpid=dpid2, port=port2 )
time.sleep( main.linkSleep )
result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
numolink=links )
if count > 5 or result:
break
utilities.assert_equals( expect=main.TRUE, actual=result,
onpass="******",
onfail="Failed to bring link up" )
def killLink(main, end1, end2, switches, links):
"""
end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.linkSleep = float(main.params['timers']['LinkDiscovery'])
main.step("Kill link between %s and %s" % (end1, end2))
LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down")
LinkDown = main.Mininet1.link(END2=end1, END1=end2, OPTION="down")
main.log.info("Waiting %s seconds for link down to be discovered" %
main.linkSleep)
time.sleep(main.linkSleep)
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links
},
attempts=10,
sleep=main.linkSleep)
result = topology & LinkDown
utilities.assert_equals(expect=main.TRUE,
actual=result,
onpass="******",
onfail="Failed to turn off link?")
def restoreLink(main, end1, end2, dpid1, dpid2, port1, port2, switches, links):
"""
Params:
end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
port1, port2: respective port of the end switches that connects to the link, ex.:'1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.step("Restore link between %s and %s" % (end1, end2))
result = False
count = 0
while True:
count += 0
main.Mininet1.link(END1=end1, END2=end2, OPTION="up")
main.Mininet1.link(END2=end1, END1=end2, OPTION="up")
main.log.info("Waiting %s seconds for link up to be discovered" % main.linkSleep)
time.sleep(main.linkSleep)
main.CLIs[main.active].portstate(dpid=dpid1, port=port1)
main.CLIs[main.active].portstate(dpid=dpid2, port=port2)
time.sleep(main.linkSleep)
result = main.CLIs[main.active].checkStatus(numoswitch=switches, numolink=links)
if count > 5 or result:
break
utilities.assert_equals(
expect=main.TRUE, actual=result, onpass="******", onfail="Failed to bring link up"
)
def killSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
Completely kill a switch and verify ONOS can see the proper change
"""
main.switchSleep = float(main.params['timers']['SwitchDiscovery'])
main.step("Kill " + switch)
main.log.info("Stopping" + switch)
main.Mininet1.switch(SW=switch, OPTION="stop")
# todo make this repeatable
main.log.info("Waiting %s seconds for switch down to be discovered" %
(main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links
},
attempts=10,
sleep=main.switchSleep)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to kill switch?")
def initTest( main ):
"""
- Construct tests variables
- GIT ( optional )
- Checkout ONOS master branch
- Pull latest ONOS code
- Building ONOS ( optional )
- Install ONOS package
- Build ONOS package
"""
main.step( "Constructing test variables" )
# Test variables
main.cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'cellApps' ]
main.diff = main.params[ 'ENV' ][ 'diffApps' ]
gitBranch = main.params[ 'GIT' ][ 'branch' ]
main.path = os.path.dirname( main.testFile )
main.dependencyPath = main.path + "/../dependencies/"
main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
# main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
main.cellData = { } # for creating cell file
main.CLIs = [ ]
main.ONOSip = [ ]
main.RESTs = [ ]
# Assigning ONOS cli handles to a list
for i in range( 1, main.maxNodes + 1 ):
main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
# -- INIT SECTION, ONLY RUNS ONCE -- #
main.startUp = imp.load_source( wrapperFile1,
main.dependencyPath +
wrapperFile1 +
".py" )
copyResult1 = main.ONOSbench.scp( main.Mininet1,
main.dependencyPath +
main.topology,
main.Mininet1.home,
direction="to" )
if main.CLIs:
stepResult = main.TRUE
else:
main.log.error( "Did not properly created list of ONOS CLI handle" )
stepResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="******" +
"test variables ",
onfail="Failed to construct test variables" )
def pingAll(main, tag=""):
main.log.report("Check full connectivity")
main.step("Check full connectivity %s" % tag)
pa = main.Mininet1.pingall()
utilities.assert_equals(
expect=main.TRUE,
actual=pa,
onpass="******",
onfail="Full connectivity failed",
)
main.ONOSbench.dumpFlows(main.ONOSip[main.active], main.logdir, "flowsOn" + tag)
main.ONOSbench.dumpGroups(main.ONOSip[main.active], main.logdir, "groupsOn" + tag)
def pingAll(main, tag=""):
main.log.report("Check full connectivity")
main.step("Check full connectivity %s" % tag)
pa = main.Mininet1.pingall()
utilities.assert_equals(expect=main.TRUE,
actual=pa,
onpass="******",
onfail="Full connectivity failed")
main.ONOSbench.dumpFlows(main.ONOSip[main.active], main.logdir,
"flowsOn" + tag)
main.ONOSbench.dumpGroups(main.ONOSip[main.active], main.logdir,
"groupsOn" + tag)
def addStaticOnosRoute(main, subnet, intf):
"""
Adds an ONOS static route with the use route-add command.
"""
main.step(
"Add static route for subnet {0} towards router interface {1}".
format(subnet, intf))
routeResult = main.Cluster.active(0).addStaticRoute(subnet, intf)
utilities.assert_equals(expect=True,
actual=(not routeResult),
onpass="******",
onfail="route-add command failed")
def startMininet(main, topology, args=""):
main.step("Starting Mininet Topology")
arg = "--onos %d %s" % (main.numCtrls, args)
main.topology = topology
topoResult = main.Mininet1.startNet(topoFile=main.dependencyPath + main.topology, args=arg)
stepResult = topoResult
utilities.assert_equals(
expect=main.TRUE, actual=stepResult, onpass="******", onfail="Failed to load topology"
)
# Exit if topology did not load properly
if not topoResult:
main.cleanup()
main.exit()
def checkDevices(main, switches, tag="", sleep=10):
main.step("Check whether the switches count is equal to %s" % switches)
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
result = utilities.retry(main.Cluster.active(0).CLI.checkStatus,
main.FALSE,
kwargs={'numoswitch': switches},
attempts=10,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=result,
onpass="******",
onfail="Failed to boot up devices?")
def initTest(main):
"""
- Construct tests variables
- GIT ( optional )
- Checkout ONOS master branch
- Pull latest ONOS code
- Building ONOS ( optional )
- Install ONOS package
- Build ONOS package
"""
main.step("Constructing test variables")
# Test variables
main.cellName = main.params["ENV"]["cellName"]
main.apps = main.params["ENV"]["cellApps"]
main.diff = main.params["ENV"]["diffApps"]
gitBranch = main.params["GIT"]["branch"]
main.path = os.path.dirname(main.testFile)
main.dependencyPath = main.path + "/../dependencies/"
main.topology = main.params["DEPENDENCY"]["topology"]
wrapperFile1 = main.params["DEPENDENCY"]["wrapper1"]
main.scale = (main.params["SCALE"]["size"]).split(",")
main.maxNodes = int(main.params["SCALE"]["max"])
# main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
main.startUpSleep = int(main.params["SLEEP"]["startup"])
main.cellData = {} # for creating cell file
main.CLIs = []
main.ONOSip = []
# Assigning ONOS cli handles to a list
for i in range(1, main.maxNodes + 1):
main.CLIs.append(getattr(main, "ONOScli" + str(i)))
main.ONOSip.append(main.CLIs[i - 1].ip_address)
# -- INIT SECTION, ONLY RUNS ONCE -- #
main.startUp = imp.load_source(wrapperFile1, main.dependencyPath + wrapperFile1 + ".py")
copyResult1 = main.ONOSbench.scp(
main.Mininet1, main.dependencyPath + main.topology, main.Mininet1.home, direction="to"
)
if main.CLIs:
stepResult = main.TRUE
else:
main.log.error("Did not properly created list of ONOS CLI handle")
stepResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
actual=stepResult,
onpass="******" + "test variables ",
onfail="Failed to construct test variables",
)
def startMininet(main, topology, args=""):
main.step("Starting Mininet Topology")
arg = "--onos %d %s" % (main.Cluster.numCtrls, args)
main.topology = topology
topoResult = main.Mininet1.startNet(topoFile=main.Mininet1.home +
main.topology,
args=arg)
stepResult = topoResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to load topology")
# Exit if topology did not load properly
if not topoResult:
main.cleanAndExit()
def installOnos(main,
vlanCfg=True,
skipPackage=False,
cliSleep=10,
parallel=True):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
if not main.apps:
main.log.error("App list is empty")
main.log.info("Cluster size: " + str(main.Cluster.numCtrls))
main.log.info("Cluster ips: " + ', '.join(main.Cluster.getIps()))
main.dynamicHosts = ['in1', 'out1']
main.testSetUp.ONOSSetUp(main.Cluster,
newCell=True,
cellName=main.cellName,
skipPack=skipPackage,
useSSH=Testcaselib.useSSH,
installParallel=parallel,
includeCaseDesc=False)
ready = utilities.retry(main.Cluster.active(0).CLI.summary,
main.FALSE,
sleep=cliSleep,
attempts=10)
if ready:
ready = main.TRUE
utilities.assert_equals(expect=main.TRUE,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
if not ready:
main.log.error("ONOS startup failed!")
main.cleanAndExit()
for ctrl in main.Cluster.active():
ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting")
ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline")
ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl")
ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def verifyNetworkHostIp(main, attempts=10, sleep=10):
"""
Verifies IP address assignment from the hosts
"""
main.step("Verify IP address assignment from hosts")
ipResult = main.TRUE
for hostName, ip in main.expectedHosts["network"].items():
ipResult = ipResult and utilities.retry(main.Network.verifyHostIp,
main.FALSE,
kwargs={
'hostList': [hostName],
'prefix': ip
},
attempts=attempts,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=ipResult,
onpass="******",
onfail="Verify network host IP failed")
def pingAll(main, tag="", dumpflows=True):
main.log.report("Check full connectivity")
print main.pingChart
for entry in main.pingChart.itervalues():
print entry
hosts, expect = entry['hosts'], entry['expect']
expect = main.TRUE if expect else main.FALSE
main.step("Connectivity for %s %s" % (str(hosts), tag))
pa = main.Mininet1.pingallHosts(hosts)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IP connectivity failed")
if dumpflows:
main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "flows",
main.logdir, "flowsOn" + tag)
main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "groups",
main.logdir, "groupsOn" + tag)
def verifyOnosHostIp(main, attempts=10, sleep=10):
"""
Verifies host IP address assignment from ONOS
"""
main.step("Verify host IP address assignment in ONOS")
ipResult = main.TRUE
for hostName, ip in main.expectedHosts["onos"].items():
ipResult = ipResult and utilities.retry(
main.Cluster.active(0).verifyHostIp,
main.FALSE,
kwargs={
'hostList': [hostName],
'prefix': ip
},
attempts=attempts,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=ipResult,
onpass="******",
onfail="Verify ONOS host IP failed")
def recoverSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
Recover a switch and verify ONOS can see the proper change
"""
main.step("Recovering " + switch)
main.log.info("Starting" + switch)
main.Mininet1.switch(SW=switch, OPTION="start")
main.log.info("Waiting %s seconds for switch up to be discovered" % (main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.switchSleep,
)
utilities.assert_equals(
expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to recover switch?"
)
def killSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
Completely kill a switch and verify ONOS can see the proper change
"""
main.switchSleep = float(main.params["timers"]["SwitchDiscovery"])
main.step("Kill " + switch)
main.log.info("Stopping" + switch)
main.Mininet1.switch(SW=switch, OPTION="stop")
main.log.info("Waiting %s seconds for switch down to be discovered" % (main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.switchSleep,
)
utilities.assert_equals(
expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to kill switch?"
)
def recoverSwitch( main, switch, switches, links ):
"""
Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
Recover a switch and verify ONOS can see the proper change
"""
# todo make this repeatable
main.step( "Recovering " + switch )
main.log.info( "Starting" + switch )
main.Mininet1.switch( SW=switch, OPTION="start" )
main.log.info( "Waiting %s seconds for switch up to be discovered" % (
main.switchSleep) )
time.sleep( main.switchSleep )
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
kwargs={ 'numoswitch': switches,
'numolink': links },
attempts=10,
sleep=main.switchSleep )
utilities.assert_equals( expect=main.TRUE, actual=topology,
onpass="******",
onfail="Failed to recover switch?" )
def killOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Killing ONOS instance")
for i in nodes:
killResult = main.ONOSbench.onosDie(main.CLIs[i].ip_address)
utilities.assert_equals(expect=main.TRUE,
actual=killResult,
onpass="******",
onfail="Error killing ONOS instance")
if i == main.active:
main.active = (i + 1) % main.numCtrls
time.sleep(12)
if len(nodes) < main.numCtrls:
nodesToCheck = []
for x in range(0, main.numCtrls):
if x not in nodes:
nodesToCheck.append(x)
nodeResults = utilities.retry(Testcaselib.nodesCheck,
False,
args=[nodesToCheck],
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
cli = main.CLIs[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
cli.name, cli.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to kill ONOS, stopping test")
main.cleanup()
main.exit()
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
else:
main.active = -1
def killLink(main, end1, end2, switches, links):
"""
end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.linkSleep = float(main.params["timers"]["LinkDiscovery"])
main.step("Kill link between %s and %s" % (end1, end2))
LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down")
main.log.info("Waiting %s seconds for link down to be discovered" % main.linkSleep)
time.sleep(main.linkSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.linkSleep,
)
result = topology & LinkDown
utilities.assert_equals(
expect=main.TRUE, actual=result, onpass="******", onfail="Failed to turn off link?"
)
def startMininet(main, topology, args=""):
copyResult = main.ONOSbench.scp(main.Mininet1,
main.topoPath + main.topology,
main.Mininet1.home,
direction="to")
if main.topologyLib:
for lib in main.topologyLib.split(","):
copyResult = copyResult and main.ONOSbench.scp(
main.Mininet1,
main.topoPath + lib,
main.Mininet1.home,
direction="to")
if main.topologyConf:
for conf in main.topologyConf.split(","):
copyResult = copyResult and main.ONOSbench.scp(
main.Mininet1,
main.configPath + main.forConfig + conf,
"~/",
direction="to")
stepResult = copyResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to copy topo files")
main.step("Starting Mininet Topology")
arg = "--onos-ip=%s %s" % (",".join(
[ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
main.topology = topology
topoResult = main.Mininet1.startNet(topoFile=main.Mininet1.home +
main.topology,
args=arg)
stepResult = topoResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to load topology")
# Exit if topology did not load properly
if not topoResult:
main.cleanAndExit()
def connectToPhysicalNetwork(main, switchNames):
main.step("Connecting to physical netowrk")
topoResult = main.NetworkBench.connectToNet()
stepResult = topoResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to load topology")
# Exit if topology did not load properly
if not topoResult:
main.cleanAndExit()
main.step("Assign switches to controllers.")
assignResult = main.TRUE
for name in switchNames:
assignResult = assignResult & main.NetworkBench.assignSwController(
sw=name, ip=main.Cluster.getIps(), port='6653')
utilities.assert_equals(
expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to assign switches to controllers")
def killOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Killing ONOS instances with index(es): {}".format(nodes))
for i in nodes:
killResult = main.ONOSbench.onosDie(
main.Cluster.runningNodes[i].ipAddress)
utilities.assert_equals(expect=main.TRUE,
actual=killResult,
onpass="******",
onfail="Error killing ONOS instance")
main.Cluster.runningNodes[i].active = False
time.sleep(12)
if len(nodes) < main.Cluster.numCtrls:
nodeResults = utilities.retry(main.Cluster.nodesCheck,
False,
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
ctrl = main.Cluster.runningNodes[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
ctrl.name,
ctrl.CLI.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to kill ONOS, stopping test")
main.cleanAndExit()
topology = utilities.retry(main.Cluster.active(0).CLI.checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
def recoverOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
Recover an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Recovering ONOS instance")
[main.ONOSbench.onosStart(main.CLIs[i].ip_address) for i in nodes]
for i in nodes:
isUp = main.ONOSbench.isup(main.ONOSip[i])
utilities.assert_equals(
expect=main.TRUE,
actual=isUp,
onpass="******",
onfail="ONOS service did not start properly")
for i in nodes:
main.step("Checking if ONOS CLI is ready")
main.CLIs[i].startCellCli()
cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i],
commandlineTimeout=60,
onosStartTimeout=100)
utilities.assert_equals(expect=main.TRUE,
actual=cliResult,
onpass="******",
onfail="ONOS CLI is not ready")
main.active = i if main.active == -1 else main.active
main.step("Checking ONOS nodes")
nodeResults = utilities.retry(Testcaselib.nodesCheck,
False,
args=[nodes],
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
cli = main.CLIs[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
cli.name, cli.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to start ONOS, stopping test")
main.cleanup()
main.exit()
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
for i in range(10):
ready = True
output = main.CLIs[main.active].summary()
if not output:
ready = False
if ready:
break
time.sleep(10)
utilities.assert_equals(expect=True,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
if not ready:
main.log.error("ONOS startup failed!")
main.cleanup()
main.exit()
def installOnos(main, vlanCfg=True):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
if main.diff:
main.apps = main.apps + "," + main.diff
else:
main.log.error("App list is empty")
main.log.info("NODE COUNT = " + str(main.Cluster.numCtrls))
main.log.info(''.join(main.Cluster.getIps()))
main.dynamicHosts = ['in1', 'out1']
main.testSetUp.createApplyCell(main.Cluster,
newCell=True,
cellName=main.cellName,
Mininet=main.Mininet1,
useSSH=Testcaselib.useSSH,
ips=main.Cluster.getIps())
# kill off all onos processes
main.log.info("Safety check, killing all ONOS processes" +
" before initiating environment setup")
for ctrl in main.Cluster.runningNodes:
main.ONOSbench.onosDie(ctrl.ipAddress)
main.testSetUp.buildOnos(main.Cluster)
main.testSetUp.installOnos(main.Cluster, False)
main.testSetUp.setupSsh(main.Cluster)
main.testSetUp.checkOnosService(main.Cluster)
cliResult = main.TRUE
main.step("Checking if ONOS CLI is ready")
for ctrl in main.Cluster.runningNodes:
ctrl.CLI.startCellCli()
cliResult = cliResult and ctrl.CLI.startOnosCli(
ctrl.ipAddress, commandlineTimeout=60, onosStartTimeout=100)
ctrl.active = True
utilities.assert_equals(expect=main.TRUE,
actual=cliResult,
onpass="******",
onfail="ONOS CLI is not ready")
ready = utilities.retry(main.Cluster.active(0).CLI.summary,
main.FALSE,
sleep=10,
attempts=10)
if ready:
ready = main.TRUE
utilities.assert_equals(expect=main.TRUE,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
with open("%s/json/%s.json" %
(main.dependencyPath, main.cfgName)) as cfg:
main.Cluster.active(0).REST.setNetCfg(json.load(cfg))
with open("%s/json/%s.chart" %
(main.dependencyPath, main.cfgName)) as chart:
main.pingChart = json.load(chart)
if not ready:
main.log.error("ONOS startup failed!")
main.cleanAndExit()
for ctrl in main.Cluster.active():
ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting")
ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline")
ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl")
ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def installOnos(main):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
apps = main.apps
if main.diff:
apps = main.apps + "," + main.diff
else:
main.log.error("App list is empty")
main.case("Package and start ONOS using apps:" + apps)
print "NODE COUNT = ", main.numCtrls
print main.ONOSip
tempOnosIp = []
for i in range(main.numCtrls):
tempOnosIp.append(main.ONOSip[i])
onosUser = main.params["ENV"]["cellUser"]
main.step("Create and Apply cell file")
main.ONOSbench.createCellFile(
main.ONOSbench.ip_address, "temp", main.Mininet1.ip_address, apps, tempOnosIp, onosUser
)
cellResult = main.ONOSbench.setCell("temp")
verifyResult = main.ONOSbench.verifyCell()
stepResult = cellResult and verifyResult
utilities.assert_equals(
expect=main.TRUE,
actual=stepResult,
onpass="******" + "environment",
onfail="Failed to apply cell to environment ",
)
# kill off all onos processes
main.log.info("Safety check, killing all ONOS processes" + " before initiating environment setup")
for i in range(main.maxNodes):
main.ONOSbench.onosDie(main.ONOSip[i])
main.step("Create and Install ONOS package")
main.ONOSbench.handle.sendline(
"cp " + main.dependencyPath + "/" + main.cfgName + ".json ~/onos/tools/package/config/network-cfg.json"
)
packageResult = main.ONOSbench.onosPackage()
onosInstallResult = main.TRUE
for i in range(main.numCtrls):
onosInstallResult = onosInstallResult and main.ONOSbench.onosInstall(node=main.ONOSip[i])
stepResult = onosInstallResult
utilities.assert_equals(
expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to install ONOS package",
)
main.step("Starting ONOS service")
stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE
for i in range(main.numCtrls):
onosIsUp = onosIsUp and main.ONOSbench.isup(main.ONOSip[i])
if onosIsUp == main.TRUE:
main.log.report("ONOS instance is up and ready")
else:
main.log.report("ONOS instance may not be up, stop and " + "start ONOS again ")
for i in range(main.numCtrls):
stopResult = stopResult and main.ONOSbench.onosStop(main.ONOSip[i])
for i in range(main.numCtrls):
startResult = startResult and main.ONOSbench.onosStart(main.ONOSip[i])
stepResult = onosIsUp and stopResult and startResult
utilities.assert_equals(
expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="ONOS service did not start properly",
)
main.step("Checking if ONOS CLI is ready")
for i in range(main.numCtrls):
main.CLIs[i].startCellCli()
cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i], commandlineTimeout=60, onosStartTimeout=100)
utilities.assert_equals(
expect=main.TRUE, actual=cliResult, onpass="******", onfail="ONOS CLI is not ready"
)
main.active = 0
for i in range(10):
ready = True
output = main.CLIs[main.active].summary()
if not output:
ready = False
if ready:
break
time.sleep(10)
utilities.assert_equals(
expect=True, actual=ready, onpass="******", onfail="ONOS summary command failed"
)
if not ready:
main.log.error("ONOS startup failed!")
main.cleanup()
main.exit()
for i in range(main.numCtrls):
main.CLIs[i].logSet("DEBUG", "org.onosproject.segmentrouting")
main.CLIs[i].logSet("DEBUG", "org.onosproject.driver.pipeline")
main.CLIs[i].logSet("DEBUG", "org.onosproject.store.group.impl")
main.CLIs[i].logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def installOnos(main, vlanCfg=True):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
apps = main.apps
if main.diff:
apps = main.apps + "," + main.diff
else:
main.log.error("App list is empty")
print "NODE COUNT = ", main.numCtrls
print main.ONOSip
tempOnosIp = []
main.dynamicHosts = ['in1', 'out1']
for i in range(main.numCtrls):
tempOnosIp.append(main.ONOSip[i])
onosUser = main.params['ENV']['cellUser']
main.step("Create and Apply cell file")
main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
"temp",
main.Mininet1.ip_address,
apps,
tempOnosIp,
onosUser,
useSSH=Testcaselib.useSSH)
cellResult = main.ONOSbench.setCell("temp")
verifyResult = main.ONOSbench.verifyCell()
stepResult = cellResult and verifyResult
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
onpass="******" + \
"environment",
onfail="Failed to apply cell to environment " )
# kill off all onos processes
main.log.info("Safety check, killing all ONOS processes" +
" before initiating environment setup")
for i in range(main.maxNodes):
main.ONOSbench.onosDie(main.ONOSip[i])
main.step("Create and Install ONOS package")
packageResult = main.ONOSbench.buckBuild()
onosInstallResult = main.TRUE
for i in range(main.numCtrls):
onosInstallResult = onosInstallResult and \
main.ONOSbench.onosInstall(
node=main.ONOSip[ i ] )
stepResult = onosInstallResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to install ONOS package")
if Testcaselib.useSSH:
for i in range(main.numCtrls):
onosInstallResult = onosInstallResult and \
main.ONOSbench.onosSecureSSH(
node=main.ONOSip[ i ] )
stepResult = onosInstallResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="Failed to secure SSH")
main.step("Starting ONOS service")
stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
for i in range(main.numCtrls):
onosIsUp = onosIsUp and main.ONOSbench.isup(main.ONOSip[i])
if onosIsUp == main.TRUE:
main.log.report("ONOS instance is up and ready")
else:
main.log.report("ONOS instance may not be up, stop and " +
"start ONOS again ")
for i in range(main.numCtrls):
stopResult = stopResult and \
main.ONOSbench.onosStop( main.ONOSip[ i ] )
for i in range(main.numCtrls):
startResult = startResult and \
main.ONOSbench.onosStart( main.ONOSip[ i ] )
stepResult = onosIsUp and stopResult and startResult
utilities.assert_equals(expect=main.TRUE,
actual=stepResult,
onpass="******",
onfail="ONOS service did not start properly")
main.step("Checking if ONOS CLI is ready")
for i in range(main.numCtrls):
main.CLIs[i].startCellCli()
cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i],
commandlineTimeout=60,
onosStartTimeout=100)
utilities.assert_equals(expect=main.TRUE,
actual=cliResult,
onpass="******",
onfail="ONOS CLI is not ready")
main.active = 0
for i in range(10):
ready = True
output = main.CLIs[main.active].summary()
if not output:
ready = False
if ready:
break
time.sleep(10)
utilities.assert_equals(expect=True,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
with open("%s/json/%s.json" %
(main.dependencyPath, main.cfgName)) as cfg:
main.RESTs[main.active].setNetCfg(json.load(cfg))
with open("%s/json/%s.chart" %
(main.dependencyPath, main.cfgName)) as chart:
main.pingChart = json.load(chart)
if not ready:
main.log.error("ONOS startup failed!")
main.cleanup()
main.exit()
for i in range(main.numCtrls):
main.CLIs[i].logSet("DEBUG", "org.onosproject.segmentrouting")
main.CLIs[i].logSet("DEBUG", "org.onosproject.driver.pipeline")
main.CLIs[i].logSet("DEBUG", "org.onosproject.store.group.impl")
main.CLIs[i].logSet("DEBUG",
"org.onosproject.net.flowobjective.impl")
def compareTopos(self, Mininet, attempts=1, includeCaseDesc=True):
"""
Description:
compares the links and hosts and switches of the onos to the mininet.
Required:
* Mininet - Mininet driver to use.
* attempts - number of attempts to compare in case
the result is different after a certain time.
Returns:
Returns main.TRUE if the results are matching else
Returns main.FALSE
"""
if includeCaseDesc:
main.case("Compare ONOS Topology view to Mininet topology")
main.caseExplanation = "Compare topology elements between Mininet" +\
" and ONOS"
main.log.info("Gathering topology information from Mininet")
devicesResults = main.FALSE # Overall Boolean for device correctness
linksResults = main.FALSE # Overall Boolean for link correctness
hostsResults = main.FALSE # Overall Boolean for host correctness
deviceFails = [] # Nodes where devices are incorrect
linkFails = [] # Nodes where links are incorrect
hostFails = [] # Nodes where hosts are incorrect
mnSwitches = Mininet.getSwitches()
mnLinks = Mininet.getLinks()
mnHosts = Mininet.getHosts()
main.step("Comparing Mininet topology to ONOS topology")
while ( attempts >= 0 ) and\
( not devicesResults or not linksResults or not hostsResults ):
main.log.info("Sleeping {} seconds".format(2))
time.sleep(2)
if not devicesResults:
devices = self.getAll("devices", False)
ports = self.getAll("ports", False)
devicesResults = main.TRUE
deviceFails = [] # Reset for each failed attempt
if not linksResults:
links = self.getAll("links", False)
linksResults = main.TRUE
linkFails = [] # Reset for each failed attempt
if not hostsResults:
hosts = self.getAll("hosts", False)
hostsResults = main.TRUE
hostFails = [] # Reset for each failed attempt
# Check for matching topology on each node
for controller in main.Cluster.getRunningPos():
controllerStr = str(controller + 1) # ONOS node number
# Compare Devices
currentDevicesResult = self.compareDevicePort(
Mininet, controller, mnSwitches, devices, ports)
if not currentDevicesResult:
deviceFails.append(controllerStr)
devicesResults = devicesResults and currentDevicesResult
# Compare Links
currentLinksResult = self.compareBase(links, controller,
Mininet.compareLinks,
[mnSwitches, mnLinks])
if not currentLinksResult:
linkFails.append(controllerStr)
linksResults = linksResults and currentLinksResult
# Compare Hosts
currentHostsResult = self.compareBase(hosts, controller,
Mininet.compareHosts,
mnHosts)
if not currentHostsResult:
hostFails.append(controllerStr)
hostsResults = hostsResults and currentHostsResult
# Decrement Attempts Remaining
attempts -= 1
utilities.assert_equals(
expect=[],
actual=deviceFails,
onpass="******",
onfail="ONOS incorrectly discovered devices on nodes: " +
str(deviceFails))
utilities.assert_equals(
expect=[],
actual=linkFails,
onpass="******",
onfail="ONOS incorrectly discovered links on nodes: " +
str(linkFails))
utilities.assert_equals(
expect=[],
actual=hostFails,
onpass="******",
onfail="ONOS incorrectly discovered hosts on nodes: " +
str(hostFails))
topoResults = hostsResults and linksResults and devicesResults
utilities.assert_equals(
expect=main.TRUE,
actual=topoResults,
onpass="******",
onfail="ONOS incorrectly discovered the topology")
return topoResults
def pingAll(main,
tag="",
dumpflows=True,
acceptableFailed=0,
basedOnIp=False,
sleep=10,
retryAttempts=1):
'''
Verify connectivity between hosts according to the ping chart
acceptableFailed: max number of acceptable failed pings.
basedOnIp: if True, run ping or ping6 based on suffix of host names
retryAttempts: the number of retry ping. Only works for IPv4 hosts.
'''
main.log.report("Check host connectivity")
main.log.debug("Ping chart: %s" % main.pingChart)
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
for entry in main.pingChart.itervalues():
main.log.debug("Entry in ping chart: %s" % entry)
expect = entry['expect']
if expect == "Unidirectional":
# Verify ping from each src host to each dst host
src = entry['src']
dst = entry['dst']
expect = main.TRUE
main.step(
"Verify unidirectional connectivity from %s to %s with tag %s"
% (str(src), str(dst), tag))
if basedOnIp:
if ("v4" in src[0]):
pa = main.Network.pingallHostsUnidirectional(
src, dst, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv4 connectivity failed")
if ("v6" in src[0]):
pa = main.Network.pingallHostsUnidirectional(
src,
dst,
ipv6=True,
acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv6 connectivity failed")
else:
pa = main.Network.pingallHostsUnidirectional(
src, dst, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IP connectivity failed")
else:
# Verify ping between each host pair
hosts = entry['hosts']
try:
expect = main.TRUE if str(
expect).lower() == 'true' else main.FALSE
except:
expect = main.FALSE
main.step("Verify full connectivity for %s with tag %s" %
(str(hosts), tag))
if basedOnIp:
if ("v4" in hosts[0]):
pa = utilities.retry(
main.Network.pingallHosts,
main.FALSE if expect else main.TRUE,
args=(hosts, ),
attempts=retryAttempts,
sleep=sleep)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv4 connectivity failed")
if ("v6" in hosts[0]):
pa = main.Network.pingIpv6Hosts(
hosts, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv6 connectivity failed")
else:
pa = main.Network.pingallHosts(hosts)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IP connectivity failed")
if dumpflows:
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "flows", main.logdir,
tag + "_FlowsOn")
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "groups", main.logdir,
tag + "_GroupsOn")
