def checkFlows(main, minFlowCount, tag="", dumpflows=True, sleep=10):
main.step("Check whether the flow count is bigger than %s" %
minFlowCount)
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
count = utilities.retry(main.Cluster.active(0).CLI.checkFlowCount,
main.FALSE,
kwargs={'min': minFlowCount},
attempts=10,
sleep=sleep)
utilities.assertEquals(expect=True,
actual=(count > 0),
onpass="******" +
str(count),
onfail="Flow count looks wrong: " + str(count))
main.step("Check whether all flow status are ADDED")
flowCheck = utilities.retry(main.Cluster.active(0).CLI.checkFlowsState,
main.FALSE,
kwargs={'isPENDING': False},
attempts=5,
sleep=sleep)
utilities.assertEquals(expect=main.TRUE,
actual=flowCheck,
onpass="******",
onfail="Flow status is wrong!")
if dumpflows:
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "flows", main.logdir,
tag + "_FlowsBefore")
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "groups", main.logdir,
tag + "_GroupsBefore")
def checkFlows(main, minFlowCount, dumpflows=True):
main.step(" Check whether the flow count is bigger than %s" %
minFlowCount)
count = utilities.retry(main.CLIs[main.active].checkFlowCount,
main.FALSE,
kwargs={'min': minFlowCount},
attempts=10,
sleep=10)
utilities.assertEquals( \
expect=True,
actual=(count > 0),
onpass="******" + str( count ),
onfail="Flow count looks wrong: " + str( count ) )
main.step("Check whether all flow status are ADDED")
flowCheck = utilities.retry(main.CLIs[main.active].checkFlowsState,
main.FALSE,
kwargs={'isPENDING': False},
attempts=2,
sleep=10)
utilities.assertEquals( \
expect=main.TRUE,
actual=flowCheck,
onpass="******",
onfail="Flow status is wrong!" )
if dumpflows:
main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "flows",
main.logdir,
"flowsBefore" + main.cfgName)
main.ONOSbench.dumpONOSCmd(main.ONOSip[main.active], "groups",
main.logdir,
"groupsBefore" + main.cfgName)
def killOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Killing ONOS instance")
for i in nodes:
killResult = main.ONOSbench.onosDie(main.CLIs[i].ip_address)
utilities.assert_equals(expect=main.TRUE,
actual=killResult,
onpass="******",
onfail="Error killing ONOS instance")
if i == main.active:
main.active = (i + 1) % main.numCtrls
time.sleep(12)
if len(nodes) < main.numCtrls:
nodesToCheck = []
for x in range(0, main.numCtrls):
if x not in nodes:
nodesToCheck.append(x)
nodeResults = utilities.retry(Testcaselib.nodesCheck,
False,
args=[nodesToCheck],
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
cli = main.CLIs[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
cli.name, cli.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to kill ONOS, stopping test")
main.cleanup()
main.exit()
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
else:
main.active = -1
def killOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Killing ONOS instances with index(es): {}".format(nodes))
for i in nodes:
killResult = main.ONOSbench.onosDie(
main.Cluster.runningNodes[i].ipAddress)
utilities.assert_equals(expect=main.TRUE,
actual=killResult,
onpass="******",
onfail="Error killing ONOS instance")
main.Cluster.runningNodes[i].active = False
time.sleep(12)
if len(nodes) < main.Cluster.numCtrls:
nodeResults = utilities.retry(main.Cluster.nodesCheck,
False,
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
ctrl = main.Cluster.runningNodes[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
ctrl.name,
ctrl.CLI.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to kill ONOS, stopping test")
main.cleanAndExit()
topology = utilities.retry(main.Cluster.active(0).CLI.checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
def killLink(main, end1, end2, switches, links):
"""
end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.linkSleep = float(main.params['timers']['LinkDiscovery'])
main.step("Kill link between %s and %s" % (end1, end2))
LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down")
LinkDown = main.Mininet1.link(END2=end1, END1=end2, OPTION="down")
main.log.info("Waiting %s seconds for link down to be discovered" %
main.linkSleep)
time.sleep(main.linkSleep)
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links
},
attempts=10,
sleep=main.linkSleep)
result = topology & LinkDown
utilities.assert_equals(expect=main.TRUE,
actual=result,
onpass="******",
onfail="Failed to turn off link?")
def killSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
Completely kill a switch and verify ONOS can see the proper change
"""
main.switchSleep = float(main.params['timers']['SwitchDiscovery'])
main.step("Kill " + switch)
main.log.info("Stopping" + switch)
main.Mininet1.switch(SW=switch, OPTION="stop")
# todo make this repeatable
main.log.info("Waiting %s seconds for switch down to be discovered" %
(main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links
},
attempts=10,
sleep=main.switchSleep)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to kill switch?")
def assignSwController(self, ip, port="6653", ptcp=""):
"""
Description:
The assignment is realized properly creating the agent.conf
for each switch and then pushing it into the device.
Required:
ip - Ip addresses of controllers. This can be a list or a string.
Optional:
port - controller port is ignored
ptcp - ptcp information is ignored
Return:
Returns main.TRUE if the switch is correctly assigned to controllers,
otherwise it will return main.FALSE or an appropriate exception(s)
"""
assignResult = main.TRUE
# Initial arguments for OFDPA
opt_args = 'OPT_ARGS="-d 2 -c 2 -c 4 '
onosIp = ""
# Parses the controller option
try:
if isinstance(ip, types.StringType):
onosIp = "-t " + str(ip)
elif isinstance(ip, types.ListType):
for ipAddress in ip:
onosIp += "-t " + str(ipAddress) + " "
else:
main.log.error(self.name + ": Invalid ip address")
return main.FALSE
# Complete the arguments adding the dpid
opt_args += onosIp + '-i %s' % self.dpid + '"'
# Create a copy of the cfg file using the template
self.createCfg()
# Load the cfg file and adds the missing option
self.updateCfg(opt_args)
# Backup the cfg on the switch
self.backupCfg()
# Push the new cfg on the device
self.pushCfg()
# Start the ofagent on the device
self.startOfAgent()
# Enable all the ports
assignResult = utilities.retry(self.enablePorts,
main.FALSE,
kwargs={},
attempts=5,
sleep=10)
# Done return true
return assignResult
# Errors handling
except pexpect.TIMEOUT:
main.log.error(self.name + ": pexpect.TIMEOUT found")
return main.FALSE
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
main.cleanAndExit()
except Exception:
main.log.exception(self.name + ": Uncaught exception!")
main.cleanAndExit()
def checkFlows(main, minFlowCount):
main.step(" Check whether the flow count is bigger than %s" % minFlowCount)
count = utilities.retry(
main.CLIs[main.active].checkFlowCount, main.FALSE, kwargs={"min": minFlowCount}, attempts=10, sleep=10
)
utilities.assertEquals(
expect=True,
actual=(count > 0),
onpass="******" + str(count),
onfail="Flow count looks wrong: " + str(count),
)
main.step("Check whether all flow status are ADDED")
flowCheck = utilities.retry(
main.CLIs[main.active].checkFlowsState, main.FALSE, kwargs={"isPENDING": False}, attempts=10, sleep=10
)
utilities.assertEquals(
expect=main.TRUE, actual=flowCheck, onpass="******", onfail="Flow status is wrong!"
)
main.ONOSbench.dumpFlows(main.ONOSip[main.active], main.logdir, "flowsBefore" + main.cfgName)
main.ONOSbench.dumpGroups(main.ONOSip[0], main.logdir, "groupsBefore" + main.cfgName)
def checkDevices(main, switches, tag="", sleep=10):
main.step("Check whether the switches count is equal to %s" % switches)
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
result = utilities.retry(main.Cluster.active(0).CLI.checkStatus,
main.FALSE,
kwargs={'numoswitch': switches},
attempts=10,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=result,
onpass="******",
onfail="Failed to boot up devices?")
def checkFlowEqualityByDpid(main, dpid, flowCount, sleep=10):
main.step(" Check whether the flow count of device %s is equal to %s" %
(dpid, flowCount))
count = utilities.retry(main.Cluster.active(0).CLI.checkFlowAddedCount,
main.FALSE,
args=(dpid, flowCount, False, 1),
attempts=5,
sleep=sleep)
utilities.assertEquals(
expect=True,
actual=(int(count) == flowCount),
onpass="******" + str(count),
onfail="Flow count looks wrong, should be " + str(flowCount))
def checkFlowsByDpid(main, dpid, minFlowCount, sleep=10):
main.step(
" Check whether the flow count of device %s is bigger than %s" %
(dpid, minFlowCount))
count = utilities.retry(main.Cluster.active(0).CLI.checkFlowAddedCount,
main.FALSE,
args=(dpid, minFlowCount),
attempts=5,
sleep=sleep)
utilities.assertEquals(expect=True,
actual=(count > minFlowCount),
onpass="******" +
str(count),
onfail="Flow count looks wrong. ")
def installOnos(main,
vlanCfg=True,
skipPackage=False,
cliSleep=10,
parallel=True):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
if not main.apps:
main.log.error("App list is empty")
main.log.info("Cluster size: " + str(main.Cluster.numCtrls))
main.log.info("Cluster ips: " + ', '.join(main.Cluster.getIps()))
main.dynamicHosts = ['in1', 'out1']
main.testSetUp.ONOSSetUp(main.Cluster,
newCell=True,
cellName=main.cellName,
skipPack=skipPackage,
useSSH=Testcaselib.useSSH,
installParallel=parallel,
includeCaseDesc=False)
ready = utilities.retry(main.Cluster.active(0).CLI.summary,
main.FALSE,
sleep=cliSleep,
attempts=10)
if ready:
ready = main.TRUE
utilities.assert_equals(expect=main.TRUE,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
if not ready:
main.log.error("ONOS startup failed!")
main.cleanAndExit()
for ctrl in main.Cluster.active():
ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting")
ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline")
ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl")
ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def sendScapyPackets(self,
sender,
receiver,
pktFilter,
pkt,
sIface=None,
dIface=None,
expect=True,
acceptableFailed=0,
collectT3=True,
t3Command=""):
"""
Description:
Send Scapy packets from sender to receiver and verify if result is as expected and retry if neccessary
If collectT3 is True and t3Command is specified, collect t3-troubleshoot output on unexpected scapy results
Options:
sender: the component of the host that is sending packets
receiver: the component of the host that is receiving packets
pktFilter: packet filter used by receiver
pkt: keyword that is expected to be conrained in the received packets
expect: expect receiver to receive the packet if True, otherwise not receiving the packet
acceptableFailed: maximum number of unexpected scapy results acceptable
collectT3: save t3-troubleshoot output for unexpected scapy results
Returns:
main.TRUE if scapy result is expected, otherwise main.FALSE
Note: It is assumed that Scapy is already started on the destination host
"""
main.log.info(
"Sending scapy packets from {} to {}, expected result is {}".
format(sender.name, receiver.name, "pass" if expect else "fail"))
scapyResult = utilities.retry(self.sendScapyPacketsHelper,
main.FALSE,
args=(sender, receiver, pktFilter, pkt,
sIface, dIface, expect),
attempts=acceptableFailed + 1,
sleep=1)
if not scapyResult and collectT3 and t3Command:
main.log.debug(
"Collecting t3 with source {} and destination {}".format(
sender.name, receiver.name))
main.log.debug("t3 command: {}".format(t3Command))
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, t3Command, main.logdir,
"t3-CASE{}-{}-{}-".format(main.CurrentTestCaseNumber,
sender.name, receiver.name))
return scapyResult
def verifyNetworkHostIp(main, attempts=10, sleep=10):
"""
Verifies IP address assignment from the hosts
"""
main.step("Verify IP address assignment from hosts")
ipResult = main.TRUE
for hostName, ip in main.expectedHosts["network"].items():
ipResult = ipResult and utilities.retry(main.Network.verifyHostIp,
main.FALSE,
kwargs={
'hostList': [hostName],
'prefix': ip
},
attempts=attempts,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=ipResult,
onpass="******",
onfail="Verify network host IP failed")
def recoverSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
Recover a switch and verify ONOS can see the proper change
"""
main.step("Recovering " + switch)
main.log.info("Starting" + switch)
main.Mininet1.switch(SW=switch, OPTION="start")
main.log.info("Waiting %s seconds for switch up to be discovered" % (main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.switchSleep,
)
utilities.assert_equals(
expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to recover switch?"
)
def verifyOnosHostIp(main, attempts=10, sleep=10):
"""
Verifies host IP address assignment from ONOS
"""
main.step("Verify host IP address assignment in ONOS")
ipResult = main.TRUE
for hostName, ip in main.expectedHosts["onos"].items():
ipResult = ipResult and utilities.retry(
main.Cluster.active(0).verifyHostIp,
main.FALSE,
kwargs={
'hostList': [hostName],
'prefix': ip
},
attempts=attempts,
sleep=sleep)
utilities.assert_equals(expect=main.TRUE,
actual=ipResult,
onpass="******",
onfail="Verify ONOS host IP failed")
def killSwitch(main, switch, switches, links):
"""
Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
Completely kill a switch and verify ONOS can see the proper change
"""
main.switchSleep = float(main.params["timers"]["SwitchDiscovery"])
main.step("Kill " + switch)
main.log.info("Stopping" + switch)
main.Mininet1.switch(SW=switch, OPTION="stop")
main.log.info("Waiting %s seconds for switch down to be discovered" % (main.switchSleep))
time.sleep(main.switchSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.switchSleep,
)
utilities.assert_equals(
expect=main.TRUE, actual=topology, onpass="******", onfail="Failed to kill switch?"
)
def recoverSwitch( main, switch, switches, links ):
"""
Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
Recover a switch and verify ONOS can see the proper change
"""
# todo make this repeatable
main.step( "Recovering " + switch )
main.log.info( "Starting" + switch )
main.Mininet1.switch( SW=switch, OPTION="start" )
main.log.info( "Waiting %s seconds for switch up to be discovered" % (
main.switchSleep) )
time.sleep( main.switchSleep )
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
kwargs={ 'numoswitch': switches,
'numolink': links },
attempts=10,
sleep=main.switchSleep )
utilities.assert_equals( expect=main.TRUE, actual=topology,
onpass="******",
onfail="Failed to recover switch?" )
def killLink(main, end1, end2, switches, links):
"""
end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
Kill a link and verify ONOS can see the proper link change
"""
main.linkSleep = float(main.params["timers"]["LinkDiscovery"])
main.step("Kill link between %s and %s" % (end1, end2))
LinkDown = main.Mininet1.link(END1=end1, END2=end2, OPTION="down")
main.log.info("Waiting %s seconds for link down to be discovered" % main.linkSleep)
time.sleep(main.linkSleep)
topology = utilities.retry(
main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={"numoswitch": switches, "numolink": links},
attempts=10,
sleep=main.linkSleep,
)
result = topology & LinkDown
utilities.assert_equals(
expect=main.TRUE, actual=result, onpass="******", onfail="Failed to turn off link?"
)
def restoreLinkBatch(main, links, linksAfter, switches=7):
"""
links = list of link (src, dst) to bring up again.
"""
main.step("Restoring a batch of links {0}".format(links))
for end1, end2 in links:
main.Network.link(END1=end1, END2=end2, OPTION="up")
main.Network.link(END1=end2, END2=end1, OPTION="up")
main.linkSleep = float(main.params['timers']['LinkDiscovery'])
main.log.info("Waiting %s seconds for links down to be discovered" %
main.linkSleep)
time.sleep(main.linkSleep)
topology = utilities.retry(main.Cluster.active(0).CLI.checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': linksAfter
},
attempts=10,
sleep=main.linkSleep)
def pingAndCapture(self,
srcHost,
dstIp,
dstHost,
dstIntf,
ipv6=False,
expect=True,
acceptableFailed=0,
collectT3=True,
t3Simple=False):
"""
Description:
Ping from src host to dst IP and capture packets at dst Host using Scapy and retry if neccessary
If collectT3 is True, collect t3-troubleshoot output on unexpected scapy results
Options:
srcHost: name of the source host
dstIp: destination IP of the ping packets
dstHost: host that runs Scapy to capture the packets
dstIntf: name of the interface on the destination host
ipv6: ping with IPv6 if True; Otherwise IPv4
expect: use True if the ping is expected to be captured at destination;
Otherwise False
acceptableFailed: maximum number of failed pings acceptable
collectT3: save t3-troubleshoot output for src and dst host that failed to ping
t3Simple: use t3-troubleshoot-simple command when collecting t3 output
Returns:
main.TRUE if packet capturing result is expected, otherwise main.FALSE
Note: It is assumed that Scapy is already started on the destination host
"""
main.log.info(
"Pinging from {} to {}, expected {} capture the packet at {}".
format(srcHost, dstIp, "to" if expect else "not to", dstHost))
# Verify host component has been created
if not hasattr(main, srcHost):
main.log.info("Creating component for host {}".format(srcHost))
main.Network.createHostComponent(srcHost)
srcHandle = getattr(main, srcHost)
main.log.info("Starting CLI on host {}".format(srcHost))
srcHandle.startHostCli()
trafficResult = utilities.retry(self.pingAndCaptureHelper,
main.FALSE,
args=(srcHost, dstIp, dstHost, dstIntf,
ipv6, expect),
attempts=acceptableFailed + 1,
sleep=1)
if not trafficResult and collectT3:
srcIp = main.Network.getIPAddress(srcHost,
proto='IPV6' if ipv6 else 'IPV4')
main.log.debug(
"Collecting t3 with source {} and destination {}".format(
srcIp, dstIp))
cmdList = main.Cluster.active(0).CLI.composeT3Command(
srcIp, dstIp, ipv6, True, t3Simple)
if not cmdList:
main.log.warn(
"Failed to compose T3 command with source {} and destination {}"
.format(srcIp, dstIp))
for i in range(0, len(cmdList)):
cmd = cmdList[i]
main.log.debug("t3 command: {}".format(cmd))
main.ONOSbench.dumpONOSCmd(main.Cluster.active(0).ipAddress,
cmd,
main.logdir,
"t3-CASE{}-{}-{}-route{}-".format(
main.CurrentTestCaseNumber,
srcIp, dstIp, i),
timeout=10)
return trafficResult
def recoverOnos(main, nodes, switches, links, expNodes):
"""
Params: nodes, integer array with position of the ONOS nodes in the CLIs array
switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
Recover an ONOS instance and verify the ONOS cluster can see the proper change
"""
main.step("Recovering ONOS instance")
[main.ONOSbench.onosStart(main.CLIs[i].ip_address) for i in nodes]
for i in nodes:
isUp = main.ONOSbench.isup(main.ONOSip[i])
utilities.assert_equals(
expect=main.TRUE,
actual=isUp,
onpass="******",
onfail="ONOS service did not start properly")
for i in nodes:
main.step("Checking if ONOS CLI is ready")
main.CLIs[i].startCellCli()
cliResult = main.CLIs[i].startOnosCli(main.ONOSip[i],
commandlineTimeout=60,
onosStartTimeout=100)
utilities.assert_equals(expect=main.TRUE,
actual=cliResult,
onpass="******",
onfail="ONOS CLI is not ready")
main.active = i if main.active == -1 else main.active
main.step("Checking ONOS nodes")
nodeResults = utilities.retry(Testcaselib.nodesCheck,
False,
args=[nodes],
attempts=5,
sleep=10)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful")
if not nodeResults:
for i in nodes:
cli = main.CLIs[i]
main.log.debug("{} components not ACTIVE: \n{}".format(
cli.name, cli.sendline("scr:list | grep -v ACTIVE")))
main.log.error("Failed to start ONOS, stopping test")
main.cleanup()
main.exit()
topology = utilities.retry(main.CLIs[main.active].checkStatus,
main.FALSE,
kwargs={
'numoswitch': switches,
'numolink': links,
'numoctrl': expNodes
},
attempts=10,
sleep=12)
utilities.assert_equals(expect=main.TRUE,
actual=topology,
onpass="******",
onfail="Failed to turn off ONOS Instance")
for i in range(10):
ready = True
output = main.CLIs[main.active].summary()
if not output:
ready = False
if ready:
break
time.sleep(10)
utilities.assert_equals(expect=True,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
if not ready:
main.log.error("ONOS startup failed!")
main.cleanup()
main.exit()
def pingAll(main,
tag="",
dumpflows=True,
acceptableFailed=0,
basedOnIp=False,
sleep=10,
retryAttempts=1):
'''
Verify connectivity between hosts according to the ping chart
acceptableFailed: max number of acceptable failed pings.
basedOnIp: if True, run ping or ping6 based on suffix of host names
retryAttempts: the number of retry ping. Only works for IPv4 hosts.
'''
main.log.report("Check host connectivity")
main.log.debug("Ping chart: %s" % main.pingChart)
if tag == "":
tag = 'CASE%d' % main.CurrentTestCaseNumber
for entry in main.pingChart.itervalues():
main.log.debug("Entry in ping chart: %s" % entry)
expect = entry['expect']
if expect == "Unidirectional":
# Verify ping from each src host to each dst host
src = entry['src']
dst = entry['dst']
expect = main.TRUE
main.step(
"Verify unidirectional connectivity from %s to %s with tag %s"
% (str(src), str(dst), tag))
if basedOnIp:
if ("v4" in src[0]):
pa = main.Network.pingallHostsUnidirectional(
src, dst, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv4 connectivity failed")
if ("v6" in src[0]):
pa = main.Network.pingallHostsUnidirectional(
src,
dst,
ipv6=True,
acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv6 connectivity failed")
else:
pa = main.Network.pingallHostsUnidirectional(
src, dst, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IP connectivity failed")
else:
# Verify ping between each host pair
hosts = entry['hosts']
try:
expect = main.TRUE if str(
expect).lower() == 'true' else main.FALSE
except:
expect = main.FALSE
main.step("Verify full connectivity for %s with tag %s" %
(str(hosts), tag))
if basedOnIp:
if ("v4" in hosts[0]):
pa = utilities.retry(
main.Network.pingallHosts,
main.FALSE if expect else main.TRUE,
args=(hosts, ),
attempts=retryAttempts,
sleep=sleep)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv4 connectivity failed")
if ("v6" in hosts[0]):
pa = main.Network.pingIpv6Hosts(
hosts, acceptableFailed=acceptableFailed)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IPv6 connectivity failed")
else:
pa = main.Network.pingallHosts(hosts)
utilities.assert_equals(
expect=expect,
actual=pa,
onpass="******",
onfail="IP connectivity failed")
if dumpflows:
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "flows", main.logdir,
tag + "_FlowsOn")
main.ONOSbench.dumpONOSCmd(
main.Cluster.active(0).ipAddress, "groups", main.logdir,
tag + "_GroupsOn")
def installOnos(main, vlanCfg=True):
"""
- Set up cell
- Create cell file
- Set cell file
- Verify cell file
- Kill ONOS process
- Uninstall ONOS cluster
- Verify ONOS start up
- Install ONOS cluster
- Connect to cli
"""
# main.scale[ 0 ] determines the current number of ONOS controller
if main.diff:
main.apps = main.apps + "," + main.diff
else:
main.log.error("App list is empty")
main.log.info("NODE COUNT = " + str(main.Cluster.numCtrls))
main.log.info(''.join(main.Cluster.getIps()))
main.dynamicHosts = ['in1', 'out1']
main.testSetUp.createApplyCell(main.Cluster,
newCell=True,
cellName=main.cellName,
Mininet=main.Mininet1,
useSSH=Testcaselib.useSSH,
ips=main.Cluster.getIps())
# kill off all onos processes
main.log.info("Safety check, killing all ONOS processes" +
" before initiating environment setup")
for ctrl in main.Cluster.runningNodes:
main.ONOSbench.onosDie(ctrl.ipAddress)
main.testSetUp.buildOnos(main.Cluster)
main.testSetUp.installOnos(main.Cluster, False)
main.testSetUp.setupSsh(main.Cluster)
main.testSetUp.checkOnosService(main.Cluster)
cliResult = main.TRUE
main.step("Checking if ONOS CLI is ready")
for ctrl in main.Cluster.runningNodes:
ctrl.CLI.startCellCli()
cliResult = cliResult and ctrl.CLI.startOnosCli(
ctrl.ipAddress, commandlineTimeout=60, onosStartTimeout=100)
ctrl.active = True
utilities.assert_equals(expect=main.TRUE,
actual=cliResult,
onpass="******",
onfail="ONOS CLI is not ready")
ready = utilities.retry(main.Cluster.active(0).CLI.summary,
main.FALSE,
sleep=10,
attempts=10)
if ready:
ready = main.TRUE
utilities.assert_equals(expect=main.TRUE,
actual=ready,
onpass="******",
onfail="ONOS summary command failed")
with open("%s/json/%s.json" %
(main.dependencyPath, main.cfgName)) as cfg:
main.Cluster.active(0).REST.setNetCfg(json.load(cfg))
with open("%s/json/%s.chart" %
(main.dependencyPath, main.cfgName)) as chart:
main.pingChart = json.load(chart)
if not ready:
main.log.error("ONOS startup failed!")
main.cleanAndExit()
for ctrl in main.Cluster.active():
ctrl.CLI.logSet("DEBUG", "org.onosproject.segmentrouting")
ctrl.CLI.logSet("DEBUG", "org.onosproject.driver.pipeline")
ctrl.CLI.logSet("DEBUG", "org.onosproject.store.group.impl")
ctrl.CLI.logSet("DEBUG", "org.onosproject.net.flowobjective.impl")
def assignSwController( self, ip, port="6653", ptcp="", updateConf=False ):
"""
Description:
The assignment is realized properly creating the agent.conf
for each switch and then pushing it into the device.
Required:
ip - Ip addresses of controllers. This can be a list or a string.
Optional:
port - controller port is ignored
ptcp - ptcp information is ignored
updateConf - create new ofagent conf file and push to the switch if
set to True; otherwise will use the existing conf file
on the switch.
Return:
Returns main.TRUE if the switch is correctly assigned to controllers,
otherwise it will return main.FALSE or an appropriate exception(s)
"""
assignResult = main.TRUE
# Initial arguments for OFDPA
opt_args = 'OPT_ARGS="-d 2 -c 2 -c 4 '
onosIp = ""
# Parses the controller option
try:
if isinstance( ip, types.StringType ):
onosIp = "-t " + str( ip )
elif isinstance( ip, types.ListType ):
for ipAddress in ip:
onosIp += "-t " + str( ipAddress ) + " "
else:
main.log.error( self.name + ": Invalid ip address" )
return main.FALSE
# Complete the arguments adding the dpid
opt_args += onosIp + '-i %s' % self.dpid + '"'
if updateConf:
# Create a copy of the cfg file using the template
self.createCfg()
# Load the cfg file and adds the missing option
self.updateCfg( opt_args )
# Backup the cfg on the switch
self.backupCfg()
# Push the new cfg on the device
self.pushCfg()
# Start the ofagent on the device
self.startOfAgent()
# Enable all the ports
assignResult = utilities.retry(
self.enablePorts,
main.FALSE,
kwargs={},
attempts=10,
sleep=10)
if not assignResult:
self.isup = False
else:
self.isup = True
# Done return true
return assignResult
# Errors handling
except pexpect.TIMEOUT:
main.log.error( self.name + ": pexpect.TIMEOUT found" )
return main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
main.cleanAndExit()
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanAndExit()