def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info("ONOS HA test: Restart all ONOS nodes - " +
"initialization")
# set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
main.Cluster.numCtrls = 1
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
from tests.HA.HAswapNodes.dependencies.Server import Server
main.Server = Server()
# load some variables from the params file
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
main.testSetUp.ONOSSetUp(main.Cluster,
cellName=cellName,
removeLog=True,
extraApply=[
main.HA.setServerForCluster,
main.HA.scalingMetadata,
main.HA.startingMininet,
main.HA.copyBackupConfig,
main.HA.setMetadataUrl
],
extraClean=main.HA.cleanUpOnosService,
installMax=True)
main.HA.initialSetUp(serviceClean=True)
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
import imp
import pexpect
import time
import json
main.log.info("ONOS HA test: Stop a minority of ONOS nodes - " +
"initialization")
# set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
# load some variables from the params file
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
main.numCtrls = int(main.params['num_controllers'])
stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
main.testSetUp.ONOSSetUp(main.Mininet1,
main.Cluster,
cellName=cellName,
removeLog=True,
extraApply=[
main.HA.startingMininet,
main.HA.customizeOnosGenPartitions
],
extraClean=main.HA.cleanUpGenPartition)
main.HA.initialSetUp()
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info("ONOS HA test: Restart a minority of ONOS nodes - " +
"initialization")
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
main.HA.generateGraph("HAkillNodes")
main.testSetUp.ONOSSetUp(main.Mininet1,
main.Cluster,
cellName=cellName,
removeLog=True,
extraApply=[
main.HA.startingMininet,
main.HA.customizeOnosGenPartitions,
main.HA.copyBackupConfig,
main.ONOSbench.preventAutoRespawn
],
extraClean=main.HA.cleanUpGenPartition)
main.HA.initialSetUp(serviceClean=True)
def CASE1( self, main ):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info( "ONOS HA Sanity test - initialization" )
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error( "ONOSSetup not found. exiting the test" )
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
cellName = main.params[ 'ENV' ][ 'cellName' ]
main.apps = main.params[ 'ENV' ][ 'appString' ]
stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException( e )
main.testSetUp.evnSetupConclusion( stepResult )
main.testSetUp.ONOSSetUp( main.Cluster, cellName=cellName, removeLog=True,
extraApply=main.HA.startingMininet )
main.HA.initialSetUp()
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info("ONOS HA test: Stop a minority of ONOS nodes - " +
"initialization")
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
stepResult = main.testSetUp.envSetup(includeCaseDesc=False)
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
applyFuncs = [main.HA.copyBackupConfig]
applyArgs = [None]
try:
if main.params['topology']['topoFile']:
main.log.info(
'Skipping start of Mininet in this case, make sure you start it elsewhere'
)
else:
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
except (KeyError, IndexError):
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
main.testSetUp.ONOSSetUp(main.Cluster,
cellName=cellName,
removeLog=True,
extraApply=applyFuncs,
applyArgs=applyArgs,
extraClean=main.HA.cleanUpGenPartition,
includeCaseDesc=False)
main.HA.initialSetUp(serviceClean=True)
main.step('Set logging levels')
logging = True
try:
logs = main.params.get('ONOS_Logging', False)
if logs:
for namespace, level in logs.items():
for ctrl in main.Cluster.active():
ctrl.CLI.logSet(level, namespace)
except AttributeError:
logging = False
utilities.assert_equals(expect=True,
actual=logging,
onpass="******",
onfail="Failed to set log levels")
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info("ONOS Single node cluster restart " +
"HA test - initialization")
# set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
# load some variables from the params file
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
main.numCtrls = int(main.params['num_controllers'])
stepResult = main.testSetUp.envSetup(includeCaseDesc=False)
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
cellApps = str(main.params["ENV"]["appString"])
cellNAme = str(main.params["ENV"]["appString"])
applyFuncs = [main.testSetUp.createApplyCell]
applyArgs = [[
main.Cluster, True, cellName, cellApps, "", True,
main.Cluster.runningNodes[0].ipAddress
]]
try:
if main.params['topology']['topoFile']:
main.log.info(
'Skipping start of Mininet in this case, make sure you start it elsewhere'
)
else:
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
except (KeyError, IndexError):
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
main.Cluster.setRunningNode(int(main.params['num_controllers']))
ip = main.Cluster.getIps(allNode=True)
main.testSetUp.ONOSSetUp(main.Cluster,
cellName="SingleHA",
extraApply=applyFuncs,
applyArgs=applyArgs,
includeCaseDesc=False)
main.HA.initialSetUp()
main.step('Set logging levels')
logging = True
try:
logs = main.params.get('ONOS_Logging', False)
if logs:
for namespace, level in logs.items():
for ctrl in main.Cluster.active():
ctrl.CLI.logSet(level, namespace)
except AttributeError:
logging = False
utilities.assert_equals(expect=True,
actual=logging,
onpass="******",
onfail="Failed to set log levels")
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
import re
main.log.info("ONOS HA test: Restart all ONOS nodes - " +
"initialization")
# set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
main.Cluster.numCtrls = 1
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
# load some variables from the params file
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
stepResult = main.testSetUp.envSetup(includeCaseDesc=False)
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
main.scaling = main.params['scaling'].split(",")
main.log.debug(main.scaling)
scale = main.scaling.pop(0)
main.log.debug(scale)
main.Cluster.setRunningNode(int(re.search("\d+", scale).group(0)))
applyFuncs = [main.HA.removeKarafConsoleLogging]
applyArgs = [None]
try:
if main.params['topology']['topoFile']:
main.log.info(
'Skipping start of Mininet in this case, make sure you start it elsewhere'
)
else:
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
except (KeyError, IndexError):
applyFuncs.append(main.HA.startingMininet)
applyArgs.append(None)
main.testSetUp.ONOSSetUp(main.Cluster,
cellName=cellName,
extraApply=applyFuncs,
applyArgs=applyArgs,
installMax=True,
atomixClusterSize=3,
includeCaseDesc=False)
main.HA.initialSetUp()
main.step('Set logging levels')
logging = True
try:
logs = main.params.get('ONOS_Logging', False)
if logs:
for namespace, level in logs.items():
for ctrl in main.Cluster.active():
ctrl.CLI.logSet(level, namespace)
except AttributeError:
logging = False
utilities.assert_equals(expect=True,
actual=logging,
onpass="******",
onfail="Failed to set log levels")
def CASE300(self, main):
"""
Stop 1 ONOS node at a time and test connectivity
"""
from tests.USECASE.VPLS.dependencies import vpls
from tests.HA.dependencies.HA import HA
assert vpls, "vpls not defined"
main.HA = HA()
main.case("Stop one ONOS node at a time and test connectivity.")
result = main.TRUE
for i in range(0, main.Cluster.numCtrls):
stri = str(i)
ip_address = main.Cluster.active(i).ipAddress
# Stop an ONOS node: i
main.step("Stop ONOS node " + stri + ".")
stopResult = main.ONOSbench.onosStop(ip_address)
main.Cluster.runningNodes[i].active = False
utilities.assert_equals(
expect=main.TRUE,
actual=stopResult,
onpass="******",
onfail="ONOS nodes NOT successfully stopped.")
# Check connectivity
connectivityResult = vpls.testConnectivityVpls(main,
isNodeUp=False)
result = result and connectivityResult
# Restart ONOS node
main.step("Restart ONOS node " + stri +
" and checking status of restart.")
startResult = main.ONOSbench.onosStart(ip_address)
utilities.assert_equals(
expect=main.TRUE,
actual=startResult,
onpass="******",
onfail="ONOS nodes NOT successfully started.")
result = result and startResult
# Check if ONOS is up yet
main.log.info("Checking if ONOS node " + stri + " is up.")
upResult = main.ONOSbench.isup(ip_address)
utilities.assert_equals(
expect=main.TRUE,
actual=upResult,
onpass="******",
onfail="ONOS did NOT successfully restart.")
# Restart CLI
main.log.info("Restarting ONOS node " + stri + "'s main.CLI.")
cliResults = main.Cluster.active(0).CLI.startOnosCli(ip_address)
main.Cluster.runningNodes[i].active = True
utilities.assert_equals(
expect=main.TRUE,
actual=cliResults,
onpass="******",
onfail="ONOS CLI did NOT successfully restart.")
# Run some basic checks to see if ONOS node truly has succesfully restarted:
# Checking if all nodes appear with status READY using 'nodes' command
main.step("Checking ONOS nodes.")
nodeResults = utilities.retry(main.Cluster.nodesCheck,
False,
sleep=main.timeSleep,
attempts=main.numAttempts)
utilities.assert_equals(expect=True,
actual=nodeResults,
onpass="******",
onfail="Nodes check NOT successful.")
# All apps that are present are active
main.log.info("Checking if apps are active.")
compareAppsResult = vpls.compareApps(main)
utilities.assert_equals(
expect=main.TRUE,
actual=compareAppsResult,
onpass="******",
onfail="Apps are NOT the same across all nodes.")
result = result and compareAppsResult
# Check connectivity
connectivityResult = vpls.testConnectivityVpls(main)
result = result and connectivityResult
if not result:
break
utilities.assert_equals(expect=main.TRUE,
actual=result,
onpass="******",
onfail="Connectivity is NOT as expected.")
def CASE2(self, main):
"""
Kill ONOS node, and measure the latency for INSTANCE_DEACTIVATED, MASTER_CHANGED, and role request
( tshark time ), then bring the node back up.
"""
import time
import datetime
import numpy
from tests.HA.dependencies.HA import HA
main.HA = HA()
main.latencyData = {
'kill_to_deactivation': [],
'deactivation_to_role_request': []
}
main.failCounter = 0
passingResult = True
criticalError = False
main.step("Gathering data starting with " + str(main.warmUp) +
" warm ups and a sample size of " + str(main.sampleSize))
for iteration in range(0, main.sampleSize + main.warmUp):
main.log.info("==========================================")
main.log.info("================iteration:{}==============".format(
str(iteration + 1)))
ip_address = main.Cluster.active(0).ipAddress
strNodeNumToKill = str(main.nodeNumToKill)
main.log.info("Assigning mastership to ONOS node " +
strNodeNumToKill)
main.Cluster.active(0).CLI.deviceRole(main.switchID, ip_address)
main.log.info("Sleeping for " + str(main.recoverySleep) +
" seconds...")
time.sleep(main.recoverySleep)
mastershipCheck = main.Cluster.active(0).CLI.getMaster(
main.switchID) == ip_address
if not mastershipCheck:
main.log.warn("Mastership is NOT as expected.")
with open(main.tsharkResultPath, "w") as tshark:
tshark.write("")
main.log.info("Starting tshark capture.")
main.ONOSbench.tsharkGrep(main.ofpRoleRequest,
main.tsharkResultPath)
time1 = time.time() * 1000.0
# Kill an ONOS node
main.log.info("Killing ONOS node " + strNodeNumToKill + ".")
killresult = main.ONOSbench.onosKill(ip_address)
main.Cluster.runningNodes[main.nodeNumToKill].active = False
# Stop an ONOS node
main.log.info("Stopping ONOS node " + strNodeNumToKill + ".")
stopresult = main.ONOSbench.onosStop(ip_address)
killStopResult = stopresult == killresult and True
if not killStopResult:
main.log.error(
"ONOS node was NOT successfully stopped and killed.")
criticalError = True
time.sleep(main.failoverSleep)
# Stop tshark and get times
main.log.info("Stopping tshark.")
main.ONOSbench.tsharkStop()
masterChangedLats = []
instanceDeactivatedLats = []
main.log.info("Obtaining latencies from 'events' output.")
for CLInum in range(0, main.Cluster.numCtrls - 1):
eventOutput = main.Cluster.active(CLInum).CLI.events(
args='-a').split("\r\n")
for line in reversed(eventOutput):
timestamp = line[:23] if line[
19] != '-' else line[:19] + '.000'
timestamp = float(
datetime.datetime.strptime(
timestamp,
"%Y-%m-%dT%H:%M:%S.%f").strftime('%s.%f')) * 1000.0
if timestamp - time1 >= 0:
if "INSTANCE_DEACTIVATED" in line:
instanceDeactivatedLats.append(timestamp - time1)
elif "MASTER_CHANGED" in line:
masterChangedLats.append(timestamp - time1)
else:
break
if instanceDeactivatedLats and masterChangedLats:
instanceDeactivatedLats.sort()
masterChangedLats.sort()
instanceDeactivated = instanceDeactivatedLats[0]
masterChanged = masterChangedLats[0]
eventLatCheck = True
else:
eventLatCheck = False
main.log.warn(
"Latencies were NOT obtained from 'events' successfully.")
main.log.info("Obtain latency from tshark output.")
tsharkLatCheck = True
with open(main.tsharkResultPath, "r") as resultFile:
resultText = resultFile.readline()
main.log.info("Capture result: " + resultText)
resultText = resultText.split()
if len(resultText) > 1:
roleRequestLat = int(float(resultText[1]) * 1000.0) - time1
resultFile.close()
else:
main.log.error("Tshark output file is NOT as expected.")
tsharkLatCheck = False
if not tsharkLatCheck:
main.log.warn(
"Latency was NOT obtained from tshark successfully.")
validDataCheck = False
if eventLatCheck and tsharkLatCheck:
main.log.info("instanceDeactivated: " +
str(instanceDeactivated))
main.log.info("masterChanged: " + str(masterChanged))
main.log.info("roleRequestLat: " + str(roleRequestLat))
if iteration >= main.warmUp:
main.log.info("Verifying that the data are valid."
) # Don't record data during a warm-up
validDataCheck = roleRequestLat >= 0 and \
instanceDeactivated >= 0 and \
masterChanged >= 0
if not validDataCheck:
main.log.warn("Data are NOT valid.")
if eventLatCheck and tsharkLatCheck and validDataCheck:
main.log.info("Saving data...")
if roleRequestLat >= instanceDeactivated:
main.latencyData['kill_to_deactivation'].append(
instanceDeactivated)
main.latencyData[
'deactivation_to_role_request'].append(
roleRequestLat - instanceDeactivated)
else:
main.latencyData['kill_to_deactivation'].append(
roleRequestLat)
main.latencyData[
'deactivation_to_role_request'].append(0)
main.log.info(
"kill_to_deactivation: " +
str(main.latencyData['kill_to_deactivation'][-1]))
main.log.info("deactivation_to_role_request: " + str(
main.latencyData['deactivation_to_role_request']
[-1]))
# Restart ONOS node
main.log.info("Restart ONOS node " + strNodeNumToKill +
" and checking status of restart.")
startResult = main.ONOSbench.onosStart(ip_address)
if not startResult:
main.log.error("ONOS nodes NOT successfully started.")
criticalError = True
# Check if ONOS is up yet
main.log.info("Checking if ONOS node " + strNodeNumToKill +
" is up.")
upResult = main.ONOSbench.isup(ip_address)
if not upResult:
main.log.error("ONOS did NOT successfully restart.")
criticalError = True
# Restart CLI
main.log.info("Restarting ONOS node " + strNodeNumToKill +
"'s main.CLI.")
cliResult = main.Cluster.active(
main.nodeNumToKill).CLI.startOnosCli(ip_address)
main.Cluster.runningNodes[main.nodeNumToKill].active = True
if not cliResult:
main.log.error("ONOS CLI did NOT successfully restart.")
criticalError = True
main.log.info("Checking ONOS nodes.")
nodeResults = utilities.retry(main.Cluster.nodesCheck,
False,
sleep=5,
attempts=50)
if not nodeResults:
main.log.error("Nodes check NOT successful.")
criticalError = True
main.log.info("Sleeping for " + str(main.recoverySleep) +
" seconds...")
time.sleep(main.recoverySleep)
if not ( mastershipCheck and
eventLatCheck and
tsharkLatCheck and
validDataCheck ) and \
iteration >= main.warmUp:
main.failCounter += 1
main.log.warn("Iteration failed. Failure count: " +
str(main.failCounter))
if float(main.failCounter) / float(
main.sampleSize) >= main.failPercent or criticalError:
main.log.error(
str(main.failPercent * 100) +
"% or more of data is invalid, or a critical error has occurred."
)
passingResult = False
break
utilities.assert_equals(
expect=True,
actual=passingResult,
onpass="******" + str(main.Cluster.numCtrls) +
" data gathering was successful.",
onfail="Node scaling " + str(main.Cluster.numCtrls) +
" data gathering FAILED. Stopping test.")
if not passingResult:
main.cleanAndExit()
def CASE1(self, main):
"""
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
cell <name>
onos-verify-cell
NOTE: temporary - onos-remove-raft-logs
onos-uninstall
start mininet
git pull
mvn clean install
onos-package
onos-install -f
onos-wait-for-start
start cli sessions
start tcpdump
"""
main.log.info("ONOS Single node cluster restart " +
"HA test - initialization")
main.case("Setting up test environment")
main.caseExplanation = "Setup the test environment including " +\
"installing ONOS, starting Mininet and ONOS" +\
"cli sessions."
# set global variables
# These are for csv plotting in jenkins
main.HAlabels = []
main.HAdata = []
try:
from tests.dependencies.ONOSSetup import ONOSSetup
main.testSetUp = ONOSSetup()
except ImportError:
main.log.error("ONOSSetup not found. exiting the test")
main.cleanAndExit()
main.testSetUp.envSetupDescription()
try:
from tests.HA.dependencies.HA import HA
main.HA = HA()
# load some variables from the params file
cellName = main.params['ENV']['cellName']
main.apps = main.params['ENV']['appString']
main.numCtrls = int(main.params['num_controllers'])
stepResult = main.testSetUp.envSetup()
except Exception as e:
main.testSetUp.envSetupException(e)
main.testSetUp.evnSetupConclusion(stepResult)
main.Cluster.setRunningNode(int(main.params['num_controllers']))
ip = main.Cluster.getIps(allNode=True)
main.testSetUp.ONOSSetUp(
main.Cluster,
cellName="SingleHA",
removeLog=True,
extraApply=[
main.testSetUp.createApplyCell, main.HA.startingMininet,
main.testSetUp.createApplyCell
],
applyArgs=[[main.Cluster, True, cellName, "", "", True, ip], None,
[
main.Cluster, True, "SingleHA", "", "", True,
main.Cluster.runningNodes[0].ipAddress
]])
main.HA.initialSetUp()
