def updateMdevices():
# Create a Back up MSW session object
backupObj = session.SSH(['root@bkupmsw'])
backupObj.assertCommand("export CALLERSCRIPT='True'")
backupObj.assertCommand('cd /usr/local/nextone/bin')
backupObj.assertCommand('nxconfig.pl -m',timeout=5)
# Disconnect and Clean up the Backup MSW session
backupObj.disconnect()
def restartSCM(mswObj,cleanRadius='OFF'):
# 29725 - Time out value changed
# Create a Back up MSW session object
backupObj = session.SSH(['root@bkupmsw'])
# Ticket 33859 - start the primary, give it some time to come up and after making sure that primary
# is UP, start the secondary
mymswConf = nxConfigInterface.MswConfig('mymsw','ON')
mymswConf.msw = mswObj
# Verify whether the msw supports nxconfig
nxConfSupport = mymswConf._isNxConfigSupported()
confObj = mswSCMConfig('eth2',nxConfSupport)
# Stop Backup MSW and Primary MSW
backupObj.assertCommand('iserver all stop',timeout=int(globalVar.iserverStopTimeout))
mswObj.assertCommand('iserver all stop',timeout= int (globalVar.iserverStopTimeout))
time.sleep(5)
# 44728 - Added code to confirm whether all the processes are stopped before starting iserver
masterStopFlag = False
slaveStopFlag = False
iterations = 0
while iterations<20 and not (masterStopFlag and slaveStopFlag):
iterations = iterations + 1
time.sleep(2)
if not masterStopFlag:
# Ticket $ 51478 added -x option to pgrep command
masterStatus = mswObj.filter("pgrep -x 'reportingdaemon|gis_sa|gis|naf|dbsync|execd|pm'")
masterStatus = masterStatus.strip('|')
if not masterStatus:
masterStopFlag = True
if not slaveStopFlag:
slaveStatus = backupObj.filter("pgrep -x 'reportingdaemon|gis_sa|gis|naf|dbsync|execd|pm'")
slaveStatus = slaveStatus.strip('|')
if not slaveStatus:
slaveStopFlag = True
if not masterStopFlag:
resString = "NEXTEST_ERROR: iserver all stop failed on Master"
raise AssertionError(resString)
elif not slaveStopFlag:
resString = "NEXTEST_ERROR: iserver all stop failed on Backup"
raise AssertionError(resString)
#clear the current and backlog directories
if (cleanRadius=='ON'):
mswObj.assertCommand('rm -rf /var/cdrs/RADACCT/')
backupObj.assertCommand('rm -rf /var/cdrs/RADACCT/')
time.sleep(2)
# Start Primary MSW
mswObj.assertCommand('iserver all start',timeout=int(globalVar.iserverStartTimeout))
retries=0
primaryUp=False
while retries<40 and not primaryUp:
retries = retries + 1
time.sleep(12)
# 34440 - Use iserver all status command to find whether primary is up
priStatus = mswObj.filter("iserver all status | grep -i 'no such process'")
if priStatus.lower().__contains__('no such process'):
scmResultString = "NEXTEST_ERROR: SCM restart on Primary MSW Failed"
primaryUp = False
else:
# 42190 - Wait for the state on the primary to be updated
priStatus = mswObj.filter("cli scm")
#48508 make cli scm check case insensitive
priStatus = priStatus.lower()
if priStatus.__contains__('active'):
primaryUp = True
else:
scmResultString = "NEXTEST_ERROR: SCM state check on Primary MSW Failed after restart"
primaryUp = False
if(not primaryUp):
raise AssertionError(scmResultString)
# PR 151232 - handling in case secondary msx takes more time to come up(same way as in case of primary MSX)
# Start Backup MSW
backupObj.assertCommand('iserver all start',timeout=int(globalVar.iserverStartTimeout))
#time.sleep(20)
retries=0
secondaryUp=False
while retries<40 and not secondaryUp:
retries = retries + 1
time.sleep(12)
# 34440 - Use iserver all status command to find whether secondary is up
secStatus = backupObj.filter("iserver all status | grep -i 'no such process'")
if secStatus.lower().__contains__('no such process'):
scmResultString = "NEXTEST_ERROR: SCM restart on Secondary MSW Failed"
secondaryUp = False
else:
# 42190 - Wait for the state on the secondary to be updated
secStatus = backupObj.filter("cli scm")
#48508 make cli scm check case insensitive
secStatus = secStatus.lower()
if secStatus.__contains__('standby'):
secondaryUp = True
else:
scmResultString = "NEXTEST_ERROR: SCM state check on backup MSW Failed after restart"
secondaryUp = False
if(not secondaryUp):
raise AssertionError(scmResultString)
# 34440 - Verify SCM status on primary
if not confObj.checkSCM(mswObj,False):
scmResultString = "NEXTEST_ERROR: SCM restart on Primary MSW Failed"
raise AssertionError(scmResultString)
time.sleep(10)
if not confObj.checkSCM(backupObj,True):
scmResultString = "NEXTEST_ERROR: SCM restart on Backup MSW Failed"
raise AssertionError(scmResultString)
# Disconnect and Clean up the Backup MSW session
backupObj.disconnect()
def Run(self):
"""Run the tests.
This method runs the tests specified in the __init__
function.
returns -- True if any tests had unexpected outcomes."""
# Write out run metadata.
self._WriteInitialAnnotations()
# Start all of the targets.
for target in self.__targets:
target.Start(self.__response_queue, self)
# Run all of the tests.
self._Trace("Starting test loop")
try:
try:
self._RunTests()
except:
self._Trace("Test loop exited with exception: %s"
% str(sys.exc_info()))
for rs in self.__result_streams:
rs.WriteAnnotation("qmtest.run.aborted", "true")
raise
finally:
self._Trace("Test loop finished.")
# Stop the targets.
self._Trace("Stopping targets.")
for target in self.__targets:
target.Stop()
# Read responses until there are no more.
self._Trace("Checking for final responses.")
while self.__CheckForResponse(wait=0):
pass
# Let all of the result streams know that the test run is
# complete.
end_time_str = qm.common.format_time_iso()
for rs in self.__result_streams:
rs.WriteAnnotation("qmtest.run.end_time", end_time_str)
rs.Summarize()
#38747 - Reverting back the original /etc/hosts at the end
# of the test and performing the iserver restart
if self.__context['nextest.scm_configuration'] and self.swap:
self.swap = False
# Restore /etc/hosts
if self.hostsFile:
self.hostsFile = False
os.system('sudo cp /etc/hosts_scm /etc/hosts')
os.system('sudo rm -rf /etc/hosts_scm')
mswObj = session.SSH(['root@mymsw'])
restartSCM(mswObj)
time.sleep(5)
mswObj.disconnect()
return self.__any_unexpected_outcomes
def Run(self, context, result):
# Attach to the log file
self.log = logging.getLogger('nextestlog')
# Output a banner to mark the start of this call
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: #### Running CallTest %s' %
context['qmtest.id'])
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: ####')
#-----------------------------------------------------------------------
# Kill any errant gen processes that may be running. Ideally,
# we need to earn why gen processes errantly persist, but for now
# we will just kill them.
#-----------------------------------------------------------------------
self.killErrantGenProcs()
if not context.has_key('mswinfo'):
raise CallTestError("CallTest requires a MSW resource")
# 18524 - Create a MSWConfig object which is used for configuring certain
# parameters of the MSW
# 26333 - Pass on the SCM configuration information
scmConf = context['nextest.scm_configuration']
context['nextest.mswconfig'] = MswConfig('mymsw', scmConf)
# 22268 - Update the iServer version information
context['nextest.mswconfig'].mswVersion = context['mswinfo'].iVersion
# For SCM testing
if (self.redundancy == "ON"):
if not context.has_key('bkupinfo'):
raise CallTestError("CallTest requires backup MSW resource")
# If configured, start CDR collection:
if (self.cdrcollect == "ON"):
self.ccap = c = data.CDRCollector(context, 'CallTestCDR')
c.start(context)
# If configured, start producing a PCAP packet trace file:
# Ticket-36712: Packet tracing is started if it is enabled either from
# the testcase or the command line
if (self.pkttrace == "ON") or (context['nextest.pkttrace'] != "OFF"):
self.pcap = p = data.PktTraceCollector(context,
'pkttracecollector', '/tmp')
p.start(context)
# If configured, collect a debug log file:
if (self.dbglog == "ON") and (context['nextest.dbglog'] != "OFF"):
self.dbg = d = data.DbgLogCollector(context, 'dbglogcollector',
'/var/log')
d.start(context)
# Ticket 13450 fix
# Instantiate the errorlog watcher
mswErrwatch = MswErrorWatch(context['mswinfo'])
globalSpace = {
'context': context,
'result': result,
'test': self,
'myglobals': globals()
}
localSpace = {}
self.runList = []
# Need a flag so that assertions are not run if test has problems
testException = False
# 36319 - Compare the pid of the gis process before and after the test
# to find out if the script has restarted the iserver.
# Check the Script Contents to see if it restarts iserver
mswObj = session.SSH(['root@mymsw'])
# 38957 - Added function names that restart iserver to the gis pid check
src = self.testsource.lower()
chk1 = src.__contains__('iserver all stop') or src.__contains__(
'ftest') or src.__contains__('checkiserver')
chk2 = src.__contains__('checkscmstatus') or src.__contains__(
'restartscm') or src.__contains__('.initialize')
if chk1 or chk2:
gispid1 = ''
else:
gispid1 = mswObj.filter('pgrep -x gis')
# 58122 - Added hung call check before the test case run
if (context['nextest.hung_calls'] != 'OFF'):
hungcalls1 = mswObj.getVPorts('Used VPORTS')
# 36319 - Find the HVal value before running the test
hVal2 = ''
hVal1 = mswObj.filter('cli test passive | grep -i hval')
self.log.debug('CallTest: hVal Value before running test is %s' %
hVal1)
try:
exec self.testsource in globalSpace, localSpace
self.log.info("CallTest: run: call test source done")
except Exception, exc:
testException = True
if sys.exc_type is AssertionError:
result.Annotate({"Assertion error": str(exc)})
result.Fail("Failed test assertion")
elif sys.exc_type is gen.EndpointError:
result.Annotate({'Endpoint error': str(exc)})
result.Fail('Call failed; an endpoint exception occurred')
else:
result.NoteException(cause="Exception executing source.")
result.NoteException(cause=str(exc))
# At this point, we have handled the exception from the
# test source. Before continuing, must tell any outstanding
# generators to stop or the next test will hang.
self._stopGens(context)
# Stop commands created via runCommand
for cmd in self.runList:
print "stopping", cmd
self.stopCommand(cmd)
class CallTest(Test):
"""
Run one or more calls and gather related data.
"""
arguments = [
qm.fields.TextField(name="testDocumentation",
title="Test documentation",
description=nexhelp.CallTestHelp.testDocumentation,
verbatim="true",
multiline="true",
default_value="""Write test description here."""),
qm.fields.TextField(
name="testsource",
title="Execution Code",
description=nexhelp.CallTestHelp.testsource,
verbatim="true",
multiline="true",
),
qm.fields.TextField(name="assertsource",
title="Assertions",
description=nexhelp.CallTestHelp.assertsource,
verbatim="true",
multiline="true",
default_value="""clist = context['cdrlist']
test.assertEquals(len(clist), 1, 'One CDR expected, unexpected number or zero captured')
test.assertEquals(clist[0]['15-call-error'],'', 'No call error')"""),
qm.fields.EnumerationField(name="pkttrace",
title="Packet Trace Enabled",
description=nexhelp.CallTestHelp.pkttrace,
enumerals=["ON", "OFF"],
default_value="OFF"),
qm.fields.EnumerationField(name="dbglog",
title="Debug Log saving Enabled",
description=nexhelp.CallTestHelp.dbglog,
enumerals=["ON", "OFF"],
default_value="ON"),
# For SCM and redundancy testing
qm.fields.EnumerationField(name="redundancy",
title="Redundancy Configuration Enabled",
description=nexhelp.CallTestHelp.redundant,
enumerals=["ON", "OFF"],
default_value="OFF"),
qm.fields.EnumerationField(name="cdrcollect",
title="CDR Collection Enabled",
description=nexhelp.CallTestHelp.cdrcollect,
enumerals=["ON", "OFF"],
default_value="ON"),
]
def Run(self, context, result):
# Attach to the log file
self.log = logging.getLogger('nextestlog')
# Output a banner to mark the start of this call
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: #### Running CallTest %s' %
context['qmtest.id'])
self.log.debug('CallTest: Run: ####')
self.log.debug('CallTest: Run: ####')
#-----------------------------------------------------------------------
# Kill any errant gen processes that may be running. Ideally,
# we need to earn why gen processes errantly persist, but for now
# we will just kill them.
#-----------------------------------------------------------------------
self.killErrantGenProcs()
if not context.has_key('mswinfo'):
raise CallTestError("CallTest requires a MSW resource")
# 18524 - Create a MSWConfig object which is used for configuring certain
# parameters of the MSW
# 26333 - Pass on the SCM configuration information
scmConf = context['nextest.scm_configuration']
context['nextest.mswconfig'] = MswConfig('mymsw', scmConf)
# 22268 - Update the iServer version information
context['nextest.mswconfig'].mswVersion = context['mswinfo'].iVersion
# For SCM testing
if (self.redundancy == "ON"):
if not context.has_key('bkupinfo'):
raise CallTestError("CallTest requires backup MSW resource")
# If configured, start CDR collection:
if (self.cdrcollect == "ON"):
self.ccap = c = data.CDRCollector(context, 'CallTestCDR')
c.start(context)
# If configured, start producing a PCAP packet trace file:
# Ticket-36712: Packet tracing is started if it is enabled either from
# the testcase or the command line
if (self.pkttrace == "ON") or (context['nextest.pkttrace'] != "OFF"):
self.pcap = p = data.PktTraceCollector(context,
'pkttracecollector', '/tmp')
p.start(context)
# If configured, collect a debug log file:
if (self.dbglog == "ON") and (context['nextest.dbglog'] != "OFF"):
self.dbg = d = data.DbgLogCollector(context, 'dbglogcollector',
'/var/log')
d.start(context)
# Ticket 13450 fix
# Instantiate the errorlog watcher
mswErrwatch = MswErrorWatch(context['mswinfo'])
globalSpace = {
'context': context,
'result': result,
'test': self,
'myglobals': globals()
}
localSpace = {}
self.runList = []
# Need a flag so that assertions are not run if test has problems
testException = False
# 36319 - Compare the pid of the gis process before and after the test
# to find out if the script has restarted the iserver.
# Check the Script Contents to see if it restarts iserver
mswObj = session.SSH(['root@mymsw'])
# 38957 - Added function names that restart iserver to the gis pid check
src = self.testsource.lower()
chk1 = src.__contains__('iserver all stop') or src.__contains__(
'ftest') or src.__contains__('checkiserver')
chk2 = src.__contains__('checkscmstatus') or src.__contains__(
'restartscm') or src.__contains__('.initialize')
if chk1 or chk2:
gispid1 = ''
else:
gispid1 = mswObj.filter('pgrep -x gis')
# 58122 - Added hung call check before the test case run
if (context['nextest.hung_calls'] != 'OFF'):
hungcalls1 = mswObj.getVPorts('Used VPORTS')
# 36319 - Find the HVal value before running the test
hVal2 = ''
hVal1 = mswObj.filter('cli test passive | grep -i hval')
self.log.debug('CallTest: hVal Value before running test is %s' %
hVal1)
try:
exec self.testsource in globalSpace, localSpace
self.log.info("CallTest: run: call test source done")
except Exception, exc:
testException = True
if sys.exc_type is AssertionError:
result.Annotate({"Assertion error": str(exc)})
result.Fail("Failed test assertion")
elif sys.exc_type is gen.EndpointError:
result.Annotate({'Endpoint error': str(exc)})
result.Fail('Call failed; an endpoint exception occurred')
else:
result.NoteException(cause="Exception executing source.")
result.NoteException(cause=str(exc))
# At this point, we have handled the exception from the
# test source. Before continuing, must tell any outstanding
# generators to stop or the next test will hang.
self._stopGens(context)
# Stop commands created via runCommand
for cmd in self.runList:
print "stopping", cmd
self.stopCommand(cmd)
# Save all test data and annotate the result with the location
if (self.cdrcollect == "ON"):
c.saveResult(context, 'cdr.dat')
result[Result.NEXTEST_CDR] = c.getRelResultFile()
# Ticket-36712: Packet tracing is started if it is enabled either from
# the testcase or the command line
if (self.pkttrace == "ON") or (context['nextest.pkttrace'] != "OFF"):
p.saveResult(context, 'pkttrace_g0_g1.pcap')
result[Result.NEXTEST_NET] = p.getRelResultFile()
ostype = posix.uname()[0]
if ostype == 'Linux':
# Change for S9 duplicate packet to maintain different behavior than s3.
if (context['userConfig.automation'] == 's9'):
context['pdmlPkts'] = pktInspect_s9.readCapture(
p.getPdmlFile())
else:
context['pdmlPkts'] = pktInspect.readCapture(
p.getPdmlFile())
elif ostype == 'SunOS':
self.log.info(
'CallTest.Run(): pdml files not supported on SunOS, don\'t run any tests using Packet Inspection Feature.'
)
#print "pkttrace file saved"
#sleep(20)
if (self.dbglog == "ON") and (context['nextest.dbglog'] != "OFF"):
d.saveResult(context, 'gisdbg.log')
result[Result.NEXTEST_GIS] = d.getRelResultFile()
mswObj = session.SSH(['root@mymsw'])
# 36319 - Verify whether gis has restarted during the test
if (gispid1 != ''):
gispid2 = mswObj.filter('pgrep -x gis')
if (gispid1 != gispid2):
annotations = {
'Assertion':
'GIS has restarted during the test, pid changed %s:%s!!!!'
% (gispid1, gispid2)
}
result.Fail('GIS has restarted', annotations)
#self.assert_((gispid1==gispid2),'GIS has restarted!!!!')
# 58122 - Verify there are no hung calls after the test case is run
if (context['nextest.hung_calls'] != 'OFF'):
hungcalls2 = mswObj.getVPorts('Used VPORTS')
self.assert_((hungcalls1 == hungcalls2),
'VPORTS are not released and calls are hung!!!!')
# Ticket 13450 fix
# Collect the errorlog from MSW and verify
mswErrwatch.watchMswErrors(
result, (context['nextest.fail_on_msw_error'] == "ON"))
# Execute the assertion code only if we didn't get a test exception
if (not testException) and (self.assertsource is not None):
try:
exec self.assertsource in globalSpace, localSpace
except AssertionError, theException:
# put the '00' in so assertion appears first in list
annotations = {'Assertion': str(theException)}
result.Fail('Test assertion(s) false.', annotations)
def setUp(self):
self.m = session.SSH(['root@mymsw'])