def initenviron(logname, maxdrones, debug=False, timeout=90, nanodebug=0, cmadebug=0):
'Initialize the test environment.'
logwatch = LogWatcher(logname, [], timeout, returnonlymatch=True, debug=debug)
logwatch.setwatch()
sysenv = SystemTestEnvironment(logname, maxdrones, nanodebug=nanodebug, cmadebug=cmadebug)
CMAinit(None, host=str(sysenv.cma.ipaddr), readonly=True,
neologin=SystemTestEnvironment.NEO4JLOGIN, neopass=SystemTestEnvironment.NEO4JPASS)
url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)
print >> sys.stderr, 'OPENING Neo4j at URL %s' % url
neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr,
SystemTestEnvironment.NEO4JLOGIN,
SystemTestEnvironment.NEO4JPASS)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
logger('$(grep MemFree: /proc/meminfo)', hardquote=False)
tq = QueryTest(store
, '''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone'''
, GN.nodeconstructor, debug=debug)
if not tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1
, delay=0.5, maxtries=20):
sysenv.cma.cleanupwhendone = False
raise RuntimeError('Query of "up" status failed. Weirdness')
return sysenv, store
def run(self, nano=None, debug=None, timeout=240):
'Actually start the nanoprobe and see if it worked'
if debug is None:
debug = self.debug
if nano is None:
nanozero = self.testenviron.select_nano_noservice()
if len(nanozero) > 0:
nano = nanozero[0]
if (nano is None or nano.status != TestSystem.RUNNING
or SystemTestEnvironment.NANOSERVICE in nano.runningservices):
return self._record(AssimSysTest.SKIPPED)
regexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename,
regexes,
timeout=timeout,
debug=debug)
watch.setwatch()
qstr = (
'''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
nano.startservice(SystemTestEnvironment.NANOSERVICE)
return self.checkresults(watch, timeout, qstr, None, nano)
def __init__(self, logname, nanocount=10
, cmaimage='assimilation/build-wily', nanoimages=('assimilation/build-wily',)
#, cmaimage='3f06b7c84030', nanoimages=('3f06b7c84030',)
, sysclass=DockerSystem, cleanupwhendone=False, nanodebug=0, cmadebug=0, chunksize=20):
'Init/constructor for our SystemTestEnvironment'
self.sysclass = sysclass
self.cmaimage = cmaimage
self.nanoimages = nanoimages
self.nanoprobes = []
self.cma = None
self.debug = 0
self.cleanupwhendone = cleanupwhendone
self.logname = logname
watch = LogWatcher(logname, [])
watch.setwatch()
nanodebug=1
cmadebug=2
self.nanodebug = nanodebug
self.cmadebug = nanodebug
self.spawncma(nanodebug=nanodebug, cmadebug=cmadebug)
regex = (' %s .* INFO: Neo4j version .* // py2neo version .*'
' // Python version .* // (java|openjdk) version.*') % self.cma.hostname
watch.setregexes((regex,))
if watch.lookforall(timeout=120) is None:
os.system("logger -s 'CMA did not start!! [[%s]]'" % (regex))
print >> sys.stderr, 'CMA did not start!! %s' % regex
raise RuntimeError('CMA did not start: %s' % regex)
print >> sys.stderr, 'nanocount is', nanocount
print >> sys.stderr, 'self.nanoimages is', self.nanoimages
# We do this in chunks to manage stress on our test environment
children_left = range(0, nanocount)
while (len(children_left) > 0):
self._create_nano_chunk(children_left[0:chunksize])
del children_left[0:chunksize]
def __init__(self, logname, nanocount=10
, cmaimage='assimilation/build-utopic', nanoimages=('assimilation/build-utopic',)
, sysclass=DockerSystem, cleanupwhendone=True, nanodebug=0, cmadebug=0, chunksize=20):
'Init/constructor for our SystemTestEnvironment'
self.sysclass = sysclass
self.cmaimage = cmaimage
self.nanoimages = nanoimages
self.nanoprobes = []
self.cma = None
self.debug = 0
self.cleanupwhendone = cleanupwhendone
self.logname = logname
watch = LogWatcher(logname, [])
watch.setwatch()
nanodebug=1
cmadebug=0
self.nanodebug = nanodebug
self.cmadebug = nanodebug
self.spawncma(nanodebug=nanodebug, cmadebug=cmadebug)
regex = (' %s .* INFO: Neo4j version .* // py2neo version .*'
' // Python version .* // java version.*') % self.cma.hostname
watch.setregexes((regex,))
if watch.lookforall(timeout=60) is None:
print >> sys.stderr, 'CMA did not start!!'
raise RuntimeError('CMA did not start')
print >> sys.stderr, 'nanocount is', nanocount
print >> sys.stderr, 'self.nanoimages is', self.nanoimages
# We do this in chunks to manage stress on our test environment
children_left = range(0, nanocount)
while (len(children_left) > 0):
self._create_nano_chunk(children_left[0:chunksize])
del children_left[0:chunksize]
def initenviron(logname,
maxdrones,
mgmtsystem,
debug=False,
cmaimage='',
nanoimages=[],
timeout=90,
nanodebug=0,
cmadebug=0):
'Initialize the test environment.'
logwatch = LogWatcher(logname, [],
timeout,
returnonlymatch=True,
debug=debug)
logwatch.setwatch()
sysenv = SystemTestEnvironment(logname,
maxdrones,
mgmtsystem,
cmaimage=cmaimage,
nanoimages=nanoimages,
nanodebug=nanodebug,
cmadebug=cmadebug)
CMAinit(None,
host=str(sysenv.cma.ipaddr),
readonly=True,
neologin=SystemTestEnvironment.NEO4JLOGIN,
neopass=SystemTestEnvironment.NEO4JPASS)
url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)
print >> sys.stderr, 'OPENING Neo4j at URL %s' % url
neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr,
SystemTestEnvironment.NEO4JLOGIN,
SystemTestEnvironment.NEO4JPASS)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
logger('$(grep MemFree: /proc/meminfo)', hardquote=False)
tq = QueryTest(
store,
'''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone''',
GN.nodeconstructor,
debug=debug)
if not tq.check([
None,
],
minrows=maxdrones + 1,
maxrows=maxdrones + 1,
delay=0.5,
maxtries=20):
sysenv.cma.cleanupwhendone = False
raise RuntimeError('Query of "up" status failed. Weirdness')
return sysenv, store
def testmain(logname, maxdrones=25, debug=False):
"A simple test main program"
regexes = []
# pylint says: [W0612:testmain] Unused variable 'j'
# pylint: disable=W0612
for j in range(0, maxdrones + 1):
regexes.append("Stored packages JSON data from *([^ ]*) ")
logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True)
logwatch.setwatch()
sysenv = SystemTestEnvironment(maxdrones)
print >> sys.stderr, "Systems all up and running."
url = "http://%s:%d/db/data/" % (sysenv.cma.ipaddr, 7474)
CMAinit(None)
store = Store(neo4j.GraphDatabaseService(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
results = logwatch.lookforall()
if debug:
print >> sys.stderr, "WATCH RESULTS:", results
tq = QueryTest(store, "START drone=node:Drone('*:*') RETURN drone", GN.nodeconstructor, debug=debug)
print >> sys.stderr, "Running Query"
if tq.check([None], minrows=maxdrones + 1, maxrows=maxdrones + 1):
print "WOOT! Systems passed query check after initial startup!"
else:
print "Systems FAILED initial startup query check"
print "Do you have a second CMA running??"
print "Rerunning query with debug=True"
tq.debug = True
tq.check([None], minrows=maxdrones + 1, maxrows=maxdrones + 1)
return 1
cma = sysenv.cma
nano = sysenv.nanoprobes[0]
regex = r"%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown" % (
cma.hostname,
nano.hostname,
nano.ipaddr,
)
# print >> sys.stderr, 'REGEX IS: [%s]' % regex
logwatch = LogWatcher(logname, [regex], timeout=30, returnonlymatch=False)
logwatch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
logwatch.look()
time.sleep(30)
tq = QueryTest(
store,
("""START drone=node:Drone('*:*') """ """WHERE drone.designation = "{0.hostname}" RETURN drone"""),
GN.nodeconstructor,
debug=debug,
)
if tq.check([nano], downbyshutdown, maxrows=1):
print "WOOT! Systems passed query check after nano shutdown!"
else:
print "Systems FAILED query check after nano shutdown"
def testmain(logname, maxdrones=25, debug=False):
'A simple test main program'
regexes = []
#pylint says: [W0612:testmain] Unused variable 'j'
#pylint: disable=W0612
for j in range(0,maxdrones+1):
regexes.append('Stored packages JSON data from *([^ ]*) ')
logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True)
logwatch.setwatch()
sysenv = SystemTestEnvironment(maxdrones)
print >> sys.stderr, 'Systems all up and running.'
url = ('http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474))
CMAinit(None)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
results = logwatch.lookforall()
if debug:
print >> sys.stderr, 'WATCH RESULTS:', results
tq = QueryTest(store
, "START drone=node:Drone('*:*') RETURN drone"
, GN.nodeconstructor, debug=debug)
print >> sys.stderr, 'Running Query'
if tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1):
print 'WOOT! Systems passed query check after initial startup!'
else:
print 'Systems FAILED initial startup query check'
print 'Do you have a second CMA running??'
print 'Rerunning query with debug=True'
tq.debug = True
tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1)
return 1
cma = sysenv.cma
nano = sysenv.nanoprobes[0]
regex = (r'%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown'
% (cma.hostname, nano.hostname, nano.ipaddr))
#print >> sys.stderr, 'REGEX IS: [%s]' % regex
logwatch = LogWatcher(logname, [regex,], timeout=30, returnonlymatch=False)
logwatch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
logwatch.look()
time.sleep(30)
tq = QueryTest(store
, ('''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" RETURN drone''')
, GN.nodeconstructor, debug=debug)
if tq.check([nano,], downbyshutdown, maxrows=1):
print 'WOOT! Systems passed query check after nano shutdown!'
else:
print 'Systems FAILED query check after nano shutdown'
def testmain(logname, maxdrones=3, debug=False):
'Test our test cases'
logger('Starting test of our test cases')
try:
sysenv, ourstore = AssimSysTest.initenviron(logname,
maxdrones,
debug,
cmadebug=5,
nanodebug=3)
except AssertionError:
print 'FAILED initial startup - which is pretty basic'
print 'Any chance you have another CMA running??'
raise RuntimeError('Another CMA is running(?)')
badregexes = (
' ERROR: ',
' CRIT: ',
' CRITICAL: '
# 'HBDEAD'
#, r'Peer at address .* is dead'
,
r'OUTALLDONE .* while in state NONE')
#for cls in [SimulCMAandNanoprobeRestart for j in range(0,20)]:
#for j in range(0,10):
#for cls in [DiscoverService for j in range(0,100)]:
for cls in AssimSysTest.testset:
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logger('CREATED LOG WATCH with %s' % str(badregexes))
badwatch.setwatch()
logger('Starting test %s' % (cls.__name__))
if cls is DiscoverService:
ret = cls(ourstore,
logname,
sysenv,
debug=debug,
service='ssh',
monitorname='check_ssh').run()
else:
ret = cls(ourstore, logname, sysenv, debug=debug).run()
#print >> sys.stderr, 'Got return of %s from test %s' % (ret, cls.__name__)
badmatch = badwatch.look(timeout=1)
if badmatch is not None:
print 'OOPS! Got bad results!', badmatch
raise RuntimeError('Test %s said bad words! [%s]' %
(cls.__name__, badmatch))
assert ret == AssimSysTest.SUCCESS or ret == AssimSysTest.SKIPPED
#assert ret == AssimSysTest.SUCCESS
logger('WOOT! All tests were successful!')
def run(self, nano=None, debug=None, timeout=60):
'Actually stop and start (restart) the CMA and see if it worked'
if debug is None:
debug = self.debug
cma = self.testenviron.cma
cma.stopservice(SystemTestEnvironment.CMASERVICE)
regexes = self.cma_start_regexes()
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
if self.delay > 0:
time.sleep(self.delay)
cma.startservice(SystemTestEnvironment.CMASERVICE)
# This just makes sure the database is still up - which it should be...
# Once we receive the CMA update message, we really should already be good to go
qstr = '''START one=node(*) RETURN one LIMIT 1'''
return self.checkresults(watch, timeout, qstr, None, nano)
def perform_tests(testset, sysenv, store, itermax, logname, debug=False):
'Actually perform the given set of tests the given number of times, etc'
badregexes = (
r' (ERROR:|CRIT:|CRITICAL:|nanoprobe\[[0-9]*]: segfault at|'
#r'Peer at address .* is dead|'
r'OUTALLDONE .* while in state NONE'
r')', )
itercount = 1
while True:
test = random.choice(testset)
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logit("STARTING test %d - %s" % (itercount, test.__name__))
os.system('logger -s "Load Avg: $(cat /proc/loadavg)"')
os.system('logger -s "$(grep MemFree: /proc/meminfo)"')
badwatch.setwatch()
if test.__name__ == 'DiscoverService':
testobj = test(store,
logname,
sysenv,
debug=debug,
service='ssh',
monitorname='check_ssh')
else:
testobj = test(store, logname, sysenv, debug=debug)
ret = testobj.run()
match = badwatch.look()
if match is not None:
logit('BAD MESSAGE from Test %d %s: %s' %
(itercount, test.__name__, match))
testobj.replace_result(AssimSysTest.FAIL)
ret = AssimSysTest.FAIL
if ret == AssimSysTest.SUCCESS:
logit('Test %d %s succeeded!' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.FAIL:
logit('Test %d %s FAILED :-(' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.SKIPPED:
logit('Test %d %s skipped' % (itercount, test.__name__))
else:
logit('Test %d %s RETURNED SOMETHING REALLY WEIRD [%s]' %
(itercount, test.__name__, str(ret)))
testobj.replace_result(AssimSysTest.FAIL)
print ''
if itercount > itermax:
break
return summarize_tests()
def run(self, nano=None, debug=None, timeout=60):
'Actually stop and start (restart) the CMA and see if it worked'
if debug is None:
debug = self.debug
cma = self.testenviron.cma
cma.stopservice(SystemTestEnvironment.CMASERVICE)
regex = (' %s .* INFO: Neo4j version .* // py2neo version .*'
' // Python version .* // java version.*') % cma.hostname
watch = LogWatcher(self.logfilename, (regex,), timeout=timeout, debug=debug)
watch.setwatch()
if self.delay > 0:
time.sleep(self.delay)
cma.startservice(SystemTestEnvironment.CMASERVICE)
# This just makes sure the database is still up - which it should be...
# Once we receive the CMA update message, we really should already be good to go
qstr = '''START one=node(*) RETURN one LIMIT 1'''
return self.checkresults(watch, timeout, qstr, None, nano)
def run(self, nano=None, debug=None, timeout=300):
'''Our default timeout is so long because we can take a while to give up shutting down
the nanoprobe - an ACK timeout might have to occur before it can shut down.
'''
if debug is None:
debug = self.debug
if nano is None:
nanozero = self.testenviron.select_nano_service()
if len(nanozero) < 1:
return self._record(AssimSysTest.SKIPPED)
nano = nanozero[0]
cma = self.testenviron.cma
regexes = self.nano_stop_regexes(nano)
regexes.extend(self.cma_stop_regexes())
regexes.extend(self.cma_start_regexes())
watch = LogWatcher(self.logfilename,
regexes,
timeout=timeout,
debug=debug)
watch.setwatch()
cma.stopservice(SystemTestEnvironment.CMASERVICE)
nano.stopservice(SystemTestEnvironment.NANOSERVICE, async=True)
cma.startservice(SystemTestEnvironment.CMASERVICE)
if self.delay > 0:
time.sleep(self.delay)
qstr = (
'''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "dead" '''
'''and drone.reason = "HBSHUTDOWN" RETURN drone''')
rc = self.checkresults(watch, timeout, qstr, None, nano)
if rc != AssimSysTest.SUCCESS:
return rc
# We have to do this in two parts because of the asynchronous shutdown above
regexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename,
regexes,
timeout=timeout,
debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
qstr = (
'''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano)
def run(self, nano=None, debug=None, timeout=180):
'Actually stop the nanoprobe and see if it worked'
if debug is None:
debug = self.debug
if nano is None:
nanozero = self.testenviron.select_nano_service()
if len(nanozero) > 0:
nano = nanozero[0]
if (nano is None or nano.status != TestSystem.RUNNING or
SystemTestEnvironment.NANOSERVICE not in nano.runningservices):
return self._record(AssimSysTest.SKIPPED)
regexes = self.nano_stop_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "dead" '''
'''and drone.reason = "HBSHUTDOWN" RETURN drone''')
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
return self.checkresults(watch, timeout, qstr, None, nano)
def perform_tests(testset, sysenv, store, itermax, logname, debug=False):
'Actually perform the given set of tests the given number of times, etc'
badregexes=(r' (ERROR:|CRIT:|CRITICAL:|nanoprobe\[[0-9]*]: segfault at|'
#r'Peer at address .* is dead|'
r'OUTALLDONE .* while in state NONE'
r')',)
itercount=1
while True:
test = random.choice(testset)
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logit("STARTING test %d - %s" % (itercount, test.__name__))
os.system('logger -s "Load Avg: $(cat /proc/loadavg)"')
os.system('logger -s "$(grep MemFree: /proc/meminfo)"')
badwatch.setwatch()
if test.__name__ == 'DiscoverService':
testobj = test(store, logname, sysenv, debug=debug
, service='ssh', monitorname='check_ssh')
else:
testobj = test(store, logname, sysenv, debug=debug)
ret = testobj.run()
match = badwatch.look()
if match is not None:
logit('BAD MESSAGE from Test %d %s: %s' % (itercount, test.__name__, match))
testobj.replace_result(AssimSysTest.FAIL)
ret = AssimSysTest.FAIL
if ret == AssimSysTest.SUCCESS:
logit('Test %d %s succeeded!' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.FAIL:
logit('Test %d %s FAILED :-(' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.SKIPPED:
logit('Test %d %s skipped' % (itercount, test.__name__))
else:
logit('Test %d %s RETURNED SOMETHING REALLY WEIRD [%s]'
% (itercount, test.__name__, str(ret)))
testobj.replace_result(AssimSysTest.FAIL)
print ''
if itercount > itermax:
break
return summarize_tests()
def _create_nano_chunk(self, childnos):
'Create a chunk of nanoprobes'
watch = LogWatcher(self.logname, [])
watch.setwatch()
regexes = []
for childcount in childnos:
childcount = childcount # Make pylint happy...
nano = self.spawnnanoprobe(debug=self.nanodebug)
regexes .extend([
r' (%s) nanoprobe\[.*]: NOTICE: Connected to CMA. Happiness :-D'
% (nano.hostname),
r' %s cma INFO: Drone %s registered from address \[::ffff:%s]'
% (self.cma.hostname, nano.hostname, nano.ipaddr),
r' %s cma INFO: Processed tcpdiscovery JSON data from (%s) into graph.'
% (self.cma.hostname, nano.hostname),
])
self.nanoprobes.append(nano)
watch.setregexes(regexes)
if watch.lookforall(timeout=30) is None:
raise RuntimeError('Nanoprobes did not start [%s, %s] - missing %s'
% (nano.hostname, nano.ipaddr, str(watch.unmatched)))
def _create_nano_chunk(self, childnos):
'Create a chunk of nanoprobes'
watch = LogWatcher(self.logname, [])
watch.setwatch()
regexes = []
for childcount in childnos:
childcount = childcount # Make pylint happy...
nano = self.spawnnanoprobe(debug=self.nanodebug)
regexes.extend([
r' (%s) nanoprobe\[.*]: NOTICE: Connected to CMA. Happiness :-D'
% (nano.hostname),
r' %s cma INFO: Drone %s registered from address \[::ffff:%s]'
% (self.cma.hostname, nano.hostname, nano.ipaddr),
r' %s cma INFO: Processed tcpdiscovery JSON data from (%s) into graph.'
% (self.cma.hostname, nano.hostname),
])
self.nanoprobes.append(nano)
watch.setregexes(regexes)
if watch.lookforall(timeout=30) is None:
raise RuntimeError(
'Nanoprobes did not start [%s, %s] - missing %s' %
(nano.hostname, nano.ipaddr, str(watch.unmatched)))
def _create_nano_chunk(self, childnos):
'Create a chunk of nanoprobes'
watch = LogWatcher(self.logname, [], debug=0)
watch.setwatch()
regexes = []
for childcount in childnos:
childcount = childcount # Make pylint happy...
nano = self.spawnnanoprobe(debug=self.nanodebug)
regexes .extend([
r' %s nanoprobe\[.*]: NOTICE: Connected to CMA. Happiness :-D'
% (nano.hostname),
r' %s cma INFO: Drone %s registered from address \[::ffff:%s]'
% (self.cma.hostname, nano.hostname, nano.ipaddr),
r' %s cma INFO: Processed u?n?changed tcpdiscovery'
r' JSON data from %s into graph.'
% (self.cma.hostname, nano.hostname),
])
self.nanoprobes.append(nano)
print >> sys.stderr, len(regexes), 'NANOPROBE REGEXES ARE:', (regexes)
watch.setregexes(regexes)
if watch.lookforall(timeout=120) is None:
raise RuntimeError('Nanoprobes did not start - missing %s'
% (str(watch.unmatched)))
def testmain(logname, maxdrones=3, debug=False):
'Test our test cases'
logger('Starting test of our test cases')
try:
sysenv, ourstore = AssimSysTest.initenviron(logname, maxdrones, debug
, cmadebug=5, nanodebug=3)
except AssertionError:
print 'FAILED initial startup - which is pretty basic'
print 'Any chance you have another CMA running??'
raise RuntimeError('Another CMA is running(?)')
badregexes=(' ERROR: ', ' CRIT: ', ' CRITICAL: '
# 'HBDEAD'
#, r'Peer at address .* is dead'
, r'OUTALLDONE .* while in state NONE'
)
#for cls in [SimulCMAandNanoprobeRestart for j in range(0,20)]:
#for j in range(0,10):
#for cls in [DiscoverService for j in range(0,100)]:
for cls in AssimSysTest.testset:
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logger('CREATED LOG WATCH with %s' % str(badregexes))
badwatch.setwatch()
logger('Starting test %s' % (cls.__name__))
if cls is DiscoverService:
ret = cls(ourstore, logname, sysenv, debug=debug
, service='ssh', monitorname='check_ssh').run()
else:
ret = cls(ourstore, logname, sysenv, debug=debug).run()
#print >> sys.stderr, 'Got return of %s from test %s' % (ret, cls.__name__)
badmatch = badwatch.look(timeout=1)
if badmatch is not None:
print 'OOPS! Got bad results!', badmatch
raise RuntimeError('Test %s said bad words! [%s]' % (cls.__name__, badmatch))
assert ret == AssimSysTest.SUCCESS or ret == AssimSysTest.SKIPPED
#assert ret == AssimSysTest.SUCCESS
logger('WOOT! All tests were successful!')
def run(self, nano=None, debug=None, timeout=300):
'''Our default timeout is so long because we can take a while to give up shutting down
the nanoprobe - an ACK timeout might have to occur before it can shut down.
'''
if debug is None:
debug = self.debug
if nano is None:
nanozero = self.testenviron.select_nano_service()
if len(nanozero) < 1:
return self._record(AssimSysTest.SKIPPED)
nano = nanozero[0]
cma = self.testenviron.cma
regexes = self.nano_stop_regexes(nano)
regexes.extend(self.cma_stop_regexes())
regexes.extend(self.cma_start_regexes())
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
cma.stopservice(SystemTestEnvironment.CMASERVICE)
nano.stopservice(SystemTestEnvironment.NANOSERVICE, async=True)
cma.startservice(SystemTestEnvironment.CMASERVICE)
if self.delay > 0:
time.sleep(self.delay)
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "dead" '''
'''and drone.reason = "HBSHUTDOWN" RETURN drone''')
rc = self.checkresults(watch, timeout, qstr, None, nano)
if rc != AssimSysTest.SUCCESS:
return rc
# We have to do this in two parts because of the asynchronous shutdown above
regexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano)
def run(self, nano=None, debug=None, timeout=240, service=None, monitorname=None):
if debug is None:
debug = self.debug
if service is None:
service = self.service
if monitorname is None:
monitorname = self.monitorname
if nano is None:
nanozero = self.testenviron.select_nano_noservice(service=service)
if nanozero is None or len(nanozero) < 1:
# bind doesn't seem to shut down properly - need to look into that...
return self._record(AssimSysTest.SKIPPED)
else:
nano = nanozero[0]
assert service not in nano.runningservices
if SystemTestEnvironment.NANOSERVICE not in nano.runningservices:
startregexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename, startregexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
match = watch.look(timeout=timeout)
if match is None:
logger('ERROR: Test %s timed out waiting for any of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.regexes), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_stop_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_start_regexes(nano)
regexes.extend(self.nano_startmonitor_regexes(nano, monitorname))
regexes.extend(self.nano_service_start_regexes(nano, monitorname))
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(service)
nano.startservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
# @TODO make a better query
# but it should be enough to let us validate the rest
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano, debug=debug)
def run(self, nano=None, debug=None, timeout=240, service=None, monitorname=None):
if debug is None:
debug = self.debug
if service is None:
service = self.service
if monitorname is None:
monitorname = self.monitorname
if nano is None:
nanozero = self.testenviron.select_nano_noservice(service=service)
if nanozero is None or len(nanozero) < 1:
# bind doesn't seem to shut down properly - need to look into that...
return self._record(AssimSysTest.SKIPPED)
else:
nano = nanozero[0]
assert service not in nano.runningservices
if SystemTestEnvironment.NANOSERVICE not in nano.runningservices:
startregexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename, startregexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
match = watch.look(timeout=timeout)
if match is None:
logger('ERROR: Test %s timed out waiting for any of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.regexes), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_stop_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_start_regexes(nano)
regexes.extend(self.nano_startmonitor_regexes(nano, monitorname))
regexes.extend(self.nano_service_start_regexes(nano, monitorname))
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(service)
nano.startservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
# @TODO make a better query
# but it should be enough to let us validate the rest
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano, debug=debug)
