def testmain(logname, maxdrones=25, debug=False):
"A simple test main program"
regexes = []
# pylint says: [W0612:testmain] Unused variable 'j'
# pylint: disable=W0612
for j in range(0, maxdrones + 1):
regexes.append("Stored packages JSON data from *([^ ]*) ")
logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True)
logwatch.setwatch()
sysenv = SystemTestEnvironment(maxdrones)
print >> sys.stderr, "Systems all up and running."
url = "http://%s:%d/db/data/" % (sysenv.cma.ipaddr, 7474)
CMAinit(None)
store = Store(neo4j.GraphDatabaseService(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
results = logwatch.lookforall()
if debug:
print >> sys.stderr, "WATCH RESULTS:", results
tq = QueryTest(store, "START drone=node:Drone('*:*') RETURN drone", GN.nodeconstructor, debug=debug)
print >> sys.stderr, "Running Query"
if tq.check([None], minrows=maxdrones + 1, maxrows=maxdrones + 1):
print "WOOT! Systems passed query check after initial startup!"
else:
print "Systems FAILED initial startup query check"
print "Do you have a second CMA running??"
print "Rerunning query with debug=True"
tq.debug = True
tq.check([None], minrows=maxdrones + 1, maxrows=maxdrones + 1)
return 1
cma = sysenv.cma
nano = sysenv.nanoprobes[0]
regex = r"%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown" % (
cma.hostname,
nano.hostname,
nano.ipaddr,
)
# print >> sys.stderr, 'REGEX IS: [%s]' % regex
logwatch = LogWatcher(logname, [regex], timeout=30, returnonlymatch=False)
logwatch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
logwatch.look()
time.sleep(30)
tq = QueryTest(
store,
("""START drone=node:Drone('*:*') """ """WHERE drone.designation = "{0.hostname}" RETURN drone"""),
GN.nodeconstructor,
debug=debug,
)
if tq.check([nano], downbyshutdown, maxrows=1):
print "WOOT! Systems passed query check after nano shutdown!"
else:
print "Systems FAILED query check after nano shutdown"
def testmain(logname, maxdrones=25, debug=False):
'A simple test main program'
regexes = []
#pylint says: [W0612:testmain] Unused variable 'j'
#pylint: disable=W0612
for j in range(0,maxdrones+1):
regexes.append('Stored packages JSON data from *([^ ]*) ')
logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True)
logwatch.setwatch()
sysenv = SystemTestEnvironment(maxdrones)
print >> sys.stderr, 'Systems all up and running.'
url = ('http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474))
CMAinit(None)
store = Store(neo4j.Graph(url), readonly=True)
for classname in GN.GraphNode.classmap:
GN.GraphNode.initclasstypeobj(store, classname)
results = logwatch.lookforall()
if debug:
print >> sys.stderr, 'WATCH RESULTS:', results
tq = QueryTest(store
, "START drone=node:Drone('*:*') RETURN drone"
, GN.nodeconstructor, debug=debug)
print >> sys.stderr, 'Running Query'
if tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1):
print 'WOOT! Systems passed query check after initial startup!'
else:
print 'Systems FAILED initial startup query check'
print 'Do you have a second CMA running??'
print 'Rerunning query with debug=True'
tq.debug = True
tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1)
return 1
cma = sysenv.cma
nano = sysenv.nanoprobes[0]
regex = (r'%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown'
% (cma.hostname, nano.hostname, nano.ipaddr))
#print >> sys.stderr, 'REGEX IS: [%s]' % regex
logwatch = LogWatcher(logname, [regex,], timeout=30, returnonlymatch=False)
logwatch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
logwatch.look()
time.sleep(30)
tq = QueryTest(store
, ('''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" RETURN drone''')
, GN.nodeconstructor, debug=debug)
if tq.check([nano,], downbyshutdown, maxrows=1):
print 'WOOT! Systems passed query check after nano shutdown!'
else:
print 'Systems FAILED query check after nano shutdown'
def run(self, nano=None, debug=None, timeout=240, service=None, monitorname=None):
if debug is None:
debug = self.debug
if service is None:
service = self.service
if monitorname is None:
monitorname = self.monitorname
if nano is None:
nanozero = self.testenviron.select_nano_noservice(service=service)
if nanozero is None or len(nanozero) < 1:
# bind doesn't seem to shut down properly - need to look into that...
return self._record(AssimSysTest.SKIPPED)
else:
nano = nanozero[0]
assert service not in nano.runningservices
if SystemTestEnvironment.NANOSERVICE not in nano.runningservices:
startregexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename, startregexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
match = watch.look(timeout=timeout)
if match is None:
logger('ERROR: Test %s timed out waiting for any of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.regexes), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_stop_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_start_regexes(nano)
regexes.extend(self.nano_startmonitor_regexes(nano, monitorname))
regexes.extend(self.nano_service_start_regexes(nano, monitorname))
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(service)
nano.startservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
# @TODO make a better query
# but it should be enough to let us validate the rest
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano, debug=debug)
def run(self, nano=None, debug=None, timeout=240, service=None, monitorname=None):
if debug is None:
debug = self.debug
if service is None:
service = self.service
if monitorname is None:
monitorname = self.monitorname
if nano is None:
nanozero = self.testenviron.select_nano_noservice(service=service)
if nanozero is None or len(nanozero) < 1:
# bind doesn't seem to shut down properly - need to look into that...
return self._record(AssimSysTest.SKIPPED)
else:
nano = nanozero[0]
assert service not in nano.runningservices
if SystemTestEnvironment.NANOSERVICE not in nano.runningservices:
startregexes = self.nano_start_regexes(nano)
watch = LogWatcher(self.logfilename, startregexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(SystemTestEnvironment.NANOSERVICE)
match = watch.look(timeout=timeout)
if match is None:
logger('ERROR: Test %s timed out waiting for any of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.regexes), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_stop_regexes(nano)
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.stopservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
regexes = self.nano_start_regexes(nano)
regexes.extend(self.nano_startmonitor_regexes(nano, monitorname))
regexes.extend(self.nano_service_start_regexes(nano, monitorname))
watch = LogWatcher(self.logfilename, regexes, timeout=timeout, debug=debug)
watch.setwatch()
nano.startservice(service)
nano.startservice(SystemTestEnvironment.NANOSERVICE)
if watch.lookforall(timeout=timeout) is None:
logger('ERROR: Test %s timed out waiting for all of %s [timeout:%s]'
% (self.__class__.__name__, str(watch.unmatched), timeout))
return self._record(AssimSysTest.FAIL)
# @TODO make a better query
# but it should be enough to let us validate the rest
qstr = ( '''START drone=node:Drone('*:*') '''
'''WHERE drone.designation = "{0.hostname}" and drone.status = "up" '''
'''RETURN drone''')
return self.checkresults(watch, timeout, qstr, None, nano, debug=debug)
def testmain(logname, maxdrones=3, debug=False):
'Test our test cases'
logger('Starting test of our test cases')
try:
sysenv, ourstore = AssimSysTest.initenviron(logname,
maxdrones,
debug,
cmadebug=5,
nanodebug=3)
except AssertionError:
print 'FAILED initial startup - which is pretty basic'
print 'Any chance you have another CMA running??'
raise RuntimeError('Another CMA is running(?)')
badregexes = (
' ERROR: ',
' CRIT: ',
' CRITICAL: '
# 'HBDEAD'
#, r'Peer at address .* is dead'
,
r'OUTALLDONE .* while in state NONE')
#for cls in [SimulCMAandNanoprobeRestart for j in range(0,20)]:
#for j in range(0,10):
#for cls in [DiscoverService for j in range(0,100)]:
for cls in AssimSysTest.testset:
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logger('CREATED LOG WATCH with %s' % str(badregexes))
badwatch.setwatch()
logger('Starting test %s' % (cls.__name__))
if cls is DiscoverService:
ret = cls(ourstore,
logname,
sysenv,
debug=debug,
service='ssh',
monitorname='check_ssh').run()
else:
ret = cls(ourstore, logname, sysenv, debug=debug).run()
#print >> sys.stderr, 'Got return of %s from test %s' % (ret, cls.__name__)
badmatch = badwatch.look(timeout=1)
if badmatch is not None:
print 'OOPS! Got bad results!', badmatch
raise RuntimeError('Test %s said bad words! [%s]' %
(cls.__name__, badmatch))
assert ret == AssimSysTest.SUCCESS or ret == AssimSysTest.SKIPPED
#assert ret == AssimSysTest.SUCCESS
logger('WOOT! All tests were successful!')
def perform_tests(testset, sysenv, store, itermax, logname, debug=False):
'Actually perform the given set of tests the given number of times, etc'
badregexes = (
r' (ERROR:|CRIT:|CRITICAL:|nanoprobe\[[0-9]*]: segfault at|'
#r'Peer at address .* is dead|'
r'OUTALLDONE .* while in state NONE'
r')', )
itercount = 1
while True:
test = random.choice(testset)
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logit("STARTING test %d - %s" % (itercount, test.__name__))
os.system('logger -s "Load Avg: $(cat /proc/loadavg)"')
os.system('logger -s "$(grep MemFree: /proc/meminfo)"')
badwatch.setwatch()
if test.__name__ == 'DiscoverService':
testobj = test(store,
logname,
sysenv,
debug=debug,
service='ssh',
monitorname='check_ssh')
else:
testobj = test(store, logname, sysenv, debug=debug)
ret = testobj.run()
match = badwatch.look()
if match is not None:
logit('BAD MESSAGE from Test %d %s: %s' %
(itercount, test.__name__, match))
testobj.replace_result(AssimSysTest.FAIL)
ret = AssimSysTest.FAIL
if ret == AssimSysTest.SUCCESS:
logit('Test %d %s succeeded!' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.FAIL:
logit('Test %d %s FAILED :-(' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.SKIPPED:
logit('Test %d %s skipped' % (itercount, test.__name__))
else:
logit('Test %d %s RETURNED SOMETHING REALLY WEIRD [%s]' %
(itercount, test.__name__, str(ret)))
testobj.replace_result(AssimSysTest.FAIL)
print ''
if itercount > itermax:
break
return summarize_tests()
def perform_tests(testset, sysenv, store, itermax, logname, debug=False):
'Actually perform the given set of tests the given number of times, etc'
badregexes=(r' (ERROR:|CRIT:|CRITICAL:|nanoprobe\[[0-9]*]: segfault at|'
#r'Peer at address .* is dead|'
r'OUTALLDONE .* while in state NONE'
r')',)
itercount=1
while True:
test = random.choice(testset)
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logit("STARTING test %d - %s" % (itercount, test.__name__))
os.system('logger -s "Load Avg: $(cat /proc/loadavg)"')
os.system('logger -s "$(grep MemFree: /proc/meminfo)"')
badwatch.setwatch()
if test.__name__ == 'DiscoverService':
testobj = test(store, logname, sysenv, debug=debug
, service='ssh', monitorname='check_ssh')
else:
testobj = test(store, logname, sysenv, debug=debug)
ret = testobj.run()
match = badwatch.look()
if match is not None:
logit('BAD MESSAGE from Test %d %s: %s' % (itercount, test.__name__, match))
testobj.replace_result(AssimSysTest.FAIL)
ret = AssimSysTest.FAIL
if ret == AssimSysTest.SUCCESS:
logit('Test %d %s succeeded!' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.FAIL:
logit('Test %d %s FAILED :-(' % (itercount, test.__name__))
itercount += 1
elif ret == AssimSysTest.SKIPPED:
logit('Test %d %s skipped' % (itercount, test.__name__))
else:
logit('Test %d %s RETURNED SOMETHING REALLY WEIRD [%s]'
% (itercount, test.__name__, str(ret)))
testobj.replace_result(AssimSysTest.FAIL)
print ''
if itercount > itermax:
break
return summarize_tests()
def testmain(logname, maxdrones=3, debug=False):
'Test our test cases'
logger('Starting test of our test cases')
try:
sysenv, ourstore = AssimSysTest.initenviron(logname, maxdrones, debug
, cmadebug=5, nanodebug=3)
except AssertionError:
print 'FAILED initial startup - which is pretty basic'
print 'Any chance you have another CMA running??'
raise RuntimeError('Another CMA is running(?)')
badregexes=(' ERROR: ', ' CRIT: ', ' CRITICAL: '
# 'HBDEAD'
#, r'Peer at address .* is dead'
, r'OUTALLDONE .* while in state NONE'
)
#for cls in [SimulCMAandNanoprobeRestart for j in range(0,20)]:
#for j in range(0,10):
#for cls in [DiscoverService for j in range(0,100)]:
for cls in AssimSysTest.testset:
badwatch = LogWatcher(logname, badregexes, timeout=1, debug=0)
logger('CREATED LOG WATCH with %s' % str(badregexes))
badwatch.setwatch()
logger('Starting test %s' % (cls.__name__))
if cls is DiscoverService:
ret = cls(ourstore, logname, sysenv, debug=debug
, service='ssh', monitorname='check_ssh').run()
else:
ret = cls(ourstore, logname, sysenv, debug=debug).run()
#print >> sys.stderr, 'Got return of %s from test %s' % (ret, cls.__name__)
badmatch = badwatch.look(timeout=1)
if badmatch is not None:
print 'OOPS! Got bad results!', badmatch
raise RuntimeError('Test %s said bad words! [%s]' % (cls.__name__, badmatch))
assert ret == AssimSysTest.SUCCESS or ret == AssimSysTest.SKIPPED
#assert ret == AssimSysTest.SUCCESS
logger('WOOT! All tests were successful!')
