def assert_no_dangling_Cclasses(doassert=None): global CheckForDanglingClasses global WorstDanglingCount sys._clear_type_cache() if doassert is None: doassert = AssertOnDanglingClasses CMAinit.uninit() gc.collect() # For good measure... count = proj_class_live_object_count() #print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count # Avoid cluttering the output up with redundant messages... if count > WorstDanglingCount and CheckForDanglingClasses: WorstDanglingCount = count if doassert: print >> sys.stderr, 'STARTING OBJECT DUMP' print 'stdout STARTING OBJECT DUMP' dump_c_objects() print >> sys.stderr, 'OBJECT DUMP COMPLETE' print 'stdout OBJECT DUMP COMPLETE' raise AssertionError("Dangling C-class objects - %d still around" % count) else: print >> sys.stderr, ( "*****ERROR: Dangling C-class objects - %d still around" % count)
def dbsetup(readonly=False): 'Set up our connection to Neo4j' ourstore = Store(neo4j.GraphDatabaseService(), uniqueindexmap={}, classkeymap={}) CMAinit(None, readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) return ourstore
def test_activate(self): io = TestIO([],0) CMAinit(io, cleanoutdb=True, debug=DEBUG) dummy = CMAdb.store.load_or_create(MonitorAction, domain='global', monitorname='DummyName' , monitorclass='OCF', monitortype='Dummy', interval=1, timeout=120, provider='heartbeat') self.assertEqual(len(CMAdb.transaction.tree['packets']), 0) CMAdb.store.commit() CMAdb.transaction.commit_trans(io) self.assertEqual(len(io.packetswritten), 0) # Shouldn't have sent out any pkts yet... CMAdb.transaction = Transaction(encryption_required=False) droneid = 1 droneip = droneipaddress(droneid) designation = dronedesignation(droneid) droneAddr = pyNetAddr((127,0,0,1),1984) droneone = CMAdb.store.load_or_create(Drone, designation=designation, port=1984 , startaddr=droneip, primary_ip_addr=droneip) self.assertTrue(not dummy.isactive) dummy.activate(droneone) CMAdb.store.commit() count=0 for obj in CMAdb.store.load_related(droneone, CMAconsts.REL_hosting, MonitorAction): self.assertTrue(obj is dummy) count += 1 self.assertEqual(count, 1) self.assertTrue(dummy.isactive) count=0 for obj in CMAdb.store.load_related(dummy, CMAconsts.REL_monitoring, Drone): self.assertTrue(obj is droneone) count += 1 self.assertEqual(count, 1) #worked if we returned at or before here CMAdb.transaction.commit_trans(io) #failed if we return here or later self.assertEqual(len(io.packetswritten), 1) # Did we send out exactly one packet? if SavePackets: #io.dumppackets() for fstuple in io.packetswritten: (dest, frameset) = fstuple self.assertEqual(frameset.get_framesettype(), FrameSetTypes.DORSCOP) for frame in frameset.iter(): self.assertEqual(frame.frametype(), FrameTypes.RSCJSON) table = pyConfigContext(init=frame.getstr()) for field in ('class', 'type', 'instance', 'repeat'): self.assertTrue(field in table) if field == 'monitorclass' and table['monitorclass'] == 'OCF': self.assertTrue('provider' in table) for tup in (('class', str), ('type', str), ('resourcename', str) , ('monitorclass', str), ('provider', str) , ('repeat_interval', (int, long)) , ('timeout', (int,long))): (n, t) = tup if n in table: self.assertTrue(isinstance(table[n], t)) # TODO: Add test for deactivating the resource(s) io.cleanio()
def initstore(): global OurStore GraphNode.clean_graphnodes() if OurStore is not None: OurStore.clean_store() db.clear() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) CreateIndexes([cls.__name__ for cls in Classes]) return OurStore
def cleanio(self): if TestIO.mainloop is not None: TestIO.mainloop.quit() if self.pipe_read >= 0: os.close(self.pipe_read) self.pipe_read = -1 TestIO.mainloop = None # Note that this having to do this implies that our I/O object persists # longer than I would have expected... # Is this because uninit needs to be done as part of the test instead of # as part of the cleanup action? del self.inframes del self.packetswritten del self.config self.timeout = None if CMAdb.store: CMAdb.store.abort() CMAdb.store.weaknoderefs = {} CMAdb.store = None CMAinit.uninit()
def dbsetup(readonly=False, url=None): 'Set up our connection to Neo4j' if url is None: url = 'localhost.com:7474' Neo4jCreds().authenticate(url) ourstore = Store(neo4j.Graph(), uniqueindexmap={}, classkeymap={}) CMAinit(DummyIO(), readonly=readonly, use_network=False) for classname in GraphNode.classmap: GraphNode.initclasstypeobj(ourstore, classname) CMAdb.store = ourstore return ourstore
def initenviron(logname, maxdrones, mgmtsystem, debug=False, cmaimage='', nanoimages=[], timeout=90, nanodebug=0, cmadebug=0): 'Initialize the test environment.' logwatch = LogWatcher(logname, [], timeout, returnonlymatch=True, debug=debug) logwatch.setwatch() sysenv = SystemTestEnvironment(logname, maxdrones, mgmtsystem, cmaimage=cmaimage, nanoimages=nanoimages, nanodebug=nanodebug, cmadebug=cmadebug) CMAinit(None, host=str(sysenv.cma.ipaddr), readonly=True, neologin=SystemTestEnvironment.NEO4JLOGIN, neopass=SystemTestEnvironment.NEO4JPASS) url = 'http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474) print >> sys.stderr, 'OPENING Neo4j at URL %s' % url neo4j.authenticate('%s:7474' % sysenv.cma.ipaddr, SystemTestEnvironment.NEO4JLOGIN, SystemTestEnvironment.NEO4JPASS) store = Store(neo4j.Graph(url), readonly=True) for classname in GN.GraphNode.classmap: GN.GraphNode.initclasstypeobj(store, classname) logger('$(grep MemFree: /proc/meminfo)', hardquote=False) tq = QueryTest( store, '''START drone=node:Drone('*:*') WHERE drone.status = "up" RETURN drone''', GN.nodeconstructor, debug=debug) if not tq.check([ None, ], minrows=maxdrones + 1, maxrows=maxdrones + 1, delay=0.5, maxtries=20): sysenv.cma.cleanupwhendone = False raise RuntimeError('Query of "up" status failed. Weirdness') return sysenv, store
def assert_no_dangling_Cclasses(doassert=None): global CheckForDanglingClasses global WorstDanglingCount sys._clear_type_cache() if doassert is None: doassert = AssertOnDanglingClasses CMAinit.uninit() gc.collect() # For good measure... count = proj_class_live_object_count() #print >>sys.stderr, "CHECKING FOR DANGLING CLASSES (%d)..." % count # Avoid cluttering the output up with redundant messages... if count > WorstDanglingCount and CheckForDanglingClasses: WorstDanglingCount = count if doassert: print >> sys.stderr, 'STARTING OBJECT DUMP' dump_c_objects() print >> sys.stderr, 'OBJECT DUMP COMPLETE' print 'stdout OBJECT DUMP COMPLETE' raise AssertionError("Dangling C-class objects - %d still around" % count) else: print >> sys.stderr, ("*****ERROR: Dangling C-class objects - %d still around" % count)
def test_eof(self): 'Get EOF with empty input' if BuildListOnly: return if DEBUG: print >> sys.stderr, 'Running test_test_eof()' framesets = [] io = TestIO(framesets, 0) CMAinit(io, cleanoutdb=True, debug=DEBUG) # just make sure it seems to do the right thing (foo, bar) = io.recvframesets() assert foo is None assert_no_dangling_Cclasses()
def testmain(logname, maxdrones=25, debug=False): 'A simple test main program' regexes = [] #pylint says: [W0612:testmain] Unused variable 'j' #pylint: disable=W0612 for j in range(0,maxdrones+1): regexes.append('Stored packages JSON data from *([^ ]*) ') logwatch = LogWatcher(logname, regexes, timeout=90, returnonlymatch=True) logwatch.setwatch() sysenv = SystemTestEnvironment(maxdrones) print >> sys.stderr, 'Systems all up and running.' url = ('http://%s:%d/db/data/' % (sysenv.cma.ipaddr, 7474)) CMAinit(None) store = Store(neo4j.Graph(url), readonly=True) for classname in GN.GraphNode.classmap: GN.GraphNode.initclasstypeobj(store, classname) results = logwatch.lookforall() if debug: print >> sys.stderr, 'WATCH RESULTS:', results tq = QueryTest(store , "START drone=node:Drone('*:*') RETURN drone" , GN.nodeconstructor, debug=debug) print >> sys.stderr, 'Running Query' if tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1): print 'WOOT! Systems passed query check after initial startup!' else: print 'Systems FAILED initial startup query check' print 'Do you have a second CMA running??' print 'Rerunning query with debug=True' tq.debug = True tq.check([None,], minrows=maxdrones+1, maxrows=maxdrones+1) return 1 cma = sysenv.cma nano = sysenv.nanoprobes[0] regex = (r'%s cma INFO: System %s at \[::ffff:%s]:1984 reports graceful shutdown' % (cma.hostname, nano.hostname, nano.ipaddr)) #print >> sys.stderr, 'REGEX IS: [%s]' % regex logwatch = LogWatcher(logname, [regex,], timeout=30, returnonlymatch=False) logwatch.setwatch() nano.stopservice(SystemTestEnvironment.NANOSERVICE) logwatch.look() time.sleep(30) tq = QueryTest(store , ('''START drone=node:Drone('*:*') ''' '''WHERE drone.designation = "{0.hostname}" RETURN drone''') , GN.nodeconstructor, debug=debug) if tq.check([nano,], downbyshutdown, maxrows=1): print 'WOOT! Systems passed query check after nano shutdown!' else: print 'Systems FAILED query check after nano shutdown'
def test_startup(self): '''A semi-interesting test: We send a STARTUP message and get back a SETCONFIG message with lots of good stuff in it.''' if BuildListOnly: return if DEBUG: print >> sys.stderr, 'Running test_startup()' droneid = 1 droneip = droneipaddress(droneid) designation = dronedesignation(droneid) designationframe = pyCstringFrame(FrameTypes.HOSTNAME, designation) dronediscovery = hostdiscoveryinfo(droneid) discoveryframe = pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery) fs = pyFrameSet(FrameSetTypes.STARTUP) fs.append(designationframe) fs.append(discoveryframe) fsin = ((droneip, (fs, )), ) io = TestIO(fsin, 0) #print >> sys.stderr, 'CMAinit: %s' % str(CMAinit) #print >> sys.stderr, 'CMAinit.__init__: %s' % str(CMAinit.__init__) CMAinit(io, cleanoutdb=True, debug=DEBUG) assimcli_check('loadqueries') OurAddr = pyNetAddr((127, 0, 0, 1), 1984) disp = MessageDispatcher({FrameSetTypes.STARTUP: DispatchSTARTUP()}, encryption_required=False) configinit = geninitconfig(OurAddr) config = pyConfigContext(init=configinit) listener = PacketListener(config, disp, io=io, encryption_required=False) io.mainloop = listener.mainloop TestIO.mainloop = listener.mainloop # We send the CMA an intial STARTUP packet listener.listen() # Let's see what happened... #print >> sys.stderr, ('WRITTEN: %s' % len(io.packetswritten)) self.assertEqual(len(io.packetswritten), 2) # Did we send out two packets? # Note that this change over time # As we change discovery... AUDITS().auditSETCONFIG(io.packetswritten[0], droneid, configinit) io.cleanio() assimcli_check("query allips", 1) assimcli_check("query allservers", 1) assimcli_check("query findip %s" % str(droneip), 1) assimcli_check("query shutdown", 0) assimcli_check("query crashed", 0) assimcli_check("query unknownips", 0)
def initstore(): global version_printed db = neo4j.Graph(None) if not version_printed: print >> sys.stderr, 'USING NEO4J VERSION %s' % str(db.neo4j_version) print >> sys.stderr, 'USING py2neo VERSION %s' % str( py2neo.__version__) version_printed = True GraphNode.clean_graphnodes() db.delete_all() CMAinit(None) OurStore = Store(db, uniqueindexmap=uniqueindexes, classkeymap=keymap) OurStore.clean_store() CreateIndexes(db, [cls.__name__ for cls in Classes]) return OurStore
def maintest(): 'test main program' from cmainit import CMAinit from droneinfo import Drone from systemnode import SystemNode print >> sys.stderr, 'Starting' CMAinit(None, cleanoutdb=True, debug=True) if CMAdb.store.transaction_pending: print 'Transaction pending in:', CMAdb.store print 'Results:', CMAdb.store.commit() print ProcessNode.__meta_labels__() print SystemNode.__meta_labels__() print Drone.__meta_labels__() print 'keys:', Drone.__meta_keyattrs__() print >> sys.stderr, 'Init done' return 0
def test_get1pkt(self): 'Read a single packet' if BuildListOnly: return if DEBUG: print >> sys.stderr, 'Running test_test_eof()' otherguy = pyNetAddr([1, 2, 3, 4], ) strframe1 = pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.") fs = pyFrameSet(42) fs.append(strframe1) framesets = ((otherguy, (strframe1, )), ) io = TestIO(framesets, 0) CMAinit(io, cleanoutdb=True, debug=DEBUG) gottenfs = io.recvframesets() self.assertEqual(len(gottenfs), 2) self.assertEqual(gottenfs, framesets[0]) gottenfs = io.recvframesets() self.assertEqual(len(gottenfs), 2) assert gottenfs[0] is None io.cleanio()
def test_echo1pkt(self): 'Read a packet and write it back out' if BuildListOnly: return if DEBUG: print >> sys.stderr, 'Running test_echo1pkt()' strframe1 = pyCstringFrame(FrameTypes.CSTRINGVAL, "Hello, world.") fs = pyFrameSet(42) fs.append(strframe1) otherguy = pyNetAddr([1, 2, 3, 4], ) framesets = ((otherguy, (strframe1, )), ) io = TestIO(framesets, 0) CMAinit(io, cleanoutdb=True, debug=DEBUG) fslist = io.recvframesets() # read in a packet self.assertEqual(len(fslist), 2) self.assertEqual(fslist, framesets[0]) io.sendframesets(fslist[0], fslist[1]) # echo it back out self.assertEqual(len(io.packetswritten), len(framesets)) gottenfs = io.recvframesets() self.assertEqual(len(gottenfs), 2) assert gottenfs[0] is None io.cleanio()
'Iterate over self.keys() - giving the names of all our *top level* attributes.' for key in self.keys(): yield key def __contains__(self, key): return key in self.map() def __len__(self): return len(self.map()) @staticmethod def __meta_keyattrs__(): 'Return our key attributes in order of significance' return ['jhash'] if __name__ == '__main__': from cmainit import CMAinit print >> sys.stderr, 'Starting' CMAinit(None, cleanoutdb=True, debug=True) if CMAdb.store.transaction_pending: print 'Transaction pending in:', CMAdb.store print 'Results:', CMAdb.store.commit() print CMAclass.__meta_labels__() print ProcessNode.__meta_labels__() print SystemNode.__meta_labels__() from droneinfo import Drone print Drone.__meta_labels__() print 'keys:', Drone.__meta_keyattrs__() print >> sys.stderr, 'Init done'
def test_startup(self): '''A semi-interesting test: We send a STARTUP message and get back a SETCONFIG message with lots of good stuff in it. and for good measure, we also send along some discovery packets. ''' if BuildListOnly: return if DEBUG: print >> sys.stderr, 'Running test_startup()' AssimEvent.disable_all_observers() from dispatchtarget import DispatchTarget droneid = 1 droneip = droneipaddress(droneid) designation = dronedesignation(droneid) designationframe=pyCstringFrame(FrameTypes.HOSTNAME, designation) dronediscovery=hostdiscoveryinfo(droneid) discoveryframe=pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery) fs = pyFrameSet(FrameSetTypes.STARTUP) fs.append(designationframe) fs.append(discoveryframe) fs2 = pyFrameSet(FrameSetTypes.JSDISCOVERY) osdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.OS_DISCOVERY) fs2.append(osdiscovery) fs3 = pyFrameSet(FrameSetTypes.JSDISCOVERY) ulimitdiscovery=pyCstringFrame(FrameTypes.JSDISCOVER, self.ULIMIT_DISCOVERY) fs3.append(ulimitdiscovery) fsin = ((droneip, (fs,)), (droneip, (fs2,)), (droneip, (fs3,))) io = TestIO(fsin,0) #print >> sys.stderr, 'CMAinit: %s' % str(CMAinit) #print >> sys.stderr, 'CMAinit.__init__: %s' % str(CMAinit.__init__) OurAddr = pyNetAddr((127,0,0,1),1984) configinit = geninitconfig(OurAddr) config = pyConfigContext(init=configinit) io.config = config CMAinit(io, cleanoutdb=True, debug=DEBUG) CMAdb.io.config = config assimcli_check('loadqueries') disp = MessageDispatcher(DispatchTarget.dispatchtable, encryption_required=False) listener = PacketListener(config, disp, io=io, encryption_required=False) io.mainloop = listener.mainloop TestIO.mainloop = listener.mainloop # We send the CMA an intial STARTUP packet listener.listen() # Let's see what happened... #print >> sys.stderr, ('READ: %s' % io.packetsread) #print >> sys.stderr, ('WRITTEN: %s' % len(io.packetswritten)) #print >> sys.stderr, ('PACKETS WRITTEN: %s' % str(io.packetswritten)) self.assertEqual(len(io.packetswritten), 2) # Did we send out four packets? # Note that this change over time # As we change discovery... self.assertEqual(io.packetsread, 3) # Did we read 3 packets? AUDITS().auditSETCONFIG(io.packetswritten[0], droneid, configinit) assimcli_check("query allips", 1) assimcli_check("query allservers", 1) assimcli_check("query findip %s" % str(droneip), 1) assimcli_check("query shutdown", 0) assimcli_check("query crashed", 0) assimcli_check("query unknownips", 0) CMAdb.io.config = config Drones = CMAdb.store.load_cypher_nodes("START n=node:Drone('*:*') RETURN n", Drone) Drones = [drone for drone in Drones] for drone in Drones: self.check_discovery(drone, (dronediscovery, self.OS_DISCOVERY, self.ULIMIT_DISCOVERY)) self.assertEqual(len(Drones), 1) # Should only be one drone io.config = None io.cleanio() del io del ulimitdiscovery, osdiscovery, Drones DispatchTarget.dispatchtable = {} del DispatchTarget
def test_several_startups(self): '''A very interesting test: We send a STARTUP message and get back a SETCONFIG message and then send back a bunch of discovery requests.''' if DEBUG: print >> sys.stderr, 'Running test_several_startups()' OurAddr = pyNetAddr((10, 10, 10, 5), 1984) configinit = geninitconfig(OurAddr) # Create the STARTUP FrameSets that our fake Drones should appear to send fsin = [] droneid = 0 for droneid in range(1, MaxDrone + 1): droneip = droneipaddress(droneid) designation = dronedesignation(droneid) designationframe = pyCstringFrame(FrameTypes.HOSTNAME, designation) dronediscovery = hostdiscoveryinfo(droneid) discoveryframe = pyCstringFrame(FrameTypes.JSDISCOVER, dronediscovery) fs = pyFrameSet(FrameSetTypes.STARTUP) fs.append(designationframe) fs.append(discoveryframe) fsin.append((droneip, (fs, ))) addrone = droneipaddress(1) maxdrones = droneid if doHBDEAD: # Create the HBDEAD FrameSets that our first fake Drone should appear to send # concerning the death of its dearly departed peers #print >> sys.stderr, 'KILLING THEM ALL!!!' for droneid in range(2, maxdrones + 1): droneip = droneipaddress(droneid) designation = dronedesignation(droneid) #deadframe=pyIpPortFrame(FrameTypes.IPPORT, addrstring=droneip) fs = pyFrameSet(FrameSetTypes.HBSHUTDOWN) #fs.append(deadframe) hostframe = pyCstringFrame(FrameTypes.HOSTNAME, designation) fs.append(hostframe) fsin.append((droneip, (fs, ))) io = TestIO(fsin) CMAinit(io, cleanoutdb=True, debug=DEBUG) assert CMAdb.io.config is not None assimcli_check('loadqueries') disp = MessageDispatcher( { FrameSetTypes.STARTUP: DispatchSTARTUP(), FrameSetTypes.HBDEAD: DispatchHBDEAD(), FrameSetTypes.HBSHUTDOWN: DispatchHBSHUTDOWN(), }, encryption_required=False) config = pyConfigContext(init=configinit) listener = PacketListener(config, disp, io=io, encryption_required=False) io.mainloop = listener.mainloop TestIO.mainloop = listener.mainloop # We send the CMA a BUNCH of intial STARTUP packets # and (optionally) a bunch of HBDEAD packets assert CMAdb.io.config is not None listener.listen() # We audit after each packet is processed # The auditing code will make sure all is well... # But it doesn't know how many drones we just registered query = neo4j.CypherQuery(CMAdb.cdb.db, "START n=node:Drone('*:*') RETURN n") Drones = CMAdb.store.load_cypher_nodes(query, Drone) Drones = [drone for drone in Drones] #print >> sys.stderr, 'WE NOW HAVE THESE DRONES:', Drones self.assertEqual(len(Drones), maxdrones) if doHBDEAD: # Verify that all drones except one are dead #livecount, partnercount, ringmemberships #self.check_live_counts(1, 0, 1) assimcli_check("query allservers", maxdrones) assimcli_check("query down", maxdrones - 1) assimcli_check("query crashed", 0) assimcli_check("query shutdown", maxdrones - 1) else: if maxdrones == 1: partnercount = 0 elif maxdrones == 2: partnercount = 2 else: partnercount = 2 * maxdrones # livecount partnercount ringmemberships #self.check_live_counts(maxdrones, partnercount, maxdrones) assimcli_check("query allservers", maxdrones) assimcli_check("query down", 0) assimcli_check("query shutdown", 0) assimcli_check("query unknownips", 0) for droneid in range(1, MaxDrone + 1): droneip = droneipaddress(droneid) assimcli_check("query findip %s" % str(droneip), 1) if DoAudit: auditalldrones() auditallrings() if DEBUG: print "The CMA read %d packets." % io.packetsread print "The CMA wrote %d packets." % io.writecount #io.dumppackets() io.config = None io.cleanio()