def dotest(type): icestorm = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, additional=iceStormArgs) icestorm.start() sys.stdout.write("creating topic... ") sys.stdout.flush() icestorm.admin("create single") print("ok") sys.stdout.write("starting subscriber... ") sys.stdout.flush() subscriberProc = TestUtil.startServer(subscriber, icestorm.reference() + subscriberArgs, count=3) print("ok") # # Start the publisher. This should publish 10 events which eventually # causes subscriber to terminate. # sys.stdout.write("starting publisher... ") sys.stdout.flush() publisherProc = TestUtil.startClient(publisher, icestorm.reference(), startReader=False) print("ok") publisherProc.startReader() subscriberProc.waitTestSuccess() publisherProc.waitTestSuccess() # # Destroy the topic. # sys.stdout.write("destroy topic... ") sys.stdout.flush() icestorm.admin("destroy single") print("ok") # # Shutdown icestorm. # icestorm.stop()
def dotest(type): icestorm = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type) icestorm.start() sys.stdout.write("creating topic... ") sys.stdout.flush() icestorm.admin("create single") print("ok") sys.stdout.write("starting subscriber... ") sys.stdout.flush() subscriberProc = TestUtil.startServer(subscriber, icestorm.reference(), count = 5) print("ok") # # Start the publisher. This should publish 10 events which eventually # causes subscriber to terminate. # sys.stdout.write("starting publisher... ") sys.stdout.flush() publisherProc = TestUtil.startClient(publisher, icestorm.reference(), startReader = False) print("ok") publisherProc.startReader() subscriberProc.waitTestSuccess() publisherProc.waitTestSuccess() # # Destroy the topic. # sys.stdout.write("destroy topic... ") sys.stdout.flush() icestorm.admin("destroy single") print("ok") # # Shutdown icestorm. # icestorm.stop()
def runtest(type, **args): icestorm = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, **args) icestorm.start() sys.stdout.write("setting up topics... ") sys.stdout.flush() icestorm.admin( "create fed1 fed2 fed3; link fed1 fed2 10; link fed2 fed3 5") print("ok") # # Test oneway subscribers. # sys.stdout.write("testing oneway subscribers... ") sys.stdout.flush() doTest(icestorm, 0) print("ok") # # Test batch oneway subscribers. # sys.stdout.write("testing batch subscribers... ") sys.stdout.flush() doTest(icestorm, 1) print("ok") # # Destroy the topics. # sys.stdout.write("destroying topics... ") sys.stdout.flush() icestorm.admin("destroy fed1 fed2 fed3") print("ok") # # Shutdown icestorm. # icestorm.stop()
def runtest(type, **args): icestorm = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, **args) icestorm.start() sys.stdout.write("setting up topics... ") sys.stdout.flush() icestorm.admin("create fed1 fed2 fed3; link fed1 fed2 10; link fed2 fed3 5") print("ok") # # Test oneway subscribers. # sys.stdout.write("testing oneway subscribers... ") sys.stdout.flush() doTest(icestorm, 0) print("ok") # # Test batch oneway subscribers. # sys.stdout.write("testing batch subscribers... ") sys.stdout.flush() doTest(icestorm, 1) print("ok") # # Destroy the topics. # sys.stdout.write("destroying topics... ") sys.stdout.flush() icestorm.admin("destroy fed1 fed2 fed3") print("ok") # # Shutdown icestorm. # icestorm.stop()
targets = [] if TestUtil.appverifier: targets = [TestUtil.getIceBox(), publisher, subscriber, control, TestUtil.getIceBoxAdmin(), \ TestUtil.getIceStormAdmin()] TestUtil.setAppVerifierSettings(targets, cwd=os.getcwd()) def runcontrol(proxy): proc = TestUtil.startClient(control, ' "%s"' % proxy) proc.waitTestSuccess() icestorm = IceStormUtil.init( TestUtil.toplevel, os.getcwd(), "replicated", replicatedPublisher=True, additional=' --IceStorm.Election.MasterTimeout=2' + ' --IceStorm.Election.ElectionTimeout=2' + ' --IceStorm.Election.ResponseTimeout=2') icestorm.start() sys.stdout.write("creating topic... ") sys.stdout.flush() icestorm.admin("create single") print("ok") sys.stdout.write("running subscriber... ") sys.stdout.flush() subscriberProc = TestUtil.startServer(subscriber, ' --Ice.ServerIdleTime=0 ' + icestorm.reference(),
def runtest(type): # Clear the idle timeout otherwise the IceBox ThreadPool will timeout. server1 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, dbDir="db", instanceName="TestIceStorm1", port=12000) server2 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, dbDir="db2", instanceName="TestIceStorm2", port=12500) global adminIceStormReference adminIceStormReference = ' --IceStormAdmin.TopicManager.Proxy="%s" --IceStormAdmin.TopicManager.Proxy2="%s"' % ( server1.proxy(), server2.proxy()) sys.stdout.write("starting icestorm services... ") sys.stdout.flush() server1.start(echo=False) server2.start(echo=False) print("ok") runAdmin("create TestIceStorm1/fed1 TestIceStorm2/fed1", "setting up the topics...") sys.stdout.write("Sending 5000 ordered events... ") sys.stdout.flush() doTest(server1, server2, '--events 5000 --qos "reliability,ordered" ' + server1.reference(), '--events 5000') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 5000 ordered events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 5000 --qos "reliability,ordered" ' + server2.reference(), '--events 5000') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 ' + server1.reference(), '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 ' + server2.reference(), '--events 20000 --oneway') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered batch events... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 --qos "reliability,batch" ' + server1.reference(), '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered batch events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 --qos "reliability,batch" ' + server2.reference(), '--events 20000 --oneway') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events with slow subscriber... ") sys.stdout.flush() doTest(server1, server2, [ '--events 2 --slow ' + server1.reference(), '--events 20000 ' + server1.reference() ], '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write( "Sending 20000 unordered events with slow subscriber & link... ") sys.stdout.flush() doTest(server1, server2, [ '--events 2 --slow' + server1.reference(), '--events 20000' + server1.reference(), '--events 2 --slow' + server2.reference(), '--events 20000' + server2.reference() ], '--events 20000 --oneway') print("ok") sys.stdout.write("shutting down icestorm services... ") sys.stdout.flush() server1.stop() server2.stop() print("ok") sys.stdout.write("starting icestorm services... ") sys.stdout.flush() # # The erratic tests emit lots of connection warnings so they are # disabled here. The IceStorm servers are stopped and restarted so the # settings will take effect. # server1.start(echo=False, additionalOptions=' --Ice.Warn.Connections=0') server2.start(echo=False, additionalOptions=' --Ice.Warn.Connections=0') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write( "Sending 20000 unordered events with erratic subscriber... ") sys.stdout.flush() doTest(server1, server2, [ '--erratic 5 --qos "reliability,ordered" --events 20000' + server1.reference(), '--erratic 5 --events 20000' + server1.reference(), '--events 20000' + server1.reference() ], '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write( "Sending 20000 unordered events with erratic subscriber across a link... " ) sys.stdout.flush() doTest(server1, server2, [ '--events 20000' + server1.reference(), '--erratic 5 --qos "reliability,ordered" --events 20000 ' + server1.reference(), '--erratic 5 --events 20000 ' + server1.reference(), '--events 20000' + server2.reference(), '--erratic 5 --qos "reliability,ordered" --events 20000 ' + server2.reference(), '--erratic 5 --events 20000 ' + server2.reference() ], '--events 20000 --oneway ') print("ok") # # Shutdown icestorm. # sys.stdout.write("shutting down icestorm services... ") sys.stdout.flush() server1.stop() server2.stop() print("ok") sys.stdout.write( "Sending 5000 ordered events with max queue size drop events... ") sys.stdout.flush() opts = " --IceStorm.Send.QueueSizeMax=2000 --IceStorm.Send.QueueSizeMaxPolicy=DropEvents" server1.start(echo=False, additionalOptions=opts) doTest( server1, server2, '--events 5000 --qos "reliability,ordered" --maxQueueDropEvents=2000 ' + server1.reference(), '--events 5000 --maxQueueTest') server1.stop() print("ok") sys.stdout.write( "Sending 5000 ordered events with max queue size remove subscriber... " ) sys.stdout.flush() opts = " --IceStorm.Send.QueueSizeMax=2000 --IceStorm.Send.QueueSizeMaxPolicy=RemoveSubscriber" server1.start(echo=False, additionalOptions=opts) doTest( server1, server2, '--events 5000 --qos "reliability,ordered" --maxQueueRemoveSub=2000 ' + server1.reference(), '--events 5000 --maxQueueTest') server1.stop() print("ok")
TestUtil.getIceBoxAdmin(), TestUtil.getIceStormAdmin(), ] TestUtil.setAppVerifierSettings(targets, cwd=os.getcwd()) def runcontrol(proxy): proc = TestUtil.startClient(control, ' "%s"' % proxy) proc.waitTestSuccess() icestorm = IceStormUtil.init( TestUtil.toplevel, os.getcwd(), "replicated", replicatedPublisher=True, additional=" --IceStorm.Election.MasterTimeout=2" + " --IceStorm.Election.ElectionTimeout=2" + " --IceStorm.Election.ResponseTimeout=2", ) icestorm.start() sys.stdout.write("creating topic... ") sys.stdout.flush() icestorm.admin("create single") print("ok") sys.stdout.write("running subscriber... ") sys.stdout.flush() subscriberProc = TestUtil.startServer(subscriber, " --Ice.ServerIdleTime=0 " + icestorm.reference(), echo=False) subscriberProc.expect("([^\n]+)\n")
def runtest(type, **args): icestorm1 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, additional = '--IceStorm.Discard.Interval=2', dbDir = "db", instanceName = "TestIceStorm1", port = 12000, **args) icestorm1.start() icestorm2 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, additional = '--IceStorm.Discard.Interval=2', dbDir = "db2", instanceName = "TestIceStorm2", port = 12500, **args) icestorm2.start() adminIceStormReference = ' --IceStormAdmin.TopicManager.Proxy="%s" --IceStormAdmin.TopicManager.Proxy2="%s"' % ( icestorm1.proxy(), icestorm2.proxy()) sys.stdout.write("setting up the topics... ") sys.stdout.flush() admin(adminIceStormReference, "create TestIceStorm1/fed1 TestIceStorm2/fed1; link TestIceStorm1/fed1 TestIceStorm2/fed1") print("ok") # # Test oneway subscribers. # sys.stdout.write("testing federation with oneway subscribers... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 0) print("ok") # # Test batch oneway subscribers. # sys.stdout.write("testing federation with batch subscribers... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 1) print("ok") # # Test #2: # # Stop and restart the service and repeat the test. This ensures that # the database is correct. # sys.stdout.write("restarting services to ensure that the database content is preserved... ") sys.stdout.flush() # # Shutdown icestorm. # icestorm1.stop() icestorm2.stop() icestorm1.start(echo=False) icestorm2.start(echo=False) print("ok") # # Test oneway subscribers. # sys.stdout.write("retesting federation with oneway subscribers... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 0) print("ok") # # Test batch oneway subscribers. # sys.stdout.write("retesting federation with batch subscribers... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 1) print("ok") # # Shutdown icestorm. # icestorm1.stop() icestorm2.stop() # # Restart the first server and publish some events. Attach a # subscriber to the channel and make sure the events are received. # # Then re-start the linked downstream server and publish the events. # Ensure they are received by the linked server. # if type != "replicated": sys.stdout.write("restarting only one IceStorm server... ") sys.stdout.flush() proc = icestorm1.start(echo=False) #proc.expect("topic.fed1.*subscriber offline") #proc.expect("connection refused") print("ok") # # Test oneway subscribers. # sys.stdout.write("testing that the federation link reports an error... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 0, icestorm1.reference()) # Give some time for the output to be sent. time.sleep(2) proc.expect("topic.fed1.*subscriber offline") print("ok") sys.stdout.write("starting downstream icestorm server... ") sys.stdout.flush() icestorm2.start(echo=False) print("ok") # # Need to sleep for at least the discard interval. # time.sleep(3) # # Test oneway subscribers. # sys.stdout.write("testing link is reestablished... ") sys.stdout.flush() doTest(icestorm1, icestorm2, 0) print("ok") try: proc.expect("topic.fed1.*subscriber offline") assert False except Expect.TIMEOUT: pass icestorm1.stop() icestorm2.stop() # # Test #4: # # Trash the TestIceStorm2 database. Then restart the servers and # verify that the link is removed. # sys.stdout.write("destroying the downstream IceStorm service database... ") sys.stdout.flush() icestorm2.clean() print("ok") sys.stdout.write("restarting IceStorm servers... ") sys.stdout.flush() icestorm1.start(echo = False) icestorm2.start(echo = False) print("ok") sys.stdout.write("checking link still exists... ") sys.stdout.flush() line = admin(adminIceStormReference, "links TestIceStorm1") if not re.compile("fed1 with cost 0").search(line): print(line) sys.exit(1) print("ok") sys.stdout.write("publishing some events... ") sys.stdout.flush() # The publisher must be run twice because all the events can be # sent out in one batch to the linked subscriber which means that # the link is not reaped until the next batch is # sent. Furthermore, with a replicated IceStorm both sets of # events must be set to the same replica. runPublisher(icestorm1, opt = " --count 2") print("ok") # Give the unsubscription time to propagate. time.sleep(1) # Verify that the link has disappeared. sys.stdout.write("verifying that the link has been destroyed... ") sys.stdout.flush() line = admin(adminIceStormReference, "links TestIceStorm1") nRetry = 5 while len(line) > 0 and nRetry > 0: line = admin(adminIceStormReference, "links TestIceStorm1") time.sleep(1) # Give more time for unsubscription to propagate. nRetry -= 1 if len(line) > 0: print(line) sys.exit(1) print("ok") # # Destroy the remaining topic. # sys.stdout.write("destroying topics... ") sys.stdout.flush() admin(adminIceStormReference, "destroy TestIceStorm1/fed1") print("ok") # # Shutdown icestorm. # sys.stdout.write("shutting down icestorm services... ") sys.stdout.flush() icestorm1.stop() icestorm2.stop() print("ok")
def runtest(type): # Clear the idle timeout otherwise the IceBox ThreadPool will timeout. server1 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, dbDir = "db", instanceName = "TestIceStorm1", port = 12000) server2 = IceStormUtil.init(TestUtil.toplevel, os.getcwd(), type, dbDir = "db2", instanceName = "TestIceStorm2", port = 12500) global adminIceStormReference adminIceStormReference = ' --IceStormAdmin.TopicManager.Proxy="%s" --IceStormAdmin.TopicManager.Proxy2="%s"' % ( server1.proxy(), server2.proxy()) sys.stdout.write("starting icestorm services... ") sys.stdout.flush() server1.start(echo=False) server2.start(echo=False) print("ok") runAdmin("create TestIceStorm1/fed1 TestIceStorm2/fed1", "setting up the topics...") sys.stdout.write("Sending 5000 ordered events... ") sys.stdout.flush() doTest(server1, server2, '--events 5000 --qos "reliability,ordered" ' + server1.reference(), '--events 5000') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 5000 ordered events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 5000 --qos "reliability,ordered" ' + server2.reference(), '--events 5000') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 ' + server1.reference(), '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 ' + server2.reference(), '--events 20000 --oneway') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered batch events... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 --qos "reliability,batch" ' + server1.reference(), '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered batch events across a link... ") sys.stdout.flush() doTest(server1, server2, '--events 20000 --qos "reliability,batch" ' + server2.reference(), '--events 20000 --oneway') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events with slow subscriber... ") sys.stdout.flush() doTest(server1, server2, ['--events 2 --slow ' + server1.reference(), '--events 20000 ' + server1.reference()], '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events with slow subscriber & link... ") sys.stdout.flush() doTest(server1, server2, ['--events 2 --slow' + server1.reference(), '--events 20000' + server1.reference(), '--events 2 --slow' + server2.reference(), '--events 20000' + server2.reference()], '--events 20000 --oneway') print("ok") sys.stdout.write("shutting down icestorm services... ") sys.stdout.flush() server1.stop() server2.stop() print("ok") sys.stdout.write("starting icestorm services... ") sys.stdout.flush() # # The erratic tests emit lots of connection warnings so they are # disabled here. The IceStorm servers are stopped and restarted so the # settings will take effect. # server1.start(echo=False, additionalOptions = ' --Ice.Warn.Connections=0') server2.start(echo=False, additionalOptions = ' --Ice.Warn.Connections=0') print("ok") runAdmin("unlink TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events with erratic subscriber... ") sys.stdout.flush() doTest(server1, server2, [ '--erratic 5 --qos "reliability,ordered" --events 20000' + server1.reference(), '--erratic 5 --events 20000' + server1.reference(), '--events 20000' + server1.reference()], '--events 20000 --oneway') print("ok") runAdmin("link TestIceStorm1/fed1 TestIceStorm2/fed1") sys.stdout.write("Sending 20000 unordered events with erratic subscriber across a link... ") sys.stdout.flush() doTest(server1, server2, [ '--events 20000' + server1.reference(), '--erratic 5 --qos "reliability,ordered" --events 20000 ' + server1.reference(), '--erratic 5 --events 20000 ' + server1.reference(), '--events 20000' + server2.reference(), '--erratic 5 --qos "reliability,ordered" --events 20000 ' + server2.reference(), '--erratic 5 --events 20000 ' + server2.reference()], '--events 20000 --oneway ') print("ok") # # Shutdown icestorm. # sys.stdout.write("shutting down icestorm services... ") sys.stdout.flush() server1.stop() server2.stop() print("ok")