class ProcessDispatcherZooKeeperStoreProxyKillsTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin): # this runs all of the ProcessDispatcherStoreTests tests plus any # ZK-specific ones, but uses a proxy in front of ZK and restarts # the proxy before each call to the store. The effect is that for each store # operation, the first call to kazoo fails with a connection error, but the # client should handle that and retry def setUp(self): self.setup_zookeeper(base_path_prefix="/processdispatcher_store_tests_", use_proxy=True) self.real_store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent) self.real_store.initialize() # have the tests use a wrapped store that restarts the connection before each call self.store = SocatProxyRestartWrapper(self.proxy, self.real_store) def tearDown(self): self.teardown_zookeeper() def test_the_fixture(self): # make sure test fixture actually works like we think def fake_operation(): self.store.kazoo.get("/") self.real_store.fake_operation = fake_operation self.assertRaises(ConnectionLoss, self.store.fake_operation)
def setup_store(self): self.setup_zookeeper(base_path_prefix="/doctor_tests_" + uuid.uuid4().hex) store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent) store.initialize() return store
class ProcessDispatcherZooKeeperStoreTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin): def setUp(self): self.setup_zookeeper("/processdispatcher_store_tests_") self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent) self.store.initialize() def tearDown(self): self.store.shutdown() self.teardown_zookeeper()
def setUp(self): self.setup_zookeeper(base_path_prefix="/processdispatcher_store_tests_", use_proxy=True) self.real_store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent) self.real_store.initialize() # have the tests use a wrapped store that restarts the connection before each call self.store = SocatProxyRestartWrapper(self.proxy, self.real_store)
def setUp(self): self.setup_zookeeper("/processdispatcher_store_tests_", use_proxy=True) self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent, timeout=2.0) self.store.initialize()
class ProcessDispatcherZooKeeperStoreProxyTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin): def setUp(self): self.setup_zookeeper("/processdispatcher_store_tests_", use_proxy=True) self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts, self.zk_base_path, use_gevent=self.use_gevent, timeout=2.0) self.store.initialize() def tearDown(self): if not self.proxy.running: self.proxy.start() self.store.shutdown() self.teardown_zookeeper() def test_elections_connection(self): matchmaker = MockLeader() doctor = MockLeader() self.store.contend_matchmaker(matchmaker) self.store.contend_doctor(doctor) matchmaker.wait_running() doctor.wait_running() # now kill the connection self.proxy.stop() matchmaker.wait_cancelled(5) doctor.wait_cancelled(5) # wait for session to expire time.sleep(3) # start connection back up. leaders should resume. eventually. self.proxy.start() matchmaker.wait_running(60) doctor.wait_running(60) def test_elections_under_siege(self, coups=10): # repeatedly kill and restart ZK connections with varying delays. # make sure we come out of it with leaders in the end. if not os.environ.get('NIGHTLYINT'): raise unittest.SkipTest("Slow integration test") matchmaker = MockLeader() doctor = MockLeader() self.store.contend_matchmaker(matchmaker) self.store.contend_doctor(doctor) for i in range(coups): sleep_time = random.uniform(0.0, 4.0) log.debug("Enjoying %s seconds of peace", sleep_time) time.sleep(sleep_time) self.proxy.stop() sleep_time = random.uniform(0.0, 5.0) log.debug("Enduring %s seconds of anarchy", sleep_time) time.sleep(sleep_time) self.proxy.start() # ensure leaders eventually recover matchmaker.wait_running(60) doctor.wait_running(60)