class ProcessDispatcherZooKeeperStoreProxyKillsTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin):
# this runs all of the ProcessDispatcherStoreTests tests plus any
# ZK-specific ones, but uses a proxy in front of ZK and restarts
# the proxy before each call to the store. The effect is that for each store
# operation, the first call to kazoo fails with a connection error, but the
# client should handle that and retry
def setUp(self):
self.setup_zookeeper(base_path_prefix="/processdispatcher_store_tests_", use_proxy=True)
self.real_store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent)
self.real_store.initialize()
# have the tests use a wrapped store that restarts the connection before each call
self.store = SocatProxyRestartWrapper(self.proxy, self.real_store)
def tearDown(self):
self.teardown_zookeeper()
def test_the_fixture(self):
# make sure test fixture actually works like we think
def fake_operation():
self.store.kazoo.get("/")
self.real_store.fake_operation = fake_operation
self.assertRaises(ConnectionLoss, self.store.fake_operation)
def setup_store(self):
self.setup_zookeeper(base_path_prefix="/doctor_tests_" + uuid.uuid4().hex)
store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent)
store.initialize()
return store
class ProcessDispatcherZooKeeperStoreTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin):
def setUp(self):
self.setup_zookeeper("/processdispatcher_store_tests_")
self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent)
self.store.initialize()
def tearDown(self):
self.store.shutdown()
self.teardown_zookeeper()
def setUp(self):
self.setup_zookeeper(base_path_prefix="/processdispatcher_store_tests_", use_proxy=True)
self.real_store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent)
self.real_store.initialize()
# have the tests use a wrapped store that restarts the connection before each call
self.store = SocatProxyRestartWrapper(self.proxy, self.real_store)
def setUp(self):
self.setup_zookeeper("/processdispatcher_store_tests_", use_proxy=True)
self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent, timeout=2.0)
self.store.initialize()
class ProcessDispatcherZooKeeperStoreProxyTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin):
def setUp(self):
self.setup_zookeeper("/processdispatcher_store_tests_", use_proxy=True)
self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent, timeout=2.0)
self.store.initialize()
def tearDown(self):
if not self.proxy.running:
self.proxy.start()
self.store.shutdown()
self.teardown_zookeeper()
def test_elections_connection(self):
matchmaker = MockLeader()
doctor = MockLeader()
self.store.contend_matchmaker(matchmaker)
self.store.contend_doctor(doctor)
matchmaker.wait_running()
doctor.wait_running()
# now kill the connection
self.proxy.stop()
matchmaker.wait_cancelled(5)
doctor.wait_cancelled(5)
# wait for session to expire
time.sleep(3)
# start connection back up. leaders should resume. eventually.
self.proxy.start()
matchmaker.wait_running(60)
doctor.wait_running(60)
def test_elections_under_siege(self, coups=10):
# repeatedly kill and restart ZK connections with varying delays.
# make sure we come out of it with leaders in the end.
if not os.environ.get('NIGHTLYINT'):
raise unittest.SkipTest("Slow integration test")
matchmaker = MockLeader()
doctor = MockLeader()
self.store.contend_matchmaker(matchmaker)
self.store.contend_doctor(doctor)
for i in range(coups):
sleep_time = random.uniform(0.0, 4.0)
log.debug("Enjoying %s seconds of peace", sleep_time)
time.sleep(sleep_time)
self.proxy.stop()
sleep_time = random.uniform(0.0, 5.0)
log.debug("Enduring %s seconds of anarchy", sleep_time)
time.sleep(sleep_time)
self.proxy.start()
# ensure leaders eventually recover
matchmaker.wait_running(60)
doctor.wait_running(60)
