class ProcessDispatcherZooKeeperStoreProxyTests(ProcessDispatcherStoreTests, ZooKeeperTestMixin):
def setUp(self):
self.setup_zookeeper("/processdispatcher_store_tests_", use_proxy=True)
self.store = ProcessDispatcherZooKeeperStore(self.zk_hosts,
self.zk_base_path, use_gevent=self.use_gevent, timeout=2.0)
self.store.initialize()
def tearDown(self):
if not self.proxy.running:
self.proxy.start()
self.store.shutdown()
self.teardown_zookeeper()
def test_elections_connection(self):
matchmaker = MockLeader()
doctor = MockLeader()
self.store.contend_matchmaker(matchmaker)
self.store.contend_doctor(doctor)
matchmaker.wait_running()
doctor.wait_running()
# now kill the connection
self.proxy.stop()
matchmaker.wait_cancelled(5)
doctor.wait_cancelled(5)
# wait for session to expire
time.sleep(3)
# start connection back up. leaders should resume. eventually.
self.proxy.start()
matchmaker.wait_running(60)
doctor.wait_running(60)
def test_elections_under_siege(self, coups=10):
# repeatedly kill and restart ZK connections with varying delays.
# make sure we come out of it with leaders in the end.
if not os.environ.get('NIGHTLYINT'):
raise unittest.SkipTest("Slow integration test")
matchmaker = MockLeader()
doctor = MockLeader()
self.store.contend_matchmaker(matchmaker)
self.store.contend_doctor(doctor)
for i in range(coups):
sleep_time = random.uniform(0.0, 4.0)
log.debug("Enjoying %s seconds of peace", sleep_time)
time.sleep(sleep_time)
self.proxy.stop()
sleep_time = random.uniform(0.0, 5.0)
log.debug("Enduring %s seconds of anarchy", sleep_time)
time.sleep(sleep_time)
self.proxy.start()
# ensure leaders eventually recover
matchmaker.wait_running(60)
doctor.wait_running(60)
