def test_mark_failed(self, client):
"""Create a plugin not in the cluster and try to mark it as failed.
Nothing should be written to etcd."""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
e.mark_node_failed()
e._client.write.assert_not_called()
e.terminate()
def test_mark_failed(self, client):
"""Create a plugin not in the cluster and try to mark it as failed.
Nothing should be written to etcd."""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
e.mark_node_failed()
e._client.write.assert_not_called()
e.terminate()
def test_leaving(self, client):
"""Create a plugin not in the cluster and try to leave the cluster.
Nothing should be written to etcd."""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
e.leave_cluster()
e._client.write.assert_not_called()
e.terminate()
def test_leaving(self, client):
"""Create a plugin not in the cluster and try to leave the cluster.
Nothing should be written to etcd."""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
e.leave_cluster()
e._client.write.assert_not_called()
e.terminate()
def test_scale_down(self):
# Start with a stable cluster of two nodes
sync1 = EtcdSynchronizer(DummyPlugin(None), '10.0.1.1')
sync2 = EtcdSynchronizer(DummyPlugin(None), '10.0.1.2')
mock_client = sync1._client
mock_client.write("/test", json.dumps({"10.0.1.1": "normal",
"10.0.1.2": "normal"}))
for s in [sync1, sync2]:
s.start_thread()
# Make the second node leave
sync2.leave_cluster()
sync2.thread.join(20)
sync2.terminate()
self.wait_for_all_normal(mock_client, required_number=1)
# Check that it's left and the cluster is stable
end = json.loads(mock_client.read("/test").value)
self.assertEqual(None, end.get("10.0.1.2"))
self.assertEqual("normal", end.get("10.0.1.1"))
sync1.terminate()
def test_watcher(self):
"""Create a new 3-node cluster with one plugin not in the cluster and
check that the main three all end up in NORMAL state"""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
self.make_and_start_synchronizers(3)
mock_client = self.syncs[0]._client
self.wait_for_all_normal(mock_client, required_number=3)
# Pause for one second - the watcher plugin might be called just after
# all other nodes enter 'normal' state
sleep(1)
self.assertTrue(self.plugin.on_stable_cluster_called)
end = json.loads(mock_client.read("/test").value)
self.assertEqual("normal", end.get("10.0.0.0"))
self.assertEqual("normal", end.get("10.0.0.1"))
self.assertEqual("normal", end.get("10.0.0.2"))
self.assertEqual(None, end.get("10.1.1.1"))
e.terminate()
def test_watcher(self):
"""Create a new 3-node cluster with one plugin not in the cluster and
check that the main three all end up in NORMAL state"""
e = EtcdSynchronizer(self.plugin, self.watcher_ip)
e.start_thread()
self.make_and_start_synchronizers(3)
mock_client = self.syncs[0]._client
self.wait_for_all_normal(mock_client, required_number=3)
# Pause for one second - the watcher plugin might be called just after
# all other nodes enter 'normal' state
sleep(1)
self.assertTrue(self.plugin.on_stable_cluster_called)
end = json.loads(mock_client.read("/test").value)
self.assertEqual("normal", end.get("10.0.0.0"))
self.assertEqual("normal", end.get("10.0.0.1"))
self.assertEqual("normal", end.get("10.0.0.2"))
self.assertEqual(None, end.get("10.1.1.1"))
e.terminate()
def test_failure(self):
# Create synchronisers, using a FailPlugin for one which will crash and
# not complete (simulating a failed node)
sync1 = EtcdSynchronizer(DummyPlugin(None), '10.0.0.1')
sync2 = EtcdSynchronizer(FailPlugin(None), '10.0.0.2')
sync3 = EtcdSynchronizer(DummyPlugin(None), '10.0.0.3')
mock_client = sync1._client
for s in [sync1, sync2, sync3]:
s.start_thread()
# After a few seconds, the scale-up will still not have completed
sleep(3)
end = json.loads(mock_client.read("/test").value)
self.assertNotEqual("normal", end.get("10.0.0.1"))
self.assertNotEqual("normal", end.get("10.0.0.2"))
self.assertNotEqual("normal", end.get("10.0.0.3"))
# Start a synchroniser to take 10.0.0.2's place
sync2.terminate()
error_syncer = EtcdSynchronizer(NullPlugin('/test'),
'10.0.0.2',
force_leave=True)
error_syncer.mark_node_failed()
error_syncer.leave_cluster()
error_syncer.start_thread()
# 10.0.0.2 will be removed from the cluster, and the cluster will
# stabilise
self.wait_for_all_normal(mock_client, required_number=2, tries=50)
end = json.loads(mock_client.read("/test").value)
self.assertEqual("normal", end.get("10.0.0.1"))
self.assertEqual("normal", end.get("10.0.0.3"))
self.assertEqual(None, end.get("10.0.0.2"))
for s in [sync1, sync3, error_syncer]:
s.terminate()
def test_failure(self):
# Create synchronisers, using a FailPlugin for one which will crash and
# not complete (simulating a failed node)
sync1 = EtcdSynchronizer(DummyPlugin(None), '10.0.0.1')
sync2 = EtcdSynchronizer(FailPlugin(None), '10.0.0.2')
sync3 = EtcdSynchronizer(DummyPlugin(None), '10.0.0.3')
mock_client = sync1._client
for s in [sync1, sync2, sync3]:
s.start_thread()
# After a few seconds, the scale-up will still not have completed
sleep(3)
end = json.loads(mock_client.read("/test").value)
self.assertNotEqual("normal", end.get("10.0.0.1"))
self.assertNotEqual("normal", end.get("10.0.0.2"))
self.assertNotEqual("normal", end.get("10.0.0.3"))
# Start a synchroniser to take 10.0.0.2's place
sync2.terminate()
error_syncer = EtcdSynchronizer(NullPlugin('/test'),
'10.0.0.2',
force_leave=True)
error_syncer.mark_node_failed()
error_syncer.leave_cluster()
error_syncer.start_thread()
# 10.0.0.2 will be removed from the cluster, and the cluster will
# stabilise
self.wait_for_all_normal(mock_client, required_number=2, tries=50)
end = json.loads(mock_client.read("/test").value)
self.assertEqual("normal", end.get("10.0.0.1"))
self.assertEqual("normal", end.get("10.0.0.3"))
self.assertEqual(None, end.get("10.0.0.2"))
for s in [sync1, sync3, error_syncer]:
s.terminate()