class TestZooKeeperStateProvider(unittest.TestCase):
def setUp(self):
self._storage = FakeStorage(SequentialThreadingHandler())
self._client = FakeClient(storage=self._storage)
self._client.start()
self._state_provider = ZooKeeperStateProvider(self._client, '/mysos')
def tearDown(self):
self._client.stop()
def test_scheduler_state(self):
expected = Scheduler(FrameworkInfo(
user='******',
name='test_fw_name',
checkpoint=True))
expected.tasks = dict(taks1='cluster1', task2='cluster2')
self._state_provider.dump_scheduler_state(expected)
actual = self._state_provider.load_scheduler_state()
assert expected.framework_info == actual.framework_info
assert expected.tasks == actual.tasks
def test_scheduler_state_errors(self):
assert not self._state_provider.load_scheduler_state() # Not an error for scheduler state to be
# not found.
self._client.ensure_path("/mysos/state")
self._client.create("/mysos/state/scheduler", cPickle.dumps(object()))
with pytest.raises(StateProvider.Error):
self._state_provider.load_scheduler_state()
def test_cluster_state(self):
expected = MySQLCluster('cluster1', 'cluster_user', 'cluster_password', 3)
expected.tasks['task1'] = MySQLTask(
'cluster1', 'task1', 'slave1', 'host1', 10000)
self._state_provider.dump_cluster_state(expected)
actual = self._state_provider.load_cluster_state('cluster1')
assert expected.user == actual.user
assert isinstance(actual.num_nodes, int)
assert expected.num_nodes == actual.num_nodes
assert len(expected.tasks) == len(actual.tasks)
assert expected.tasks['task1'].port == actual.tasks['task1'].port
def test_cluster_state_errors(self):
assert not self._state_provider.load_cluster_state('nonexistent')
self._client.ensure_path("/mysos/state/clusters")
self._client.create("/mysos/state/clusters/cluster1", cPickle.dumps(object()))
with pytest.raises(StateProvider.Error):
self._state_provider.load_cluster_state('cluster1')
class TestCluster(unittest.TestCase):
def setUp(self):
self.storage = FakeStorage(SequentialThreadingHandler())
self.client = FakeClient(storage=self.storage)
self.client.start()
def tearDown(self):
self.client.stop()
def test_add_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
assert member1 == manager.add_member(instance1) # Second insertion is ignored.
instance2 = ServiceInstance(Endpoint("host2", 10000))
manager.add_member(instance2)
assert len(manager._cluster.members) == 2
assert self.storage.paths["/home/my_cluster/slaves/member_0000000000"]["data"] == ServiceInstance.pack(
instance1
)
assert self.storage.paths["/home/my_cluster/slaves/member_0000000001"]["data"] == ServiceInstance.pack(
instance2
)
def test_promote_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.promote_member(member)
assert not manager.promote_member(member) # The 2nd promotion is a no-op.
assert self.storage.paths["/home/my_cluster/master/member_0000000000"]["data"] == ServiceInstance.pack(instance)
def test_remove_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.remove_member(member)
assert not manager.remove_member(member) # The second deletion is ignored.
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
def test_callbacks(self):
manager = ClusterManager(self.client, "/home/my_cluster")
# Set up 2 listeners.
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(
self.client,
"/home/my_cluster",
instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback,
handler1.termination_callback,
)
listener1.start()
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
handler2 = CallbackHandler()
listener2 = ClusterListener(
self.client,
"/home/my_cluster",
instance2,
handler2.promotion_callback,
handler2.demotion_callback,
handler2.master_callback,
)
listener2.start()
member2 = manager.add_member(instance2)
# Test promotion.
manager.promote_member(member1)
assert handler1.promoted.wait(1)
assert handler2.detected.get(True, 1) == instance1
assert self.storage.paths["/home/my_cluster/master/member_0000000000"]["data"] == ServiceInstance.pack(
instance1
)
assert self.storage.paths["/home/my_cluster/slaves/member_0000000001"]["data"] == ServiceInstance.pack(
instance2
)
manager.promote_member(member2)
assert handler1.demoted.wait(1)
assert handler2.promoted.wait(1)
assert self.storage.paths["/home/my_cluster/master/member_0000000001"]["data"] == ServiceInstance.pack(
instance2
)
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
manager.remove_member(member2)
assert handler2.demoted.wait(1)
# Test removing cluster.
manager.remove_member(member1)
manager.delete_cluster()
assert handler1.terminated.wait(1)
def test_invalid_arguments(self):
client = FakeClient()
client.start()
manager = ClusterManager(client, "/home/my_cluster")
with pytest.raises(ValueError) as e:
manager.promote_member("123")
assert e.value.message == "Invalid member_id: 123"
def test_invalid_znode(self):
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(
self.client,
"/home/my_cluster",
instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback,
)
listener1.start()
self.client.ensure_path("/home/my_cluster/master")
self.client.create("/home/my_cluster/master/member_", "Invalid Data", sequence=True)
# Invalid ZNode data translates into a 'None' return.
assert handler1.detected.get(True, 1) is None
def test_existing_zk(self):
"""
ClusterManager needs to be able to recover from an existing ZK group for scheduler failover.
"""
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
assert self.storage.paths["/home/my_cluster/slaves/member_0000000000"]["data"] == ServiceInstance.pack(
instance1
)
assert self.storage.paths["/home/my_cluster/slaves/member_0000000001"]["data"] == ServiceInstance.pack(
instance2
)
manager.promote_member(member1)
# Test the new ClusterManager.
manager2 = ClusterManager(self.client, "/home/my_cluster")
assert len(manager2._cluster.members) == 2
assert member1 in manager2._cluster.members
assert member2 in manager2._cluster.members
assert manager2._cluster.members[member1] == ServiceInstance.pack(instance1)
def test_remove_cluster(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
manager.promote_member(member1)
with pytest.raises(ClusterManager.Error):
manager.delete_cluster()
manager.remove_member(member1)
manager.remove_member(member2)
manager.delete_cluster()
assert "/home/my_cluster" not in self.storage.paths
class TestCluster(unittest.TestCase):
def setUp(self):
self.storage = FakeStorage(SequentialThreadingHandler())
self.client = FakeClient(storage=self.storage)
self.client.start()
def tearDown(self):
self.client.stop()
def test_add_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
assert member1 == manager.add_member(
instance1) # Second insertion is ignored.
instance2 = ServiceInstance(Endpoint("host2", 10000))
manager.add_member(instance2)
assert len(manager._cluster.members) == 2
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
def test_promote_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.promote_member(member)
assert not manager.promote_member(
member) # The 2nd promotion is a no-op.
assert (self.storage.paths["/home/my_cluster/master/member_0000000000"]
["data"] == ServiceInstance.pack(instance))
def test_remove_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.remove_member(member)
assert not manager.remove_member(
member) # The second deletion is ignored.
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
def test_callbacks(self):
manager = ClusterManager(self.client, "/home/my_cluster")
# Set up 2 listeners.
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(self.client, "/home/my_cluster", instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback,
handler1.termination_callback)
listener1.start()
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
handler2 = CallbackHandler()
listener2 = ClusterListener(self.client, "/home/my_cluster", instance2,
handler2.promotion_callback,
handler2.demotion_callback,
handler2.master_callback)
listener2.start()
member2 = manager.add_member(instance2)
# Test promotion.
manager.promote_member(member1)
assert handler1.promoted.wait(1)
assert handler2.detected.get(True, 1) == instance1
assert (self.storage.paths["/home/my_cluster/master/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
manager.promote_member(member2)
assert handler1.demoted.wait(1)
assert handler2.promoted.wait(1)
assert (self.storage.paths["/home/my_cluster/master/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
manager.remove_member(member2)
assert handler2.demoted.wait(1)
# Test removing cluster.
manager.remove_member(member1)
manager.delete_cluster()
assert handler1.terminated.wait(1)
def test_invalid_arguments(self):
client = FakeClient()
client.start()
manager = ClusterManager(client, "/home/my_cluster")
with pytest.raises(ValueError) as e:
manager.promote_member("123")
assert e.value.message == 'Invalid member_id: 123'
def test_invalid_znode(self):
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(self.client, "/home/my_cluster", instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback)
listener1.start()
self.client.ensure_path("/home/my_cluster/master")
self.client.create("/home/my_cluster/master/member_",
"Invalid Data",
sequence=True)
# Invalid ZNode data translates into a 'None' return.
assert handler1.detected.get(True, 1) is None
def test_existing_zk(self):
"""
ClusterManager needs to be able to recover from an existing ZK group for scheduler failover.
"""
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
manager.promote_member(member1)
# Test the new ClusterManager.
manager2 = ClusterManager(self.client, "/home/my_cluster")
assert len(manager2._cluster.members) == 2
assert member1 in manager2._cluster.members
assert member2 in manager2._cluster.members
assert manager2._cluster.members[member1] == ServiceInstance.pack(
instance1)
def test_remove_cluster(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
manager.promote_member(member1)
with pytest.raises(ClusterManager.Error):
manager.delete_cluster()
manager.remove_member(member1)
manager.remove_member(member2)
manager.delete_cluster()
assert "/home/my_cluster" not in self.storage.paths
