def test_existing_zk(self):
"""
ClusterManager needs to be able to recover from an existing ZK group for scheduler failover.
"""
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
manager.promote_member(member1)
# Test the new ClusterManager.
manager2 = ClusterManager(self.client, "/home/my_cluster")
assert len(manager2._cluster.members) == 2
assert member1 in manager2._cluster.members
assert member2 in manager2._cluster.members
assert manager2._cluster.members[member1] == ServiceInstance.pack(
instance1)
def test_callbacks(self):
manager = ClusterManager(self.client, "/home/my_cluster")
# Set up 2 listeners.
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(self.client, "/home/my_cluster", instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback,
handler1.termination_callback)
listener1.start()
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
handler2 = CallbackHandler()
listener2 = ClusterListener(self.client, "/home/my_cluster", instance2,
handler2.promotion_callback,
handler2.demotion_callback,
handler2.master_callback)
listener2.start()
member2 = manager.add_member(instance2)
# Test promotion.
manager.promote_member(member1)
assert handler1.promoted.wait(1)
assert handler2.detected.get(True, 1) == instance1
assert (self.storage.paths["/home/my_cluster/master/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
manager.promote_member(member2)
assert handler1.demoted.wait(1)
assert handler2.promoted.wait(1)
assert (self.storage.paths["/home/my_cluster/master/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
manager.remove_member(member2)
assert handler2.demoted.wait(1)
# Test removing cluster.
manager.remove_member(member1)
manager.delete_cluster()
assert handler1.terminated.wait(1)
def test_demote(self):
task_control = FakeTaskControl()
runner = MysosTaskRunner(
self._self_instance,
self._client,
"/home/test/my_cluster",
NoopPackageInstaller(),
task_control,
self._state_manager)
manager = ClusterManager(self._client, "/home/test/my_cluster")
runner.start()
self_member = manager.add_member(self._self_instance)
# 'self_instance' becomes the master.
manager.promote_member(self_member)
runner.promoted.wait(1)
another_member = manager.add_member(ServiceInstance(Endpoint("another_host", 10000)))
# This demotes 'self_instance', which should cause runner to stop.
manager.promote_member(another_member)
assert deadline(runner.join, Amount(1, Time.SECONDS))
def test_add_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
assert member1 == manager.add_member(
instance1) # Second insertion is ignored.
instance2 = ServiceInstance(Endpoint("host2", 10000))
manager.add_member(instance2)
assert len(manager._cluster.members) == 2
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000000"]
["data"] == ServiceInstance.pack(instance1))
assert (self.storage.paths["/home/my_cluster/slaves/member_0000000001"]
["data"] == ServiceInstance.pack(instance2))
def test_remove_cluster(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance1 = ServiceInstance(Endpoint("host1", 10000))
member1 = manager.add_member(instance1)
instance2 = ServiceInstance(Endpoint("host2", 10000))
member2 = manager.add_member(instance2)
manager.promote_member(member1)
with pytest.raises(ClusterManager.Error):
manager.delete_cluster()
manager.remove_member(member1)
manager.remove_member(member2)
manager.delete_cluster()
assert "/home/my_cluster" not in self.storage.paths
def test_remove_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.remove_member(member)
assert not manager.remove_member(
member) # The second deletion is ignored.
assert "/home/my_cluster/master/member_0000000000" not in self.storage.paths
def test_promote_member(self):
manager = ClusterManager(self.client, "/home/my_cluster")
instance = ServiceInstance(Endpoint("host", 10000))
member = manager.add_member(instance)
assert manager.promote_member(member)
assert not manager.promote_member(
member) # The 2nd promotion is a no-op.
assert (self.storage.paths["/home/my_cluster/master/member_0000000000"]
["data"] == ServiceInstance.pack(instance))
def from_task(self, task, sandbox):
data = json.loads(task.data)
cluster_name, port, zk_url = data['cluster'], data['port'], data[
'zk_url']
_, servers, path = zookeeper.parse(zk_url)
zk_client = FakeClient()
zk_client.start()
self_instance = ServiceInstance(
Endpoint(socket.gethostbyname(socket.gethostname()), port))
task_control = self._task_control_provider.from_task(task, sandbox)
return MysosTaskRunner(self_instance, zk_client,
posixpath.join(path, cluster_name),
NoopPackageInstaller(), task_control, Fake())
def test_invalid_znode(self):
instance1 = ServiceInstance(Endpoint("host1", 10000))
handler1 = CallbackHandler()
listener1 = ClusterListener(self.client, "/home/my_cluster", instance1,
handler1.promotion_callback,
handler1.demotion_callback,
handler1.master_callback)
listener1.start()
self.client.ensure_path("/home/my_cluster/master")
self.client.create("/home/my_cluster/master/member_",
"Invalid Data",
sequence=True)
# Invalid ZNode data translates into a 'None' return.
assert handler1.detected.get(True, 1) is None
def test_service_instance_to_json():
json = """{
"additionalEndpoints": {
"aurora": {
"host": "hostname",
"inet6": "2001:db8:1234:ffff:ffff:ffff:ffff:ffff",
"port": 22
},
"health": {
"host": "hostname",
"inet": "1.2.3.4",
"port": 23
},
"http": {
"host": "hostname",
"inet": "1.2.3.4",
"inet6": "2001:db8:1234:ffff:ffff:ffff:ffff:ffff",
"port": 23
}
},
"serviceEndpoint": {
"host": "hostname",
"port": 24
},
"shard": 1,
"status": "ALIVE"
}"""
service_instance = ServiceInstance(
Endpoint("hostname", 24), {
"aurora":
Endpoint("hostname", 22, "1.2.3.4"),
"health":
Endpoint("hostname", 23, None,
"2001:db8:1234:ffff:ffff:ffff:ffff:ffff"),
"http":
Endpoint("hostname", 23, "1.2.3.4",
"2001:db8:1234:ffff:ffff:ffff:ffff:ffff"),
}, 'ALIVE', 1)
assert ServiceInstance.unpack(json) == service_instance
assert ServiceInstance.unpack(
ServiceInstance.pack(service_instance)) == service_instance
def from_task(self, task, sandbox):
data = json.loads(task.data)
cluster_name, host, port, zk_url = data['cluster'], data['host'], data[
'port'], data['zk_url']
_, servers, path = parse(zk_url)
kazoo = KazooClient(servers)
kazoo.start()
self_instance = ServiceInstance(Endpoint(host, port))
try:
task_control = self._task_control_provider.from_task(task, sandbox)
installer = self._installer_provider.from_task(task, sandbox)
backup_store = self._backup_store_provider.from_task(task, sandbox)
except (TaskControl.Error, PackageInstaller.Error) as e:
raise TaskError(e.message)
state_manager = StateManager(sandbox, backup_store)
return MysosTaskRunner(self_instance, kazoo,
get_cluster_path(path, cluster_name), installer,
task_control, state_manager)
def test_reparent(self):
task_control = FakeTaskControl()
runner = MysosTaskRunner(
self._self_instance,
self._client,
"/home/test/my_cluster",
NoopPackageInstaller(),
task_control,
self._state_manager)
manager = ClusterManager(self._client, "/home/test/my_cluster")
runner.start()
# Promote another instance.
master = ServiceInstance(Endpoint("another_host", 10000))
another_member = manager.add_member(master)
manager.promote_member(another_member)
assert runner.master.get(True, 1) == master
assert runner.stop()
assert deadline(runner.join, Amount(1, Time.SECONDS))
def status_update(self, status):
"""
Handle the status update for a task of this cluster.
NOTE:
Duplicate status updates may be handled by either the same scheduler instance or a new
instance with the restored state.
"""
with self._lock:
task_id = status.task_id.value
if task_id not in self._cluster.tasks:
log.warn("Ignoring status update for unknown task %s" %
task_id)
return
task = self._cluster.tasks[task_id]
previous_state = task.state
# We don't want to ignore a duplicate update if the previous one was not successfully handled.
# Therefore, we should not checkpoint the status change until we have finished all operations.
if previous_state == status.state:
log.info('Ignoring duplicate status update %s for task %s' %
(mesos_pb2.TaskState.Name(status.state), task_id))
return
if is_terminal(previous_state):
log.info(
'Ignoring status update %s for task %s as it is in terminal state %s'
% (mesos_pb2.TaskState.Name(status.state), task_id,
mesos_pb2.TaskState.Name(previous_state)))
return
log.info('Updating state of task %s of cluster %s from %s to %s' %
(status.task_id.value, self.cluster_name,
mesos_pb2.TaskState.Name(previous_state),
mesos_pb2.TaskState.Name(status.state)))
task.state = status.state
if status.state == mesos_pb2.TASK_RUNNING:
# Register this cluster member.
endpoint = Endpoint(self._cluster.tasks[task_id].hostname,
self._cluster.tasks[task_id].port)
# If the scheduler fails over after ZK is updated but before the state change is
# checkpointed, it will receive the same status update again and try to publish a duplicate
# member to ZK. ClusterManager.add_member() is idempotent and doesn't update ZK in this
# case.
member_id = self._cluster_manager.add_member(
ServiceInstance(endpoint))
log.info('Added %s (member id=%s) to cluster %s' %
(endpoint, member_id, self.cluster_name))
self._cluster.members[task_id] = member_id
# Checkpoint the status update here. It's OK if the elector fails to launch later because
# the new scheduler instance will retry based on the fact that there are running instances
# of the cluster but no master.
self._state_provider.dump_cluster_state(self._cluster)
# If MySQL master is already elected for this cluster don't bother adding it to the elector.
if self._cluster.master_id:
log.info(
"MySQL slave task %s on %s started after a master is already elected for cluster %s"
% (task_id, endpoint.host, self.cluster_name))
return
if not self._elector:
self._elector = self._new_elector()
# Add current slaves.
for t in self._cluster.running_tasks:
self._elector.add_slave(t.task_id, t.mesos_slave_id)
self._elector.start()
else:
self._elector.add_slave(task_id, status.slave_id.value)
elif status.state == mesos_pb2.TASK_FINISHED:
raise self.Error(
"Task %s is in unexpected state %s with message '%s'" %
(status.task_id.value,
mesos_pb2.TaskState.Name(status.state), status.message))
elif is_terminal(status.state):
if status.state == mesos_pb2.TASK_KILLED:
log.info("Task %s was successfully killed" %
status.task_id.value)
else:
log.error(
"Task %s is now in terminal state %s with message '%s'"
% (status.task_id.value,
mesos_pb2.TaskState.Name(
status.state), status.message))
del self._cluster.tasks[task_id]
if task_id in self._cluster.members:
member_id = self._cluster.members[task_id]
del self._cluster.members[task_id]
# If the scheduler fails over after ZK is updated but before its result is persisted, it
# will receive the same status update and try to remove the non-existent member.
# Removing a non-existent member is a no-op for ClusterManager.remove_member().
# Note that if the order is reversed, the scheduler will fail to clean up the orphan ZK
# entry.
self._cluster_manager.remove_member(member_id)
if member_id == self._cluster.master_id:
self._cluster.master_id = None
log.info(
"Master of cluster %s has terminated. Restarting election"
% self.cluster_name)
assert not self._elector, "Election must not be running since there is a current master"
self._elector = self._new_elector()
# Add current slaves after removing the terminated task.
for t in self._cluster.running_tasks:
self._elector.add_slave(t.task_id,
t.mesos_slave_id)
self._elector.start()
else:
# It will be rescheduled next time the launcher is given an offer.
log.info("Slave %s of cluster %s has terminated" %
(task_id, self.cluster_name))
else:
assert previous_state != mesos_pb2.TASK_RUNNING, (
"Task must exist in ClusterManager if it was running")
log.warn("Slave %s of cluster %s failed to start running" %
(task_id, self.cluster_name))
if self.terminated:
log.info("Shutting down launcher for cluster %s" %
self.cluster_name)
self._shutdown()
return
# Finally, checkpoint the status update.
self._state_provider.dump_cluster_state(self._cluster)
log.info(
"Checkpointed the status update for task %s of cluster %s"
% (task_id, self.cluster_name))
def setUp(self):
self._storage = FakeStorage(SequentialThreadingHandler())
self._client = FakeClient(storage=self._storage)
self._client.start()
self._self_instance = ServiceInstance(Endpoint("host", 10000))
self._state_manager = FakeStateManager()