def test_launcher_recovery_corrupted_password(self): # 1. Launch a single instance for a cluster on the running launcher. task_id, remaining = self._launcher.launch(self._offer) del self._offer.resources[:] self._offer.resources.extend(remaining) assert task_id == "mysos-cluster0-0" # The task has successfully started. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING status.slave_id.value = self._offer.slave_id.value status.task_id.value = "mysos-cluster0-0" self._launcher.status_update(status) # 2. Recover the launcher. self._cluster = self._state_provider.load_cluster_state(self._cluster.name) self._cluster.encrypted_password = "******" # The corrupted password causes the launcher constructor to fail. with pytest.raises(ValueError): self._launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", self._scheduler_key, query_interval=Amount(150, Time.MILLISECONDS))
def test_invalid_status_update(self): """Launcher raises an exception when an invalid status is received.""" self._cluster.num_nodes = 1 launcher = MySQLClusterLauncher(self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml") self._launchers.append(launcher) resources = create_resources(cpus=4, mem=512 * 3, ports=set([10000])) self._offer.resources.extend(resources) task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-0" tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == self._cluster.num_nodes status = mesos_pb2.TaskStatus() status.task_id.value = task_id status.state = mesos_pb2.TASK_RUNNING # Valid state. launcher.status_update(status) status.state = mesos_pb2.TASK_FINISHED # An invalid state. with pytest.raises(MySQLClusterLauncher.Error): launcher.status_update(status)
def setup(self, request): self._driver = FakeDriver() self._storage = FakeStorage(SequentialThreadingHandler()) self._zk_client = FakeClient(storage=self._storage) self._zk_client.start() self._offer = mesos_pb2.Offer() self._offer.id.value = "offer_id_0" self._offer.framework_id.value = "framework_id_0" self._offer.slave_id.value = "slave_id_0" self._offer.hostname = "localhost" # Enough memory and ports to fit three tasks. resources = create_resources(cpus=4, mem=512 * 3, ports=set([10000, 10001, 10002])) self._offer.resources.extend(resources) self._framework_user = "******" # Some tests use the default launcher; some don't. self._zk_url = "zk://host/mysos/test" self._cluster = MySQLCluster("cluster0", "user", "pass", 3) # Construct the state provider based on the test parameter. if request.param == LocalStateProvider: tmpdir = tempfile.mkdtemp() self._state_provider = LocalStateProvider(tmpdir) request.addfinalizer(lambda: shutil.rmtree(tmpdir, True) ) # Clean up after ourselves. elif request.param == ZooKeeperStateProvider: self._state_provider = ZooKeeperStateProvider( self._zk_client, "/mysos/test") self._launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", query_interval=Amount(150, Time.MILLISECONDS)) # Short interval. self._elected = threading.Event() self._launchers = [self._launcher] # See teardown(). request.addfinalizer(self.teardown)
def test_two_launchers(self): """Two launchers share resources and launch their clusters successfully.""" launchers = [ MySQLClusterLauncher(self._driver, MySQLCluster("cluster0", "user0", "pass0", 1), self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml"), MySQLClusterLauncher(self._driver, MySQLCluster("cluster1", "user1", "pass1", 2), self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml") ] self._launchers.extend(launchers) resources = create_resources(cpus=4, mem=512 * 3, ports=set([10000, 10001, 10002])) self._offer.resources.extend(resources) # Three nodes in total across two clusters. # Simulate the scheduler. for i in range(3): for launcher in launchers: task_id, remaining = launcher.launch(self._offer) if task_id: # Update the offer so other launchers will use its remaining resources. del self._offer.resources[:] self._offer.resources.extend(remaining) break tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == 3
def test_terminal_status_update(self): """Launcher reacts to terminated task by launching a new one.""" self._cluster.num_nodes = 1 launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(1, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", self._scheduler_key) self._launchers.append(launcher) resources = create_resources( cpus=DEFAULT_TASK_CPUS, mem=DEFAULT_TASK_MEM, disk=DEFAULT_TASK_DISK, ports=set([10000])) self._offer.resources.extend(resources) task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-0" launched = self._driver.method_calls["launchTasks"] assert len(launched) == self._cluster.num_nodes status = mesos_pb2.TaskStatus() status.task_id.value = task_id status.state = mesos_pb2.TASK_RUNNING launcher.status_update(status) assert len(launcher._cluster.running_tasks) == 1 status.state = mesos_pb2.TASK_LOST launcher.status_update(status) assert len(launcher._cluster.running_tasks) == 0 task_id, _ = launcher.launch(self._offer) assert task_id == "mysos-cluster0-1" launched = self._driver.method_calls["launchTasks"] # One task is relaunched to make up for the lost one. assert len(launched) == self._cluster.num_nodes + 1
def test_launcher_recovery_before_election_completed(self): # 1. Launch a cluster on the running launcher. for i in range(self._cluster.num_nodes): task_id, remaining = self._launcher.launch(self._offer) del self._offer.resources[:] self._offer.resources.extend(remaining) assert task_id == "mysos-cluster0-%s" % i tasks = self._driver.method_calls["launchTasks"] assert len(tasks) == self._cluster.num_nodes # No new tasks are launched. assert self._launcher.launch(self._offer)[0] is None assert len(self._driver.method_calls["launchTasks"]) == self._cluster.num_nodes # All 3 nodes have successfully started. status = mesos_pb2.TaskStatus() status.state = mesos_pb2.TASK_RUNNING status.slave_id.value = self._offer.slave_id.value for i in range(self._cluster.num_nodes): status.task_id.value = "mysos-cluster0-%s" % i self._launcher.status_update(status) deadline( lambda: wait_for_master( get_cluster_path(self._zk_url, self._cluster.name), self._zk_client), Amount(5, Time.SECONDS)) # The first slave is elected. assert "/mysos/test/cluster0/master/member_0000000000" in self._storage.paths # Two slaves. assert len([x for x in self._storage.paths.keys() if x.startswith( "/mysos/test/cluster0/slaves/member_")]) == 2 # Now fail the master task which leads to re-election. status.task_id.value = "mysos-cluster0-0" status.state = mesos_pb2.TASK_FAILED self._launcher.status_update(status) # 2. Recover the launcher. self._cluster = self._state_provider.load_cluster_state(self._cluster.name) self._launcher = MySQLClusterLauncher( self._driver, self._cluster, self._state_provider, self._zk_url, self._zk_client, self._framework_user, "./executor.pex", "cmd.sh", Amount(5, Time.SECONDS), "/etc/mysos/admin_keyfile.yml", self._scheduler_key, query_interval=Amount(150, Time.MILLISECONDS)) for i in range(1, self._cluster.num_nodes): self._launcher.framework_message( "mysos-cluster0-%s" % i, self._offer.slave_id.value, json.dumps(dict(epoch=2, position=str(i)))) deadline( lambda: wait_for_master( get_cluster_path(self._zk_url, self._cluster.name), self._zk_client), Amount(5, Time.SECONDS)) # The second slave has the larger position and is elected. assert "/mysos/test/cluster0/master/member_0000000002" in self._storage.paths