def __wait_for_containers(self, condition, timeout=TIMEOUT): """ Wait for the agent's '/containers' endpoint to return data subject to 'condition'. """ try: data = http.get_json(self.flags["master"], "slaves") except Exception as exception: raise CLIException("Could not get '/slaves' endpoint" " as JSON: {error}" .format(error=exception)) if len(data["slaves"]) != 1: raise CLIException("More than one agent detected when" " reading from '/slaves' endpoint") try: agent = parse.parse( "slave({id})@{addr}", data["slaves"][0]["pid"]) except Exception as exception: raise CLIException("Unable to parse agent info: {error}" .format(error=exception)) try: data = http.get_json( agent["addr"], "containers", condition, timeout) except Exception as exception: raise CLIException("Could not get '/containers' endpoint as" " JSON subject to condition: {error}" .format(error=exception))
def list(self, argv): """ List the agents in a cluster by checking the /slaves endpoint. """ # pylint: disable=unused-argument try: master = self.config.master() except Exception as exception: raise CLIException("Unable to get leading master address: {error}" .format(error=exception)) try: agents = http.get_json(master, "slaves")["slaves"] except Exception as exception: raise CLIException("Could not open '/slaves'" " endpoint at '{addr}': {error}" .format(addr=master, error=exception)) if not agents: print("The cluster does not have any agents.") return try: table = Table(["Agent ID", "Hostname", "Active"]) for agent in agents: table.add_row([agent["id"], agent["hostname"], str(agent["active"])]) except Exception as exception: raise CLIException("Unable to build table of agents: {error}" .format(error=exception)) print(str(table))
def get_tasks(master, config, query=None): """ Get the tasks in a Mesos cluster. """ endpoint = "tasks" key = "tasks" if query is None: query = {'order':'asc'} try: data = http.get_json(master, endpoint, config, query=query) except Exception as exception: raise CLIException( "Could not open '/{endpoint}' with query parameters: {query}" "on master: {error}" .format(endpoint=endpoint, query=query, error=exception)) if not key in data: raise CLIException( "Missing '{key}' key in data retrieved" " from master on '/{endpoint}' with query parameters: {query}" .format(key=key, endpoint=endpoint, query=query)) return data[key]
def list(self, argv): """ List the agents in a cluster by checking the /slaves endpoint. """ # pylint: disable=unused-argument try: master = self.config.master() except Exception as exception: raise CLIException( "Unable to get leading master address: {error}".format( error=exception)) try: agents = http.get_json(master, "slaves")["slaves"] except Exception as exception: raise CLIException("Could not open '/slaves'" " endpoint at '{addr}': {error}".format( addr=master, error=exception)) if not agents: print("The cluster does not have any agents.") return try: table = Table(["Agent ID", "Hostname", "Active"]) for agent in agents: table.add_row( [agent["id"], agent["hostname"], str(agent["active"])]) except Exception as exception: raise CLIException( "Unable to build table of agents: {error}".format( error=exception)) print(str(table))
def _wait_for_task(): tasks = http.get_json(master.addr, "tasks", config.Config(None))["tasks"] for task in tasks: if task["name"] == name and task["state"] == state: return task raise Exception()
def list(self, argv): """ List the tasks running in a cluster by checking the /tasks endpoint. """ # pylint: disable=unused-argument try: master = self.config.master() except Exception as exception: raise CLIException( "Unable to get leading master address: {error}".format( error=exception)) try: tasks = http.get_json(master, "tasks")["tasks"] except Exception as exception: raise CLIException("Could not open '/tasks'" " endpoint at '{addr}': {error}".format( addr=master, error=exception)) if not tasks: print("There are no tasks running in the cluster.") return try: table = Table(["Task ID", "Framework ID", "Executor ID"]) for task in tasks: table.add_row( [task["id"], task["framework_id"], task["executor_id"]]) except Exception as exception: raise CLIException( "Unable to build table of tasks: {error}".format( error=exception)) print(str(table))
def list(self, argv): """ List the tasks running in a cluster by checking the /tasks endpoint. """ # pylint: disable=unused-argument try: master = self.config.master() except Exception as exception: raise CLIException("Unable to get leading master address: {error}" .format(error=exception)) try: tasks = http.get_json(master, "tasks")["tasks"] except Exception as exception: raise CLIException("Could not open '/tasks'" " endpoint at '{addr}': {error}" .format(addr=master, error=exception)) if len(tasks) == 0: print "There are no tasks running in the cluster." return try: table = Table(["Task ID", "Framework ID", "Executor ID"]) for task in tasks: table.add_row([task["id"], task["framework_id"], task["executor_id"]]) except Exception as exception: raise CLIException("Unable to build table of tasks: {error}" .format(error=exception)) print str(table)
def updated_tasks(): # Open the master's `/tasks` endpoint and # read the task information ourselves. tasks = http.get_json(master.addr, "tasks")["tasks"] if tasks[0]["state"] == "TASK_RUNNING": return tasks raise CLIException("Unable to find running task in master state" " '{master}'".format(master=master))
def test_list(self): """ Basic test for the task `list()` sub-command. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() task = Task({"command": "sleep 1000"}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException("Error waiting for task '{name}' to" " reach state '{state}': {error}".format( name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}". format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) # Invoke the task plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = TaskPlugin(None, test_config) output = capture_output(plugin.list, {"--all": False}) table = Table.parse(output) # Verify there are two rows in the table # and that they are formatted as expected, # with the proper task info in them. self.assertEqual(table.dimensions()[0], 2) self.assertEqual(table.dimensions()[1], 4) self.assertEqual("ID", table[0][0]) self.assertEqual("State", table[0][1]) self.assertEqual("Framework ID", table[0][2]) self.assertEqual("Executor ID", table[0][3]) self.assertEqual(tasks[0]["id"], table[1][0]) self.assertEqual(tasks[0]["statuses"][-1]["state"], table[1][1]) self.assertEqual(tasks[0]["framework_id"], table[1][2]) self.assertEqual(tasks[0]["executor_id"], table[1][3]) # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def test_list(self): """ Basic test for the task `list()` sub-command. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() task = Task({"command": "sleep 1000"}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}" .format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) # Invoke the task plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = TaskPlugin(None, test_config) output = capture_output(plugin.list, {"--all": False}) table = Table.parse(output) # Verify there are two rows in the table # and that they are formatted as expected, # with the proper task info in them. self.assertEqual(table.dimensions()[0], 2) self.assertEqual(table.dimensions()[1], 4) self.assertEqual("ID", table[0][0]) self.assertEqual("State", table[0][1]) self.assertEqual("Framework ID", table[0][2]) self.assertEqual("Executor ID", table[0][3]) self.assertEqual(tasks[0]["id"], table[1][0]) self.assertEqual(tasks[0]["statuses"][-1]["state"], table[1][1]) self.assertEqual(tasks[0]["framework_id"], table[1][2]) self.assertEqual(tasks[0]["executor_id"], table[1][3]) # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def kill(self, timeout=TIMEOUT): """ After killing the agent, we need to make sure it has successfully unregistered from the master before proceeding. """ super(Agent, self).kill() if self.proc is None: return try: # pylint: disable=missing-docstring def no_slaves(data): return len(data["slaves"]) == 0 http.get_json(self.flags["master"], "slaves", no_slaves, timeout) except Exception as exception: raise CLIException("Could not get '/slaves' endpoint as" " JSON with 0 agents in it: {error}" .format(error=exception))
def kill(self, timeout=TIMEOUT): """ After killing the agent, we need to make sure it has successfully unregistered from the master before proceeding. """ super(Agent, self).kill() try: # pylint: disable=missing-docstring def one_inactive_slave(data): slaves = data["slaves"] return len(slaves) == 1 and not slaves[0]["active"] http.get_json(self.flags["master"], "slaves", one_inactive_slave, timeout) except Exception as exception: raise CLIException( "Could not get '/slaves' endpoint as" " JSON with 0 agents in it: {error}".format(error=exception)) Agent.count -= 1
def kill(self, timeout=TIMEOUT): """ After killing the agent, we need to make sure it has successfully unregistered from the master before proceeding. """ super(Agent, self).kill() try: # pylint: disable=missing-docstring def one_inactive_slave(data): slaves = data["slaves"] return len(slaves) == 1 and not slaves[0]["active"] http.get_json( self.flags["master"], "slaves", one_inactive_slave, timeout) except Exception as exception: raise CLIException("Could not get '/slaves' endpoint as" " JSON with 0 agents in it: {error}" .format(error=exception)) Agent.count -= 1
def launch(self, timeout=TIMEOUT): """ After starting the agent, we need to make sure it has successfully registered with the master before proceeding. """ super(Agent, self).launch() try: # pylint: disable=missing-docstring def single_slave(data): return len(data["slaves"]) == 1 http.get_json(self.flags["master"], "slaves", single_slave, timeout) except Exception as exception: stdout = "" if self.proc.poll(): stdout = "\n{output}".format(output=self.proc.stdout.read()) self.proc = None raise CLIException("Could not get '/slaves' endpoint as JSON with" " only 1 agent in it: {error}{stdout}" .format(error=exception, stdout=stdout))
def get_agent_address(agent_id, master): """ Given a master and an agent id, return the agent address by checking the /slaves endpoint of the master. """ try: agents = http.get_json(master, "slaves")["slaves"] except Exception as exception: raise CLIException("Could not open '/slaves'" " endpoint at '{addr}': {error}".format( addr=master, error=exception)) for agent in agents: if agent["id"] == agent_id: return agent["pid"].split("@")[1] raise CLIException("Unable to find agent '{id}'".format(id=agent_id))
def get_agent_address(agent_id, master): """ Given a master and an agent id, return the agent address by checking the /slaves endpoint of the master. """ try: agents = http.get_json(master, "slaves")["slaves"] except Exception as exception: raise CLIException("Could not open '/slaves'" " endpoint at '{addr}': {error}" .format(addr=master, error=exception)) for agent in agents: if agent["id"] == agent_id: return agent["pid"].split("@")[1] raise CLIException("Unable to find agent '{id}'".format(id=agent_id))
def test_exec(self): """ Basic test for the task `exec()` sub-command. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() with open(LOREM_IPSUM) as text: content = text.read() command = "printf '{data}' > a.txt && sleep 1000".format(data=content) task = Task({"command": command}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException("Error waiting for task '{name}' to" " reach state '{state}': {error}".format( name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}". format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) returncode, stdout, stderr = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "cat", "a.txt"]) self.assertEqual(returncode, 0) self.assertEqual(stdout, content) self.assertEqual(stderr, "") # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def test_exec(self): """ Basic test for the task `exec()` sub-command. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() with open(LOREM_IPSUM) as text: content = text.read() command = "printf '{data}' > a.txt && sleep 1000".format(data=content) task = Task({"command": command}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}" .format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) returncode, stdout, stderr = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "cat", "a.txt"]) self.assertEqual(returncode, 0) self.assertEqual(stdout, content) self.assertEqual(stderr, "") # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def get_key_endpoint(key, endpoint, master, config): """ Get the json key of the given endpoint """ try: data = http.get_json(master, endpoint, config) except Exception as exception: raise CLIException( "Could not open '/{endpoint}' on master: {error}".format( endpoint=endpoint, error=exception)) if not key in data: raise CLIException("Missing '{key}' key in data retrieved" " from master on '/{endpoint}'".format( key=key, endpoint=endpoint)) return data[key]
def test_exec_exit_status(self): """ Basic test for the task `exec()` sub-command exit status. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() task = Task({"command": "sleep 1000"}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException("Error waiting for task '{name}' to" " reach state '{state}': {error}".format( name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}". format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) returncode, _, _ = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "true"]) self.assertEqual(returncode, 0) returncode, _, _ = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "bash", "-c", "exit 10"]) self.assertEqual(returncode, 10) # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def launch(self): """ After starting the agent, we first need to make sure its reference count is increased and then check that it has successfully registered with the master before proceeding. """ super(Agent, self).launch() Agent.count += 1 data = http.get_json(self.flags["master"], "slaves", self.config) if len(data["slaves"]) == 1: stdout = "" if self.proc.poll(): stdout = "\n{output}".format(output=self.proc.stdout.read()) raise CLIException("Could not get '/slaves' endpoint as JSON with" " only 1 agent in it: {stdout}" .format(stdout=stdout))
def kill(self): """ After killing the agent, we need to make sure it has successfully unregistered from the master before proceeding. """ super(Agent, self).kill() data = http.get_json(self.flags["master"], "slaves", self.config) if len(data["slaves"]) == 1 and not data["slaves"][0]["active"]: stdout = "" if self.proc.poll(): stdout = "\n{output}".format(output=self.proc.stdout.read()) raise CLIException("Could not get '/slaves' endpoint as" " JSON with 0 agents in it: {stdout}" .format(stdout=stdout)) Agent.count -= 1
def test_list(self): """ Basic test for the task `list()` sub-command. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() task = Task({"command": "sleep 1000"}) task.launch() # Open the master's `/tasks` endpoint and read the # task information ourselves. tasks = http.get_json(master.addr, 'tasks')["tasks"] self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) # Invoke the task plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = TaskPlugin(None, test_config) output = capture_output(plugin.list, {}) table = Table.parse(output) # Verify there are two rows in the table # and that they are formatted as expected, # with the proper task info in them. self.assertEqual(table.dimensions()[0], 2) self.assertEqual(table.dimensions()[1], 3) self.assertEqual("Task ID", table[0][0]) self.assertEqual("Framework ID", table[0][1]) self.assertEqual("Executor ID", table[0][2]) self.assertEqual(tasks[0]["id"], table[1][0]) self.assertEqual(tasks[0]["framework_id"], table[1][1]) self.assertEqual(tasks[0]["executor_id"], table[1][2]) # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def test_exec_exit_status(self): """ Basic test for the task `exec()` sub-command exit status. """ # Launch a master, agent, and task. master = Master() master.launch() agent = Agent() agent.launch() task = Task({"command": "sleep 1000"}) task.launch() try: wait_for_task(master, task.name, "TASK_RUNNING") except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task.name, state="TASK_RUNNING", error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}" .format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 1) returncode, _, _ = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "true"]) self.assertEqual(returncode, 0) returncode, _, _ = exec_command( ["mesos", "task", "exec", tasks[0]["id"], "bash", "-c", "exit 10"]) self.assertEqual(returncode, 10) # Kill the task, agent, and master. task.kill() agent.kill() master.kill()
def get_agents(master, config): """ Get the agents in a Mesos cluster. """ endpoint = "slaves" key = "slaves" try: data = http.get_json(master, endpoint, config) except Exception as exception: raise CLIException( "Could not open '/{endpoint}' on master: {error}" .format(endpoint=endpoint, error=exception)) if not key in data: raise CLIException( "Missing '{key}' key in data retrieved" " from master on '/{endpoint}'" .format(key=key, endpoint=endpoint)) return data[key]
def test_list(self): """ Basic test for the agent `list()` sub-command. """ # Launch a master and agent. master = Master() master.launch() agent = Agent() agent.launch() # Open the master's `/slaves` endpoint and read the # agents' information ourselves. agents = http.get_json(master.addr, 'slaves', config.Config(None))["slaves"] self.assertEqual(type(agents), list) self.assertEqual(len(agents), 1) # Invoke the agent plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = AgentPlugin(None, test_config) output = capture_output(plugin.list, {}) table = Table.parse(output) # Verify there are two rows in the table # and that they are formatted as expected, # with the proper agent info in them. self.assertEqual(table.dimensions()[0], 2) self.assertEqual(table.dimensions()[1], 3) self.assertEqual("Agent ID", table[0][0]) self.assertEqual("Hostname", table[0][1]) self.assertEqual("Active", table[0][2]) self.assertEqual(agents[0]["id"], table[1][0]) self.assertEqual(agents[0]["hostname"], table[1][1]) self.assertEqual(str(agents[0]["active"]), table[1][2]) # Kill the agent and master. agent.kill() master.kill()
def get_agents(master): """ Get the agents in a Mesos cluster. """ endpoint = "slaves" key = "slaves" try: data = http.get_json(master, endpoint) except Exception as exception: raise CLIException( "Could not open '/{endpoint}' on master: {error}" .format(endpoint=endpoint, error=exception)) if not key in data: raise CLIException( "Missing '{key}' key in data retrieved" " from master on '/{endpoint}'" .format(key=key, endpoint=endpoint)) return data[key]
def test_list(self): """ Basic test for the agent `list()` sub-command. """ # Launch a master and agent. master = Master() master.launch() agent = Agent() agent.launch() # Open the master's `/slaves` endpoint and read the # agents' information ourselves. agents = http.get_json(master.addr, 'slaves')["slaves"] self.assertEqual(type(agents), list) self.assertEqual(len(agents), 1) # Invoke the agent plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = AgentPlugin(None, test_config) output = capture_output(plugin.list, {}) table = Table.parse(output) # Verify there are two rows in the table # and that they are formatted as expected, # with the proper agent info in them. self.assertEqual(table.dimensions()[0], 2) self.assertEqual(table.dimensions()[1], 3) self.assertEqual("Agent ID", table[0][0]) self.assertEqual("Hostname", table[0][1]) self.assertEqual("Active", table[0][2]) self.assertEqual(agents[0]["id"], table[1][0]) self.assertEqual(agents[0]["hostname"], table[1][1]) self.assertEqual(str(agents[0]["active"]), table[1][2]) # Kill the agent and master. agent.kill() master.kill()
def test_list_all(self): """ Basic test for the task `list()` sub-command with flag `--all`. """ # Launch a master, agent, and two tasks. master = Master() master.launch() agent = Agent() agent.launch() task1 = Task({"command": "true"}) task1.launch() task1_state = "TASK_FINISHED" try: wait_for_task(master, task1.name, task1_state) except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task1.name, state=task1_state, error=exception)) task2 = Task({"command": "sleep 1000"}) task2.launch() task2_state = "TASK_RUNNING" try: wait_for_task(master, task2.name, task2_state) except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task2.name, state=task2_state, error=exception)) try: tasks = http.get_json(master.addr, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}" .format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 2) # Invoke the task plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = TaskPlugin(None, test_config) output = capture_output(plugin.list, {"--all": True}) table = Table.parse(output) # Verify that there are two rows in the table, one for the running task # and one for the finished task. We do verify the information in the # table as this is already covered in the test `test_list`. self.assertEqual(table.dimensions()[0], 3) self.assertEqual(table.dimensions()[1], 4) # Kill the task1, task2, agent, and master. task1.kill() task2.kill() agent.kill() master.kill()
def test_list_all(self): """ Basic test for the task `list()` sub-command with flag `--all`. """ # Launch a master, agent, and two tasks. master = Master() master.launch() agent = Agent() agent.launch() task1 = Task({"command": "true"}) task1.launch() task1_state = "TASK_FINISHED" try: wait_for_task(master, task1.name, task1_state) except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task1.name, state=task1_state, error=exception)) task2 = Task({"command": "sleep 1000"}) task2.launch() task2_state = "TASK_RUNNING" try: wait_for_task(master, task2.name, task2_state) except Exception as exception: raise CLIException( "Error waiting for task '{name}' to" " reach state '{state}': {error}" .format(name=task2.name, state=task2_state, error=exception)) try: tasks = http.get_json(master.addr, None, "tasks")["tasks"] except Exception as exception: raise CLIException( "Could not get tasks from '/{endpoint}' on master: {error}" .format(endpoint="tasks", error=exception)) self.assertEqual(type(tasks), list) self.assertEqual(len(tasks), 2) # Invoke the task plugin `list()` command # and parse its output as a table. test_config = config.Config(None) plugin = TaskPlugin(None, test_config) output = capture_output(plugin.list, {"--all": True}) table = Table.parse(output) # Verify that there are two rows in the table, one for the running task # and one for the finished task. We do verify the information in the # table as this is already covered in the test `test_list`. self.assertEqual(table.dimensions()[0], 3) self.assertEqual(table.dimensions()[1], 4) # Kill the task1, task2, agent, and master. task1.kill() task2.kill() agent.kill() master.kill()
def _wait_for_task(): tasks = http.get_json(master.addr, "tasks")["tasks"] for task in tasks: if task["name"] == name and task["state"] == state: return task raise Exception()
def _running_tasks(): tasks = http.get_json(master.addr, "tasks")["tasks"] if tasks[0]["state"] == "TASK_RUNNING": return tasks raise Exception()