def read_endpoint(addr, endpoint, config, query=None):
"""
Read the specified endpoint and return the results.
"""
try:
addr = cli.util.sanitize_address(addr)
except Exception as exception:
raise CLIException("Unable to sanitize address '{addr}': {error}"
.format(addr=addr, error=str(exception)))
try:
url = "{addr}/{endpoint}".format(addr=addr, endpoint=endpoint)
if query is not None:
url += "?{query}".format(query=urlencode(query))
if config.principal() is not None and config.secret() is not None:
headers = urllib3.make_headers(
basic_auth=config.principal() + ":" + config.secret()
)
else:
headers = None
http = urllib3.PoolManager()
http_response = http.request(
'GET',
url,
headers=headers,
timeout=config.agent_timeout()
)
return http_response.data.decode('utf-8')
except Exception as exception:
raise CLIException("Unable to open url '{url}': {error}"
.format(url=url, error=str(exception)))
def list(self, argv):
"""
List the agents in a cluster by checking the /slaves endpoint.
"""
# pylint: disable=unused-argument
try:
master = self.config.master()
except Exception as exception:
raise CLIException("Unable to get leading master address: {error}"
.format(error=exception))
try:
agents = http.get_json(master, "slaves")["slaves"]
except Exception as exception:
raise CLIException("Could not open '/slaves'"
" endpoint at '{addr}': {error}"
.format(addr=master, error=exception))
if len(agents) == 0:
print "The cluster does not have any agents."
return
try:
table = Table(["Agent ID", "Hostname", "Active"])
for agent in agents:
table.add_row([agent["id"],
agent["hostname"],
str(agent["active"])])
except Exception as exception:
raise CLIException("Unable to build table of agents: {error}"
.format(error=exception))
print str(table)
def verify_address_format(address):
"""
Verify that an address ip and port are correct.
"""
# We use 'basestring' as the type of address because it can be
# 'str' or 'unicode' depending on the source of the address (e.g.
# a config file or a flag). Both types inherit from basestring.
if not isinstance(address, basestring):
raise CLIException("The address must be a string")
address_pattern = re.compile(r'[0-9]+(?:\.[0-9]+){3}:[0-9]+')
if not address_pattern.match(address):
raise CLIException("The address '{address}' does not match"
" the expected format '<ip>:<port>'"
.format(address=address))
colon_pos = address.rfind(':')
ip = address[:colon_pos]
port = int(address[colon_pos+1:])
try:
socket.inet_aton(ip)
except socket.error as err:
raise CLIException("The IP '{ip}' is not valid: {error}"
.format(ip=ip, error=err))
# A correct port number is between these two values.
if port < 0 or port > 65535:
raise CLIException("The port '{port}' is not valid")
def get_json(addr, endpoint, condition=None, timeout=5):
"""
Return the contents of the 'endpoint' at 'addr' as JSON data
subject to the condition specified in 'condition'. If we are
unable to read the data or unable to meet the condition within
'timeout' seconds we throw an error.
"""
start_time = time.time()
while True:
data = None
try:
data = read_endpoint(addr, endpoint)
except Exception as exception:
pass
if data:
try:
data = json.loads(data)
except Exception as exception:
raise CLIException("Could not load JSON from '{data}': {error}"
.format(data=data, error=str(exception)))
if not condition:
return data
if condition(data):
return data
if time.time() - start_time > timeout:
raise CLIException("Failed to get data within {seconds} seconds"
.format(seconds=str(timeout)))
time.sleep(0.1)
def list(self, argv):
"""
List the agents in a cluster by checking the /slaves endpoint.
"""
# pylint: disable=unused-argument
try:
master = self.config.master()
except Exception as exception:
raise CLIException(
"Unable to get leading master address: {error}".format(
error=exception))
try:
agents = get_agents(master)
except Exception as exception:
raise CLIException("Unable to get agents from leading"
" master '{master}': {error}".format(
master=master, error=exception))
if not agents:
print("The cluster does not have any agents.")
return
try:
table = Table(["Agent ID", "Hostname", "Active"])
for agent in agents:
table.add_row(
[agent["id"], agent["hostname"],
str(agent["active"])])
except Exception as exception:
raise CLIException(
"Unable to build table of agents: {error}".format(
error=exception))
print(str(table))
def __wait_for_containers(self, condition, timeout=TIMEOUT):
"""
Wait for the agent's '/containers' endpoint
to return data subject to 'condition'.
"""
try:
data = http.get_json(self.flags["master"], "slaves")
except Exception as exception:
raise CLIException("Could not get '/slaves' endpoint"
" as JSON: {error}".format(error=exception))
if len(data["slaves"]) != 1:
raise CLIException("More than one agent detected when"
" reading from '/slaves' endpoint")
try:
agent = parse.parse("slave({id})@{addr}", data["slaves"][0]["pid"])
except Exception as exception:
raise CLIException(
"Unable to parse agent info: {error}".format(error=exception))
try:
data = http.get_json(agent["addr"], "containers", condition,
timeout)
except Exception as exception:
raise CLIException(
"Could not get '/containers' endpoint as"
" JSON subject to condition: {error}".format(error=exception))
def _output_thread(self):
"""
Reads from the output_queue and writes the data
to the appropriate STDOUT or STDERR.
"""
while True:
# Get a message from the output queue and decode it.
# Then write the data to the appropriate stdout or stderr.
output = self.output_queue.get()
if not output.get('data'):
raise CLIException("Error no 'data' field in output message")
data = output['data']
data = base64.b64decode(data.encode('utf-8'))
if output.get('type') and output['type'] == 'STDOUT':
sys.stdout.buffer.write(data)
sys.stdout.flush()
elif output.get('type') and output['type'] == 'STDERR':
sys.stderr.buffer.write(data)
sys.stderr.flush()
else:
raise CLIException("Unsupported data type in output stream")
self.output_queue.task_done()
def launch(self):
"""
Launch 'self.executable'. We assume it is in the system PATH.
"""
if self.proc is not None:
raise CLIException(
"{name} already launched".format(name=self.name.capitalize()))
try:
flags = [
"--{key}={value}".format(key=key, value=value)
for key, value in self.flags.iteritems()
]
proc = subprocess.Popen([self.executable] + flags,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except Exception as exception:
raise CLIException(
"Failed to launch '{executable}': {error}".format(
executable=self.executable, error=exception))
if proc.poll():
raise CLIException(
"Failed to launch '{executable}': {error}".format(
executable=self.executable, error=proc.stdout.read()))
self.proc = proc
def get_tasks(master, config, query=None):
"""
Get the tasks in a Mesos cluster.
"""
endpoint = "tasks"
key = "tasks"
if query is None:
query = {'order': 'asc', 'limit': '-1'}
try:
data = http.get_json(master, endpoint, config, query=query)
except Exception as exception:
raise CLIException(
"Could not open '/{endpoint}' with query parameters: {query}"
"on master: {error}".format(endpoint=endpoint,
query=query,
error=exception))
if not key in data:
raise CLIException(
"Missing '{key}' key in data retrieved"
" from master on '/{endpoint}' with query parameters: {query}".
format(key=key, endpoint=endpoint, query=query))
return data[key]
def get_container_id(task):
"""
Get the container ID of a task.
"""
if 'statuses' not in task:
raise CLIException("Unable to obtain status information for task")
statuses = task['statuses']
if not statuses:
raise CLIException("No status updates available for task")
# It doesn't matter which status we use to get the `container_id`, if the
# `container_id` has been set for the task, all statuses will contain it.
if not 'container_status' in statuses[0]:
raise CLIException(
"Task status does not contain container information")
container_status = statuses[0]['container_status']
if 'container_id' in container_status:
container_id = container_status['container_id']
if 'value' in container_id:
return container_id
raise CLIException("No container found for the specified task."
" It might still be spinning up."
" Please try again.")
def list(self, argv):
"""
List the tasks running in a cluster by checking the /tasks endpoint.
"""
# pylint: disable=unused-argument
try:
master = self.config.master()
except Exception as exception:
raise CLIException(
"Unable to get leading master address: {error}".format(
error=exception))
try:
tasks = http.get_json(master, "tasks")["tasks"]
except Exception as exception:
raise CLIException("Could not open '/tasks'"
" endpoint at '{addr}': {error}".format(
addr=master, error=exception))
if not tasks:
print("There are no tasks running in the cluster.")
return
try:
table = Table(["Task ID", "Framework ID", "Executor ID"])
for task in tasks:
table.add_row(
[task["id"], task["framework_id"], task["executor_id"]])
except Exception as exception:
raise CLIException(
"Unable to build table of tasks: {error}".format(
error=exception))
print(str(table))
def _run(self):
"""
Run the helper threads in this class which enable streaming
of STDIN/STDOUT/STDERR between the CLI and the Mesos Agent API.
If a tty is requested, we take over the current terminal and
put it into raw mode. We make sure to reset the terminal back
to its original settings before exiting.
"""
# Without a TTY.
if not self.tty:
try:
self._start_threads()
self.exit_event.wait()
except Exception as e:
self.exception = e
if self.exception:
# Known Pylint issue: https://github.com/PyCQA/pylint/issues/157
# pylint: disable=raising-bad-type
raise self.exception
return
# With a TTY.
if platform.system() == "Windows":
raise CLIException(
"Running with the '--tty' flag is not supported on windows")
if not sys.stdin.isatty():
raise CLIException(
"Must be running in a tty to pass the '--tty flag'")
fd = sys.stdin.fileno()
oldtermios = termios.tcgetattr(fd)
try:
if self.interactive:
self.supports_exit_sequence = True
tty.setraw(fd, when=termios.TCSANOW)
# To force a redraw of the remote terminal, we first resize it
# to 0 before setting it to the actual size of our local
# terminal. After that, all terminal resizing is handled in our
# SIGWINCH handler.
self._window_resize(signal.SIGWINCH, dimensions=[0, 0])
self._window_resize(signal.SIGWINCH)
signal.signal(signal.SIGWINCH, self._window_resize)
self._start_threads()
self.exit_event.wait()
except Exception as e:
self.exception = e
termios.tcsetattr(sys.stdin.fileno(), termios.TCSAFLUSH, oldtermios)
if self.exception:
# Known Pylint issue: https://github.com/PyCQA/pylint/issues/157
# pylint: disable=raising-bad-type
raise self.exception
def test_list(self):
"""
Basic test for the task `list()` sub-command.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
task = Task({"command": "sleep 1000"})
task.launch()
try:
wait_for_task(master, task.name, "TASK_RUNNING")
except Exception as exception:
raise CLIException("Error waiting for task '{name}' to"
" reach state '{state}': {error}".format(
name=task.name,
state="TASK_RUNNING",
error=exception))
try:
tasks = http.get_json(master.addr, "tasks")["tasks"]
except Exception as exception:
raise CLIException(
"Could not get tasks from '/{endpoint}' on master: {error}".
format(endpoint="tasks", error=exception))
self.assertEqual(type(tasks), list)
self.assertEqual(len(tasks), 1)
# Invoke the task plugin `list()` command
# and parse its output as a table.
test_config = config.Config(None)
plugin = TaskPlugin(None, test_config)
output = capture_output(plugin.list, {"--all": False})
table = Table.parse(output)
# Verify there are two rows in the table
# and that they are formatted as expected,
# with the proper task info in them.
self.assertEqual(table.dimensions()[0], 2)
self.assertEqual(table.dimensions()[1], 4)
self.assertEqual("ID", table[0][0])
self.assertEqual("State", table[0][1])
self.assertEqual("Framework ID", table[0][2])
self.assertEqual("Executor ID", table[0][3])
self.assertEqual(tasks[0]["id"], table[1][0])
self.assertEqual(tasks[0]["statuses"][-1]["state"], table[1][1])
self.assertEqual(tasks[0]["framework_id"], table[1][2])
self.assertEqual(tasks[0]["executor_id"], table[1][3])
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def get_agent_address(agent_id, master):
"""
Given a master and an agent id, return the agent address
by checking the /slaves endpoint of the master.
"""
try:
agents = http.get_json(master, "slaves")["slaves"]
except Exception as exception:
raise CLIException("Could not open '/slaves'"
" endpoint at '{addr}': {error}".format(
addr=master, error=exception))
for agent in agents:
if agent["id"] == agent_id:
return agent["pid"].split("@")[1]
raise CLIException("Unable to find agent '{id}'".format(id=agent_id))
def __init__(self, columns):
"""
Initialize a table with a list of column names
to act as headers for each column in the table.
"""
if not isinstance(columns, list):
raise CLIException("Column headers must be supplied as a list")
for column in columns:
if re.search(r"(\s)\1{2,}", column):
raise CLIException("Column headers cannot have more"
" than one space between words")
self.table = [columns]
self.padding = [len(column) for column in columns]
def _attach_container_output(self):
"""
Streams all output data (e.g. STDOUT/STDERR) to the
client from the agent.
"""
message = {
'type': 'ATTACH_CONTAINER_OUTPUT',
'attach_container_output': {
'container_id': self.container_id}}
req_extra_args = {
'stream': True,
'additional_headers': {
'Content-Type': 'application/json',
'Accept': 'application/recordio',
'Message-Accept': 'application/json'}}
try:
resource = mesos.http.Resource(self.agent_url)
response = resource.request(
mesos.http.METHOD_POST,
data=json.dumps(message),
retry=False,
timeout=None,
**req_extra_args)
except MesosHTTPException as e:
text = "I/O switchboard server was disabled for this container"
if e.response.status_code == 500 and e.response.text == text:
raise CLIException("Unable to attach to a task"
" launched without a TTY")
raise e
self._process_output_stream(response)
def list(self, argv):
"""
Show a list of running frameworks
"""
try:
master = self.config.master()
except Exception as exception:
raise CLIException(
"Unable to get leading master address: {error}".format(
error=exception))
data = get_frameworks(master, self.config)
table = Table(["ID", "Active", "Hostname", "Name"])
for framework in data:
if (not argv["--all"] and not framework["active"]):
continue
active = "False"
if framework["active"]:
active = "True"
table.add_row([
framework["id"], active, framework["hostname"],
framework["name"]
])
print(str(table))
def launch(self, timeout=TIMEOUT):
"""
After starting the task, we need to make sure its container
has actually been added to the agent before proceeding.
"""
super(Task, self).launch()
try:
# pylint: disable=missing-docstring
def container_exists(data):
return any(container["executor_id"] == self.flags["name"]
for container in data)
self.__wait_for_containers(container_exists, timeout)
except Exception as exception:
stdout = ""
if self.proc.poll():
stdout = "\n{output}".format(output=self.proc.stdout.read())
self.proc = None
raise CLIException("Waiting for container '{name}'"
" failed: {error}{stdout}"
.format(name=self.flags["name"],
error=exception,
stdout=stdout))
def __init__(self, flags=None):
super(Master, self).__init__()
if Master.count > 0:
raise CLIException("Creating more than one master"
" is currently not possible")
Master.count += 1
if flags is None:
flags = {}
if "ip" not in flags:
flags["ip"] = DEFAULT_MASTER_IP
if "port" not in flags:
flags["port"] = DEFAULT_MASTER_PORT
if "work_dir" not in flags:
flags["work_dir"] = tempfile.mkdtemp()
self.flags = flags
self.name = "master"
self.addr = "{ip}:{port}".format(ip=flags["ip"], port=flags["port"])
self.executable = os.path.join(
CLITestCase.MESOS_BUILD_DIR,
"bin",
"mesos-{name}.sh".format(name=self.name))
def __init__(self, flags=None):
super(Agent, self).__init__()
if Agent.count > 0:
raise CLIException("Creating more than one agent"
" is currently not possible")
Agent.count += 1
if flags is None:
flags = {}
if "ip" not in flags:
flags["ip"] = DEFAULT_AGENT_IP
if "port" not in flags:
flags["port"] = DEFAULT_AGENT_PORT
if "master" not in flags:
flags["master"] = "{ip}:{port}".format(
ip=DEFAULT_MASTER_IP,
port=DEFAULT_MASTER_PORT)
if "work_dir" not in flags:
flags["work_dir"] = tempfile.mkdtemp()
if "runtime_dir" not in flags:
flags["runtime_dir"] = tempfile.mkdtemp()
# Disabling systemd support on Linux to run without sudo.
if "linux" in sys.platform and "systemd_enable_support" not in flags:
flags["systemd_enable_support"] = "false"
self.flags = flags
self.name = "agent"
self.addr = "{ip}:{port}".format(ip=flags["ip"], port=flags["port"])
self.executable = os.path.join(
CLITestCase.MESOS_BUILD_DIR,
"bin",
"mesos-{name}.sh".format(name=self.name))
def test_exec(self):
"""
Basic test for the task `exec()` sub-command.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
with open(LOREM_IPSUM) as text:
content = text.read()
command = "printf '{data}' > a.txt && sleep 1000".format(data=content)
task = Task({"command": command})
task.launch()
tasks = running_tasks(master)
if not tasks:
raise CLIException("Unable to find running tasks on master"
" '{master}'".format(master=master.addr))
returncode, stdout, stderr = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "cat", "a.txt"])
self.assertEqual(returncode, 0)
self.assertEqual(stdout, content)
self.assertEqual(stderr, "")
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def test_exec_exit_status(self):
"""
Basic test for the task `exec()` sub-command exit status.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
task = Task({"command": "sleep 1000"})
task.launch()
tasks = running_tasks(master)
if not tasks:
raise CLIException("Unable to find running tasks on master"
" '{master}'".format(master=master.addr))
returncode, _, _ = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "true"])
self.assertEqual(returncode, 0)
returncode, _, _ = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "bash", "-c", "exit 10"])
self.assertEqual(returncode, 10)
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def test_exec_interactive(self):
"""
Test for the task `exec()` sub-command, using `--interactive`.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
task = Task({"command": "sleep 1000"})
task.launch()
tasks = running_tasks(master)
if not tasks:
raise CLIException("Unable to find running tasks on master"
" '{master}'".format(master=master.addr))
with open(LOREM_IPSUM) as text:
returncode, stdout, stderr = exec_command(
["mesos", "task", "exec", "-i", tasks[0]["id"], "cat"],
stdin=text)
self.assertEqual(returncode, 0)
with open(LOREM_IPSUM) as text:
self.assertEqual(stdout, text.read())
self.assertEqual(stderr, "")
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def _process_output_stream(self, response):
"""
Gets data streamed over the given response and places the
returned messages into our output_queue. Only expects to
receive data messages.
:param response: Response from an http post
:type response: requests.models.Response
"""
# Now that we are ready to process the output stream (meaning
# our output connection has been established), allow the input
# stream to be attached by setting an event.
self.attach_input_event.set()
# If we are running in interactive mode, wait to make sure that
# our input connection succeeds before pushing any output to the
# output queue.
if self.interactive:
self.print_output_event.wait()
try:
for chunk in response.iter_content(chunk_size=None):
records = self.decoder.decode(chunk)
for r in records:
if r.get('type') and r['type'] == 'DATA':
self.output_queue.put(r['data'])
except Exception as e:
raise CLIException(
"Error parsing output stream: {error}".format(error=e))
self.output_queue.join()
self.exit_event.set()
def _wait(self):
"""
Wait for the container associated with this class (through
'container_id') to exit and return its exit status.
"""
message = {
'type': 'WAIT_CONTAINER',
'wait_container': {
'container_id': self.container_id}}
req_extra_args = {
'verify': self.config.agent_ssl_verify(),
'additional_headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'}}
try:
resource = mesos.http.Resource(self.agent_url)
response = resource.request(
mesos.http.METHOD_POST,
data=json.dumps(message),
auth=self.config.authentication_header(),
retry=False,
timeout=None,
**req_extra_args)
except MesosException as exception:
raise CLIException(
"Error waiting for command to complete: {error}"
.format(error=exception))
exit_status = response.json()["wait_container"]["exit_status"]
if os.WIFSIGNALED(exit_status):
return os.WTERMSIG(exit_status) + 128
return os.WEXITSTATUS(exit_status)
def test_exec(self):
"""
Basic test for the task `exec()` sub-command.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
with open(LOREM_IPSUM) as text:
content = text.read()
command = "printf '{data}' > a.txt && sleep 1000".format(data=content)
task = Task({"command": command})
task.launch()
try:
wait_for_task(master, task.name, "TASK_RUNNING")
except Exception as exception:
raise CLIException("Error waiting for task '{name}' to"
" reach state '{state}': {error}".format(
name=task.name,
state="TASK_RUNNING",
error=exception))
try:
tasks = http.get_json(master.addr, "tasks")["tasks"]
except Exception as exception:
raise CLIException(
"Could not get tasks from '/{endpoint}' on master: {error}".
format(endpoint="tasks", error=exception))
self.assertEqual(type(tasks), list)
self.assertEqual(len(tasks), 1)
returncode, stdout, stderr = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "cat", "a.txt"])
self.assertEqual(returncode, 0)
self.assertEqual(stdout, content)
self.assertEqual(stderr, "")
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def updated_tasks():
# Open the master's `/tasks` endpoint and
# read the task information ourselves.
tasks = http.get_json(master.addr, "tasks")["tasks"]
if tasks[0]["state"] == "TASK_RUNNING":
return tasks
raise CLIException("Unable to find running task in master state"
" '{master}'".format(master=master))
def get_key_endpoint(key, endpoint, master, config):
"""
Get the json key of the given endpoint
"""
try:
data = http.get_json(master, endpoint, config)
except Exception as exception:
raise CLIException(
"Could not open '/{endpoint}' on master: {error}".format(
endpoint=endpoint, error=exception))
if not key in data:
raise CLIException("Missing '{key}' key in data retrieved"
" from master on '/{endpoint}'".format(
key=key, endpoint=endpoint))
return data[key]
def test_exec_exit_status(self):
"""
Basic test for the task `exec()` sub-command exit status.
"""
# Launch a master, agent, and task.
master = Master()
master.launch()
agent = Agent()
agent.launch()
task = Task({"command": "sleep 1000"})
task.launch()
try:
wait_for_task(master, task.name, "TASK_RUNNING")
except Exception as exception:
raise CLIException("Error waiting for task '{name}' to"
" reach state '{state}': {error}".format(
name=task.name,
state="TASK_RUNNING",
error=exception))
try:
tasks = http.get_json(master.addr, "tasks")["tasks"]
except Exception as exception:
raise CLIException(
"Could not get tasks from '/{endpoint}' on master: {error}".
format(endpoint="tasks", error=exception))
self.assertEqual(type(tasks), list)
self.assertEqual(len(tasks), 1)
returncode, _, _ = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "true"])
self.assertEqual(returncode, 0)
returncode, _, _ = exec_command(
["mesos", "task", "exec", tasks[0]["id"], "bash", "-c", "exit 10"])
self.assertEqual(returncode, 10)
# Kill the task, agent, and master.
task.kill()
agent.kill()
master.kill()
def plugins(self):
"""
Parse the plugins listed in the configuration file and return them.
"""
# Allow extra plugins to be pulled in from the configuration file.
if "plugins" in self.data:
if not isinstance(self.data["plugins"], list):
raise CLIException("Unable to parse config file '{path}':"
" 'plugins' field must be a list"
.format(path=self.path))
for plugin in self.data["plugins"]:
if not os.path.exists(plugin):
raise CLIException("Plugin path not found: {path}"
.format(path=plugin))
return self.data["plugins"]
return []
