def task(self, fltr, completed=False):
"""Returns the task with `fltr` in its ID. Raises a MesosException if
there is not exactly one such task.
:param fltr: filter string
:type fltr: str
:returns: the task that has `fltr` in its ID
:param completed: also include completed tasks
:type completed: bool
:rtype: Task
"""
tasks = self.tasks(fltr, completed)
if len(tasks) == 0:
raise MesosException(
'Cannot find a task with ID containing "{}"'.format(fltr))
elif len(tasks) > 1:
msg = [("There are multiple tasks with ID matching [{}]. " +
"Please choose one:").format(fltr)]
msg += ["\t{0}".format(t["id"]) for t in tasks]
raise MesosException('\n'.join(msg))
else:
return tasks[0]
def slave(self, fltr):
"""Returns the slave that has `fltr` in its ID. If any slaves
are an exact match, returns that task, id not raises a
MesosException if there is not exactly one such slave.
:param fltr: filter string
:type fltr: str
:returns: the slave that has `fltr` in its ID
:rtype: Slave
"""
slaves = self.slaves(fltr)
if len(slaves) == 0:
raise MesosException('No agent found with ID "{}".'.format(fltr))
elif len(slaves) > 1:
exact_matches = [s for s in slaves if s['id'] == fltr]
if len(exact_matches) == 1:
return exact_matches[0]
else:
matches = ['\t{0}'.format(s['id']) for s in slaves]
raise MesosException(
"There are multiple agents with that ID. " +
"Please choose one:\n{}".format('\n'.join(matches)))
else:
return slaves[0]
def enforce_file_permissions(path):
"""Enforce 400 or 600 permissions on file
:param path: Path to the TOML file
:type path: str
:rtype: None
"""
if not os.path.isfile(path):
raise MesosException('Path [{}] is not a file'.format(path))
# Unix permissions are incompatible with windows
# TODO: https://github.com/dcos/dcos-cli/issues/662
if sys.platform == 'win32':
return
else:
permissions = oct(stat.S_IMODE(os.stat(path).st_mode))
if permissions not in ['0o600', '0600', '0o400', '0400']:
if os.path.realpath(path) != path:
path = '%s (pointed to by %s)' % (os.path.realpath(path), path)
msg = (
"Permissions '{}' for configuration file '{}' are too open. "
"File must only be accessible by owner. "
"Aborting...".format(permissions, path))
raise MesosException(msg)
def _output_thread(self):
"""Reads from the output_queue and writes the data
to the appropriate STDOUT or STDERR.
"""
while True:
# Get a message from the output queue and decode it.
# Then write the data to the appropriate stdout or stderr.
output = self.output_queue.get()
if not output.get('data'):
raise MesosException("Error no 'data' field in output message")
data = output['data']
data = base64.b64decode(data.encode('utf-8'))
if output.get('type') and output['type'] == 'STDOUT':
sys.stdout.buffer.write(data)
sys.stdout.flush()
elif output.get('type') and output['type'] == 'STDERR':
sys.stderr.buffer.write(data)
sys.stderr.flush()
else:
raise MesosException("Unsupported data type in output stream")
self.output_queue.task_done()
def decode(self, data):
"""Decode a 'RecordIO' formatted message to its original type.
:param data: an array of 'UTF-8' encoded bytes that make up a
partial 'RecordIO' message. Subsequent calls to this
function maintain state to build up a full 'RecordIO'
message and decode it
:type data: bytes
:returns: a list of deserialized messages
:rtype: list
"""
if not isinstance(data, bytes):
raise MesosException("Parameter 'data' must of of type 'bytes'")
if self.state == self.FAILED:
raise MesosException("Decoder is in a FAILED state")
records = []
for c in data:
if self.state == self.HEADER:
if c != ord('\n'):
self.buffer += bytes([c])
continue
try:
self.length = int(self.buffer.decode("UTF-8"))
except Exception as exception:
self.state = self.FAILED
raise MesosException("Failed to decode length"
"'{buffer}': {error}".format(
buffer=self.buffer,
error=exception))
self.buffer = bytes("", "UTF-8")
self.state = self.RECORD
# Note that for 0 length records, we immediately decode.
if self.length <= 0:
records.append(self.deserialize(self.buffer))
self.state = self.HEADER
elif self.state == self.RECORD:
assert self.length
assert len(self.buffer) < self.length
self.buffer += bytes([c])
if len(self.buffer) == self.length:
records.append(self.deserialize(self.buffer))
self.buffer = bytes("", "UTF-8")
self.state = self.HEADER
return records
def run(self):
"""Run the helper threads in this class which enable streaming
of STDIN/STDOUT/STDERR between the CLI and the Mesos Agent API.
If a tty is requested, we take over the current terminal and
put it into raw mode. We make sure to reset the terminal back
to its original settings before exiting.
"""
# Without a TTY.
if not self.tty:
try:
self._start_threads()
self.exit_event.wait()
except Exception as e:
self.exception = e
if self.exception:
raise self.exception
return
# With a TTY.
if util.is_windows_platform():
raise MesosException(
"Running with the '--tty' flag is not supported on windows.")
if not sys.stdin.isatty():
raise MesosException(
"Must be running in a tty to pass the '--tty flag'.")
fd = sys.stdin.fileno()
oldtermios = termios.tcgetattr(fd)
try:
if self.interactive:
tty.setraw(fd, when=termios.TCSANOW)
self._window_resize(signal.SIGWINCH, None)
signal.signal(signal.SIGWINCH, self._window_resize)
self._start_threads()
self.exit_event.wait()
except Exception as e:
self.exception = e
termios.tcsetattr(
sys.stdin.fileno(),
termios.TCSAFLUSH,
oldtermios)
if self.exception:
raise self.exception
def _process_output_stream(self, response):
"""Gets data streamed over the given response and places the
returned messages into our output_queue. Only expects to
receive data messages.
:param response: Response from an http post
:type response: requests.models.Response
"""
# Now that we are ready to process the output stream (meaning
# our output connection has been established), allow the input
# stream to be attached by setting an event.
self.attach_input_event.set()
# If we are running in interactive mode, wait to make sure that
# our input connection succeeds before pushing any output to the
# output queue.
if self.interactive:
self.print_output_event.wait()
try:
for chunk in response.iter_content(chunk_size=None):
records = self.decoder.decode(chunk)
for r in records:
if r.get('type') and r['type'] == 'DATA':
self.output_queue.put(r['data'])
except Exception as e:
raise MesosException(
"Error parsing output stream: {error}".format(error=e))
self.output_queue.join()
self.exit_event.set()
def _get_container_id(container_status):
if 'container_id' in container_status:
if 'value' in container_status['container_id']:
return container_status['container_id']
raise MesosException("No container found for the specified task."
" It might still be spinning up."
" Please try again.")
def _get_container_status(task):
if 'statuses' in task:
if len(task['statuses']) > 0:
if 'container_status' in task['statuses'][0]:
return task['statuses'][0]['container_status']
raise MesosException(
"Unable to obtain container status for task '{}'".format(
task['id']))
def read_file(path):
"""
:param path: path to file
:type path: str
:returns: contents of file
:rtype: str
"""
if not os.path.isfile(path):
raise MesosException('path [{}] is not a file'.format(path))
with open_file(path) as file_:
return file_.read()
def sh_move(src, dst):
"""Move file src to the file or directory dst.
:param src: source file
:type src: str
:param dst: destination file or directory
:type dst: str
:rtype: None
"""
try:
shutil.move(src, dst)
except EnvironmentError as e:
logger.exception('Unable to move [%s] to [%s]', src, dst)
if e.strerror:
if e.filename:
raise MesosException("{}: {}".format(e.strerror, e.filename))
else:
raise MesosException(e.strerror)
else:
raise MesosException(e)
except Exception as e:
logger.exception('Unknown error while moving [%s] to [%s]', src, dst)
raise MesosException(e)
def get_container_id(self, task_id):
"""Returns the container ID for a task ID matching `task_id`
:param task_id: The task ID which will be mapped to container ID
:type task_id: str
:returns: The container ID associated with 'task_id'
:rtype: str
"""
def _get_task(task_id):
candidates = []
if 'frameworks' in self.state():
for framework in self.state()['frameworks']:
if 'tasks' in framework:
for task in framework['tasks']:
if 'id' in task:
if task['id'].startswith(task_id):
candidates.append(task)
if len(candidates) == 1:
return candidates[0]
raise MesosException(
"More than one task matching '{}' found: {}".format(
task_id, candidates))
def _get_container_status(task):
if 'statuses' in task:
if len(task['statuses']) > 0:
if 'container_status' in task['statuses'][0]:
return task['statuses'][0]['container_status']
raise MesosException(
"Unable to obtain container status for task '{}'".format(
task['id']))
def _get_container_id(container_status):
if 'container_id' in container_status:
if 'value' in container_status['container_id']:
return container_status['container_id']
raise MesosException("No container found for the specified task."
" It might still be spinning up."
" Please try again.")
if not task_id:
raise MesosException("Invalid task ID")
task = _get_task(task_id)
container_status = _get_container_status(task)
return _get_container_id(container_status)
def ensure_file_exists(path):
""" Create file if it doesn't exist
:param path: path of file to create
:type path: str
:rtype: None
"""
if not os.path.exists(path):
try:
open(path, 'w').close()
os.chmod(path, 0o600)
except IOError as e:
raise MesosException('Cannot create file [{}]: {}'.format(path, e))
def io_exception(path, errno):
"""Returns a MesosException for when there is an error opening the
file at `path`
:param path: file path
:type path: str
:param errno: IO error number
:type errno: int
:returns: MesosException
:rtype: MesosException
"""
return MesosException('Error opening file [{}]: {}'.format(
path, os.strerror(errno)))
def load_jsons(value):
"""Deserialize a string to a python object
:param value: The JSON string
:type value: str
:returns: The deserialized JSON object
:rtype: dict | list | str | int | float | bool
"""
try:
return json.loads(value)
except Exception:
logger.exception('Unhandled exception while loading JSON: %r', value)
raise MesosException('Error loading JSON.')
def _get_task(task_id):
candidates = []
if 'frameworks' in self.state():
for framework in self.state()['frameworks']:
if 'tasks' in framework:
for task in framework['tasks']:
if 'id' in task:
if task['id'].startswith(task_id):
candidates.append(task)
if len(candidates) == 1:
return candidates[0]
raise MesosException(
"More than one task matching '{}' found: {}".format(
task_id, candidates))
def parse_float(string):
"""Parse string and an float
:param string: string to parse as an float
:type string: str
:returns: the float value of the string
:rtype: float
"""
try:
return float(string)
except ValueError:
logger.error('Unhandled exception while parsing string as float: %r',
string)
raise MesosException('Error parsing string as float')
def parse_int(string):
"""Parse string and an integer
:param string: string to parse as an integer
:type string: str
:returns: the interger value of the string
:rtype: int
"""
try:
return int(string)
except ValueError:
logger.error('Unhandled exception while parsing string as int: %r',
string)
raise MesosException('Error parsing string as int')
def ensure_dir_exists(directory):
"""If `directory` does not exist, create it.
:param directory: path to the directory
:type directory: string
:rtype: None
"""
if not os.path.exists(directory):
logger.info('Creating directory: %r', directory)
try:
os.makedirs(directory, 0o775)
except os.error as e:
raise MesosException('Cannot create directory [{}]: {}'.format(
directory, e))
def encode(self, message):
"""Encode a message into 'RecordIO' format.
:param message: a message to serialize and then wrap in
a 'RecordIO' frame.
:type message: object
:returns: a serialized message wrapped in a 'RecordIO' frame
:rtype: bytes
"""
s = self.serialize(message)
if not isinstance(s, bytes):
raise MesosException("Calling 'serialize(message)' must"
" return a 'bytes' object")
return bytes(str(len(s)) + "\n", "UTF-8") + s
def load_json(reader, keep_order=False):
"""Deserialize a reader into a python object
:param reader: the json reader
:type reader: a :code:`.read()`-supporting object
:param keep_order: whether the return should be an ordered dictionary
:type keep_order: bool
:returns: the deserialized JSON object
:rtype: dict | list | str | int | float | bool
"""
try:
if keep_order:
return json.load(reader, object_pairs_hook=collections.OrderedDict)
else:
return json.load(reader)
except Exception as error:
logger.error('Unhandled exception while loading JSON: %r', error)
raise MesosException('Error loading JSON: {}'.format(error))
def configure_logger(log_level):
"""Configure the program's logger.
:param log_level: Log level for configuring logging
:type log_level: str
:rtype: None
"""
if log_level is None:
logging.disable(logging.CRITICAL)
return None
if log_level in constants.VALID_LOG_LEVEL_VALUES:
logging.basicConfig(format=('%(threadName)s: '
'%(asctime)s '
'%(pathname)s:%(funcName)s:%(lineno)d - '
'%(message)s'),
stream=sys.stderr,
level=log_level.upper())
return None
msg = 'Log level set to an unknown value {!r}. Valid values are {!r}'
raise MesosException(
msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES))
def _request(method,
url,
is_success=_default_is_success,
timeout=DEFAULT_TIMEOUT,
auth=None,
verify=None,
toml_config=None,
**kwargs):
"""Sends an HTTP request.
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param auth: authentication
:type auth: AuthBase
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
if 'headers' not in kwargs:
kwargs['headers'] = {'Accept': 'application/json'}
verify = False
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs
if verify is not None:
silence_requests_warnings()
logger.info('Sending HTTP [%r] to [%r]: %r', method, url,
kwargs.get('headers'))
try:
response = requests.request(method=method,
url=url,
timeout=timeout,
auth=auth,
verify=verify,
**kwargs)
except requests.exceptions.SSLError as e:
logger.exception("HTTP SSL Error")
msg = ("An SSL error occurred.")
if description is not None:
msg += "\n<value>: {}".format(description)
raise MesosException(msg)
except requests.exceptions.ConnectionError as e:
logger.exception("HTTP Connection Error")
raise MesosConnectionError(url)
except requests.exceptions.Timeout as e:
logger.exception("HTTP Timeout")
raise MesosException('Request to URL [{0}] timed out.'.format(url))
except requests.exceptions.RequestException as e:
logger.exception("HTTP Exception")
raise MesosException('HTTP Exception: {}'.format(e))
logger.info('Received HTTP response [%r]: %r', response.status_code,
response.headers)
return response
def __init__(self,
mesos_master_url,
task_id,
cmd=None,
args=None,
interactive=False,
tty=False):
# Store relevant parameters of the call for later.
self.cmd = cmd
self.interactive = interactive
self.tty = tty
self.args = args
self._mesos_master_url = mesos_master_url
master = Master(get_master_state(self._mesos_master_url))
# Get the task and make sure its container was launched by the UCR.
# Since task's containers are launched by the UCR by default, we want
# to allow most tasks to pass through unchecked. The only exception is
# when a task has an explicit container specified and it is not of type
# "MESOS". Having a type of "MESOS" implies that it was launched by the
# UCR -- all other types imply it was not.
task_obj = master.task(task_id)
if "container" in task_obj.dict():
if "type" in task_obj.dict()["container"]:
if task_obj.dict()["container"]["type"] != "MESOS":
raise MesosException(
"This command is only supported for tasks"
" launched by the Universal Container Runtime (UCR).")
# Get the URL to the agent running the task.
self.agent_url = urllib.parse.urljoin(task_obj.slave().http_url(),
'api/v1')
# Grab a reference to the container ID for the task.
self.parent_id = master.get_container_id(task_id)
if "user" in task_obj.dict():
self.user = task_obj.dict()['user']
else:
self.user = None
# Generate a new UUID for the nested container
# used to run commands passed to `task exec`.
self.container_id = str(uuid.uuid4())
# Set up a recordio encoder and decoder
# for any incoming and outgoing messages.
self.encoder = recordio.Encoder(
lambda s: bytes(json.dumps(s, ensure_ascii=False), "UTF-8"))
self.decoder = recordio.Decoder(
lambda s: json.loads(s.decode("UTF-8")))
# Set up queues to send messages between threads used for
# reading/writing to STDIN/STDOUT/STDERR and threads
# sending/receiving data over the network.
self.input_queue = Queue()
self.output_queue = Queue()
# Set up an event to block attaching
# input until attaching output is complete.
self.attach_input_event = threading.Event()
self.attach_input_event.clear()
# Set up an event to block printing the output
# until an attach input event has successfully
# been established.
self.print_output_event = threading.Event()
self.print_output_event.clear()
# Set up an event to block the main thread
# from exiting until signaled to do so.
self.exit_event = threading.Event()
self.exit_event.clear()
# Use a class variable to store exceptions thrown on
# other threads and raise them on the main thread before
# exiting.
self.exception = None
