def _deployment_list(app_id, json_):
"""
:param app_id: the application id
:type app_id: str
:param json_: output json if True
:type json_: bool
:returns: process return code
:rtype: int
"""
client = marathon.create_client()
deployments = client.get_deployments(app_id)
if not deployments and not json_:
msg = "There are no deployments"
if app_id:
msg += " for '{}'".format(app_id)
raise DCOSException(msg)
emitting.publish_table(emitter, deployments, tables.deployment_table,
json_)
return 0
def _update_job(job_file):
"""
:param job_file: filename for the application resource
:type job_file: str
:returns: process return code
:rtype: int
"""
# only updates the job (does NOT update schedules)
full_json = _get_resource(job_file)
if full_json is None:
raise DCOSException("No JSON provided.")
job_id = full_json['id']
if 'schedules' in full_json:
del full_json['schedules']
try:
_put_job(job_id, full_json)
except DCOSHTTPException as e:
emitter.publish("Error updating job: '{}'".format(job_id))
return 0
def _search(json_, query):
"""Search for matching packages.
:param json_: output json if True
:type json_: bool
:param query: The search term
:type query: str
:returns: Process status
:rtype: int
"""
if not query:
query = ''
package_manager = get_package_manager()
results = package_manager.search_sources(query)
if json_ or results['packages']:
emitting.publish_table(emitter, results, tables.package_search_table,
json_)
else:
raise DCOSException('No packages found.')
return 0
def render_mustache_json(template, data):
"""Render the supplied mustache template and data as a JSON value
:param template: the mustache template to render
:type template: str
:param data: the data to use as a rendering context
:type data: dict
:returns: the rendered template
:rtype: dict | list | str | int | float | bool
"""
try:
r = CustomJsonRenderer()
rendered = r.render(template, data)
except Exception as e:
logger.exception('Error rendering mustache template [%r] [%r]',
template, data)
raise DCOSException(e)
logger.debug('Rendered mustache template: %s', rendered)
return load_jsons(rendered)
def _list(json_flag=False):
"""
:returns: process return code
:rtype: int
"""
response = None
url = _get_api_url('v1/jobs' + METRONOME_EMBEDDED)
try:
response = _do_request(url, 'GET')
except DCOSException as e:
raise DCOSException(e)
json_list = _read_http_response_body(response)
if json_flag:
emitter.publish(json_list)
else:
table = tables.job_table(json_list)
output = six.text_type(table)
if output:
emitter.publish(output)
return 0
def configure_logger(log_level):
"""Configure the program's logger.
:param log_level: Log level for configuring logging
:type log_level: str
:rtype: None
"""
if log_level is None:
logging.disable(logging.CRITICAL)
return None
if log_level in constants.VALID_LOG_LEVEL_VALUES:
logging.basicConfig(format=('%(asctime)s '
'%(pathname)s:%(funcName)s:%(lineno)d - '
'%(message)s'),
stream=sys.stderr,
level=log_level.upper())
return None
msg = 'Log level set to an unknown value {!r}. Valid values are {!r}'
raise DCOSException(msg.format(log_level,
constants.VALID_LOG_LEVEL_VALUES))
def _get_unit_type(unit_name):
""" Get the full unit name including the type postfix
or default to service.
:param unit_name: unit name with or without type
:type unit_name: str
:return: unit name with type
:rtype: str
"""
if not unit_name:
raise DCOSException('Empty systemd unit parameter')
# https://www.freedesktop.org/software/systemd/man/systemd.unit.html
unit_types = [
'service', 'socket', 'device', 'mount', 'automount', 'swap', 'target',
'path', 'timer', 'slice', 'scope'
]
for unit_type in unit_types:
if unit_name.endswith('.{}'.format(unit_type)):
return unit_name
return '{}.service'.format(unit_name)
def _bundle_create(nodes):
"""
Create a diagnostics bundle.
:param nodes: a list of nodes to collect the logs from.
:type nodes: list
:returns: process return code
:rtype: int
"""
_check_3dt_version()
url = urllib.parse.urljoin(DIAGNOSTICS_BASE_URL, 'create')
response = _do_diagnostics_request(url, 'POST', json={'nodes': nodes})
if ('status' not in response or 'extra' not in response
or 'bundle_name' not in response['extra']):
raise DCOSException(
'Request to create a diagnostics bundle {} returned an '
'unexpected response {}'.format(url, response))
emitter.publish('\n{}, available bundle: {}'.format(
response['status'], response['extra']['bundle_name']))
return 0
def _get_bundle_list():
"""
Get a list of tuples (bundle_file_name, file_size), ..
:return: list of diagnostic bundles
:rtype: list of tuples
"""
available_bundles = []
for _, bundle_files in _get_bundles_json().items():
if bundle_files is None:
continue
for bundle_file_obj in bundle_files:
if ('file_name' not in bundle_file_obj
or 'file_size' not in bundle_file_obj):
raise DCOSException(
'Request to get a list of available diagnostic bundles '
'returned unexpected response {}'.format(bundle_file_obj))
available_bundles.append(
(os.path.basename(bundle_file_obj['file_name']),
bundle_file_obj['file_size']))
return available_bundles
def uninstall(package_name, remove_all, app_id, cli, app):
"""Uninstalls a package.
:param package_name: The package to uninstall
:type package_name: str
:param remove_all: Whether to remove all instances of the named app
:type remove_all: boolean
:param app_id: App ID of the app instance to uninstall
:type app_id: str
:param init_client: The program to use to run the app
:type init_client: object
:rtype: None
"""
if cli is False and app is False:
cli = app = True
uninstalled = False
if cli:
if subcommand.uninstall(package_name):
uninstalled = True
if app:
num_apps = uninstall_app(package_name, remove_all, app_id,
marathon.create_client(), mesos.DCOSClient())
if num_apps > 0:
uninstalled = True
if uninstalled:
return None
else:
msg = 'Package [{}]'.format(package_name)
if app_id is not None:
msg += " with id [{}]".format(app_id)
msg += " is not installed."
raise DCOSException(msg)
def _ls(task, path, long_):
""" List files in a task's sandbox.
:param task: task pattern to match
:type task: str
:param path: file path to read
:type path: str
:param long_: whether to use a long listing format
:type long_: bool
:returns: process return code
:rtype: int
"""
if path is None:
path = '.'
if path.startswith('/'):
path = path[1:]
dcos_client = mesos.DCOSClient()
task_obj = mesos.get_master(dcos_client).task(task)
dir_ = posixpath.join(task_obj.directory(), path)
try:
files = dcos_client.browse(task_obj.slave(), dir_)
except DCOSHTTPException as e:
if e.response.status_code == 404:
raise DCOSException(
'Cannot access [{}]: No such file or directory'.format(path))
else:
raise
if files:
if long_:
emitter.publish(tables.ls_long_table(files))
else:
emitter.publish(' '.join(
posixpath.basename(file_['path']) for file_ in files))
def _dcos_log_v2(follow, tasks, lines, file_):
""" a client to dcos-log v2
:param follow: same as unix tail's -f
:type follow: bool
:param tasks: tasks pattern to match
:type tasks: list of str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
"""
if len(tasks) != 1:
raise DCOSException(
"found more than one task with the same name: {}. Please provide "
"a unique task name.".format([task['id'] for task in tasks]))
task = tasks[0]
endpoint = '/system/v1/logs/v2/task/{}/file/{}'.format(task['id'], file_)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
if lines:
# according to dcos-log v2 API the skip parameter must be negative
# integer. dcos-cli uses positive int to limit the number of last lines
# use "lines * -1" to make it negative.
if lines > 0:
lines *= -1
endpoint += '?cursor=END&skip={}'.format(lines)
url = dcos_url + endpoint
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def _list_repos(is_json):
"""List configured package repositories.
:param is_json: output json if True
:type is_json: bool
:returns: Process status
:rtype: int
"""
package_manager = get_package_manager()
repos = package_manager.get_repos()
if is_json:
return emitter.publish(repos)
elif repos.get("repositories"):
repos = ["{}: {}".format(repo.get("name"), repo.get("uri"))
for repo in repos.get("repositories")]
emitter.publish("\n".join(repos))
else:
msg = ("There are currently no repos configured. "
"Please use `dcos package repo add` to add a repo")
raise DCOSException(msg)
return 0
def run_dcos_command(command, raise_on_error=False, print_output=True):
""" Run `dcos {command}` via DC/OS CLI
:param command: the command to execute
:type command: str
:param raise_on_error: whether to raise a DCOSException if the return code is nonzero
:type raise_on_error: bool
:param print_output: whether to print the resulting stdout/stderr from running the command
:type print_output: bool
:return: (stdout, stderr, return_code)
:rtype: tuple
"""
call = shlex.split(command)
call.insert(0, 'dcos')
print("\n{}{}\n".format(shakedown.fchr('>>'), ' '.join(call)))
proc = subprocess.Popen(call,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output, error = proc.communicate()
return_code = proc.wait()
stdout = output.decode('utf-8')
stderr = error.decode('utf-8')
if print_output:
print(stdout, stderr, return_code)
if return_code != 0 and raise_on_error:
raise DCOSException(
'Got error code {} when running command "dcos {}":\nstdout: "{}"\nstderr: "{}"'
.format(return_code, command, stdout, stderr))
return stdout, stderr, return_code
def package_add_local(self, dcos_package):
"""
Adds a locally stored DC/OS package to DC/OS
:param dcos_package: path to the DC/OS package
:type dcos_package: None | str
:return: Response to the package add request
:rtype: requests.Response
"""
try:
with util.open_file(dcos_package, 'rb') as pkg:
extra_headers = {
'Content-Type': 'application/vnd.dcos.'
'universe.package+zip;version=v1',
'X-Dcos-Content-MD5': util.md5_hash_file(pkg)
}
return self._post('add', headers=extra_headers, data=pkg)
except DCOSHTTPException as e:
if e.status() == 404:
message = 'Your version of DC/OS ' \
'does not support this operation'
raise DCOSException(message)
else:
raise e
def __call__(self, value):
"""
:param value: String to try and parse
:type value: str
:returns: The parse value
:rtype: str | int | float | bool | list | dict
"""
value = clean_value(value)
if self.schema['type'] == 'string':
return _parse_string(value)
elif self.schema['type'] == 'object':
return _parse_object(value)
elif self.schema['type'] == 'number':
return _parse_number(value)
elif self.schema['type'] == 'integer':
return _parse_integer(value)
elif self.schema['type'] == 'boolean':
return _parse_boolean(value)
elif self.schema['type'] == 'array':
return _parse_array(value)
else:
raise DCOSException('Unknown type {!r}'.format(self._value_type))
def execute(cmds, args):
"""Executes one of the commands based on the arguments passed.
:param cmds: commands to try to execute; the order determines the order of
evaluation
:type cmds: list of Command
:param args: command line arguments
:type args: dict
:returns: the process status
:rtype: int
"""
for hierarchy, arg_keys, function in cmds:
# Let's find if the function matches the command
match = True
for positional in hierarchy:
if not args[positional]:
match = False
if match:
params = [args[name] for name in arg_keys]
return function(*params)
raise DCOSException('Could not find a command with the passed arguments')
def _search(json_, query):
"""Search for matching packages.
:param json_: output json if True
:type json_: bool
:param query: The search term
:type query: str
:returns: Process status
:rtype: int
"""
if not query:
query = ''
config = util.get_config()
results = [
index_entry.as_dict() for index_entry in package.search(query, config)
]
if any(result['packages'] for result in results) or json_:
emitting.publish_table(emitter, results, tables.package_search_table,
json_)
else:
raise DCOSException('No packages found.')
return 0
def _install_env(pkg, revision, options):
""" Install subcommand virtual env.
:param pkg: the package to install
:type pkg: Package
:param revision: the package revision to install
:type revision: str
:param options: package parameters
:type options: dict
:rtype: None
"""
pkg_dir = package_dir(pkg.name())
install_operation = pkg.command_json(revision, options)
env_dir = os.path.join(pkg_dir,
constants.DCOS_SUBCOMMAND_VIRTUALENV_SUBDIR)
if 'pip' in install_operation:
_install_with_pip(pkg.name(), env_dir, install_operation['pip'])
else:
raise DCOSException("Installation methods '{}' not supported".format(
install_operation.keys()))
def _dcos_log(follow, tasks, lines, file_, completed):
""" a client to dcos-log
:param follow: same as unix tail's -f
:type follow: bool
:param task: task pattern to match
:type task: str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
:param completed: whether to include completed tasks
:type completed: bool
"""
# only stdout and stderr is supported
if file_ not in ('stdout', 'stderr'):
raise DCOSException('Expect file stdout or stderr. '
'Got {}'.format(file_))
# state json may container tasks and completed_tasks fields. Based on
# user request we should traverse the appropriate field.
tasks_field = 'tasks'
if completed:
tasks_field = 'completed_tasks'
for task in tasks:
executor_info = task.executor()
if not executor_info:
continue
if (tasks_field not in executor_info and
not isinstance(executor_info[tasks_field], list)):
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException('Invalid executor info. '
'Missing field {}'.format(tasks_field))
for t in executor_info[tasks_field]:
container_id = get_nested_container_id(t)
if not container_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing container id')
# get slave_id field
slave_id = t.get('slave_id')
if not slave_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `slave_id`')
framework_id = t.get('framework_id')
if not framework_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `framework_id`')
# try `executor_id` first.
executor_id = t.get('executor_id')
if not executor_id:
# if `executor_id` is an empty string, default to `id`.
executor_id = t.get('id')
if not executor_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing executor id')
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
# dcos-log provides 2 base endpoints /range/ and /stream/
# for range and streaming requests.
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
endpoint = ('/system/v1/agent/{}/logs/v1/{}/framework/{}'
'/executor/{}/container/{}'.format(slave_id,
endpoint_type,
framework_id,
executor_id,
container_id))
# append request parameters.
# `skip_prev` will move the cursor to -n lines.
# `filter=STREAM:{STDOUT,STDERR}` will filter logs by label.
url = (dcos_url + endpoint +
'?skip_prev={}&filter=STREAM:{}'.format(lines,
file_.upper()))
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def get_nested_container_id(task):
""" Get the nested container id from mesos state.
:param task: task definition
:type task: dict
:return: comma separated string of nested containers
:rtype: string
"""
# get current task state
task_state = task.get('state')
if not task_state:
logger.debug('Full task state: {}'.format(task))
raise DCOSException('Invalid executor info. '
'Missing field `state`')
container_ids = []
statuses = task.get('statuses')
if not statuses:
logger.debug('Full task state: {}'.format(task))
raise DCOSException('Invalid executor info. Missing field `statuses`')
for status in statuses:
if 'state' not in status:
logger.debug('Full task state: {}'.format(task))
raise DCOSException('Invalid executor info. Missing field `state`')
if status['state'] != task_state:
continue
container_status = status.get('container_status')
if not container_status:
logger.debug('Full task state: {}'.format(task))
# if task status is TASK_FAILED and no container_id
# available then the executor has never started and no
# logs available for this task.
if status.get('state') == 'TASK_FAILED':
raise DCOSException('No available logs found. '
'Please check your executor status')
raise DCOSException('Invalid executor info. '
'Missing field `container_status`')
container_id = container_status.get('container_id')
if not container_id:
logger.debug('Full task state: {}'.format(task))
raise DCOSException('Invalid executor info. '
'Missing field `container_id`')
# traverse nested container_id field
while True:
value = container_id.get('value')
if not value:
logger.debug('Full task state: {}'.format(task))
raise DCOSException('Invalid executor info. Missing field'
'`value` for nested container ids')
container_ids.append(value)
if 'parent' not in container_id:
break
container_id = container_id['parent']
return '.'.join(reversed(container_ids))
def _log(all_, follow, completed, lines, task, file_):
""" Tail a file in the task's sandbox.
:param all_: If True, include all tasks
:type all_: bool
:param follow: same as unix tail's -f
:type follow: bool
:param completed: whether to include completed tasks
:type completed: bool
:param lines: number of lines to print
:type lines: int
:param task: task pattern to match
:type task: str
:param file_: file path to read
:type file_: str
:returns: process return code
:rtype: int
"""
fltr = task
if file_ is None:
file_ = 'stdout'
if lines is None:
lines = 10
lines = util.parse_int(lines)
# get tasks
client = mesos.DCOSClient()
master = mesos.Master(client.get_master_state())
tasks = master.tasks(fltr=fltr, completed=completed, all_=all_)
if not tasks:
if not fltr:
raise DCOSException("No tasks found. Exiting.")
elif not completed:
completed_tasks = master.tasks(completed=True, fltr=fltr)
if completed_tasks:
msg = 'No running tasks match ID [{}]; however, there '.format(
fltr)
if len(completed_tasks) > 1:
msg += 'are {} matching completed tasks. '.format(
len(completed_tasks))
else:
msg += 'is 1 matching completed task. '
msg += 'Run with --completed to see these logs.'
raise DCOSException(msg)
raise DCOSException('No matching tasks. Exiting.')
# if journald logging is disabled, read files API and exit.
if not log.dcos_log_enabled():
mesos_files = _mesos_files(tasks, file_, client)
if not mesos_files:
if fltr is None:
msg = "No tasks found. Exiting."
else:
msg = "No matching tasks. Exiting."
raise DCOSException(msg)
log.log_files(mesos_files, follow, lines)
return 0
# otherwise
if file_ in ('stdout', 'stderr'):
_dcos_log(follow, tasks, lines, file_, completed)
return 0
raise DCOSException('Invalid file {}. dcos-log only '
'supports stdout/stderr'.format(file_))
return 1
def _install_with_pip(
package_name,
env_directory,
requirements):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's virtual env
:type env_directory: str
:param requirements: the list of pip requirements
:type requirements: list of str
:rtype: None
"""
bin_directory = util.dcos_bin_path()
new_package_dir = not os.path.exists(env_directory)
pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip')
if not os.path.exists(pip_path):
virtualenv_path = _find_virtualenv(bin_directory)
virtualenv_version = _execute_command(
[virtualenv_path, '--version'])[0].strip().decode('utf-8')
if LooseVersion("12") > LooseVersion(virtualenv_version):
msg = ("Unable to install CLI subcommand. "
"Required program 'virtualenv' must be version 12+, "
"currently version {}\n"
"Please see installation instructions: "
"https://virtualenv.pypa.io/en/latest/installation.html"
"".format(virtualenv_version))
raise DCOSException(msg)
cmd = [_find_virtualenv(bin_directory), env_directory]
if _execute_command(cmd)[2] != 0:
raise _generic_error(package_name)
# Do not replace util.temptext NamedTemporaryFile
# otherwise bad things will happen on Windows
with util.temptext() as text_file:
fd, requirement_path = text_file
# Write the requirements to the file
with os.fdopen(fd, 'w') as requirements_file:
for line in requirements:
print(line, file=requirements_file)
cmd = [
os.path.join(env_directory, BIN_DIRECTORY, 'pip'),
'install',
'--requirement',
requirement_path,
]
if _execute_command(cmd)[2] != 0:
# We should remove the directory that we just created
if new_package_dir:
shutil.rmtree(env_directory)
raise _generic_error(package_name)
return None
def _install_with_binary(
package_name,
env_directory,
binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
binary_url = _rewrite_binary_url(
binary_url,
config.get_config_val("core.dcos_url"))
try:
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
if util.is_windows_platform():
binary_name += '.exe'
binary_file = os.path.join(env_bin_dir, binary_name)
# copy to avoid windows error of moving open file
# binary_tmp will be removed by context manager
shutil.copy(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# check contents for package_name/env/bin folder structure
if not os.path.exists(env_bin_dir):
msg = (
"CLI subcommand for [{}] has an unexpected format. "
"Please contact the package maintainer".format(
package_name))
raise DCOSException(msg)
else:
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
"Please contact the package maintainer".format(
package_name, kind))
raise DCOSException(msg)
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name, e.message)
return None
def _request(method,
url,
is_success=_default_is_success,
timeout=DEFAULT_TIMEOUT,
auth=None,
verify=None,
**kwargs):
"""Sends an HTTP request.
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param auth: authentication
:type auth: AuthBase
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
if 'headers' not in kwargs:
kwargs['headers'] = {'Accept': 'application/json'}
verify = _verify_ssl(verify)
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs
if verify is not None:
silence_requests_warnings()
logger.info('Sending HTTP [%r] to [%r]: %r', method, url,
kwargs.get('headers'))
try:
response = requests.request(method=method,
url=url,
timeout=timeout,
auth=auth,
verify=verify,
**kwargs)
except requests.exceptions.SSLError as e:
logger.exception("HTTP SSL Error")
msg = ("An SSL error occurred. To configure your SSL settings, "
"please run: `dcos config set core.ssl_verify <value>`")
description = config.get_property_description("core", "ssl_verify")
if description is not None:
msg += "\n<value>: {}".format(description)
raise DCOSException(msg)
except requests.exceptions.ConnectionError as e:
logger.exception("HTTP Connection Error")
raise DCOSException('URL [{0}] is unreachable: {1}'.format(url, e))
except requests.exceptions.Timeout as e:
logger.exception("HTTP Timeout")
raise DCOSException('Request to URL [{0}] timed out.'.format(url))
except requests.exceptions.RequestException as e:
logger.exception("HTTP Exception")
raise DCOSException('HTTP Exception: {}'.format(e))
logger.info('Received HTTP response [%r]: %r', response.status_code,
response.headers)
return response
def _mock_exception(contents='exception'):
return MagicMock(side_effect=DCOSException(contents))
def _ssh(leader, slave, option, config_file, user, master_proxy, proxy_ip,
command):
"""SSH into a DC/OS node using the IP addresses found in master's
state.json
:param leader: True if the user has opted to SSH into the leading
master
:type leader: bool | None
:param slave: The slave ID if the user has opted to SSH into a slave
:type slave: str | None
:param option: SSH option
:type option: [str]
:param config_file: SSH config file
:type config_file: str | None
:param user: SSH user
:type user: str | None
:param master_proxy: If True, SSH-hop from a master
:type master_proxy: bool | None
:param proxy_ip: If set, SSH-hop from this IP address
:type proxy_ip: str | None
:param command: Command to run on the node
:type command: str | None
:rtype: int
:returns: process return code
"""
ssh_options = util.get_ssh_options(config_file, option)
dcos_client = mesos.DCOSClient()
if leader:
host = mesos.MesosDNSClient().hosts('leader.mesos.')[0]['ip']
else:
summary = dcos_client.get_state_summary()
slave_obj = next((slave_ for slave_ in summary['slaves']
if slave_['id'] == slave),
None)
if slave_obj:
host = mesos.parse_pid(slave_obj['pid'])[1]
else:
raise DCOSException('No slave found with ID [{}]'.format(slave))
if command is None:
command = ''
master_public_ip = dcos_client.metadata().get('PUBLIC_IPV4')
if master_proxy:
if not master_public_ip:
raise DCOSException(("Cannot use --master-proxy. Failed to find "
"'PUBLIC_IPV4' at {}").format(
dcos_client.get_dcos_url('metadata')))
proxy_ip = master_public_ip
if proxy_ip:
if not os.environ.get('SSH_AUTH_SOCK'):
raise DCOSException(
"There is no SSH_AUTH_SOCK env variable, which likely means "
"you aren't running `ssh-agent`. `dcos node ssh "
"--master-proxy/--proxy-ip` depends on `ssh-agent` to safely "
"use your private key to hop between nodes in your cluster. "
"Please run `ssh-agent`, then add your private key with "
"`ssh-add`.")
cmd = "ssh -A -t {0}{1}@{2} ssh -A -t {0}{1}@{3} {4}".format(
ssh_options,
user,
proxy_ip,
host,
command)
else:
cmd = "ssh -t {0}{1}@{2} {3}".format(
ssh_options,
user,
host,
command)
emitter.publish(DefaultError("Running `{}`".format(cmd)))
if (not master_proxy and not proxy_ip) and master_public_ip:
emitter.publish(
DefaultError("If you are running this command from a separate "
"network than DC/OS, consider using "
"`--master-proxy` or `--proxy-ip`"))
return subprocess.Subproc().call(cmd, shell=True)
def uninstall(pkg, package_name, remove_all, app_id, cli, app):
"""Uninstalls a package.
:param pkg: package manager to uninstall with
:type pkg: PackageManager
:param package_name: The package to uninstall
:type package_name: str
:param remove_all: Whether to remove all instances of the named app
:type remove_all: boolean
:param app_id: App ID of the app instance to uninstall
:type app_id: str
:param cli: Whether to remove the CLI only
:type cli: boolean
:param app: Whether to remove app only
:type app: boolean
:rtype: None
"""
installed = installed_packages(pkg, None, package_name, cli_only=False)
installed_pkg = next(iter(installed), None)
if installed_pkg:
installed_cli = installed_pkg.get("command")
installed_app = installed_pkg.get("apps") or []
else:
msg = 'Package [{}]'.format(package_name)
if app_id is not None:
app_id = util.normalize_marathon_id_path(app_id)
msg += " with id [{}]".format(app_id)
msg += " is not installed"
raise DCOSException(msg)
# Having `app == True` means that the user supplied an explicit `--app`
# flag on the command line. Having `cli == True` means that the user
# supplied an explicit `--cli` flag on the command line. If either of these
# is `True`, run the following.
if app or cli:
# This forces an unconditional uninstall of the app associated with the
# supplied package (with different semantics depending on the values of
# `remove_all` and `app_id` as described in the docstring for this
# function).
if app and installed_app:
if not pkg.uninstall_app(package_name, remove_all, app_id):
raise DCOSException("Couldn't uninstall package")
# This forces an unconditional uninstall of the CLI associated with the
# supplied package.
if cli and installed_cli:
if not subcommand.uninstall(package_name):
raise DCOSException("Couldn't uninstall subcommand")
return
# Having both `app == False` and `cli == False` means that the user didn't
# supply either `--app` or `--cli` on the command line. In this situation
# we uninstall the app associated with the supplied package (if it exists)
# just as if the user had explicitly passed `--app` on the command line.
# However, we only uninstall the CLI associated with the package if the app
# being uninstalled is the last one remaining on the system. Otherwise, we
# leave the CLI in place so other instances of the app can continue to
# interact with it.
if installed_app:
if not pkg.uninstall_app(package_name, remove_all, app_id):
raise DCOSException("Couldn't uninstall package")
if installed_cli and (remove_all or len(installed_app) <= 1):
if not subcommand.uninstall(package_name):
raise DCOSException("Couldn't uninstall subcommand")
def _build(output_json, build_definition, output_directory):
""" Creates a DC/OS Package from a DC/OS Package Build Definition
:param output_json: whether to output json
:type output_json: None | bool
:param build_definition: The path to a DC/OS Package Build Definition
:type build_definition: str
:param output_directory: The directory where the DC/OS Package
will be stored
:type output_directory: str
:returns: The process status
:rtype: int
"""
# get the path of the build definition
cwd = os.getcwd()
build_definition_path = build_definition
if not os.path.isabs(build_definition_path):
build_definition_path = os.path.join(cwd, build_definition_path)
build_definition_directory = os.path.dirname(build_definition_path)
if not os.path.exists(build_definition_path):
raise DCOSException(
"The file [{}] does not exist".format(build_definition_path))
# get the path to the output directory
if output_directory is None:
output_directory = cwd
if not os.path.exists(output_directory):
raise DCOSException("The output directory [{}]"
" does not exist".format(output_directory))
logger.debug("Using [%s] as output directory", output_directory)
# load raw build definition
with util.open_file(build_definition_path) as bd:
build_definition_raw = util.load_json(bd, keep_order=True)
# validate DC/OS Package Build Definition with local references
build_definition_schema_path = "data/schemas/build-definition-schema.json"
build_definition_schema = util.load_jsons(
pkg_resources.resource_string("dcoscli",
build_definition_schema_path).decode())
errs = util.validate_json(build_definition_raw, build_definition_schema)
if errs:
logger.debug("Failed before resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_raw))
raise DCOSException(_validation_error(build_definition_path))
# resolve local references in build definition
_resolve_local_references(build_definition_raw, build_definition_schema,
build_definition_directory)
# at this point all the local references have been resolved
build_definition_resolved = build_definition_raw
# validate resolved build definition
metadata_schema_path = "data/schemas/metadata-schema.json"
metadata_schema = util.load_jsons(
pkg_resources.resource_string("dcoscli",
metadata_schema_path).decode())
errs = util.validate_json(build_definition_resolved, metadata_schema)
if errs:
logger.debug("Failed after resolution: \n"
"\tbuild definition: {}"
"".format(build_definition_resolved))
raise DCOSException('Error validating package: '
'there was a problem resolving '
'the local references in '
'[{}]'.format(build_definition_path))
# create the manifest
manifest_json = {'built-by': "dcoscli.version={}".format(dcoscli.version)}
# create the metadata
metadata_json = build_definition_resolved
# create zip file
with tempfile.NamedTemporaryFile() as temp_file:
with zipfile.ZipFile(temp_file.file,
mode='w',
compression=zipfile.ZIP_DEFLATED,
allowZip64=True) as zip_file:
metadata = json.dumps(metadata_json, indent=2).encode()
zip_file.writestr("metadata.json", metadata)
manifest = json.dumps(manifest_json, indent=2).encode()
zip_file.writestr("manifest.json", manifest)
# name the package appropriately
temp_file.file.seek(0)
dcos_package_name = '{}-{}-{}.dcos'.format(
metadata_json['name'], metadata_json['version'],
md5_hash_file(temp_file.file))
# get the dcos package path
dcos_package_path = os.path.join(output_directory, dcos_package_name)
if os.path.exists(dcos_package_path):
raise DCOSException(
'Output file [{}] already exists'.format(dcos_package_path))
# create a new file to contain the package
temp_file.file.seek(0)
with util.open_file(dcos_package_path, 'w+b') as dcos_package:
shutil.copyfileobj(temp_file.file, dcos_package)
if output_json:
message = {'package_path': dcos_package_path}
else:
message = 'Created DC/OS Universe Package [{}]'.format(
dcos_package_path)
emitter.publish(message)
return 0
def _no_file_exception():
return DCOSException('No files exist. Exiting.')
