def _check_hash(filename, content_hashes): """Validates whether downloaded binary matches expected hash :param filename: path to binary :type filename: str :param content_hashes: list of hash algorithms/value :type content_hashes: [{"algo": <str>, "value": <str>}] :returns: None if valid hash, else throws exception :rtype: None """ content_hash = next( (contents for contents in content_hashes if contents.get("algo") == "sha256"), None) if content_hash: expected_value = content_hash.get("value") actual_value = _hashfile(filename) if expected_value != actual_value: raise DCOSException( "The hash for the downloaded subcommand [{}] " "does not match the expected value [{}]. Aborting...".format( actual_value, expected_value)) else: return else: raise DCOSException( "Hash algorithm specified is unsupported. " "Please contact the package maintainer. Aborting...")
def _install_cli(pkg, pkg_dir): """Install subcommand cli :param pkg: the package to install :type pkg: PackageVersion :param pkg_dir: directory to install package :type pkg_dir: str :rtype: None """ with util.remove_path_on_error(pkg_dir) as pkg_dir: env_dir = os.path.join(pkg_dir, constants.DCOS_SUBCOMMAND_ENV_SUBDIR) resources = pkg.resource_json() if resources and resources.get("cli") is not None: binary = resources["cli"] binary_cli = _get_cli_binary_info(binary) _install_with_binary(pkg.name(), env_dir, binary_cli) elif pkg.command_json() is not None: install_operation = pkg.command_json() if 'pip' in install_operation: _install_with_pip(pkg.name(), env_dir, install_operation['pip']) else: raise DCOSException( "Installation methods '{}' not supported".format( install_operation.keys())) else: raise DCOSException( "Could not find a CLI subcommand for your platform")
def unset(name): """ :param name: name of config value to unset :type name: str :returns: message of property removed :rtype: str """ toml_config = get_config(True) toml_config_pre = copy.deepcopy(toml_config) section = name.split(".", 1)[0] if section not in toml_config_pre._dictionary: toml_config_pre._dictionary[section] = {} value = toml_config.pop(name, None) if value is None: raise DCOSException("Property {!r} doesn't exist".format(name)) elif isinstance(value, collections.Mapping): raise DCOSException(_generate_choice_msg(name, value)) else: msg = "Removed [{}]".format(name) # dcos_acs_token is coupled to a specific dcos_url if name == "core.dcos_url": unset_token = bool(toml_config.pop("core.dcos_acs_token", None)) if unset_token: msg += " and [core.dcos_acs_token]" save(toml_config) return msg
def _get_cli_binary_info(cli_resources): """Find compatible cli binary, if one exists :param cli_resources: cli property of resource.json :type resources: {} :returns: {"url": <str>, "kind": <str>, "contentHash": [{}]} :rtype: {} | None """ if "binaries" in cli_resources: binaries = cli_resources["binaries"] arch = platform.architecture()[0] if arch != "64bit": raise DCOSException( "There is no compatible subcommand for your architecture [{}] " "We only support x86-64. Aborting...".format(arch)) system = platform.system().lower() binary = binaries.get(system) if binary is None: raise DCOSException( "There is not compatible subcommand for your system [{}] " "Aborting...".format(system)) elif "x86-64" in binary: return binary["x86-64"] raise DCOSException( "The CLI subcommand has unexpected format [{}]. " "Please contact the package maintainer. Aborting...".format( cli_resources))
def _output_thread(self): """Reads from the output_queue and writes the data to the appropriate STDOUT or STDERR. """ while True: # Get a message from the output queue and decode it. # Then write the data to the appropriate stdout or stderr. output = self.output_queue.get() if not output.get('data'): raise DCOSException("Error no 'data' field in output message") data = output['data'] data = base64.b64decode(data.encode('utf-8')) if output.get('type') and output['type'] == 'STDOUT': sys.stdout.buffer.write(data) sys.stdout.flush() elif output.get('type') and output['type'] == 'STDERR': sys.stderr.buffer.write(data) sys.stderr.flush() else: raise DCOSException("Unsupported data type in output stream") self.output_queue.task_done()
def enforce_file_permissions(path): """Enforce 400 or 600 permissions on file :param path: Path to the TOML file :type path: str :rtype: None """ if not os.path.isfile(path): raise DCOSException('Path [{}] is not a file'.format(path)) # Unix permissions are incompatible with windows # TODO: https://github.com/dcos/dcos-cli/issues/662 if sys.platform == 'win32': return else: permissions = oct(stat.S_IMODE(os.stat(path).st_mode)) if permissions not in ['0o600', '0600', '0o400', '0400']: if os.path.realpath(path) != path: path = '%s (pointed to by %s)' % (os.path.realpath(path), path) msg = ( "Permissions '{}' for configuration file '{}' are too open. " "File must only be accessible by owner. " "Aborting...".format(permissions, path)) raise DCOSException(msg)
def check_config(toml_config_pre, toml_config_post, section): """ :param toml_config_pre: dictionary for the value before change :type toml_config_pre: dcos.api.config.Toml :param toml_config_post: dictionary for the value with change :type toml_config_post: dcos.api.config.Toml :param section: section of the config to check :type section: str :returns: process status :rtype: int """ errors_pre = util.validate_json(toml_config_pre._dictionary[section], get_config_schema(section)) errors_post = util.validate_json(toml_config_post._dictionary[section], get_config_schema(section)) logger.info('Comparing changes in the configuration...') logger.info('Errors before the config command: %r', errors_pre) logger.info('Errors after the config command: %r', errors_post) if len(errors_post) != 0: if len(errors_pre) == 0: raise DCOSException(util.list_to_err(errors_post)) def _errs(errs): return set([e.split('\n')[0] for e in errs]) diff_errors = _errs(errors_post) - _errs(errors_pre) if len(diff_errors) != 0: raise DCOSException(util.list_to_err(errors_post))
def task(self, fltr, completed=False): """Returns the task with `fltr` in its ID. Raises a DCOSException if there is not exactly one such task. :param fltr: filter string :type fltr: str :returns: the task that has `fltr` in its ID :param completed: also include completed tasks :type completed: bool :rtype: Task """ tasks = self.tasks(fltr, completed) if len(tasks) == 0: raise DCOSException( 'Cannot find a task with ID containing "{}"'.format(fltr)) elif len(tasks) > 1: msg = [("There are multiple tasks with ID matching [{}]. " + "Please choose one:").format(fltr)] msg += ["\t{0}".format(t["id"]) for t in tasks] raise DCOSException('\n'.join(msg)) else: return tasks[0]
def slave(self, fltr): """Returns the slave that has `fltr` in its ID. If any slaves are an exact match, returns that task, id not raises a DCOSException if there is not exactly one such slave. :param fltr: filter string :type fltr: str :returns: the slave that has `fltr` in its ID :rtype: Slave """ slaves = self.slaves(fltr) if len(slaves) == 0: raise DCOSException('No agent found with ID "{}".'.format(fltr)) elif len(slaves) > 1: exact_matches = [s for s in slaves if s['id'] == fltr] if len(exact_matches) == 1: return exact_matches[0] else: matches = ['\t{0}'.format(s['id']) for s in slaves] raise DCOSException( "There are multiple agents with that ID. " + "Please choose one:\n{}".format('\n'.join(matches))) else: return slaves[0]
def command_executables(subcommand): """List the real path to executable dcos program for specified subcommand. :param subcommand: name of subcommand. E.g. marathon :type subcommand: str :returns: the dcos program path :rtype: str """ executables = [] if subcommand in default_subcommands(): executables += [default_list_paths()] executables += [ command_path for command_path in list_paths() if noun(command_path) == subcommand ] if len(executables) > 1: msg = 'Found more than one executable for command {!r}. {!r}' raise DCOSException(msg.format(subcommand, executables)) if len(executables) == 0: msg = "{!r} is not a dcos command." raise DCOSException(msg.format(subcommand)) return executables[0]
def get_config_schema(command): """ :param command: the subcommand name :type command: str :returns: the subcommand's configuration schema :rtype: dict """ # import here to avoid circular import from dcos.subcommand import ( command_executables, config_schema, default_subcommands) # handle config schema for core.* properties and built-in subcommands if command == "core" or command in default_subcommands(): try: schema = pkg_resources.resource_string( 'dcos', 'data/config-schema/{}.json'.format(command)) except FileNotFoundError: msg = "Subcommand '{}' is not configurable.".format(command) raise DCOSException(msg) return json.loads(schema.decode('utf-8')) try: executable = command_executables(command) except DCOSException as e: msg = "Config section '{}' is invalid: {}".format(command, e) raise DCOSException(msg) return config_schema(executable, command)
def decode(self, data): """Decode a 'RecordIO' formatted message to its original type. :param data: an array of 'UTF-8' encoded bytes that make up a partial 'RecordIO' message. Subsequent calls to this function maintain state to build up a full 'RecordIO' message and decode it :type data: bytes :returns: a list of deserialized messages :rtype: list """ if not isinstance(data, bytes): raise DCOSException("Parameter 'data' must of of type 'bytes'") if self.state == self.FAILED: raise DCOSException("Decoder is in a FAILED state") records = [] for c in data: if self.state == self.HEADER: if c != ord('\n'): self.buffer += bytes([c]) continue try: self.length = int(self.buffer.decode("UTF-8")) except Exception as exception: self.state = self.FAILED raise DCOSException("Failed to decode length" "'{buffer}': {error}".format( buffer=self.buffer, error=exception)) self.buffer = bytes("", "UTF-8") self.state = self.RECORD # Note that for 0 length records, we immediately decode. if self.length <= 0: records.append(self.deserialize(self.buffer)) self.state = self.HEADER elif self.state == self.RECORD: assert self.length assert len(self.buffer) < self.length self.buffer += bytes([c]) if len(self.buffer) == self.length: records.append(self.deserialize(self.buffer)) self.buffer = bytes("", "UTF-8") self.state = self.HEADER return records
def run(self): """Run the helper threads in this class which enable streaming of STDIN/STDOUT/STDERR between the CLI and the Mesos Agent API. If a tty is requested, we take over the current terminal and put it into raw mode. We make sure to reset the terminal back to its original settings before exiting. """ # Without a TTY. if not self.tty: try: self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e if self.exception: raise self.exception return # With a TTY. if util.is_windows_platform(): raise DCOSException( "Running with the '--tty' flag is not supported on windows.") if not sys.stdin.isatty(): raise DCOSException( "Must be running in a tty to pass the '--tty flag'.") fd = sys.stdin.fileno() oldtermios = termios.tcgetattr(fd) try: if self.interactive: tty.setraw(fd, when=termios.TCSANOW) self._window_resize(signal.SIGWINCH, None) signal.signal(signal.SIGWINCH, self._window_resize) self._start_threads() self.exit_event.wait() except Exception as e: self.exception = e termios.tcsetattr( sys.stdin.fileno(), termios.TCSAFLUSH, oldtermios) if self.exception: raise self.exception
def start_service(self, package_name, package_version, options): """ Starts a service that has been added to the cluster via cosmos' package/add endpoint. :param package_name: the name of the package to start :type package_name: str :param package_version: the version of the package to start :type package_version: None | str :param options: the options for the service :type options: None | dict :return: the response of cosmos' service/start endpoint :rtype: requests.Response """ endpoint = 'service/start' json = {'packageName': package_name} if package_version is not None: json['packageVersion'] = package_version if options is not None: json['options'] = options try: return self.cosmos.call_endpoint(endpoint, json=json) except (DCOSAuthenticationException, DCOSAuthorizationException): raise except DCOSHTTPException as e: if e.status() == 404: message = 'Your version of DC/OS ' \ 'does not support this operation' raise DCOSException(message) else: return e.response
def auth_type_description(provider_info): """ Returns human readable description of auth type :param provider_info: info about auth provider :type provider_info: dict :returns: human readable description of auth type :rtype: str """ auth_type = provider_info.get("authentication-type") if auth_type == "dcos-uid-password": msg = ("Authenticate using a standard DC/OS user account " "(using username and password)") elif auth_type == "dcos-uid-servicekey": msg = ("Authenticate using a DC/OS service user account " "(using username and private key)") elif auth_type == "dcos-uid-password-ldap": msg = ("Authenticate using an LDAP user account " "(using username and password)") elif auth_type == "saml-sp-initiated": msg = "Authenticate using SAML 2.0 ({})".format( provider_info["description"]) elif auth_type in ["oidc-authorization-code-flow", "oidc-implicit-flow"]: msg = "Authenticate using OpenID Connect ({})".format( provider_info["description"]) else: raise DCOSException("Unknown authentication type") return msg
def browser_prompt_auth(dcos_url, provider_info): """ Get DC/OS Authentication token by browser prompt :param dcos_url: url to cluster :type dcos_url: str :param provider_info: info about provider to auth with :param provider_info: str :rtype: None """ start_flow_url = provider_info["config"]["start_flow_url"].lstrip('/') if not urlparse(start_flow_url).netloc: start_flow_url = dcos_url.rstrip('/') + start_flow_url dcos_token = _prompt_user_for_token( start_flow_url, "DC/OS Authentication Token") # verify token endpoint = '/pkgpanda/active.buildinfo.full.json' url = urllib.parse.urljoin(dcos_url, endpoint) response = http._request('HEAD', url, auth=http.DCOSAcsAuth(dcos_token)) if response.status_code in [200, 403]: config.set_val("core.dcos_acs_token", dcos_token) else: raise DCOSException("Authentication failed")
def configure_logger(log_level): """Configure the program's logger. :param log_level: Log level for configuring logging :type log_level: str :rtype: None """ if log_level is None: logging.disable(logging.CRITICAL) return None if log_level in constants.VALID_LOG_LEVEL_VALUES: logging.basicConfig( format=('%(threadName)s: ' '%(asctime)s ' '%(pathname)s:%(funcName)s:%(lineno)d - ' '%(message)s'), stream=sys.stderr, level=log_level.upper()) return None msg = 'Log level set to an unknown value {!r}. Valid values are {!r}' raise DCOSException( msg.format(log_level, constants.VALID_LOG_LEVEL_VALUES))
def _update(self, resource_type, resource_id, resource_json, force=False): """Update an application or group. The HTTP response is handled differently for pods; see `update_pod`. :param resource_type: either 'apps' or 'groups' :type resource_type: str :param resource_id: the app or group ID :type resource_id: str :param resource_json: the json payload :type resource_json: {} :param force: whether to override running deployments :type force: bool :returns: the resulting deployment ID :rtype: str """ response = self._update_req( resource_type, resource_id, resource_json, force) body_json = self._parse_json(response) try: return body_json.get('deploymentId') except KeyError: template = ('Error: missing "deploymentId" field in the following ' 'JSON response from Job:\n{}') rendered_json = json.dumps(body_json, indent=2, sort_keys=True) raise DCOSException(template.format(rendered_json))
def _process_output_stream(self, response): """Gets data streamed over the given response and places the returned messages into our output_queue. Only expects to receive data messages. :param response: Response from an http post :type response: requests.models.Response """ # Now that we are ready to process the output stream (meaning # our output connection has been established), allow the input # stream to be attached by setting an event. self.attach_input_event.set() # If we are running in interactive mode, wait to make sure that # our input connection succeeds before pushing any output to the # output queue. if self.interactive: self.print_output_event.wait() try: for chunk in response.iter_content(chunk_size=None): records = self.decoder.decode(chunk) for r in records: if r.get('type') and r['type'] == 'DATA': self.output_queue.put(r['data']) except Exception as e: raise DCOSException( "Error parsing output stream: {error}".format(error=e)) self.output_queue.join() self.exit_event.set()
def _get_auth_scheme(response): """Return authentication scheme requested by server for 'acsjwt' (DC/OS acs auth), 'oauthjwt' (DC/OS acs oauth), or None (no auth) :param response: requests.response :type response: requests.Response :returns: auth_scheme :rtype: str | None """ if 'WWW-Authenticate' in response.headers: auths = response.headers['WWW-Authenticate'].split(',') scheme = next((auth_type.rstrip().lower() for auth_type in auths if auth_type.rstrip().lower().startswith("acsjwt") or auth_type.rstrip().lower().startswith("oauthjwt")), None) if scheme: scheme_info = scheme.split("=") auth_scheme = scheme_info[0].split(" ")[0].lower() return auth_scheme else: msg = ("Server responded with an HTTP 'www-authenticate' field of " "'{}', DC/OS only supports ['oauthjwt', 'acsjwt']".format( response.headers['WWW-Authenticate'])) raise DCOSException(msg) else: logger.debug("HTTP response: no www-authenticate field found") return
def servicecred_auth(dcos_url, username, key_path): """ Get DC/OS Authentication token by browser prompt :param dcos_url: url to cluster :type dcos_url: str :param username: username user for authentication :type username: str :param key_path: path to service key :param key_path: str :rtype: None """ # 'token' below contains a short lived service login token. This requires # the local machine to be in sync with DC/OS nodes enough that the 5min # padding here is enough time to validate the token. creds = { 'uid': username, 'token': jwt.encode( { 'exp': int(time.time()+5*60), 'uid': username }, util.read_file_secure(key_path), algorithm='RS256') .decode('ascii') } dcos_token = _get_dcostoken_by_post_with_creds(dcos_url, creds) if not dcos_token: raise DCOSException("Authentication failed") else: return
def wait_for_service_tasks_all_unchanged(service_name, old_task_ids, task_predicate=None, timeout_sec=30): """ Returns after verifying that NONE of old_task_ids have been removed or replaced from the service :param service_name: the service name :type service_name: str :param old_task_ids: list of original task ids as returned by get_service_task_ids :type old_task_ids: [str] :param task_predicate: filter to use when searching for tasks :type task_predicate: func :param timeout_sec: duration to wait until assuming tasks are unchanged :type timeout_sec: int :return: the duration waited in seconds (the timeout value) :rtype: int """ try: time_wait(lambda: tasks_missing_predicate(service_name, old_task_ids, task_predicate), timeout_seconds=timeout_sec) # shouldn't have exited successfully: raise below except TimeoutExpired: return timeout_sec # no changes occurred within timeout, as expected raise DCOSException( "One or more of the following tasks were no longer found: {}".format( old_task_ids))
def get_resource(resource): """:param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is None: return None if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: http.silence_requests_warnings() req = http.get(resource) if req.status_code == 200: data = b'' for chunk in req.iter_content(1024): data += chunk return util.load_jsons(data.decode('utf-8')) else: raise Exception except Exception: raise DCOSException( "Can't read from resource: {0}. Please check that it exists.". format(resource))
def get_resource(resource): """:param resource: optional filename or http(s) url for the application or group resource :type resource: str :returns: resource :rtype: dict """ if resource is None: return None if os.path.isfile(resource): with util.open_file(resource) as resource_file: return util.load_json(resource_file) else: try: auth = DCOSAcsAuth(dcos_acs_token()) req = requests.get(resource, auth=auth, verify=verify_ssl()) if req.status_code == 200: return req.json() else: raise Exception except Exception: raise DCOSException( "Can't read from resource: {0}. Please check that it exists.". format(resource))
def get_app_versions(self, app_id, max_count=None): """Asks Marathon for all the versions of the Application up to a maximum count. :param app_id: the ID of the application or group :type app_id: str :param max_count: the maximum number of version to fetch :type max_count: int :returns: a list of all the version of the application :rtype: [str] """ if max_count is not None and max_count <= 0: raise DCOSException( 'Maximum count must be a positive number: {}'.format( max_count)) app_id = util.normalize_marathon_id_path(app_id) path = 'v2/apps{}/versions'.format(app_id) response = self._rpc.http_req(http.get, path) if max_count is None: return response.json().get('versions') else: return response.json().get('versions')[:max_count]
def __call__(self, value): """ :param value: String to try and parse :type value: str :returns: The parse value :rtype: str | int | float | bool | list | dict """ value = clean_value(value) if self.schema['type'] == 'string': if self.schema.get('format') == 'uri': return _parse_url(value) else: return _parse_string(value) elif self.schema['type'] == 'object': return _parse_object(value) elif self.schema['type'] == 'number': return _parse_number(value) elif self.schema['type'] == 'integer': return _parse_integer(value) elif self.schema['type'] == 'boolean': return _parse_boolean(value) elif self.schema['type'] == 'array': return _parse_array(value) else: raise DCOSException('Unknown type {!r}'.format(self._value_type))
def _parse_url(value): """ :param value: The url to parse :type url: str :returns: The parsed value :rtype: str """ scheme_pattern = r'^(?P<scheme>(?:(?:https?)://))' domain_pattern = ( r'(?P<hostname>(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.?)+' '(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)?|') # domain, value_regex = re.match( scheme_pattern + # http:// or https:// r'(([^:])+(:[^:]+)?@){0,1}' + # auth credentials domain_pattern + r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}))' # or ip r'(?P<port>(?::\d+))?' # port r'(?P<path>(?:/?|[/?]\S+))$', # resource path value, re.IGNORECASE) if value_regex is None: scheme_match = re.match(scheme_pattern, value, re.IGNORECASE) if scheme_match is None: logger.debug("Defaulting URL to https scheme") return "https://" + value else: raise DCOSException('Unable to parse {!r} as a url'.format(value)) else: return value
def parse_json_item(json_item, schema): """Parse the json item (optionally based on a schema). :param json_item: A JSON item in the form 'key=value' :type json_item: str :param schema: The JSON schema to use for parsing :type schema: dict | None :returns: A tuple for the parsed JSON item :rtype: (str, any) where any is one of str, int, float, bool, list or dict """ terms = json_item.split('=', 1) if len(terms) != 2: raise DCOSException('{!r} is not a valid json-item'.format(json_item)) # Check that it is a valid key in our jsonschema key = terms[0] # Use the schema if we have it else, guess the type if schema: value = parse_json_value(key, terms[1], schema) else: value = _find_type(clean_value(terms[1])) return (json.dumps(key), value)
def _install_with_pip(package_name, env_directory, requirements): """ :param package_name: the name of the package :type package_name: str :param env_directory: the path to the directory in which to install the package's virtual env :type env_directory: str :param requirements: the list of pip requirements :type requirements: list of str :rtype: None """ bin_directory = util.dcos_bin_path() new_package_dir = not os.path.exists(env_directory) pip_path = os.path.join(env_directory, BIN_DIRECTORY, 'pip') if not os.path.exists(pip_path): virtualenv_path = _find_virtualenv(bin_directory) virtualenv_version = _execute_command([virtualenv_path, '--version' ])[0].strip().decode('utf-8') if LooseVersion("12") > LooseVersion(virtualenv_version): msg = ("Unable to install CLI subcommand. " "Required program 'virtualenv' must be version 12+, " "currently version {}\n" "Please see installation instructions: " "https://virtualenv.pypa.io/en/latest/installation.html" "".format(virtualenv_version)) raise DCOSException(msg) cmd = [_find_virtualenv(bin_directory), env_directory] if _execute_command(cmd)[2] != 0: raise _generic_error(package_name) # Do not replace util.temptext NamedTemporaryFile # otherwise bad things will happen on Windows with util.temptext() as text_file: fd, requirement_path = text_file # Write the requirements to the file with os.fdopen(fd, 'w') as requirements_file: for line in requirements: print(line, file=requirements_file) cmd = [ os.path.join(env_directory, BIN_DIRECTORY, 'pip'), 'install', '--requirement', requirement_path, ] if _execute_command(cmd)[2] != 0: # We should remove the directory that we just created if new_package_dir: shutil.rmtree(env_directory) raise _generic_error(package_name) return None
def is_enterprise_cli_package_installed(): """Returns `True` if `dcos-enterprise-cli` package is installed.""" stdout, stderr, return_code = shakedown.run_dcos_command('package list --json') logger.info('package list command returned code:{}, stderr:{}, stdout: {}'.format(return_code, stderr, stdout)) try: result_json = json.loads(stdout) except JSONDecodeError as error: raise DCOSException('Could not parse: "{}"'.format(stdout))(error) return any(cmd['name'] == 'dcos-enterprise-cli' for cmd in result_json)