def print_components(ip, use_json): """ Print components for a given node ip. The data is taked from 3dt endpoint: /system/health/v1/nodes/<ip>/units :param ip: DC/OS node ip address :type ip: str :param use_json: print components in json format :type use_json: bool """ dcos_url = config.get_config_val('core.dcos_url').rstrip("/") if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = dcos_url + '/system/health/v1/nodes/{}/units'.format(ip) response = http.get(url).json() if 'units' not in response: raise DCOSException( 'Invalid response. Missing field `units`. {}'.format(response)) if use_json: emitter.publish(response['units']) else: for component in response['units']: emitter.publish(component['id'])
def logging_strategy(): """ function returns logging strategy :return: logging strategy. :rtype: str """ # default strategy is sandbox logging. strategy = 'logrotate' if not has_journald_capability(): return strategy base_url = config.get_config_val("core.dcos_url") url = urllib.parse.urljoin(base_url, '/dcos-metadata/ui-config.json') if not base_url: raise config.missing_config_exception(['core.dcos_url']) try: response = http.get(url).json() except (DCOSAuthenticationException, DCOSAuthorizationException): raise except DCOSException: emitter.publish('Unable to determine logging mechanism for ' 'your cluster. Defaulting to files API.') return strategy try: strategy = response['uiConfiguration']['plugins']['mesos'][ 'logging-strategy'] # noqa: ignore=F403,E501 except Exception: pass return strategy
def _metrics(summary, task_id, json_): """ Get metrics from the specified task. :param summary: summarise output if true, output all if false :type summary: bool :param task_id: mesos task id :type task_id: str :param json: print raw JSON :type json: bool :return: Process status :rtype: int """ master = mesos.get_master() task = master.task(task_id) if 'slave_id' not in task: raise DCOSException( 'Error finding agent associated with task: {}'.format(task_id)) slave_id = task['slave_id'] container_id = master.get_container_id(task_id) endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format( slave_id, container_id) dcos_url = config.get_config_val('core.dcos_url').rstrip('/') if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = dcos_url + endpoint app_url = url + '/app' return metrics.print_task_metrics(url, app_url, summary, json_)
def _metrics(summary, task_id, json_): """ Get metrics from the specified task. :param summary: summarise output if true, output all if false :type summary: bool :param task_id: mesos task id :type task_id: str :param json: print raw JSON :type json: bool :return: Process status :rtype: int """ master = mesos.get_master() task = master.task(task_id) if 'slave_id' not in task: raise DCOSException( 'Error finding agent associated with task: {}'.format(task_id)) slave_id = task['slave_id'] container_id = master.get_container_id(task_id) endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format( slave_id, container_id ) dcos_url = config.get_config_val('core.dcos_url').rstrip('/') if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = dcos_url + endpoint app_url = url + '/app' return metrics.print_task_metrics(url, app_url, summary, json_)
def __init__(self): toml_config = config.get_config() self._dcos_url = toml_config.get("core.dcos_url") if self._dcos_url is None: raise config.missing_config_exception(['core.dcos_url']) self._mesos_master_url = toml_config.get('core.mesos_master_url') self._timeout = toml_config.get('core.timeout')
def __init__(self): toml_config = config.get_config() self._dcos_url = config.get_config_val("core.dcos_url", toml_config) if self._dcos_url is None: raise config.missing_config_exception(['core.dcos_url']) self._mesos_master_url = config.get_config_val('core.mesos_master_url', toml_config) self._timeout = config.get_config_val('core.timeout', toml_config)
def get_cosmos_url(): """ :returns: cosmos base url :rtype: str """ toml_config = config.get_config() cosmos_url = config.get_config_val("package.cosmos_url", toml_config) if cosmos_url is None: cosmos_url = config.get_config_val("core.dcos_url", toml_config) if cosmos_url is None: raise config.missing_config_exception(["core.dcos_url"]) return cosmos_url
def _dcos_log(follow, lines, leader, slave, component, filters): """ Print logs from dcos-log backend. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param leader: whether to print the leading master's log :type leader: bool :param slave: the slave ID to print :type slave: str | None :param component: DC/OS component name :type component: string :param filters: a list of filters ["key:value", ...] :type filters: list """ if not log.dcos_log_enabled(): raise DCOSException('dcos-log is not supported') filter_query = '' if component: filters.append('_SYSTEMD_UNIT:{}'.format(_get_unit_type(component))) for f in filters: key_value = f.split(':') if len(key_value) != 2: raise SystemExit('Invalid filter parameter {}. ' 'Must be --filter=key:value'.format(f)) filter_query += '&filter={}'.format(f) endpoint = '/system/v1' if leader: endpoint += '/logs/v1/' if slave: endpoint += '/agent/{}/logs/v1/'.format(slave) endpoint_type = 'range' if follow: endpoint_type = 'stream' dcos_url = config.get_config_val('core.dcos_url').rstrip("/") if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = (dcos_url + endpoint + endpoint_type + '/?skip_prev={}'.format(lines) + filter_query) if follow: return log.follow_logs(url) return log.print_logs_range(url)
def get_cosmos_url(): """ Gets the cosmos url :returns: cosmos base url :rtype: str """ toml_config = config.get_config() cosmos_url = config.get_config_val('package.cosmos_url', toml_config) if cosmos_url is None: cosmos_url = config.get_config_val('core.dcos_url', toml_config) if cosmos_url is None: raise config.missing_config_exception(['core.dcos_url']) return cosmos_url
def _dcos_log(follow, lines, leader, slave, component, filters): """ Print logs from dcos-log backend. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param leader: whether to print the leading master's log :type leader: bool :param slave: the slave ID to print :type slave: str | None :param component: DC/OS component name :type component: string :param filters: a list of filters ["key:value", ...] :type filters: list """ filter_query = '' if component: filters.append('_SYSTEMD_UNIT:{}'.format(_get_unit_type(component))) for f in filters: key_value = f.split(':') if len(key_value) != 2: raise SystemExit('Invalid filter parameter {}. ' 'Must be --filter=key:value'.format(f)) filter_query += '&filter={}'.format(f) endpoint = '/system/v1' if leader: endpoint += _build_leader_url(component) elif slave: endpoint += '/agent/{}/logs/v1/'.format(slave) endpoint_type = 'range' if follow: endpoint_type = 'stream' dcos_url = config.get_config_val('core.dcos_url').rstrip("/") if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = (dcos_url + endpoint + endpoint_type + '/?skip_prev={}'.format(lines) + filter_query) if follow: return log.follow_logs(url) return log.print_logs_range(url)
def _do_request(url, method, timeout=None, stream=False, **kwargs): """ make HTTP request :param url: url :type url: string :param method: HTTP method, GET or POST :type method: string :param timeout: HTTP request timeout, default 3 seconds :type timeout: integer :param stream: stream parameter for requests lib :type stream: bool :return: http response :rtype: requests.Response """ def _is_success(status_code): # consider 400 and 503 to be successful status codes. # API will return the error message. if status_code in [200, 400, 503]: return True return False # if timeout is not passed, try to read `core.timeout` # if `core.timeout` is not set, default to 3 min. if timeout is None: timeout = config.get_config_val('core.timeout') if not timeout: timeout = 180 base_url = config.get_config_val("core.dcos_url") if not base_url: raise config.missing_config_exception(['core.dcos_url']) url = urllib.parse.urljoin(base_url, url) if method.lower() == 'get': http_response = http.get(url, is_success=_is_success, timeout=timeout, **kwargs) elif method.lower() == 'post': http_response = http.post(url, is_success=_is_success, timeout=timeout, stream=stream, **kwargs) else: raise DCOSException('Unsupported HTTP method: ' + method) return http_response
def _get_default_base_url(): """ Gets the default service manager URL :returns: cosmos base url :rtype: str """ toml_config = config.get_config() base_url = config.get_config_val('package.cosmos_url', toml_config) if base_url is None: base_url = config.get_config_val('core.dcos_url', toml_config) if base_url is None: raise config.missing_config_exception(['core.dcos_url']) else: base_url = urllib.parse.urljoin(base_url, 'cosmos/') return base_url
def _get_marathon_url(toml_config): """ :param toml_config: configuration dictionary :type toml_config: config.Toml :returns: marathon base url :rtype: str """ marathon_url = config.get_config_val('marathon.url', toml_config) if marathon_url is None: dcos_url = config.get_config_val('core.dcos_url', toml_config) if dcos_url is None: raise config.missing_config_exception(['core.dcos_url']) marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/') return marathon_url
def _dcos_log_v2(follow, lines, leader, slave, component, filters): """ Print logs from dcos-log v2 backend. :param follow: same as unix tail's -f :type follow: bool :param lines: number of lines to print :type lines: int :param leader: whether to print the leading master's log :type leader: bool :param slave: the slave ID to print :type slave: str | None :param component: DC/OS component name :type component: string :param filters: a list of filters ["key:value", ...] :type filters: list """ filter_query = '' for f in filters: key_value = f.split(':') if len(key_value) != 2: raise DCOSException('Invalid filter parameter {}. ' 'Must be --filter=key:value'.format(f)) filter_query += '&filter={}'.format(f) endpoint = '/system/v1' if leader: endpoint += _build_leader_url(component, version=2) elif slave: endpoint += '/agent/{}/logs/v2/component'.format(slave) if component: component_with_type = _get_unit_type(component) endpoint += '/{}'.format(component_with_type) dcos_url = config.get_config_val('core.dcos_url').rstrip("/") if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) # dcos-log v2 required the skip option be a negative integer if lines > 0: lines *= -1 url = dcos_url + endpoint + '?skip={}'.format(lines) + filter_query if follow: return log.follow_logs(url) return log.print_logs_range(url)
def _do_request(url, method, timeout=None, stream=False, **kwargs): """ make HTTP request :param url: url :type url: string :param method: HTTP method, GET or POST :type method: string :param timeout: HTTP request timeout, default 3 seconds :type timeout: integer :param stream: stream parameter for requests lib :type stream: bool :return: http response :rtype: requests.Response """ def _is_success(status_code): # consider 400 and 503 to be successful status codes. # API will return the error message. if status_code in [200, 400, 503]: return True return False # if timeout is not passed, try to read `core.timeout` # if `core.timeout` is not set, default to 3 min. if timeout is None: timeout = config.get_config_val('core.timeout') if not timeout: timeout = 180 # POST to snapshot api base_url = config.get_config_val("core.dcos_url") if not base_url: raise config.missing_config_exception(['core.dcos_url']) url = urllib.parse.urljoin(base_url, url) if method.lower() == 'get': http_response = http.get(url, is_success=_is_success, timeout=timeout, **kwargs) elif method.lower() == 'post': http_response = http.post(url, is_success=_is_success, timeout=timeout, stream=stream, **kwargs) else: raise DCOSException('Unsupported HTTP method: ' + method) return http_response
def _get_metronome_url(toml_config=None): """ :param toml_config: configuration dictionary :type toml_config: config.Toml :returns: metronome base url :rtype: str """ if toml_config is None: toml_config = config.get_config() metronome_url = config.get_config_val('metronome.url', toml_config) if metronome_url is None: # dcos must be capable to use dcos_url _check_capability() dcos_url = config.get_config_val('core.dcos_url', toml_config) if dcos_url is None: raise config.missing_config_exception(['core.dcos_url']) metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/') return metronome_url
def _metrics(summary, mesos_id, json_): """ Get metrics from the specified agent. :param summary: summarise output if true, output all if false :type summary: bool :param mesos_id: mesos node id :type mesos_id: str :param json_: print raw JSON :type json_: bool :returns: Process status :rtype: int """ endpoint = '/system/v1/agent/{}/metrics/v0/node'.format(mesos_id) dcos_url = config.get_config_val('core.dcos_url').rstrip('/') if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) url = dcos_url + endpoint return metrics.print_node_metrics(url, summary, json_)
def _dcos_log_v2(follow, tasks, lines, file_): """ a client to dcos-log v2 :param follow: same as unix tail's -f :type follow: bool :param tasks: tasks pattern to match :type tasks: list of str :param lines: number of lines to print :type lines: int :param file_: file path to read :type file_: str """ if len(tasks) != 1: raise DCOSException( "found more than one task with the same name: {}. Please provide " "a unique task name.".format([task['id'] for task in tasks])) task = tasks[0] endpoint = '/system/v1/logs/v2/task/{}/file/{}'.format(task['id'], file_) dcos_url = config.get_config_val('core.dcos_url').rstrip('/') if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) if lines: # according to dcos-log v2 API the skip parameter must be negative # integer. dcos-cli uses positive int to limit the number of last lines # use "lines * -1" to make it negative. if lines > 0: lines *= -1 endpoint += '?cursor=END&skip={}'.format(lines) url = dcos_url + endpoint if follow: return log.follow_logs(url) return log.print_logs_range(url)
def _dcos_log(follow, tasks, lines, file_, completed): """ a client to dcos-log :param follow: same as unix tail's -f :type follow: bool :param task: task pattern to match :type task: str :param lines: number of lines to print :type lines: int :param file_: file path to read :type file_: str :param completed: whether to include completed tasks :type completed: bool """ # only stdout and stderr is supported if file_ not in ('stdout', 'stderr'): raise DCOSException('Expect file stdout or stderr. ' 'Got {}'.format(file_)) # state json may container tasks and completed_tasks fields. Based on # user request we should traverse the appropriate field. tasks_field = 'tasks' if completed: tasks_field = 'completed_tasks' for task in tasks: executor_info = task.executor() if not executor_info: continue if (tasks_field not in executor_info and not isinstance(executor_info[tasks_field], list)): logger.debug('Executor info: {}'.format(executor_info)) raise DCOSException('Invalid executor info. ' 'Missing field {}'.format(tasks_field)) for t in executor_info[tasks_field]: container_id = get_nested_container_id(t) if not container_id: logger.debug('Executor info: {}'.format(executor_info)) raise DCOSException( 'Invalid executor info. Missing container id') # get slave_id field slave_id = t.get('slave_id') if not slave_id: logger.debug('Executor info: {}'.format(executor_info)) raise DCOSException( 'Invalid executor info. Missing field `slave_id`') framework_id = t.get('framework_id') if not framework_id: logger.debug('Executor info: {}'.format(executor_info)) raise DCOSException( 'Invalid executor info. Missing field `framework_id`') # try `executor_id` first. executor_id = t.get('executor_id') if not executor_id: # if `executor_id` is an empty string, default to `id`. executor_id = t.get('id') if not executor_id: logger.debug('Executor info: {}'.format(executor_info)) raise DCOSException( 'Invalid executor info. Missing executor id') dcos_url = config.get_config_val('core.dcos_url').rstrip('/') if not dcos_url: raise config.missing_config_exception(['core.dcos_url']) # dcos-log provides 2 base endpoints /range/ and /stream/ # for range and streaming requests. endpoint_type = 'range' if follow: endpoint_type = 'stream' endpoint = ('/system/v1/agent/{}/logs/v1/{}/framework/{}' '/executor/{}/container/{}'.format(slave_id, endpoint_type, framework_id, executor_id, container_id)) # append request parameters. # `skip_prev` will move the cursor to -n lines. # `filter=STREAM:{STDOUT,STDERR}` will filter logs by label. url = (dcos_url + endpoint + '?skip_prev={}&filter=STREAM:{}'.format(lines, file_.upper())) if follow: return log.follow_logs(url) return log.print_logs_range(url)