def print_components(ip, use_json):
""" Print components for a given node ip.
The data is taked from 3dt endpoint:
/system/health/v1/nodes/<ip>/units
:param ip: DC/OS node ip address
:type ip: str
:param use_json: print components in json format
:type use_json: bool
"""
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + '/system/health/v1/nodes/{}/units'.format(ip)
response = http.get(url).json()
if 'units' not in response:
raise DCOSException(
'Invalid response. Missing field `units`. {}'.format(response))
if use_json:
emitter.publish(response['units'])
else:
for component in response['units']:
emitter.publish(component['id'])
def logging_strategy():
""" function returns logging strategy
:return: logging strategy.
:rtype: str
"""
# default strategy is sandbox logging.
strategy = 'logrotate'
if not has_journald_capability():
return strategy
base_url = config.get_config_val("core.dcos_url")
url = urllib.parse.urljoin(base_url, '/dcos-metadata/ui-config.json')
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
try:
response = http.get(url).json()
except (DCOSAuthenticationException, DCOSAuthorizationException):
raise
except DCOSException:
emitter.publish('Unable to determine logging mechanism for '
'your cluster. Defaulting to files API.')
return strategy
try:
strategy = response['uiConfiguration']['plugins']['mesos'][
'logging-strategy'] # noqa: ignore=F403,E501
except Exception:
pass
return strategy
def print_components(ip, use_json):
""" Print components for a given node ip.
The data is taked from 3dt endpoint:
/system/health/v1/nodes/<ip>/units
:param ip: DC/OS node ip address
:type ip: str
:param use_json: print components in json format
:type use_json: bool
"""
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + '/system/health/v1/nodes/{}/units'.format(ip)
response = http.get(url).json()
if 'units' not in response:
raise DCOSException(
'Invalid response. Missing field `units`. {}'.format(response))
if use_json:
emitter.publish(response['units'])
else:
for component in response['units']:
emitter.publish(component['id'])
def _metrics(summary, task_id, json_):
"""
Get metrics from the specified task.
:param summary: summarise output if true, output all if false
:type summary: bool
:param task_id: mesos task id
:type task_id: str
:param json: print raw JSON
:type json: bool
:return: Process status
:rtype: int
"""
master = mesos.get_master()
task = master.task(task_id)
if 'slave_id' not in task:
raise DCOSException(
'Error finding agent associated with task: {}'.format(task_id))
slave_id = task['slave_id']
container_id = master.get_container_id(task_id)
endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format(
slave_id, container_id)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
app_url = url + '/app'
return metrics.print_task_metrics(url, app_url, summary, json_)
def _metrics(summary, task_id, json_):
"""
Get metrics from the specified task.
:param summary: summarise output if true, output all if false
:type summary: bool
:param task_id: mesos task id
:type task_id: str
:param json: print raw JSON
:type json: bool
:return: Process status
:rtype: int
"""
master = mesos.get_master()
task = master.task(task_id)
if 'slave_id' not in task:
raise DCOSException(
'Error finding agent associated with task: {}'.format(task_id))
slave_id = task['slave_id']
container_id = master.get_container_id(task_id)
endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format(
slave_id, container_id
)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
app_url = url + '/app'
return metrics.print_task_metrics(url, app_url, summary, json_)
def __init__(self):
toml_config = config.get_config()
self._dcos_url = toml_config.get("core.dcos_url")
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = toml_config.get('core.mesos_master_url')
self._timeout = toml_config.get('core.timeout')
def __init__(self):
toml_config = config.get_config()
self._dcos_url = config.get_config_val("core.dcos_url", toml_config)
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = config.get_config_val('core.mesos_master_url',
toml_config)
self._timeout = config.get_config_val('core.timeout', toml_config)
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def _dcos_log(follow, lines, leader, slave, component, filters):
""" Print logs from dcos-log backend.
:param follow: same as unix tail's -f
:type follow: bool
:param lines: number of lines to print
:type lines: int
:param leader: whether to print the leading master's log
:type leader: bool
:param slave: the slave ID to print
:type slave: str | None
:param component: DC/OS component name
:type component: string
:param filters: a list of filters ["key:value", ...]
:type filters: list
"""
if not log.dcos_log_enabled():
raise DCOSException('dcos-log is not supported')
filter_query = ''
if component:
filters.append('_SYSTEMD_UNIT:{}'.format(_get_unit_type(component)))
for f in filters:
key_value = f.split(':')
if len(key_value) != 2:
raise SystemExit('Invalid filter parameter {}. '
'Must be --filter=key:value'.format(f))
filter_query += '&filter={}'.format(f)
endpoint = '/system/v1'
if leader:
endpoint += '/logs/v1/'
if slave:
endpoint += '/agent/{}/logs/v1/'.format(slave)
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = (dcos_url + endpoint + endpoint_type +
'/?skip_prev={}'.format(lines) + filter_query)
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def _dcos_log(follow, lines, leader, slave, component, filters):
""" Print logs from dcos-log backend.
:param follow: same as unix tail's -f
:type follow: bool
:param lines: number of lines to print
:type lines: int
:param leader: whether to print the leading master's log
:type leader: bool
:param slave: the slave ID to print
:type slave: str | None
:param component: DC/OS component name
:type component: string
:param filters: a list of filters ["key:value", ...]
:type filters: list
"""
filter_query = ''
if component:
filters.append('_SYSTEMD_UNIT:{}'.format(_get_unit_type(component)))
for f in filters:
key_value = f.split(':')
if len(key_value) != 2:
raise SystemExit('Invalid filter parameter {}. '
'Must be --filter=key:value'.format(f))
filter_query += '&filter={}'.format(f)
endpoint = '/system/v1'
if leader:
endpoint += _build_leader_url(component)
elif slave:
endpoint += '/agent/{}/logs/v1/'.format(slave)
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = (dcos_url + endpoint + endpoint_type +
'/?skip_prev={}'.format(lines) + filter_query)
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def _do_request(url, method, timeout=None, stream=False, **kwargs):
"""
make HTTP request
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:param timeout: HTTP request timeout, default 3 seconds
:type timeout: integer
:param stream: stream parameter for requests lib
:type stream: bool
:return: http response
:rtype: requests.Response
"""
def _is_success(status_code):
# consider 400 and 503 to be successful status codes.
# API will return the error message.
if status_code in [200, 400, 503]:
return True
return False
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
if timeout is None:
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = 180
base_url = config.get_config_val("core.dcos_url")
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
url = urllib.parse.urljoin(base_url, url)
if method.lower() == 'get':
http_response = http.get(url,
is_success=_is_success,
timeout=timeout,
**kwargs)
elif method.lower() == 'post':
http_response = http.post(url,
is_success=_is_success,
timeout=timeout,
stream=stream,
**kwargs)
else:
raise DCOSException('Unsupported HTTP method: ' + method)
return http_response
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def _get_marathon_url(toml_config):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: marathon base url
:rtype: str
"""
marathon_url = config.get_config_val('marathon.url', toml_config)
if marathon_url is None:
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/')
return marathon_url
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def _get_marathon_url(toml_config):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: marathon base url
:rtype: str
"""
marathon_url = config.get_config_val('marathon.url', toml_config)
if marathon_url is None:
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/')
return marathon_url
def _dcos_log_v2(follow, lines, leader, slave, component, filters):
""" Print logs from dcos-log v2 backend.
:param follow: same as unix tail's -f
:type follow: bool
:param lines: number of lines to print
:type lines: int
:param leader: whether to print the leading master's log
:type leader: bool
:param slave: the slave ID to print
:type slave: str | None
:param component: DC/OS component name
:type component: string
:param filters: a list of filters ["key:value", ...]
:type filters: list
"""
filter_query = ''
for f in filters:
key_value = f.split(':')
if len(key_value) != 2:
raise DCOSException('Invalid filter parameter {}. '
'Must be --filter=key:value'.format(f))
filter_query += '&filter={}'.format(f)
endpoint = '/system/v1'
if leader:
endpoint += _build_leader_url(component, version=2)
elif slave:
endpoint += '/agent/{}/logs/v2/component'.format(slave)
if component:
component_with_type = _get_unit_type(component)
endpoint += '/{}'.format(component_with_type)
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
# dcos-log v2 required the skip option be a negative integer
if lines > 0:
lines *= -1
url = dcos_url + endpoint + '?skip={}'.format(lines) + filter_query
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def _do_request(url, method, timeout=None, stream=False, **kwargs):
"""
make HTTP request
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:param timeout: HTTP request timeout, default 3 seconds
:type timeout: integer
:param stream: stream parameter for requests lib
:type stream: bool
:return: http response
:rtype: requests.Response
"""
def _is_success(status_code):
# consider 400 and 503 to be successful status codes.
# API will return the error message.
if status_code in [200, 400, 503]:
return True
return False
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
if timeout is None:
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = 180
# POST to snapshot api
base_url = config.get_config_val("core.dcos_url")
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
url = urllib.parse.urljoin(base_url, url)
if method.lower() == 'get':
http_response = http.get(url, is_success=_is_success, timeout=timeout,
**kwargs)
elif method.lower() == 'post':
http_response = http.post(url, is_success=_is_success, timeout=timeout,
stream=stream, **kwargs)
else:
raise DCOSException('Unsupported HTTP method: ' + method)
return http_response
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def _metrics(summary, mesos_id, json_):
""" Get metrics from the specified agent.
:param summary: summarise output if true, output all if false
:type summary: bool
:param mesos_id: mesos node id
:type mesos_id: str
:param json_: print raw JSON
:type json_: bool
:returns: Process status
:rtype: int
"""
endpoint = '/system/v1/agent/{}/metrics/v0/node'.format(mesos_id)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
return metrics.print_node_metrics(url, summary, json_)
def _metrics(summary, mesos_id, json_):
""" Get metrics from the specified agent.
:param summary: summarise output if true, output all if false
:type summary: bool
:param mesos_id: mesos node id
:type mesos_id: str
:param json_: print raw JSON
:type json_: bool
:returns: Process status
:rtype: int
"""
endpoint = '/system/v1/agent/{}/metrics/v0/node'.format(mesos_id)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
return metrics.print_node_metrics(url, summary, json_)
def _dcos_log_v2(follow, tasks, lines, file_):
""" a client to dcos-log v2
:param follow: same as unix tail's -f
:type follow: bool
:param tasks: tasks pattern to match
:type tasks: list of str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
"""
if len(tasks) != 1:
raise DCOSException(
"found more than one task with the same name: {}. Please provide "
"a unique task name.".format([task['id'] for task in tasks]))
task = tasks[0]
endpoint = '/system/v1/logs/v2/task/{}/file/{}'.format(task['id'], file_)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
if lines:
# according to dcos-log v2 API the skip parameter must be negative
# integer. dcos-cli uses positive int to limit the number of last lines
# use "lines * -1" to make it negative.
if lines > 0:
lines *= -1
endpoint += '?cursor=END&skip={}'.format(lines)
url = dcos_url + endpoint
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def _dcos_log(follow, tasks, lines, file_, completed):
""" a client to dcos-log
:param follow: same as unix tail's -f
:type follow: bool
:param task: task pattern to match
:type task: str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
:param completed: whether to include completed tasks
:type completed: bool
"""
# only stdout and stderr is supported
if file_ not in ('stdout', 'stderr'):
raise DCOSException('Expect file stdout or stderr. '
'Got {}'.format(file_))
# state json may container tasks and completed_tasks fields. Based on
# user request we should traverse the appropriate field.
tasks_field = 'tasks'
if completed:
tasks_field = 'completed_tasks'
for task in tasks:
executor_info = task.executor()
if not executor_info:
continue
if (tasks_field not in executor_info and
not isinstance(executor_info[tasks_field], list)):
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException('Invalid executor info. '
'Missing field {}'.format(tasks_field))
for t in executor_info[tasks_field]:
container_id = get_nested_container_id(t)
if not container_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing container id')
# get slave_id field
slave_id = t.get('slave_id')
if not slave_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `slave_id`')
framework_id = t.get('framework_id')
if not framework_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `framework_id`')
# try `executor_id` first.
executor_id = t.get('executor_id')
if not executor_id:
# if `executor_id` is an empty string, default to `id`.
executor_id = t.get('id')
if not executor_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing executor id')
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
# dcos-log provides 2 base endpoints /range/ and /stream/
# for range and streaming requests.
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
endpoint = ('/system/v1/agent/{}/logs/v1/{}/framework/{}'
'/executor/{}/container/{}'.format(slave_id,
endpoint_type,
framework_id,
executor_id,
container_id))
# append request parameters.
# `skip_prev` will move the cursor to -n lines.
# `filter=STREAM:{STDOUT,STDERR}` will filter logs by label.
url = (dcos_url + endpoint +
'?skip_prev={}&filter=STREAM:{}'.format(lines,
file_.upper()))
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def _dcos_log(follow, tasks, lines, file_, completed):
""" a client to dcos-log
:param follow: same as unix tail's -f
:type follow: bool
:param task: task pattern to match
:type task: str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
:param completed: whether to include completed tasks
:type completed: bool
"""
# only stdout and stderr is supported
if file_ not in ('stdout', 'stderr'):
raise DCOSException('Expect file stdout or stderr. '
'Got {}'.format(file_))
# state json may container tasks and completed_tasks fields. Based on
# user request we should traverse the appropriate field.
tasks_field = 'tasks'
if completed:
tasks_field = 'completed_tasks'
for task in tasks:
executor_info = task.executor()
if not executor_info:
continue
if (tasks_field not in executor_info and
not isinstance(executor_info[tasks_field], list)):
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException('Invalid executor info. '
'Missing field {}'.format(tasks_field))
for t in executor_info[tasks_field]:
container_id = get_nested_container_id(t)
if not container_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing container id')
# get slave_id field
slave_id = t.get('slave_id')
if not slave_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `slave_id`')
framework_id = t.get('framework_id')
if not framework_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `framework_id`')
# try `executor_id` first.
executor_id = t.get('executor_id')
if not executor_id:
# if `executor_id` is an empty string, default to `id`.
executor_id = t.get('id')
if not executor_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing executor id')
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
# dcos-log provides 2 base endpoints /range/ and /stream/
# for range and streaming requests.
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
endpoint = ('/system/v1/agent/{}/logs/v1/{}/framework/{}'
'/executor/{}/container/{}'.format(slave_id,
endpoint_type,
framework_id,
executor_id,
container_id))
# append request parameters.
# `skip_prev` will move the cursor to -n lines.
# `filter=STREAM:{STDOUT,STDERR}` will filter logs by label.
url = (dcos_url + endpoint +
'?skip_prev={}&filter=STREAM:{}'.format(lines,
file_.upper()))
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
