def _get_dcos_auth(auth_scheme, username, password, hostname):
"""Get authentication flow for dcos acs auth and dcos oauth
:param auth_scheme: authentication_scheme
:type auth_scheme: str
:param username: username user for authentication
:type username: str
:param password: password for authentication
:type password: str
:param hostname: hostname for credentials
:type hostname: str
:returns: DCOSAcsAuth
:rtype: AuthBase
"""
toml_config = config.get_config()
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:
creds = _get_dcos_oauth_creds(dcos_url)
url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login')
response = _request('post', url, json=creds)
if response.status_code == 200:
token = response.json()['token']
config.set_val("core.dcos_acs_token", token)
return DCOSAcsAuth(token)
def _is_request_to_dcos(url, toml_config=None):
"""Checks if a request is for the DC/OS cluster.
:param url: URL of the request
:type url: str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether the request is for the cluster
:rtype: bool
"""
if toml_config is None:
toml_config = config.get_config()
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
cosmos_url = urlparse(
config.get_config_val("package.cosmos_url", toml_config))
parsed_url = urlparse(url)
# request should match scheme + netloc
def _request_match(expected_url, actual_url):
return expected_url.scheme == actual_url.scheme and \
expected_url.netloc == actual_url.netloc
is_request_to_cluster = _request_match(dcos_url, parsed_url) or \
_request_match(cosmos_url, parsed_url)
return is_request_to_cluster
def request(method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
toml_config = config.get_config()
auth_token = config.get_config_val("core.dcos_acs_token", toml_config)
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
parsed_url = urlparse(url)
# only request with DC/OS Auth if request is to DC/OS cluster
# request should match scheme + netloc
scheme_eq = parsed_url.scheme == dcos_url.scheme
netloc_eq = parsed_url.netloc == dcos_url.netloc
if auth_token and scheme_eq and netloc_eq:
auth = DCOSAcsAuth(auth_token)
else:
auth = None
response = _request(method, url, is_success, timeout,
auth=auth, verify=verify, **kwargs)
if is_success(response.status_code):
return response
elif response.status_code == 401:
if auth_token is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSAuthenticationException(msg)
else:
raise DCOSAuthenticationException(response)
elif response.status_code == 422:
raise DCOSUnprocessableException(response)
elif response.status_code == 403:
raise DCOSAuthorizationException(response)
elif response.status_code == 400:
raise DCOSBadRequest(response)
else:
raise DCOSHTTPException(response)
def __init__(self):
toml_config = config.get_config()
self._dcos_url = config.get_config_val("core.dcos_url", toml_config)
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = config.get_config_val(
'core.mesos_master_url', toml_config)
self._timeout = config.get_config_val('core.timeout', toml_config)
def __init__(self):
toml_config = config.get_config()
self._dcos_url = config.get_config_val("core.dcos_url", toml_config)
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = config.get_config_val('core.mesos_master_url',
toml_config)
self._timeout = config.get_config_val('core.timeout', toml_config)
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def _do_request(url, method, timeout=None, stream=False, **kwargs):
"""
make HTTP request
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:param timeout: HTTP request timeout, default 3 seconds
:type timeout: integer
:param stream: stream parameter for requests lib
:type stream: bool
:return: http response
:rtype: requests.Response
"""
def _is_success(status_code):
# consider 400 and 503 to be successful status codes.
# API will return the error message.
if status_code in [200, 400, 503]:
return True
return False
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
if timeout is None:
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = 180
base_url = config.get_config_val("core.dcos_url")
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
url = urllib.parse.urljoin(base_url, url)
if method.lower() == 'get':
http_response = http.get(url,
is_success=_is_success,
timeout=timeout,
**kwargs)
elif method.lower() == 'post':
http_response = http.post(url,
is_success=_is_success,
timeout=timeout,
stream=stream,
**kwargs)
else:
raise DCOSException('Unsupported HTTP method: ' + method)
return http_response
def overlay_info(self):
# XXX This methodology is copied from the dcos.mesos package, this can
# be removed once the upstream package has support for this field.
url = self.dcos_client.master_url('overlay-agent/overlay')
timeout = dcosconfig.get_config_val('core.timeout', dcosconfig.get_config())
return dcoshttp.get(url, timeout=timeout).json()
def get_arangodb_framework(name):
url = config.get_config_val('core.dcos_url') + "/mesos/state.json"
try:
response = http.get(url, timeout=15)
except DCOSException as e:
print("cannot connect to '" + url + "'", e)
sys.exit(1)
if response.status_code >= 200 and response.status_code < 300:
json = response.json()
if 'frameworks' not in json:
print(json)
sys.exit(1)
frameworks = json['frameworks']
for framework in frameworks:
if name == framework['name']:
return framework
print("ArangoDB framework '" + name + "' is not running yet.")
sys.exit(1)
else:
print("Bad response getting master state. Status code: "
+ str(response.status_code))
sys.exit(1)
def get_ssh_user(ssh_config_file, user):
"""Returns the SSH user name to use accessing cluster nodes. If an ssh
config file provided, expects the username to be specified in the
file and ignores any provided username. Username resolution
follows this logic, use any username explicitly provided, else use
the one in dcos cli config, else use default ssh user from constants.
:param ssh_config_file: SSH options
:type ssh_config_file: [str]
:param user: SSH user
:type user: str | None
:rtype: str | None
"""
if ssh_config_file:
return None
if user:
return user
dcos_config_ssh_user = config.get_config_val("core.ssh_user")
if dcos_config_ssh_user:
return dcos_config_ssh_user
return constants.DEFAULT_SSH_USER
def print_components(ip, use_json):
""" Print components for a given node ip.
The data is taked from 3dt endpoint:
/system/health/v1/nodes/<ip>/units
:param ip: DC/OS node ip address
:type ip: str
:param use_json: print components in json format
:type use_json: bool
"""
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + '/system/health/v1/nodes/{}/units'.format(ip)
response = http.get(url).json()
if 'units' not in response:
raise DCOSException(
'Invalid response. Missing field `units`. {}'.format(response))
if use_json:
emitter.publish(response['units'])
else:
for component in response['units']:
emitter.publish(component['id'])
def _metrics(summary, task_id, json_):
"""
Get metrics from the specified task.
:param summary: summarise output if true, output all if false
:type summary: bool
:param task_id: mesos task id
:type task_id: str
:param json: print raw JSON
:type json: bool
:return: Process status
:rtype: int
"""
master = mesos.get_master()
task = master.task(task_id)
if 'slave_id' not in task:
raise DCOSException(
'Error finding agent associated with task: {}'.format(task_id))
slave_id = task['slave_id']
container_id = master.get_container_id(task_id)
endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format(
slave_id, container_id
)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
app_url = url + '/app'
return metrics.print_task_metrics(url, app_url, summary, json_)
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def print_components(ip, use_json):
""" Print components for a given node ip.
The data is taked from 3dt endpoint:
/system/health/v1/nodes/<ip>/units
:param ip: DC/OS node ip address
:type ip: str
:param use_json: print components in json format
:type use_json: bool
"""
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + '/system/health/v1/nodes/{}/units'.format(ip)
response = http.get(url).json()
if 'units' not in response:
raise DCOSException(
'Invalid response. Missing field `units`. {}'.format(response))
if use_json:
emitter.publish(response['units'])
else:
for component in response['units']:
emitter.publish(component['id'])
def logging_strategy():
""" function returns logging strategy
:return: logging strategy.
:rtype: str
"""
# default strategy is sandbox logging.
strategy = 'logrotate'
if not has_journald_capability():
return strategy
base_url = config.get_config_val("core.dcos_url")
url = urllib.parse.urljoin(base_url, '/dcos-metadata/ui-config.json')
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
try:
response = http.get(url).json()
except (DCOSAuthenticationException, DCOSAuthorizationException):
raise
except DCOSException:
emitter.publish('Unable to determine logging mechanism for '
'your cluster. Defaulting to files API.')
return strategy
try:
strategy = response['uiConfiguration']['plugins']['mesos'][
'logging-strategy'] # noqa: ignore=F403,E501
except Exception:
pass
return strategy
def _main():
signal.signal(signal.SIGINT, signal_handler)
http.silence_requests_warnings()
args = docopt.docopt(default_doc("dcos"), options_first=True)
log_level = args['--log-level']
if log_level and not _config_log_level_environ(log_level):
return 1
if args['--debug']:
os.environ[constants.DCOS_DEBUG_ENV] = 'true'
util.configure_process_from_environ()
if args['--version']:
return _get_versions(config.get_config_val("core.dcos_url"))
command = args['<command>']
if not command:
command = "help"
if command in subcommand.default_subcommands():
sc = SubcommandMain(command, args['<args>'])
else:
executable = subcommand.command_executables(command)
sc = subcommand.SubcommandProcess(
executable, command, args['<args>'])
exitcode, _ = sc.run_and_capture()
return exitcode
def _metrics(summary, task_id, json_):
"""
Get metrics from the specified task.
:param summary: summarise output if true, output all if false
:type summary: bool
:param task_id: mesos task id
:type task_id: str
:param json: print raw JSON
:type json: bool
:return: Process status
:rtype: int
"""
master = mesos.get_master()
task = master.task(task_id)
if 'slave_id' not in task:
raise DCOSException(
'Error finding agent associated with task: {}'.format(task_id))
slave_id = task['slave_id']
container_id = master.get_container_id(task_id)
endpoint = '/system/v1/agent/{}/metrics/v0/containers/{}'.format(
slave_id, container_id)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
app_url = url + '/app'
return metrics.print_task_metrics(url, app_url, summary, json_)
def __init__(self, toml_config: Optional[Any] = None):
super(DCOSClient, self).__init__()
self.toml_config = toml_config
if toml_config is None:
self.toml_config = config.get_config()
self._dcos_url = cast(ParseResult, urlparse(config.get_config_val("core.dcos_url", toml_config)))
def _login():
"""
:returns: process status
:rtype: int
"""
# every call to login will generate a new token if applicable
_logout()
dcos_url = config.get_config_val("core.dcos_url")
if dcos_url is None:
msg = ("Please provide the url to your DC/OS cluster: "
"`dcos config set core.dcos_url`")
raise DCOSException(msg)
# hit protected endpoint which will prompt for auth if cluster has auth
try:
url = urllib.parse.urljoin(dcos_url, 'exhibitor')
http.get(url)
# if the user is authenticated, they have effectively "logged in" even if
# they are not authorized for this endpoint
except DCOSAuthorizationException:
pass
emitter.publish("Login successful!")
return 0
def _page(output, pager_command=None):
"""Conditionally pipes the supplied output through a pager.
:param output:
:type output: object
:param pager_command:
:type pager_command: str
"""
output = six.text_type(output)
if not sys.stdout.isatty() or util.is_windows_platform():
print(output)
return
num_lines = output.count('\n')
exceeds_tty_height = pager.getheight() - 1 < num_lines
if pager_command is None:
pager_command = 'less -R'
paginate = config.get_config_val("core.pagination") or True
if exceeds_tty_height and paginate and \
spawn.find_executable(pager_command.split(' ')[0]) is not None:
pydoc.pipepager(output, cmd=pager_command)
else:
print(output)
def _verify_ssl(url, verify=None, toml_config=None):
"""Returns whether to verify ssl for the given url
:param url: the target URL
:type url: str
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether to verify SSL certs or path to cert(s)
:rtype: bool | str
"""
if not _is_request_to_dcos(url, toml_config):
# Leave verify to None if URL is outside the DC/OS cluster
# https://jira.mesosphere.com/browse/DCOS_OSS-618
return None
if toml_config is None:
toml_config = config.get_config()
if verify is None:
verify = config.get_config_val("core.ssl_verify", toml_config)
if verify and verify.lower() == "true":
verify = True
elif verify and verify.lower() == "false":
verify = False
return verify
def _main():
signal.signal(signal.SIGINT, signal_handler)
http.silence_requests_warnings()
args = docopt.docopt(default_doc("dcos"), options_first=True)
log_level = args['--log-level']
if log_level and not _config_log_level_environ(log_level):
return 1
if args['--debug']:
os.environ[constants.DCOS_DEBUG_ENV] = 'true'
util.configure_process_from_environ()
if args['--version']:
return _get_versions(config.get_config_val("core.dcos_url"))
command = args['<command>']
if not command:
if args['--help']:
command = "help"
else:
return dcos_help()
if command in subcommand.default_subcommands():
sc = SubcommandMain(command, args['<args>'])
else:
executable = subcommand.command_executables(command)
sc = subcommand.SubcommandProcess(executable, command, args['<args>'])
exitcode, _ = sc.run_and_capture()
return exitcode
def _get_marathon_url(toml_config):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: marathon base url
:rtype: str
"""
marathon_url = config.get_config_val('marathon.url', toml_config)
if marathon_url is None:
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/')
return marathon_url
def _get_marathon_url(toml_config):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: marathon base url
:rtype: str
"""
marathon_url = config.get_config_val('marathon.url', toml_config)
if marathon_url is None:
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon/')
return marathon_url
def _page(output, pager_command=None):
"""Conditionally pipes the supplied output through a pager.
:param output:
:type output: object
:param pager_command:
:type pager_command: str
"""
output = six.text_type(output)
if not sys.stdout.isatty() or util.is_windows_platform():
print(output)
sys.stdout.flush()
return
num_lines = output.count('\n')
exceeds_tty_height = pager.getheight() - 1 < num_lines
if pager_command is None:
pager_command = 'less -R'
try:
paginate = config.get_config_val("core.pagination")
except Exception:
paginate = True
if exceeds_tty_height and paginate and \
spawn.find_executable(pager_command.split(' ')[0]) is not None:
pydoc.pipepager(output, cmd=pager_command)
else:
print(output)
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def _login(password_str, password_env, password_file, provider, username,
key_path):
"""
:param password_str: password
:type password_str: str
:param password_env: name of environment variable with password
:type password_env: str
:param password_file: path to file with password
:type password_file: bool
:param provider: name of provider to authentication with
:type provider: str
:param username: username
:type username: str
:param key_path: path to file with private key
:type param: str
:rtype: int
"""
dcos_url = config.get_config_val("core.dcos_url")
if dcos_url is None:
msg = ("Please provide the url to your DC/OS cluster: "
"`dcos config set core.dcos_url`")
raise DCOSException(msg)
# every call to login will generate a new token if applicable
_logout()
login(dcos_url, password_str, password_env, password_file, provider,
username, key_path)
emitter.publish("Login successful!")
return 0
def _unlink(name):
"""
Unlink a DC/OS cluster.
:param name: ID or name of the cluster
:type name: str
:returns: process status
:rtype: int
"""
c = cluster.get_cluster(name)
if c:
name = c.get_cluster_id()
dcos_url = config.get_config_val('core.dcos_url')
endpoint = urllib.parse.urljoin(dcos_url, '/cluster/v1/links/' + name)
try:
http.delete(endpoint)
except DCOSHTTPException as e:
if e.status() == 404:
raise DCOSException('Unknown cluster link {}.'.format(name))
raise
return 0
def overlay_info(self):
# XXX This methodology is copied from the dcos.mesos package, this can
# be removed once the upstream package has support for this field.
url = self.dcos_client.master_url('overlay-agent/overlay')
timeout = dcosconfig.get_config_val('core.timeout',
dcosconfig.get_config())
return dcoshttp.get(url, timeout=timeout).json()
def _do_request(url, method, timeout=None, stream=False, **kwargs):
"""
make HTTP request
:param url: url
:type url: string
:param method: HTTP method, GET or POST
:type method: string
:param timeout: HTTP request timeout, default 3 seconds
:type timeout: integer
:param stream: stream parameter for requests lib
:type stream: bool
:return: http response
:rtype: requests.Response
"""
def _is_success(status_code):
# consider 400 and 503 to be successful status codes.
# API will return the error message.
if status_code in [200, 400, 503]:
return True
return False
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
if timeout is None:
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = 180
# POST to snapshot api
base_url = config.get_config_val("core.dcos_url")
if not base_url:
raise config.missing_config_exception(['core.dcos_url'])
url = urllib.parse.urljoin(base_url, url)
if method.lower() == 'get':
http_response = http.get(url, is_success=_is_success, timeout=timeout,
**kwargs)
elif method.lower() == 'post':
http_response = http.post(url, is_success=_is_success, timeout=timeout,
stream=stream, **kwargs)
else:
raise DCOSException('Unsupported HTTP method: ' + method)
return http_response
def update_marathon_client():
global client
global toml_config_o
toml_config_o = config.get_config()
dcos_url = config.get_config_val('core.dcos_url', toml_config_o)
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon-user/')
config.set_val('marathon.url', marathon_url)
toml_config_m = config.get_config()
client = marathon.create_client(toml_config_m)
def setup_env(env):
# token will be removed when we change dcos_url
token = config.get_config_val('core.dcos_acs_token')
config_set("core.dcos_url", "https://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
try:
yield
finally:
config_set("core.dcos_url", "http://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
def setup_env(env):
# token will be removed when we change dcos_url
token = config.get_config_val('core.dcos_acs_token')
config_set("core.dcos_url", "https://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
try:
yield
finally:
config_set("core.dcos_url", "http://dcos.snakeoil.mesosphere.com", env)
config_set("core.dcos_acs_token", token, env)
def _request_with_auth(response,
method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Request with credentials
:param response: requests.response
:type response: requests.Response
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: requests.Response
"""
parsed_url = urlparse(url)
hostname = parsed_url.hostname
auth_scheme, realm = get_auth_scheme(response)
creds = (hostname, auth_scheme, realm)
with lock:
if creds not in AUTH_CREDS:
auth = _get_http_auth(response, parsed_url, auth_scheme)
else:
auth = AUTH_CREDS[creds]
response = _request(method, url, is_success, timeout, auth,
verify, **kwargs)
# only store credentials if they're valid
with lock:
if creds not in AUTH_CREDS and response.status_code == 200:
AUTH_CREDS[creds] = auth
# acs invalid token
elif response.status_code == 401 and \
auth_scheme in ["acsjwt", "oauthjwt"]:
if config.get_config_val("core.dcos_acs_token") is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
return response
def _request_with_auth(response,
method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Request with credentials
:param response: requests.response
:type response: requests.Response
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: requests.Response
"""
parsed_url = urlparse(url)
hostname = parsed_url.hostname
auth_scheme, realm = get_auth_scheme(response)
creds = (hostname, auth_scheme, realm)
with lock:
if creds not in AUTH_CREDS:
auth = _get_http_auth(response, parsed_url, auth_scheme)
else:
auth = AUTH_CREDS[creds]
response = _request(method, url, is_success, timeout, auth, verify,
**kwargs)
# only store credentials if they're valid
with lock:
if creds not in AUTH_CREDS and response.status_code == 200:
AUTH_CREDS[creds] = auth
# acs invalid token
elif response.status_code == 401 and \
auth_scheme in ["acsjwt", "oauthjwt"]:
if config.get_config_val("core.dcos_acs_token") is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
return response
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def test_repo_list():
repo_list = bytes(("test-universe: {test-universe}\n"
"helloworld-universe: {helloworld-universe}\n").format(
**UNIVERSE_TEST_REPOS), 'utf-8')
assert_command(['dcos', 'package', 'repo', 'list'], stdout=repo_list)
# test again, but override the dcos_url with a cosmos_url config
dcos_url = config.get_config_val("core.dcos_url")
with update_config('package.cosmos_url', dcos_url):
assert_command(['dcos', 'package', 'repo', 'list'], stdout=repo_list)
def _logout():
"""
Logout the user from dcos acs auth or oauth
:returns: process status
:rtype: int
"""
if config.get_config_val("core.dcos_acs_token") is not None:
config.unset("core.dcos_acs_token")
return 0
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def test_get_cluster_name_ignore_env():
with env():
os.environ['DCOS_CLUSTER_NAME'] = 'fake-name'
cluster_conf = config.Toml({
'cluster': {'name': 'real-name'},
})
cluster_name = config.get_config_val('cluster.name', cluster_conf)
assert cluster_name == 'real-name'
def _get_timeout():
"""
:returns: timout value for API calls
:rtype: str
"""
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = DEFAULT_TIMEOUT
return timeout
def _get_timeout():
"""
:returns: timout value for API calls
:rtype: str
"""
# if timeout is not passed, try to read `core.timeout`
# if `core.timeout` is not set, default to 3 min.
timeout = config.get_config_val('core.timeout')
if not timeout:
timeout = DEFAULT_TIMEOUT
return timeout
def delete_zk_node(znode):
"""Delete Zookeeper node
:param znode: znode to delete
:type znode: str
:rtype: None
"""
dcos_url = config.get_config_val('core.dcos_url')
znode_url = urllib.parse.urljoin(
dcos_url,
'/exhibitor/exhibitor/v1/explorer/znode/{}'.format(znode))
http.delete(znode_url)
def _get_dcos_auth(auth_scheme, username, password, hostname):
"""Get authentication flow for dcos acs auth and dcos oauth
:param auth_scheme: authentication_scheme
:type auth_scheme: str
:param username: username user for authentication
:type username: str
:param password: password for authentication
:type password: str
:param hostname: hostname for credentials
:type hostname: str
:returns: DCOSAcsAuth
:rtype: AuthBase
"""
toml_config = config.get_config()
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:
creds = _get_dcos_oauth_creds(dcos_url)
verify = _verify_ssl()
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad cert
if verify is not None:
silence_requests_warnings()
url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login')
# using private method here, so we don't retry on this request
# error here will be bubbled up to _request_with_auth
response = _request('post', url, json=creds, verify=verify)
if response.status_code == 200:
token = response.json()['token']
config.set_val("core.dcos_acs_token", token)
return DCOSAcsAuth(token)
def request(method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
auth_token = config.get_config_val(
"core.dcos_acs_token", config.get_config())
if auth_token is None:
auth = None
else:
auth = DCOSAcsAuth(auth_token)
response = _request(method, url, is_success, timeout,
auth=auth, verify=verify, **kwargs)
if is_success(response.status_code):
return response
elif response.status_code == 401:
if auth_token is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
else:
raise DCOSAuthenticationException(response)
elif response.status_code == 422:
raise DCOSUnprocessableException(response)
elif response.status_code == 403:
raise DCOSAuthorizationException(response)
elif response.status_code == 400:
raise DCOSBadRequest(response)
else:
raise DCOSHTTPException(response)
def _dcos_log(follow, lines, leader, slave, component, filters):
""" Print logs from dcos-log backend.
:param follow: same as unix tail's -f
:type follow: bool
:param lines: number of lines to print
:type lines: int
:param leader: whether to print the leading master's log
:type leader: bool
:param slave: the slave ID to print
:type slave: str | None
:param component: DC/OS component name
:type component: string
:param filters: a list of filters ["key:value", ...]
:type filters: list
"""
filter_query = ''
if component:
filters.append('_SYSTEMD_UNIT:{}'.format(_get_unit_type(component)))
for f in filters:
key_value = f.split(':')
if len(key_value) != 2:
raise SystemExit('Invalid filter parameter {}. '
'Must be --filter=key:value'.format(f))
filter_query += '&filter={}'.format(f)
endpoint = '/system/v1'
if leader:
endpoint += _build_leader_url(component)
elif slave:
endpoint += '/agent/{}/logs/v1/'.format(slave)
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
dcos_url = config.get_config_val('core.dcos_url').rstrip("/")
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = (dcos_url + endpoint + endpoint_type +
'/?skip_prev={}'.format(lines) + filter_query)
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)
def create_client(toml_config=None):
"""Creates a Marathon client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Marathon client
:rtype: dcos.marathon.Client
"""
if toml_config is None:
toml_config = config.get_config()
marathon_url = _get_marathon_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
logger.info('Creating marathon client with: %r', marathon_url)
return Client(marathon_url, timeout=timeout)
def _verify_ssl(verify=None):
"""Returns whether to verify ssl
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:return: whether to verify SSL certs or path to cert(s)
:rtype: bool | str
"""
if verify is None:
verify = config.get_config_val("core.ssl_verify")
if verify and verify.lower() == "true":
verify = True
elif verify and verify.lower() == "false":
verify = False
return verify
def marathon_on_marathon(name='marathon-user'):
""" Context manager for altering the marathon client for MoM
:param name: service name of MoM to use
:type name: str
"""
toml_config_o = config.get_config()
dcos_url = config.get_config_val('core.dcos_url', toml_config_o)
service_name = 'service/{}/'.format(name)
marathon_url = urllib.parse.urljoin(dcos_url, service_name)
config.set_val('marathon.url', marathon_url)
try:
yield
finally:
# return config to previous state
config.save(toml_config_o)
def create_client(toml_config=None):
"""Creates a Metronome client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Metronome client
:rtype: dcos.metronome.Client
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = _get_metronome_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
rpc_client = rpcclient.create_client(metronome_url, timeout)
logger.info('Creating metronome client with: %r', metronome_url)
return Client(rpc_client)
def _metrics(summary, mesos_id, json_):
""" Get metrics from the specified agent.
:param summary: summarise output if true, output all if false
:type summary: bool
:param mesos_id: mesos node id
:type mesos_id: str
:param json_: print raw JSON
:type json_: bool
:returns: Process status
:rtype: int
"""
endpoint = '/system/v1/agent/{}/metrics/v0/node'.format(mesos_id)
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
url = dcos_url + endpoint
return metrics.print_node_metrics(url, summary, json_)
def get_providers():
"""
Returns dict of providers configured on cluster
:returns: configured providers
:rtype: {}
"""
dcos_url = config.get_config_val("core.dcos_url").rstrip('/')
endpoint = '/acs/api/v1/auth/providers'
url = urllib.parse.urljoin(dcos_url, endpoint)
try:
providers = http.get(url)
return providers.json()
# this endpoint should only have authentication in DC/OS 1.8
except DCOSAuthenticationException:
msg = "This command is not supported for your cluster"
raise DCOSException(msg)
except DCOSHTTPException as e:
if e.response.status_code == 404:
msg = "This command is not supported for your cluster"
raise DCOSException(msg)
return {}
def __init__(self, url=None):
self.url = url or urllib.parse.urljoin(
config.get_config_val('core.dcos_url'), '/mesos_dns/')
def run(_args=[], configure_logging=True):
# If we're being invoked via DC/OS then route our http
# calls via its extension to the requests library. In
# addition remove the 'conduct-dcos' and 'conduct' arg so that the conduct
# sub-commands are positioned correctly, along with their
# arguments.
if sys.argv and Path(sys.argv[0]).name == constants.DCOS_COMMAND_PREFIX + 'conduct':
dcos_mode = True
_args = sys.argv[2:]
else:
dcos_mode = False
if not _args:
# Remove the 'conduct' arg so that we start with the sub command directly
_args = sys.argv[1:]
# Parse arguments
parser = build_parser(dcos_mode)
argcomplete.autocomplete(parser)
args = parser.parse_args(_args)
args.dcos_mode = dcos_mode
if not vars(args).get('func'):
if vars(args).get('dcos_info'):
print('Lightbend ConductR sub commands. Type \'dcos conduct\' to see more.')
exit(0)
else:
parser.print_help()
else:
# Offline functions are the functions which do not require network to run, e.g. `conduct version` or
# `conduct setup-dcos`.
offline_functions = ['version', 'setup']
# Only setup network related args (i.e. host, bundle resolvers, basic auth, etc) for functions which requires
# connectivity to ConductR.
current_function = vars(args).get('func').__name__
if current_function not in offline_functions:
# Add custom plugin dir to import path
custom_plugins_dir = vars(args).get('custom_plugins_dir')
if custom_plugins_dir:
sys.path.append(custom_plugins_dir)
# DC/OS provides the location of ConductR...
if dcos_mode:
args.command = 'dcos conduct'
dcos_url = urlparse(config.get_config_val('core.dcos_url'))
args.scheme = dcos_url.scheme
args.ip = dcos_url.hostname
default_http_port = 80 if dcos_url.scheme == 'http' else 443
args.port = dcos_url.port if dcos_url.port else default_http_port
dcos_url_path = dcos_url.path if dcos_url.path else '/'
args.base_path = dcos_url_path + 'service/{}/'.format(DEFAULT_DCOS_SERVICE)
else:
args.command = 'conduct'
# Ensure ConductR host is not empty
host_from_args = conduct_url.conductr_host(args)
if not host_from_args:
host_from_env = host.resolve_default_host()
if host_from_env:
args.host = host_from_env
else:
# Configure logging so error message can be logged properly before exiting with failure
logging_setup.configure_logging(args)
log = logging.getLogger(__name__)
log.error('ConductR host address is not specified')
log.error('Please ensure either `{}` environment is specified,'
' or specify the ConductR host using `--host` argument'.format(CONDUCTR_HOST))
exit(1)
else:
args.local_connection = False
args.cli_parameters = get_cli_parameters(args)
args.custom_settings = custom_settings.load_from_file(args)
args.conductr_auth = custom_settings.load_conductr_credentials(args)
# Ensure HTTPS is used if authentication is configured
if args.conductr_auth and not args.scheme == 'https':
# Configure logging so error message can be logged properly before exiting with failure
logging_setup.configure_logging(args)
log = logging.getLogger(__name__)
log.error('Unable to use Basic Auth over {}'.format(args.scheme))
log.error('Please ensure either `{}` environment is set to `https`,'
' or specify https using `--scheme https` argument'.format(CONDUCTR_SCHEME))
exit(1)
args.server_verification_file = custom_settings.load_server_ssl_verification_file(args)
# Ensure verification file exists if specified
if args.server_verification_file \
and not os.path.exists(args.server_verification_file):
# Configure logging so error message can be logged properly before exiting with failure
logging_setup.configure_logging(args)
log = logging.getLogger(__name__)
log.error('Ensure server SSL verification file exists: {}'.format(args.server_verification_file))
exit(1)
if not args.dcos_mode and args.scheme == 'https':
disable_urllib3_warnings()
if configure_logging:
logging_setup.configure_logging(args)
is_completed_without_error = args.func(args)
if not is_completed_without_error:
exit(1)
def _dcos_log(follow, tasks, lines, file_, completed):
""" a client to dcos-log
:param follow: same as unix tail's -f
:type follow: bool
:param task: task pattern to match
:type task: str
:param lines: number of lines to print
:type lines: int
:param file_: file path to read
:type file_: str
:param completed: whether to include completed tasks
:type completed: bool
"""
# only stdout and stderr is supported
if file_ not in ('stdout', 'stderr'):
raise DCOSException('Expect file stdout or stderr. '
'Got {}'.format(file_))
# state json may container tasks and completed_tasks fields. Based on
# user request we should traverse the appropriate field.
tasks_field = 'tasks'
if completed:
tasks_field = 'completed_tasks'
for task in tasks:
executor_info = task.executor()
if not executor_info:
continue
if (tasks_field not in executor_info and
not isinstance(executor_info[tasks_field], list)):
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException('Invalid executor info. '
'Missing field {}'.format(tasks_field))
for t in executor_info[tasks_field]:
container_id = get_nested_container_id(t)
if not container_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing container id')
# get slave_id field
slave_id = t.get('slave_id')
if not slave_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `slave_id`')
framework_id = t.get('framework_id')
if not framework_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing field `framework_id`')
# try `executor_id` first.
executor_id = t.get('executor_id')
if not executor_id:
# if `executor_id` is an empty string, default to `id`.
executor_id = t.get('id')
if not executor_id:
logger.debug('Executor info: {}'.format(executor_info))
raise DCOSException(
'Invalid executor info. Missing executor id')
dcos_url = config.get_config_val('core.dcos_url').rstrip('/')
if not dcos_url:
raise config.missing_config_exception(['core.dcos_url'])
# dcos-log provides 2 base endpoints /range/ and /stream/
# for range and streaming requests.
endpoint_type = 'range'
if follow:
endpoint_type = 'stream'
endpoint = ('/system/v1/agent/{}/logs/v1/{}/framework/{}'
'/executor/{}/container/{}'.format(slave_id,
endpoint_type,
framework_id,
executor_id,
container_id))
# append request parameters.
# `skip_prev` will move the cursor to -n lines.
# `filter=STREAM:{STDOUT,STDERR}` will filter logs by label.
url = (dcos_url + endpoint +
'?skip_prev={}&filter=STREAM:{}'.format(lines,
file_.upper()))
if follow:
return log.follow_logs(url)
return log.print_logs_range(url)