def update_marathon_client():
global client
global toml_config_o
toml_config_o = config.get_config()
dcos_url = config.get_config_val('core.dcos_url', toml_config_o)
marathon_url = urllib.parse.urljoin(dcos_url, 'service/marathon-user/')
config.set_val('marathon.url', marathon_url)
toml_config_m = config.get_config()
client = marathon.create_client(toml_config_m)
def test_get_config(load_path_mock):
with env(), util.tempdir() as tempdir:
os.environ.pop('DCOS_CONFIG', None)
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
# this should create the global config
config.get_config()
global_toml = os.path.join(tempdir, "dcos.toml")
load_path_mock.assert_called_once_with(global_toml, False)
load_path_mock.reset_mock()
# create old global config toml
global_toml = create_global_config(tempdir)
config.get_config()
load_path_mock.assert_called_once_with(global_toml, False)
load_path_mock.reset_mock()
# clusters dir, no clusters
_create_clusters_dir(tempdir)
config.get_config()
load_path_mock.assert_called_once_with(global_toml, False)
load_path_mock.reset_mock()
cluster_id = "fake-cluster"
cluster_path = add_cluster_dir(cluster_id, tempdir)
cluster_toml = os.path.join(cluster_path, "dcos.toml")
config.get_config(True)
load_path_mock.assert_called_with(cluster_toml, True)
def test_get_config(load_path_mock):
with env(), util.tempdir() as tempdir:
os.environ.pop('DCOS_CONFIG', None)
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
with pytest.raises(DCOSException) as e:
config.get_config()
msg = ("No cluster is attached. "
"Please run `dcos cluster attach <cluster-name>`")
assert str(e.value) == msg
load_path_mock.assert_not_called()
# create old global config toml
global_toml = create_global_config(tempdir)
config.get_config()
load_path_mock.assert_called_once_with(global_toml, False)
# clusters dir, no clusters
_create_clusters_dir(tempdir)
with pytest.raises(DCOSException) as e:
config.get_config()
assert str(e.value) == msg
cluster_id = "fake-cluster"
cluster_path = add_cluster_dir(cluster_id, tempdir)
cluster_toml = os.path.join(cluster_path, "dcos.toml")
config.get_config(True)
load_path_mock.assert_any_call(cluster_toml, True)
def _main():
signal.signal(signal.SIGINT, signal_handler)
http.silence_requests_warnings()
toml_config = config.get_config()
set_ssl_info_env_vars(toml_config)
args = docopt.docopt(default_doc("dcos"), options_first=True)
log_level = args['--log-level']
if log_level and not _config_log_level_environ(log_level):
return 1
if args['--debug']:
os.environ[constants.DCOS_DEBUG_ENV] = 'true'
util.configure_process_from_environ()
if args['--version']:
return _get_versions(toml_config.get("core.dcos_url"))
command = args['<command>']
if not command:
command = "help"
if command in subcommand.default_subcommands():
sc = SubcommandMain(command, args['<args>'])
else:
executable = subcommand.command_executables(command)
sc = subcommand.SubcommandProcess(
executable, command, args['<args>'])
exitcode, _ = sc.run_and_capture()
return exitcode
def get_spark_dispatcher():
dcos_spark_url = os.getenv("DCOS_SPARK_URL")
if dcos_spark_url is not None:
return dcos_spark_url
base_url = config.get_config().get('core.dcos_url')
return urllib.parse.urljoin(base_url, '/service/' + service.app_id() + '/')
def _is_request_to_dcos(url, toml_config=None):
"""Checks if a request is for the DC/OS cluster.
:param url: URL of the request
:type url: str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether the request is for the cluster
:rtype: bool
"""
if toml_config is None:
toml_config = config.get_config()
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
cosmos_url = urlparse(
config.get_config_val("package.cosmos_url", toml_config))
parsed_url = urlparse(url)
# request should match scheme + netloc
def _request_match(expected_url, actual_url):
return expected_url.scheme == actual_url.scheme and \
expected_url.netloc == actual_url.netloc
is_request_to_cluster = _request_match(dcos_url, parsed_url) or \
_request_match(cosmos_url, parsed_url)
return is_request_to_cluster
def get_spark_dispatcher():
dcos_spark_url = os.getenv("DCOS_SPARK_URL")
if dcos_spark_url is not None:
return dcos_spark_url
base_url = config.get_config().get("core.dcos_url")
return urllib.parse.urljoin(base_url, "/service/" + service.app_id() + "/")
def _login():
"""
:returns: process status
:rtype: int
"""
# every call to login will generate a new token if applicable
_logout()
conf = config.get_config()
dcos_url = conf.get("core.dcos_url")
if dcos_url is None:
msg = ("Please provide the url to your DCOS cluster: "
"`dcos config set core.dcos_url`")
raise DCOSException(msg)
# hit protected endpoint which will prompt for auth if cluster has auth
try:
url = urllib.parse.urljoin(dcos_url, 'exhibitor')
http.get(url)
# if the user is authenticated, they have effectively "logged in" even if
# they are not authorized for this endpoint
except DCOSAuthorizationException:
pass
emitter.publish("Login successful!")
return 0
def __init__(self, toml_config: Optional[Any] = None):
super(DCOSClient, self).__init__()
self.toml_config = toml_config
if toml_config is None:
self.toml_config = config.get_config()
self._dcos_url = cast(ParseResult, urlparse(config.get_config_val("core.dcos_url", toml_config)))
def _get_dcos_auth(auth_scheme, username, password, hostname):
"""Get authentication flow for dcos acs auth and dcos oauth
:param auth_scheme: authentication_scheme
:type auth_scheme: str
:param username: username user for authentication
:type username: str
:param password: password for authentication
:type password: str
:param hostname: hostname for credentials
:type hostname: str
:returns: DCOSAcsAuth
:rtype: AuthBase
"""
toml_config = config.get_config()
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:
creds = _get_dcos_oauth_creds(dcos_url)
url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login')
response = _request('post', url, json=creds)
if response.status_code == 200:
token = response.json()['token']
config.set_val("core.dcos_acs_token", token)
return DCOSAcsAuth(token)
def overlay_info(self):
# XXX This methodology is copied from the dcos.mesos package, this can
# be removed once the upstream package has support for this field.
url = self.dcos_client.master_url('overlay-agent/overlay')
timeout = dcosconfig.get_config_val('core.timeout', dcosconfig.get_config())
return dcoshttp.get(url, timeout=timeout).json()
def _page(output, pager_command=None):
"""Conditionally pipes the supplied output through a pager.
:param output:
:type output: object
:param pager_command:
:type pager_command: str
"""
output = six.text_type(output)
if pager_command is None:
pager_command = 'less -R'
if not sys.stdout.isatty() or util.is_windows_platform():
print(output)
return
num_lines = output.count('\n')
exceeds_tty_height = pager.getheight() - 1 < num_lines
paginate = config.get_config().get("core.pagination", True)
if exceeds_tty_height and paginate:
pydoc.pipepager(output, cmd=pager_command)
else:
print(output)
def _verify_ssl(url, verify=None, toml_config=None):
"""Returns whether to verify ssl for the given url
:param url: the target URL
:type url: str
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether to verify SSL certs or path to cert(s)
:rtype: bool | str
"""
if not _is_request_to_dcos(url, toml_config):
# Leave verify to None if URL is outside the DC/OS cluster
# https://jira.mesosphere.com/browse/DCOS_OSS-618
return None
if toml_config is None:
toml_config = config.get_config()
if verify is None:
verify = config.get_config_val("core.ssl_verify", toml_config)
if verify and verify.lower() == "true":
verify = True
elif verify and verify.lower() == "false":
verify = False
return verify
def overlay_info(self):
# XXX This methodology is copied from the dcos.mesos package, this can
# be removed once the upstream package has support for this field.
url = self.dcos_client.master_url('overlay-agent/overlay')
timeout = dcosconfig.get_config_val('core.timeout',
dcosconfig.get_config())
return dcoshttp.get(url, timeout=timeout).json()
def request(method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
toml_config = config.get_config()
auth_token = config.get_config_val("core.dcos_acs_token", toml_config)
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
parsed_url = urlparse(url)
# only request with DC/OS Auth if request is to DC/OS cluster
# request should match scheme + netloc
scheme_eq = parsed_url.scheme == dcos_url.scheme
netloc_eq = parsed_url.netloc == dcos_url.netloc
if auth_token and scheme_eq and netloc_eq:
auth = DCOSAcsAuth(auth_token)
else:
auth = None
response = _request(method, url, is_success, timeout,
auth=auth, verify=verify, **kwargs)
if is_success(response.status_code):
return response
elif response.status_code == 401:
if auth_token is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSAuthenticationException(msg)
else:
raise DCOSAuthenticationException(response)
elif response.status_code == 422:
raise DCOSUnprocessableException(response)
elif response.status_code == 403:
raise DCOSAuthorizationException(response)
elif response.status_code == 400:
raise DCOSBadRequest(response)
else:
raise DCOSHTTPException(response)
def __init__(self):
toml_config = config.get_config()
self._dcos_url = toml_config.get("core.dcos_url")
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = toml_config.get('core.mesos_master_url')
self._timeout = toml_config.get('core.timeout')
def dcos_config():
""" Context manager for altering the toml
"""
toml_config_o = config.get_config()
try:
yield
finally:
# return config to previous state
config.save(toml_config_o)
def __init__(self):
toml_config = config.get_config()
self._dcos_url = config.get_config_val("core.dcos_url", toml_config)
if self._dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
self._mesos_master_url = config.get_config_val('core.mesos_master_url',
toml_config)
self._timeout = config.get_config_val('core.timeout', toml_config)
def _logout():
"""
Logout the user from dcos acs auth or oauth
:returns: process status
:rtype: int
"""
if config.get_config().get("core.dcos_acs_token") is not None:
config.unset("core.dcos_acs_token")
return 0
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def get_cosmos_url():
"""
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val("package.cosmos_url", toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val("core.dcos_url", toml_config)
if cosmos_url is None:
raise config.missing_config_exception(["core.dcos_url"])
return cosmos_url
def request(method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
auth_token = config.get_config_val(
"core.dcos_acs_token", config.get_config())
if auth_token is None:
auth = None
else:
auth = DCOSAcsAuth(auth_token)
response = _request(method, url, is_success, timeout,
auth=auth, verify=verify, **kwargs)
if is_success(response.status_code):
return response
elif response.status_code == 401:
if auth_token is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
else:
raise DCOSAuthenticationException(response)
elif response.status_code == 422:
raise DCOSUnprocessableException(response)
elif response.status_code == 403:
raise DCOSAuthorizationException(response)
elif response.status_code == 400:
raise DCOSBadRequest(response)
else:
raise DCOSHTTPException(response)
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def get_cosmos_url():
"""
Gets the cosmos url
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
cosmos_url = config.get_config_val('package.cosmos_url', toml_config)
if cosmos_url is None:
cosmos_url = config.get_config_val('core.dcos_url', toml_config)
if cosmos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
return cosmos_url
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def _get_default_base_url():
"""
Gets the default service manager URL
:returns: cosmos base url
:rtype: str
"""
toml_config = config.get_config()
base_url = config.get_config_val('package.cosmos_url', toml_config)
if base_url is None:
base_url = config.get_config_val('core.dcos_url', toml_config)
if base_url is None:
raise config.missing_config_exception(['core.dcos_url'])
else:
base_url = urllib.parse.urljoin(base_url, 'cosmos/')
return base_url
def _validate():
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
errs = util.validate_json(toml_config._dictionary,
config.generate_root_schema(toml_config))
if len(errs) != 0:
emitter.publish(util.list_to_err(errs))
return 1
emitter.publish("Congratulations, your configuration is valid!")
return 0
def _validate():
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
errs = util.validate_json(toml_config._dictionary,
config.generate_root_schema(toml_config))
if len(errs) != 0:
emitter.publish(util.list_to_err(errs))
return 1
emitter.publish("Congratulations, your configuration is valid!")
return 0
def create_client(toml_config=None):
"""Creates a Marathon client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Marathon client
:rtype: dcos.marathon.Client
"""
if toml_config is None:
toml_config = config.get_config()
marathon_url = _get_marathon_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
logger.info('Creating marathon client with: %r', marathon_url)
return Client(marathon_url, timeout=timeout)
def marathon_on_marathon(name='marathon-user'):
""" Context manager for altering the marathon client for MoM
:param name: service name of MoM to use
:type name: str
"""
toml_config_o = config.get_config()
dcos_url = config.get_config_val('core.dcos_url', toml_config_o)
service_name = 'service/{}/'.format(name)
marathon_url = urllib.parse.urljoin(dcos_url, service_name)
config.set_val('marathon.url', marathon_url)
try:
yield
finally:
# return config to previous state
config.save(toml_config_o)
def create_client(toml_config=None):
"""Creates a Marathon client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Marathon client
:rtype: dcos.marathon.Client
"""
if toml_config is None:
toml_config = config.get_config()
marathon_url = _get_marathon_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
logger.info('Creating marathon client with: %r', marathon_url)
return Client(marathon_url, timeout=timeout)
def create_client(toml_config=None):
"""Creates a Metronome client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Metronome client
:rtype: dcos.metronome.Client
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = _get_metronome_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
rpc_client = rpcclient.create_client(metronome_url, timeout)
logger.info('Creating metronome client with: %r', metronome_url)
return Client(rpc_client)
def create_client(toml_config=None):
"""Creates a Metronome client with the supplied configuration.
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: Metronome client
:rtype: dcos.metronome.Client
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = _get_metronome_url(toml_config)
timeout = config.get_config_val('core.timeout') or http.DEFAULT_TIMEOUT
rpc_client = rpcclient.create_client(metronome_url, timeout)
logger.info('Creating metronome client with: %r', metronome_url)
return Client(rpc_client)
def _show(name):
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
if name is not None:
file_value = toml_config.get(name)
try:
# If the user presented a partial key name, eg 'core' when
# we have 'core.xyz'; we will get an exception here
effective_value, envvar_name = config.get_config_val_envvar(name)
except DCOSException as e:
# The expected case of a partial key name has special
# handling via this mechanism.
if isinstance(file_value, collections.Mapping):
exc_msg = config.generate_choice_msg(name, file_value)
raise DCOSException(exc_msg)
raise # Unexpected errors, pass right along
if effective_value is None:
raise DCOSException("Property {!r} doesn't exist".format(name))
else:
msg = _format_config(file_value,
effective_value,
envvar_name=envvar_name)
emitter.publish(msg)
else:
# Let's list all of the values
for key, value in sorted(toml_config.property_items()):
file_value = toml_config.get(key)
effective_value, envvar_name = config.get_config_val_envvar(key)
msg = _format_config(file_value,
effective_value,
key,
envvar_name=envvar_name)
emitter.publish(msg)
return 0
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def _get_metronome_url(toml_config=None):
"""
:param toml_config: configuration dictionary
:type toml_config: config.Toml
:returns: metronome base url
:rtype: str
"""
if toml_config is None:
toml_config = config.get_config()
metronome_url = config.get_config_val('metronome.url', toml_config)
if metronome_url is None:
# dcos must be capable to use dcos_url
_check_capability()
dcos_url = config.get_config_val('core.dcos_url', toml_config)
if dcos_url is None:
raise config.missing_config_exception(['core.dcos_url'])
metronome_url = urllib.parse.urljoin(dcos_url, 'service/metronome/')
return metronome_url
def _show(name):
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
if name is not None:
file_value = toml_config.get(name)
try:
# If the user presented a partial key name, eg 'core' when
# we have 'core.xyz'; we will get an exception here
effective_value, envvar_name = config.get_config_val_envvar(name)
except DCOSException as e:
# The expected case of a partial key name has special
# handling via this mechanism.
if isinstance(file_value, collections.Mapping):
exc_msg = config.generate_choice_msg(name, file_value)
raise DCOSException(exc_msg)
raise # Unexpected errors, pass right along
if effective_value is None:
raise DCOSException("Property {!r} doesn't exist".format(name))
else:
msg = _format_config(file_value, effective_value,
envvar_name=envvar_name)
emitter.publish(msg)
else:
# Let's list all of the values
for key, value in sorted(toml_config.property_items()):
file_value = toml_config.get(key)
effective_value, envvar_name = config.get_config_val_envvar(key)
msg = _format_config(file_value, effective_value, key,
envvar_name=envvar_name)
emitter.publish(msg)
return 0
def update_config(name, value, env=None):
""" Context manager for altering config for tests
:param key: <key>
:type key: str
:param value: <value>
:type value: str
;param env: env vars
:type env: dict
:rtype: None
"""
toml_config = config.get_config(True)
result = toml_config.get(name)
# when we change the dcos_url we remove the acs_token
# we need to also restore the token if this occurs
token = None
if name == "core.dcos_url":
token = toml_config.get("core.dcos_acs_token")
# if we are setting a value
if value is not None:
config_set(name, value, env)
# only unset if the config param already exists
elif result is not None:
config_unset(name, env)
try:
yield
finally:
# return config to previous state
if result is not None:
config_set(name, str(result), env)
else:
exec_command(['dcos', 'config', 'unset', name], env)
if token:
config_set("core.dcos_acs_token", token, env)
def _verify_ssl(verify=None, toml_config=None):
"""Returns whether to verify ssl
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:return: whether to verify SSL certs or path to cert(s)
:rtype: bool | str
"""
if toml_config is None:
toml_config = config.get_config()
if verify is None:
verify = config.get_config_val("core.ssl_verify", toml_config)
if verify and verify.lower() == "true":
verify = True
elif verify and verify.lower() == "false":
verify = False
return verify
def __init__(self, ctx):
self._master_url = None
self._zk_url = None
self._marathon_url = None
self._framework_url = None
self.ctx = ctx
try:
self.ctx.vlog("Attempting to create DCOSClient")
dcos_client = mesos.DCOSClient()
self.client = dcos_client
dcos_url = self.client.get_dcos_url("")
ssl_verify = dcos_config.get_config().get("core.ssl_verify")
self.ctx.vlog("DCOS core.ssl_verify value = " + ssl_verify)
if ssl_verify is not None and (not ssl_verify or ssl_verify == "false"):
self.ctx.vlog("Setting insecure_ssl to True")
self.ctx.insecure_ssl = True
if dcos_url is None:
raise Exception("Unable to find DCOS server URL")
dcos_url.rstrip("/")
self.dcos_url = dcos_url
except dcos_errors.DCOSException:
raise Exception("DCOS is not configured properly")
def _get_dcos_auth(auth_scheme, username, password, hostname):
"""Get authentication flow for dcos acs auth and dcos oauth
:param auth_scheme: authentication_scheme
:type auth_scheme: str
:param username: username user for authentication
:type username: str
:param password: password for authentication
:type password: str
:param hostname: hostname for credentials
:type hostname: str
:returns: DCOSAcsAuth
:rtype: AuthBase
"""
toml_config = config.get_config()
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:
creds = _get_dcos_oauth_creds(dcos_url)
verify = _verify_ssl()
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad cert
if verify is not None:
silence_requests_warnings()
url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login')
# using private method here, so we don't retry on this request
# error here will be bubbled up to _request_with_auth
response = _request('post', url, json=creds, verify=verify)
if response.status_code == 200:
token = response.json()['token']
config.set_val("core.dcos_acs_token", token)
return DCOSAcsAuth(token)
def _get_dcos_auth(auth_scheme, username, password, hostname):
"""Get authentication flow for dcos acs auth and dcos oauth
:param auth_scheme: authentication_scheme
:type auth_scheme: str
:param username: username user for authentication
:type username: str
:param password: password for authentication
:type password: str
:param hostname: hostname for credentials
:type hostname: str
:returns: DCOSAcsAuth
:rtype: AuthBase
"""
toml_config = config.get_config()
token = config.get_config_val("core.dcos_acs_token", toml_config)
if token is None:
dcos_url = config.get_config_val("core.dcos_url", toml_config)
if auth_scheme == "acsjwt":
creds = _get_dcos_acs_auth_creds(username, password, hostname)
else:
creds = _get_dcos_oauth_creds(dcos_url)
verify = _verify_ssl()
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad cert
if verify is not None:
silence_requests_warnings()
url = urllib.parse.urljoin(dcos_url, 'acs/api/v1/auth/login')
# using private method here, so we don't retry on this request
# error here will be bubbled up to _request_with_auth
response = _request('post', url, json=creds, verify=verify)
if response.status_code == 200:
token = response.json()['token']
config.set_val("core.dcos_acs_token", token)
return DCOSAcsAuth(token)
def __init__(self, ctx):
self._master_url = None
self._zk_url = None
self._marathon_url = None
self._framework_url = None
self.ctx = ctx
try:
self.ctx.vlog('Attempting to create DCOSClient')
dcos_client = mesos.DCOSClient()
self.client = dcos_client
dcos_url = self.client.get_dcos_url('')
ssl_verify = dcos_config.get_config().get('core.ssl_verify')
self.ctx.vlog('DCOS core.ssl_verify value = ' + ssl_verify)
if ssl_verify is not None and (not ssl_verify
or ssl_verify == 'false'):
self.ctx.vlog('Setting insecure_ssl to True')
self.ctx.insecure_ssl = True
if dcos_url is None:
raise Exception("Unable to find DCOS server URL")
dcos_url.rstrip('/')
self.dcos_url = dcos_url
except dcos_errors.DCOSException:
raise Exception("DCOS is not configured properly")
def _show(name):
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
if name is not None:
value = toml_config.get(name)
if value is None:
raise DCOSException("Property {!r} doesn't exist".format(name))
elif isinstance(value, collections.Mapping):
raise DCOSException(config.generate_choice_msg(name, value))
else:
emitter.publish(value)
else:
# Let's list all of the values
for key, value in sorted(toml_config.property_items()):
if key == "core.dcos_acs_token":
value = "*" * 8
emitter.publish('{} {}'.format(key, value))
return 0
def _show(name):
"""
:returns: process status
:rtype: int
"""
toml_config = config.get_config(True)
if name is not None:
value = toml_config.get(name)
if value is None:
raise DCOSException("Property {!r} doesn't exist".format(name))
elif isinstance(value, collections.Mapping):
raise DCOSException(config.generate_choice_msg(name, value))
else:
emitter.publish(value)
else:
# Let's list all of the values
for key, value in sorted(toml_config.property_items()):
if key == "core.dcos_acs_token":
value = "*"*8
emitter.publish('{}={}'.format(key, value))
return 0
def _cert_verification():
try:
core_verify_ssl = config.get_config()['core.ssl_verify']
return str(core_verify_ssl).lower() in ['true', 'yes', '1']
except:
return True
def get_spark_webui():
base_url = config.get_config().get("core.dcos_url")
return base_url + "/service/" + service.app_id() + "/ui"
def _request_with_auth(response,
method,
url,
is_success=_default_is_success,
timeout=None,
verify=None,
**kwargs):
"""Try request (3 times) with credentials if 401 returned from server
:param response: requests.response
:type response: requests.Response
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: requests.Response
"""
i = 0
while i < 3 and response.status_code == 401:
parsed_url = urlparse(url)
hostname = parsed_url.hostname
auth_scheme, realm = get_auth_scheme(response)
creds = (hostname, auth_scheme, realm)
with lock:
if creds not in AUTH_CREDS:
auth = _get_http_auth(response, parsed_url, auth_scheme)
else:
auth = AUTH_CREDS[creds]
# try request again, with auth
response = _request(method, url, is_success, timeout, auth,
verify, **kwargs)
# only store credentials if they're valid
with lock:
if creds not in AUTH_CREDS and response.status_code == 200:
AUTH_CREDS[creds] = auth
# acs invalid token
elif response.status_code == 401 and \
auth_scheme in ["acsjwt", "oauthjwt"]:
if config.get_config().get("core.dcos_acs_token") is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSException(msg)
i += 1
if response.status_code == 401:
raise DCOSAuthenticationException(response)
return response
def app_id():
try:
return config.get_config()["spark.app_id"]
except KeyError:
return "spark"
def _cert_verification():
try:
core_verify_ssl = config.get_config()['core.ssl_verify']
return str(core_verify_ssl).lower() in ['true', 'yes', '1']
except:
return True
def request(method,
url,
is_success=_default_is_success,
timeout=DEFAULT_TIMEOUT,
verify=None,
toml_config=None,
**kwargs):
"""Sends an HTTP request. If the server responds with a 401, ask the
user for their credentials, and try request again (up to 3 times).
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: request timeout
:type timeout: int
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
if toml_config is None:
toml_config = config.get_config()
auth_token = config.get_config_val("core.dcos_acs_token", toml_config)
prompt_login = config.get_config_val("core.prompt_login", toml_config)
dcos_url = urlparse(config.get_config_val("core.dcos_url", toml_config))
# only request with DC/OS Auth if request is to DC/OS cluster
if auth_token and _is_request_to_dcos(url):
auth = DCOSAcsAuth(auth_token)
else:
auth = None
response = _request(method,
url,
is_success,
timeout,
auth=auth,
verify=verify,
toml_config=toml_config,
**kwargs)
if is_success(response.status_code):
return response
elif response.status_code == 401:
if prompt_login:
# I don't like having imports that aren't at the top level, but
# this is to resolve a circular import issue between shakedown.http and
# dcos.auth
from dcos.auth import header_challenge_auth
header_challenge_auth(dcos_url.geturl())
# if header_challenge_auth succeeded, then we auth-ed correctly and
# thus can safely recursively call ourselves and not have to worry
# about an infinite loop
return request(method=method,
url=url,
is_success=is_success,
timeout=timeout,
verify=verify,
**kwargs)
else:
if auth_token is not None:
msg = ("Your core.dcos_acs_token is invalid. "
"Please run: `dcos auth login`")
raise DCOSAuthenticationException(response, msg)
else:
raise DCOSAuthenticationException(response)
elif response.status_code == 422:
raise DCOSUnprocessableException(response)
elif response.status_code == 403:
raise DCOSAuthorizationException(response)
elif response.status_code == 400:
raise DCOSBadRequest(response)
else:
raise DCOSHTTPException(response)
def _get_token():
dcos_url = config.get_config().get('core.dcos_url')
hostname = urllib.parse.urlparse(dcos_url).hostname
return http._get_dcos_auth(None, None, None, hostname).token
def _spark_dist_dir():
dist_dir = config.get_config().get('spark.distribution_directory',
'~/.dcos/spark/dist')
return os.path.expanduser(dist_dir)
def _request(method,
url,
is_success=_default_is_success,
timeout=True,
auth=None,
verify=None,
toml_config=None,
**kwargs):
"""Sends an HTTP request.
:param method: method for the new Request object
:type method: str
:param url: URL for the new Request object
:type url: str
:param is_success: Defines successful status codes for the request
:type is_success: Function from int to bool
:param timeout: How many seconds to wait for the server to send data
before aborting and raising an exception. A timeout is
either a float, a (connect timeout, read timeout) tuple
or `None`. `None` means unlimited timeout. If no timeout
is passed it defaults to the DEFAULT_TIMEOUT tuple, where
the connect timeout can optionally be overridden by the
`core.timeout` config.
:type timeout: int | float | None | bool |
(int | float | None, int | float | None)
:param auth: authentication
:type auth: AuthBase
:param verify: whether to verify SSL certs or path to cert(s)
:type verify: bool | str
:param toml_config: cluster config to use
:type toml_config: Toml
:param kwargs: Additional arguments to requests.request
(see http://docs.python-requests.org/en/latest/api/#requests.request)
:type kwargs: dict
:rtype: Response
"""
if timeout is True:
if toml_config is None:
toml_config = config.get_config()
timeout = config.get_config_val("core.timeout", toml_config)
timeout = (DEFAULT_CONNECT_TIMEOUT, timeout or DEFAULT_READ_TIMEOUT)
elif type(timeout) in (float, int):
timeout = (DEFAULT_CONNECT_TIMEOUT, timeout)
if 'headers' not in kwargs:
kwargs['headers'] = {'Accept': 'application/json'}
verify = _verify_ssl(url, verify, toml_config)
# Silence 'Unverified HTTPS request' and 'SecurityWarning' for bad certs
if verify is not None:
silence_requests_warnings()
logger.info('Sending HTTP [%r] to [%r]: %r', method, url,
kwargs.get('headers'))
try:
response = requests.request(method=method,
url=url,
timeout=timeout,
auth=auth,
verify=verify,
**kwargs)
except requests.exceptions.SSLError as e:
logger.exception("HTTP SSL Error")
msg = ("An SSL error occurred. To configure your SSL settings, "
"please run: `dcos config set core.ssl_verify <value>`")
description = config.get_property_description("core", "ssl_verify")
if description is not None:
msg += "\n<value>: {}".format(description)
raise DCOSException(msg)
except requests.exceptions.ConnectionError as e:
logger.exception("HTTP Connection Error")
raise DCOSConnectionError(url)
except requests.exceptions.Timeout as e:
logger.exception("HTTP Timeout")
raise DCOSException('Request to URL [{0}] timed out.'.format(url))
except requests.exceptions.RequestException as e:
logger.exception("HTTP Exception")
raise DCOSException('HTTP Exception: {}'.format(e))
logger.info('Received HTTP response [%r]: %r', response.status_code,
response.headers)
return response
def _get_token():
dcos_url = config.get_config().get('core.dcos_url')
hostname = urllib.parse.urlparse(dcos_url).hostname
return http._get_dcos_auth(None, None, None, hostname).token
def _spark_dist_dir():
dist_dir = config.get_config().get('spark.distribution_directory',
'~/.dcos/spark/dist')
return os.path.expanduser(dist_dir)
