def get_clusters(include_linked=False):
"""
Get configured or linked clusters.
:param include_linked: whether to look for linked clusters
:type include_linked: bool
:returns: list of Clusters
:rtype: [Clusters]
"""
clusters_path = config.get_clusters_path()
util.ensure_dir_exists(clusters_path)
clusters = set()
uuid_regex = re.compile((r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-'
r'[89ab][a-f0-9]{3}-[a-f0-9]{12}\Z'))
for entry in os.listdir(clusters_path):
entry_path = os.path.join(clusters_path, entry)
if os.path.isdir(entry_path) and uuid_regex.match(entry):
c = Cluster(entry)
clusters.add(c)
# Search for linked clusters after configured clusters, this makes sure
# the set gives priority to Cluster over LinkedCluster.
if include_linked:
try:
linked_clusters = get_linked_clusters()
except DCOSException:
pass
else:
clusters.update(linked_clusters)
return list(clusters)
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is None
found and `DCOS_CONFIG` set to default value. Only use to get the config,
not to resolve a specific config parameter. This should be done with
`get_config_val`.
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Configuration object
:rtype: Toml | MutableToml
"""
cluster_path = get_attached_cluster_path()
if cluster_path is None:
if uses_deprecated_config():
return get_global_config(mutable)
msg = ("No cluster is attached. "
"Please run `dcos cluster attach <cluster-name>`")
raise DCOSException(msg)
util.ensure_dir_exists(os.path.dirname(cluster_path))
path = os.path.join(cluster_path, "dcos.toml")
return load_from_path(path, mutable)
def _install_with_binary(package_name, env_directory, binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
try:
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
binary_file = os.path.join(env_bin_dir, binary_name)
shutil.move(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# check contents for package_name/env/bin folder structure
if not os.path.exists(env_bin_dir):
msg = (
"CLI subcommand for [{}] has an unexpected format. "
"Please contact the package maintainer".format(package_name)
)
raise DCOSException(msg)
else:
msg = "CLI subcommand for [{}] is an unsupported type: {}" "Please contact the package maintainer".format(
package_name, kind
)
raise DCOSException(msg)
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if f.startswith(constants.DCOS_COMMAND_PREFIX):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name)
return None
def setup_cluster_config(dcos_url, temp_path, stored_cert):
"""
Create a cluster directory for cluster specified in "temp_path"
directory.
:param dcos_url: url to DC/OS cluster
:type dcos_url: str
:param temp_path: path to temporary config dir
:type temp_path: str
:param stored_cert: whether we stored cert bundle in 'setup' dir
:type stored_cert: bool
:returns: path to cluster specific directory
:rtype: str
"""
try:
# find cluster id
cluster_url = dcos_url.rstrip('/') + '/metadata'
res = http.get(cluster_url, timeout=1)
cluster_id = res.json().get("CLUSTER_ID")
except DCOSException as e:
msg = ("Error trying to find cluster id: {}\n "
"Please make sure the provided DC/OS URL is valid: {}".format(
e, dcos_url))
raise DCOSException(msg)
# create cluster id dir
cluster_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR, cluster_id)
if os.path.exists(cluster_path):
raise DCOSException("Cluster [{}] is already setup".format(dcos_url))
util.ensure_dir_exists(cluster_path)
# move contents of setup dir to new location
for (path, dirnames, filenames) in os.walk(temp_path):
for f in filenames:
util.sh_copy(os.path.join(path, f), cluster_path)
cluster = Cluster(cluster_id)
config_path = cluster.get_config_path()
if stored_cert:
cert_path = os.path.join(cluster_path, "dcos_ca.crt")
config.set_val("core.ssl_verify", cert_path, config_path=config_path)
cluster_name = cluster_id
try:
url = dcos_url.rstrip('/') + '/mesos/state-summary'
name_query = http.get(url, toml_config=cluster.get_config())
cluster_name = name_query.json().get("cluster")
except DCOSException:
pass
config.set_val("cluster.name", cluster_name, config_path=config_path)
return cluster_path
def _install_with_binary(package_name, env_directory, binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
try:
env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
binary_file = os.path.join(env_bin_dir, binary_name)
shutil.move(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# check contents for package_name/env/bin folder structure
if not os.path.exists(env_bin_dir):
msg = ("CLI subcommand for [{}] has an unexpected format. "
"Please contact the package maintainer".format(
package_name))
raise DCOSException(msg)
else:
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
"Please contact the package maintainer".format(
package_name, kind))
raise DCOSException(msg)
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name)
return None
def add_cluster_dir(cluster_id, dcos_dir):
clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)
cluster_path = os.path.join(clusters_dir, cluster_id)
util.ensure_dir_exists(cluster_path)
os.path.join(cluster_path, "dcos.toml")
return cluster_path
def get_clusters():
"""
:returns: list of configured Clusters
:rtype: [Clusters]
"""
clusters_path = config.get_clusters_path()
util.ensure_dir_exists(clusters_path)
clusters = os.listdir(clusters_path)
return [Cluster(cluster_id) for cluster_id in clusters]
def install(pkg):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg)
_install_cli(pkg)
def install(pkg):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg)
_install_cli(pkg)
def install(pkg, options):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:param options: package parameters
:type options: dict
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg)
_install_env(pkg, options)
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is none
found and `DCOS_CONFIG` set to default value
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Configuration object
:rtype: Toml | MutableToml
"""
path = get_config_path()
default = get_default_config_path()
if path == default:
util.ensure_dir_exists(os.path.dirname(default))
return load_from_path(path, mutable)
def test_get_clusters():
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
assert cluster.get_clusters() == []
# cluster dir exists, no cluster
clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)
assert cluster.get_clusters() == []
# one cluster
cluster_id = "fake_cluster"
add_cluster_dir(cluster_id, tempdir)
assert cluster.get_clusters() == [_cluster(cluster_id)]
def install(pkg, options):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:param options: package parameters
:type options: dict
:rtype: None
"""
pkg_dir = _package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg)
_install_env(pkg, options)
def setup_directory():
"""
A context manager for the temporary setup directory created as a
placeholder before we find the cluster's CLUSTER_ID.
:returns: path of setup directory
:rtype: str
"""
try:
temp_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR, "setup")
util.ensure_dir_exists(temp_path)
yield temp_path
finally:
shutil.rmtree(temp_path, ignore_errors=True)
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is None
found and `DCOS_CONFIG` set to default value. Only use to get the config,
not to resolve a specific config parameter. This should be done with
`get_config_val`.
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Configuration object
:rtype: Toml | MutableToml
"""
path = get_config_path()
default = get_default_config_path()
if path == default:
util.ensure_dir_exists(os.path.dirname(default))
return load_from_path(path, mutable)
def get_clusters():
"""
:returns: list of configured Clusters
:rtype: [Clusters]
"""
clusters_path = config.get_clusters_path()
util.ensure_dir_exists(clusters_path)
clusters = []
uuid_regex = re.compile((r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-'
r'[89ab][a-f0-9]{3}-[a-f0-9]{12}\Z'))
for entry in os.listdir(clusters_path):
entry_path = os.path.join(clusters_path, entry)
if os.path.isdir(entry_path) and uuid_regex.match(entry):
clusters.append(entry)
return [Cluster(cluster_id) for cluster_id in clusters]
def get_config(mutable=False):
"""Returns the DCOS configuration object and creates config file is None
found and `DCOS_CONFIG` set to default value. Only use to get the config,
not to resolve a specific config parameter. This should be done with
`get_config_val`.
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Configuration object
:rtype: Toml | MutableToml
"""
path = get_config_path()
default = get_default_config_path()
if path == default:
util.ensure_dir_exists(os.path.dirname(default))
return load_from_path(path, mutable)
def install(pkg, global_=False):
"""Installs the dcos cli subcommand
:param pkg: the package to install
:type pkg: Package
:param global_: whether to install the CLI globally
:type global_: bool
:rtype: None
"""
if global_ or config.uses_deprecated_config():
pkg_dir = global_package_dir(pkg.name())
else:
pkg_dir = _cluster_package_dir(pkg.name())
util.ensure_dir_exists(pkg_dir)
_write_package_json(pkg, pkg_dir)
_install_cli(pkg, pkg_dir)
def move_to_cluster_config():
"""Create a cluster specific config file + directory
from a global config file. This will move users from global config
structure (~/.dcos/dcos.toml) to the cluster specific one
(~/.dcos/clusters/CLUSTER_ID/dcos.toml) and set that cluster as
the "attached" cluster.
:rtype: None
"""
global_config = config.get_global_config()
dcos_url = config.get_config_val("core.dcos_url", global_config)
# if no cluster is set, do not move the cluster yet
if dcos_url is None:
return
try:
# find cluster id
cluster_url = dcos_url.rstrip('/') + '/metadata'
res = http.get(cluster_url)
cluster_id = res.json().get("CLUSTER_ID")
# don't move cluster if dcos_url is not valid
except DCOSException as e:
logger.error("Error trying to find cluster id: {}".format(e))
return
# create cluster id dir
cluster_path = os.path.join(config.get_config_dir_path(),
constants.DCOS_CLUSTERS_SUBDIR, cluster_id)
util.ensure_dir_exists(cluster_path)
# move config file to new location
global_config_path = config.get_global_config_path()
util.sh_copy(global_config_path, cluster_path)
# set cluster as attached
util.ensure_file_exists(
os.path.join(cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
def load_from_path(path, mutable=False):
"""Loads a TOML file from the path
:param path: Path to the TOML file
:type path: str
:param mutable: True if the returned Toml object should be mutable
:type mutable: boolean
:returns: Map for the configuration file
:rtype: Toml | MutableToml
"""
util.ensure_dir_exists(os.path.dirname(path))
util.ensure_file_exists(path)
util.enforce_file_permissions(path)
with util.open_file(path, 'r') as config_file:
try:
toml_obj = toml.loads(config_file.read())
except Exception as e:
raise DCOSException('Error parsing config file at [{}]: {}'.format(
path, e))
return (MutableToml if mutable else Toml)(toml_obj)
def test_get_clusters():
with env(), util.tempdir() as tempdir:
os.environ[constants.DCOS_DIR_ENV] = tempdir
# no config file of any type
assert cluster.get_clusters() == []
# cluster dir exists, no cluster
clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)
assert cluster.get_clusters() == []
# a valid cluster
cluster_id = "a8b53513-63d4-4059-8b08-fde4fe1f1a83"
add_cluster_dir(cluster_id, tempdir)
# Make sure clusters dir can contain random files / folders
# cf. https://jira.mesosphere.com/browse/DCOS_OSS-1782
util.ensure_file_exists(os.path.join(clusters_dir, '.DS_Store'))
util.ensure_dir_exists(os.path.join(clusters_dir, 'not_a_cluster'))
assert cluster.get_clusters() == [_cluster(cluster_id)]
def _install_with_binary(
package_name,
env_directory,
binary_cli):
"""
:param package_name: the name of the package
:type package_name: str
:param env_directory: the path to the directory in which to install the
package's binary_cli
:type env_directory: str
:param binary_cli: binary cli to install
:type binary_cli: str
:rtype: None
"""
binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")
binary_url = _rewrite_binary_url(
binary_url,
config.get_config_val("core.dcos_url"))
try:
env_bin_dir = os.path.join(env_directory, 'bin')
if not os.path.exists(env_bin_dir) and util.is_windows_platform():
env_bin_dir = os.path.join(env_directory, "Scripts")
if kind in ["executable", "zip"]:
with util.temptext() as file_tmp:
_, binary_tmp = file_tmp
_download_and_store(binary_url, binary_tmp)
_check_hash(binary_tmp, binary_cli.get("contentHash"))
if kind == "executable":
util.ensure_dir_exists(env_bin_dir)
binary_name = "dcos-{}".format(package_name)
if util.is_windows_platform():
binary_name += '.exe'
binary_file = os.path.join(env_bin_dir, binary_name)
# copy to avoid windows error of moving open file
# binary_tmp will be removed by context manager
shutil.copy(binary_tmp, binary_file)
else:
# kind == "zip"
with zipfile.ZipFile(binary_tmp) as zf:
zf.extractall(env_directory)
# Check contents for package_name/env/bin folder structure.
# Don't fail if the env or Scripts folder doesn't exist,
# it is possible that a plugin.toml file specifies a custom
# path. The package format should be validated in a more thorough
# way during the refactoring of this command in Go.
if os.path.exists(env_bin_dir):
# make binar(ies) executable
for f in os.listdir(env_bin_dir):
binary = os.path.join(env_bin_dir, f)
if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
st = os.stat(binary)
os.chmod(binary, st.st_mode | stat.S_IEXEC)
else:
msg = ("CLI subcommand for [{}] is an unsupported type: {}"
"Please contact the package maintainer".format(
package_name, kind))
raise DCOSException(msg)
except DCOSException:
raise
except Exception as e:
logger.exception(e)
raise _generic_error(package_name, e.message)
return None
def _create_clusters_dir(dcos_dir):
clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
util.ensure_dir_exists(clusters_dir)