Beispiel #1
0
def get_clusters(include_linked=False):
    """
    Get configured or linked clusters.

    :param include_linked: whether to look for linked clusters
    :type include_linked: bool
    :returns: list of Clusters
    :rtype: [Clusters]
    """

    clusters_path = config.get_clusters_path()
    util.ensure_dir_exists(clusters_path)
    clusters = set()

    uuid_regex = re.compile((r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-'
                             r'[89ab][a-f0-9]{3}-[a-f0-9]{12}\Z'))

    for entry in os.listdir(clusters_path):
        entry_path = os.path.join(clusters_path, entry)
        if os.path.isdir(entry_path) and uuid_regex.match(entry):
            c = Cluster(entry)
            clusters.add(c)

    # Search for linked clusters after configured clusters, this makes sure
    # the set gives priority to Cluster over LinkedCluster.
    if include_linked:
        try:
            linked_clusters = get_linked_clusters()
        except DCOSException:
            pass
        else:
            clusters.update(linked_clusters)

    return list(clusters)
Beispiel #2
0
def get_config(mutable=False):
    """Returns the DCOS configuration object and creates config file is None
    found and `DCOS_CONFIG` set to default value. Only use to get the config,
    not to resolve a specific config parameter. This should be done with
    `get_config_val`.

    :param mutable: True if the returned Toml object should be mutable
    :type mutable: boolean
    :returns: Configuration object
    :rtype: Toml | MutableToml
    """

    cluster_path = get_attached_cluster_path()
    if cluster_path is None:
        if uses_deprecated_config():
            return get_global_config(mutable)

        msg = ("No cluster is attached. "
               "Please run `dcos cluster attach <cluster-name>`")
        raise DCOSException(msg)

    util.ensure_dir_exists(os.path.dirname(cluster_path))

    path = os.path.join(cluster_path, "dcos.toml")
    return load_from_path(path, mutable)
Beispiel #3
0
def _install_with_binary(package_name, env_directory, binary_cli):
    """
    :param package_name: the name of the package
    :type package_name: str
    :param env_directory: the path to the directory in which to install the
                          package's binary_cli
    :type env_directory: str
    :param binary_cli: binary cli to install
    :type binary_cli: str
    :rtype: None
    """

    binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")

    try:
        env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)

        if kind in ["executable", "zip"]:
            with util.temptext() as file_tmp:
                _, binary_tmp = file_tmp
                _download_and_store(binary_url, binary_tmp)
                _check_hash(binary_tmp, binary_cli.get("contentHash"))

                if kind == "executable":
                    util.ensure_dir_exists(env_bin_dir)
                    binary_name = "dcos-{}".format(package_name)
                    binary_file = os.path.join(env_bin_dir, binary_name)
                    shutil.move(binary_tmp, binary_file)
                else:
                    # kind == "zip"
                    with zipfile.ZipFile(binary_tmp) as zf:
                        zf.extractall(env_directory)

            # check contents for package_name/env/bin folder structure
            if not os.path.exists(env_bin_dir):
                msg = (
                    "CLI subcommand for [{}] has an unexpected format. "
                    "Please contact the package maintainer".format(package_name)
                )
                raise DCOSException(msg)
        else:
            msg = "CLI subcommand for [{}] is an unsupported type: {}" "Please contact the package maintainer".format(
                package_name, kind
            )
            raise DCOSException(msg)

        # make binar(ies) executable
        for f in os.listdir(env_bin_dir):
            binary = os.path.join(env_bin_dir, f)
            if f.startswith(constants.DCOS_COMMAND_PREFIX):
                st = os.stat(binary)
                os.chmod(binary, st.st_mode | stat.S_IEXEC)
    except DCOSException:
        raise
    except Exception as e:
        logger.exception(e)
        raise _generic_error(package_name)

    return None
Beispiel #4
0
def setup_cluster_config(dcos_url, temp_path, stored_cert):
    """
    Create a cluster directory for cluster specified in "temp_path"
    directory.

    :param dcos_url: url to DC/OS cluster
    :type dcos_url: str
    :param temp_path: path to temporary config dir
    :type temp_path: str
    :param stored_cert: whether we stored cert bundle in 'setup' dir
    :type stored_cert: bool
    :returns: path to cluster specific directory
    :rtype: str
    """

    try:
        # find cluster id
        cluster_url = dcos_url.rstrip('/') + '/metadata'
        res = http.get(cluster_url, timeout=1)
        cluster_id = res.json().get("CLUSTER_ID")

    except DCOSException as e:
        msg = ("Error trying to find cluster id: {}\n "
               "Please make sure the provided DC/OS URL is valid: {}".format(
                   e, dcos_url))
        raise DCOSException(msg)

    # create cluster id dir
    cluster_path = os.path.join(config.get_config_dir_path(),
                                constants.DCOS_CLUSTERS_SUBDIR, cluster_id)
    if os.path.exists(cluster_path):
        raise DCOSException("Cluster [{}] is already setup".format(dcos_url))

    util.ensure_dir_exists(cluster_path)

    # move contents of setup dir to new location
    for (path, dirnames, filenames) in os.walk(temp_path):
        for f in filenames:
            util.sh_copy(os.path.join(path, f), cluster_path)

    cluster = Cluster(cluster_id)
    config_path = cluster.get_config_path()
    if stored_cert:
        cert_path = os.path.join(cluster_path, "dcos_ca.crt")
        config.set_val("core.ssl_verify", cert_path, config_path=config_path)

    cluster_name = cluster_id
    try:
        url = dcos_url.rstrip('/') + '/mesos/state-summary'
        name_query = http.get(url, toml_config=cluster.get_config())
        cluster_name = name_query.json().get("cluster")

    except DCOSException:
        pass

    config.set_val("cluster.name", cluster_name, config_path=config_path)

    return cluster_path
Beispiel #5
0
def _install_with_binary(package_name, env_directory, binary_cli):
    """
    :param package_name: the name of the package
    :type package_name: str
    :param env_directory: the path to the directory in which to install the
                          package's binary_cli
    :type env_directory: str
    :param binary_cli: binary cli to install
    :type binary_cli: str
    :rtype: None
    """

    binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")

    try:
        env_bin_dir = os.path.join(env_directory, BIN_DIRECTORY)

        if kind in ["executable", "zip"]:
            with util.temptext() as file_tmp:
                _, binary_tmp = file_tmp
                _download_and_store(binary_url, binary_tmp)
                _check_hash(binary_tmp, binary_cli.get("contentHash"))

                if kind == "executable":
                    util.ensure_dir_exists(env_bin_dir)
                    binary_name = "dcos-{}".format(package_name)
                    binary_file = os.path.join(env_bin_dir, binary_name)
                    shutil.move(binary_tmp, binary_file)
                else:
                    # kind == "zip"
                    with zipfile.ZipFile(binary_tmp) as zf:
                        zf.extractall(env_directory)

            # check contents for package_name/env/bin folder structure
            if not os.path.exists(env_bin_dir):
                msg = ("CLI subcommand for [{}] has an unexpected format. "
                       "Please contact the package maintainer".format(
                           package_name))
                raise DCOSException(msg)
        else:
            msg = ("CLI subcommand for [{}] is an unsupported type: {}"
                   "Please contact the package maintainer".format(
                       package_name, kind))
            raise DCOSException(msg)

        # make binar(ies) executable
        for f in os.listdir(env_bin_dir):
            binary = os.path.join(env_bin_dir, f)
            if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
                st = os.stat(binary)
                os.chmod(binary, st.st_mode | stat.S_IEXEC)
    except DCOSException:
        raise
    except Exception as e:
        logger.exception(e)
        raise _generic_error(package_name)

    return None
Beispiel #6
0
def add_cluster_dir(cluster_id, dcos_dir):
    clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
    util.ensure_dir_exists(clusters_dir)

    cluster_path = os.path.join(clusters_dir, cluster_id)
    util.ensure_dir_exists(cluster_path)

    os.path.join(cluster_path, "dcos.toml")
    return cluster_path
Beispiel #7
0
def get_clusters():
    """
    :returns: list of configured Clusters
    :rtype: [Clusters]
    """

    clusters_path = config.get_clusters_path()
    util.ensure_dir_exists(clusters_path)
    clusters = os.listdir(clusters_path)
    return [Cluster(cluster_id) for cluster_id in clusters]
Beispiel #8
0
def install(pkg):
    """Installs the dcos cli subcommand

    :param pkg: the package to install
    :type pkg: Package
    :rtype: None
    """

    pkg_dir = _package_dir(pkg.name())
    util.ensure_dir_exists(pkg_dir)

    _write_package_json(pkg)

    _install_cli(pkg)
Beispiel #9
0
def install(pkg):
    """Installs the dcos cli subcommand

    :param pkg: the package to install
    :type pkg: Package
    :rtype: None
    """

    pkg_dir = _package_dir(pkg.name())
    util.ensure_dir_exists(pkg_dir)

    _write_package_json(pkg)

    _install_cli(pkg)
Beispiel #10
0
def install(pkg, options):
    """Installs the dcos cli subcommand

    :param pkg: the package to install
    :type pkg: Package
    :param options: package parameters
    :type options: dict
    :rtype: None
    """

    pkg_dir = _package_dir(pkg.name())
    util.ensure_dir_exists(pkg_dir)

    _write_package_json(pkg)

    _install_env(pkg, options)
Beispiel #11
0
def get_config(mutable=False):
    """Returns the DCOS configuration object and creates config file is none
    found and `DCOS_CONFIG` set to default value

    :param mutable: True if the returned Toml object should be mutable
    :type mutable: boolean
    :returns: Configuration object
    :rtype: Toml | MutableToml
    """

    path = get_config_path()
    default = get_default_config_path()

    if path == default:
        util.ensure_dir_exists(os.path.dirname(default))
    return load_from_path(path, mutable)
def test_get_clusters():
    with env(), util.tempdir() as tempdir:
        os.environ[constants.DCOS_DIR_ENV] = tempdir

        # no config file of any type
        assert cluster.get_clusters() == []

        # cluster dir exists, no cluster
        clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
        util.ensure_dir_exists(clusters_dir)
        assert cluster.get_clusters() == []

        # one cluster
        cluster_id = "fake_cluster"
        add_cluster_dir(cluster_id, tempdir)
        assert cluster.get_clusters() == [_cluster(cluster_id)]
Beispiel #13
0
def install(pkg, options):
    """Installs the dcos cli subcommand

    :param pkg: the package to install
    :type pkg: Package
    :param options: package parameters
    :type options: dict
    :rtype: None
    """

    pkg_dir = _package_dir(pkg.name())
    util.ensure_dir_exists(pkg_dir)

    _write_package_json(pkg)

    _install_env(pkg, options)
Beispiel #14
0
def setup_directory():
    """
    A context manager for the temporary setup directory created as a
    placeholder before we find the cluster's CLUSTER_ID.

    :returns: path of setup directory
    :rtype: str
    """

    try:
        temp_path = os.path.join(config.get_config_dir_path(),
                                 constants.DCOS_CLUSTERS_SUBDIR, "setup")
        util.ensure_dir_exists(temp_path)

        yield temp_path
    finally:
        shutil.rmtree(temp_path, ignore_errors=True)
Beispiel #15
0
def get_config(mutable=False):
    """Returns the DCOS configuration object and creates config file is None
    found and `DCOS_CONFIG` set to default value. Only use to get the config,
    not to resolve a specific config parameter. This should be done with
    `get_config_val`.

    :param mutable: True if the returned Toml object should be mutable
    :type mutable: boolean
    :returns: Configuration object
    :rtype: Toml | MutableToml
    """

    path = get_config_path()
    default = get_default_config_path()

    if path == default:
        util.ensure_dir_exists(os.path.dirname(default))
    return load_from_path(path, mutable)
Beispiel #16
0
def get_clusters():
    """
    :returns: list of configured Clusters
    :rtype: [Clusters]
    """

    clusters_path = config.get_clusters_path()
    util.ensure_dir_exists(clusters_path)
    clusters = []

    uuid_regex = re.compile((r'^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-'
                             r'[89ab][a-f0-9]{3}-[a-f0-9]{12}\Z'))
    for entry in os.listdir(clusters_path):
        entry_path = os.path.join(clusters_path, entry)
        if os.path.isdir(entry_path) and uuid_regex.match(entry):
            clusters.append(entry)

    return [Cluster(cluster_id) for cluster_id in clusters]
Beispiel #17
0
def get_config(mutable=False):
    """Returns the DCOS configuration object and creates config file is None
    found and `DCOS_CONFIG` set to default value. Only use to get the config,
    not to resolve a specific config parameter. This should be done with
    `get_config_val`.

    :param mutable: True if the returned Toml object should be mutable
    :type mutable: boolean
    :returns: Configuration object
    :rtype: Toml | MutableToml
    """

    path = get_config_path()
    default = get_default_config_path()

    if path == default:
        util.ensure_dir_exists(os.path.dirname(default))
    return load_from_path(path, mutable)
Beispiel #18
0
def install(pkg, global_=False):
    """Installs the dcos cli subcommand

    :param pkg: the package to install
    :type pkg: Package
    :param global_: whether to install the CLI globally
    :type global_: bool
    :rtype: None
    """

    if global_ or config.uses_deprecated_config():
        pkg_dir = global_package_dir(pkg.name())
    else:
        pkg_dir = _cluster_package_dir(pkg.name())

    util.ensure_dir_exists(pkg_dir)

    _write_package_json(pkg, pkg_dir)

    _install_cli(pkg, pkg_dir)
Beispiel #19
0
def move_to_cluster_config():
    """Create a cluster specific config file + directory
    from a global config file. This will move users from global config
    structure (~/.dcos/dcos.toml) to the cluster specific one
    (~/.dcos/clusters/CLUSTER_ID/dcos.toml) and set that cluster as
    the "attached" cluster.

    :rtype: None
    """

    global_config = config.get_global_config()
    dcos_url = config.get_config_val("core.dcos_url", global_config)

    # if no cluster is set, do not move the cluster yet
    if dcos_url is None:
        return

    try:
        # find cluster id
        cluster_url = dcos_url.rstrip('/') + '/metadata'
        res = http.get(cluster_url)
        cluster_id = res.json().get("CLUSTER_ID")

    # don't move cluster if dcos_url is not valid
    except DCOSException as e:
        logger.error("Error trying to find cluster id: {}".format(e))
        return

    # create cluster id dir
    cluster_path = os.path.join(config.get_config_dir_path(),
                                constants.DCOS_CLUSTERS_SUBDIR, cluster_id)

    util.ensure_dir_exists(cluster_path)

    # move config file to new location
    global_config_path = config.get_global_config_path()
    util.sh_copy(global_config_path, cluster_path)

    # set cluster as attached
    util.ensure_file_exists(
        os.path.join(cluster_path, constants.DCOS_CLUSTER_ATTACHED_FILE))
Beispiel #20
0
def load_from_path(path, mutable=False):
    """Loads a TOML file from the path

    :param path: Path to the TOML file
    :type path: str
    :param mutable: True if the returned Toml object should be mutable
    :type mutable: boolean
    :returns: Map for the configuration file
    :rtype: Toml | MutableToml
    """

    util.ensure_dir_exists(os.path.dirname(path))
    util.ensure_file_exists(path)
    util.enforce_file_permissions(path)
    with util.open_file(path, 'r') as config_file:
        try:
            toml_obj = toml.loads(config_file.read())
        except Exception as e:
            raise DCOSException('Error parsing config file at [{}]: {}'.format(
                path, e))
        return (MutableToml if mutable else Toml)(toml_obj)
Beispiel #21
0
def test_get_clusters():
    with env(), util.tempdir() as tempdir:
        os.environ[constants.DCOS_DIR_ENV] = tempdir

        # no config file of any type
        assert cluster.get_clusters() == []

        # cluster dir exists, no cluster
        clusters_dir = os.path.join(tempdir, constants.DCOS_CLUSTERS_SUBDIR)
        util.ensure_dir_exists(clusters_dir)
        assert cluster.get_clusters() == []

        # a valid cluster
        cluster_id = "a8b53513-63d4-4059-8b08-fde4fe1f1a83"
        add_cluster_dir(cluster_id, tempdir)

        # Make sure clusters dir can contain random files / folders
        # cf. https://jira.mesosphere.com/browse/DCOS_OSS-1782
        util.ensure_file_exists(os.path.join(clusters_dir, '.DS_Store'))
        util.ensure_dir_exists(os.path.join(clusters_dir, 'not_a_cluster'))

        assert cluster.get_clusters() == [_cluster(cluster_id)]
Beispiel #22
0
def _install_with_binary(
        package_name,
        env_directory,
        binary_cli):
    """
    :param package_name: the name of the package
    :type package_name: str
    :param env_directory: the path to the directory in which to install the
                          package's binary_cli
    :type env_directory: str
    :param binary_cli: binary cli to install
    :type binary_cli: str
    :rtype: None
    """

    binary_url, kind = binary_cli.get("url"), binary_cli.get("kind")

    binary_url = _rewrite_binary_url(
        binary_url,
        config.get_config_val("core.dcos_url"))

    try:
        env_bin_dir = os.path.join(env_directory, 'bin')
        if not os.path.exists(env_bin_dir) and util.is_windows_platform():
            env_bin_dir = os.path.join(env_directory, "Scripts")

        if kind in ["executable", "zip"]:
            with util.temptext() as file_tmp:
                _, binary_tmp = file_tmp
                _download_and_store(binary_url, binary_tmp)
                _check_hash(binary_tmp, binary_cli.get("contentHash"))

                if kind == "executable":
                    util.ensure_dir_exists(env_bin_dir)
                    binary_name = "dcos-{}".format(package_name)
                    if util.is_windows_platform():
                        binary_name += '.exe'
                    binary_file = os.path.join(env_bin_dir, binary_name)

                    # copy to avoid windows error of moving open file
                    # binary_tmp will be removed by context manager
                    shutil.copy(binary_tmp, binary_file)
                else:
                    # kind == "zip"
                    with zipfile.ZipFile(binary_tmp) as zf:
                        zf.extractall(env_directory)

            # Check contents for package_name/env/bin folder structure.
            # Don't fail if the env or Scripts folder doesn't exist,
            # it is possible that a plugin.toml file specifies a custom
            # path. The package format should be validated in a more thorough
            # way during the refactoring of this command in Go.
            if os.path.exists(env_bin_dir):
                # make binar(ies) executable
                for f in os.listdir(env_bin_dir):
                    binary = os.path.join(env_bin_dir, f)
                    if (f.startswith(constants.DCOS_COMMAND_PREFIX)):
                        st = os.stat(binary)
                        os.chmod(binary, st.st_mode | stat.S_IEXEC)
        else:
            msg = ("CLI subcommand for [{}] is an unsupported type: {}"
                   "Please contact the package maintainer".format(
                       package_name, kind))
            raise DCOSException(msg)

    except DCOSException:
        raise
    except Exception as e:
        logger.exception(e)
        raise _generic_error(package_name, e.message)

    return None
Beispiel #23
0
def _create_clusters_dir(dcos_dir):
    clusters_dir = os.path.join(dcos_dir, constants.DCOS_CLUSTERS_SUBDIR)
    util.ensure_dir_exists(clusters_dir)