Пример #1
0
    def __init__(
            self,
            root,
            config_dir,
            rooted_systemd,
            manage_systemd,
            block_systemd,
            fake_path=False,
            skip_systemd_dirs=False):
        assert type(rooted_systemd) == bool
        assert type(fake_path) == bool
        self.__root = os.path.abspath(root)
        self.__config_dir = os.path.abspath(config_dir) if config_dir else None
        if not skip_systemd_dirs:
            if rooted_systemd:
                self.__systemd_dir = "{}/dcos.target.wants".format(root)
            else:
                self.__systemd_dir = "/etc/systemd/system/dcos.target.wants"
        self.__manage_systemd = manage_systemd
        self.__block_systemd = block_systemd

        # Look up the machine roles
        self.__roles = []
        if self.__config_dir:
            self.__roles = if_exists(os.listdir, os.path.join(self.__config_dir, "roles"))
            if self.__roles is None:
                self.__roles = []

        self.__well_known_dirs = ["bin", "etc", "include", "lib"]
        if not skip_systemd_dirs:
            self.__well_known_dirs.append(self.__systemd_dir)

        self.__fake_path = fake_path
        self.__skip_systemd_dirs = skip_systemd_dirs
Пример #2
0
    def __init__(self,
                 root,
                 config_dir,
                 rooted_systemd,
                 manage_systemd,
                 block_systemd,
                 fake_path=False,
                 skip_systemd_dirs=False):
        assert type(rooted_systemd) == bool
        assert type(fake_path) == bool
        self.__root = os.path.abspath(root)
        self.__config_dir = os.path.abspath(config_dir) if config_dir else None
        if not skip_systemd_dirs:
            if rooted_systemd:
                self.__systemd_dir = "{}/dcos.target.wants".format(root)
            else:
                self.__systemd_dir = "/etc/systemd/system/dcos.target.wants"
        self.__manage_systemd = manage_systemd
        self.__block_systemd = block_systemd

        # Look up the machine roles
        self.__roles = []
        if self.__config_dir:
            self.__roles = if_exists(os.listdir,
                                     os.path.join(self.__config_dir, "roles"))
            if self.__roles is None:
                self.__roles = []

        self.__well_known_dirs = ["bin", "etc", "include", "lib"]
        if not skip_systemd_dirs:
            self.__well_known_dirs.append(self.__systemd_dir)

        self.__fake_path = fake_path
        self.__skip_systemd_dirs = skip_systemd_dirs
Пример #3
0
def _do_bootstrap(install, repository):
    # These files should be set by the environment which initially builds
    # the host (cloud-init).
    repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url"))

    def fetcher(id, target):
        if repository_url is None:
            raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(id))
        return requests_fetcher(repository_url, id, target, os.getcwd())

    setup_pkg_dir = install.get_config_filename("setup-packages")
    if os.path.exists(setup_pkg_dir):
        raise ValidationError(
            "setup-packages is no longer supported. It's functionality has been replaced with late "
            "binding packages. Found setup packages dir: {}".format(setup_pkg_dir))

    setup_packages_to_activate = []

    # If the host has late config values, build the late config package from them.
    late_config = if_exists(load_yaml, install.get_config_filename("setup-flags/late-config.yaml"))
    if late_config:
        pkg_id_str = late_config['late_bound_package_id']
        late_values = late_config['bound_values']
        print("Binding late config to late package {}".format(pkg_id_str))
        print("Bound values: {}".format(late_values))

        if not PackageId.is_id(pkg_id_str):
            raise ValidationError("Invalid late package id: {}".format(pkg_id_str))
        pkg_id = PackageId(pkg_id_str)
        if pkg_id.version != "setup":
            raise ValidationError("Late package must have the version setup. Bad package: {}".format(pkg_id_str))

        # Collect the late config package.
        with tempfile.NamedTemporaryFile() as f:
            download(
                f.name,
                repository_url + '/packages/{0}/{1}.dcos_config'.format(pkg_id.name, pkg_id_str),
                os.getcwd(),
                rm_on_error=False,
            )
            late_package = load_yaml(f.name)

        # Resolve the late package using the bound late config values.
        final_late_package = resolve_late_package(late_package, late_values)

        # Render the package onto the filesystem and add it to the package
        # repository.
        with tempfile.NamedTemporaryFile() as f:
            do_gen_package(final_late_package, f.name)
            repository.add(lambda _, target: extract_tarball(f.name, target), pkg_id_str)
        setup_packages_to_activate.append(pkg_id_str)

    # If active.json is set on the host, use that as the set of packages to
    # activate. Otherwise just use the set of currently active packages (those
    # active in the bootstrap tarball)
    to_activate = None
    active_path = install.get_config_filename("setup-flags/active.json")
    if os.path.exists(active_path):
        print("Loaded active packages from", active_path)
        to_activate = load_json(active_path)

        # Ensure all packages are local
        print("Ensuring all packages in active set {} are local".format(",".join(to_activate)))
        for package in to_activate:
            repository.add(fetcher, package)
    else:
        print("Calculated active packages from bootstrap tarball")
        to_activate = list(install.get_active())

        package_list_filename = install.get_config_filename("setup-flags/cluster-package-list")
        print("Checking for cluster packages in:", package_list_filename)
        package_list_id = if_exists(load_string, package_list_filename)
        if package_list_id:
            print("Cluster package list:", package_list_id)
            cluster_packages = _get_package_list(package_list_id, repository_url)
            print("Loading cluster-packages: {}".format(cluster_packages))

            for package_id_str in cluster_packages:
                # Validate the package ids
                pkg_id = PackageId(package_id_str)

                # Fetch the packages if not local
                if not repository.has_package(package_id_str):
                    repository.add(fetcher, package_id_str)

                # Add the package to the set to activate
                setup_packages_to_activate.append(package_id_str)
        else:
            print("No cluster-packages specified")

    # Calculate the full set of final packages (Explicit activations + setup packages).
    # De-duplicate using a set.
    to_activate = list(set(to_activate + setup_packages_to_activate))

    print("Activating packages")
    install.activate(repository.load_packages(to_activate))
Пример #4
0
def _do_bootstrap(install, repository):
    # These files should be set by the environment which initially builds
    # the host (cloud-init).
    repository_url = if_exists(
        load_string, install.get_config_filename("setup-flags/repository-url"))

    # TODO(cmaloney): If there is 1+ master, grab the active config from a master.
    # If the config can't be grabbed from any of them, fail.
    def fetcher(id, target):
        if repository_url is None:
            raise ValidationError(
                "ERROR: Non-local package {} but no repository url given.".
                format(repository_url))
        return requests_fetcher(repository_url, id, target, os.getcwd())

    # Copy host/cluster-specific packages written to the filesystem manually
    # from the setup-packages folder into the repository. Do not overwrite or
    # merge existing packages, hard fail instead.
    setup_packages_to_activate = []
    setup_pkg_dir = install.get_config_filename("setup-packages")
    copy_fetcher = partial(_copy_fetcher, setup_pkg_dir)
    if os.path.exists(setup_pkg_dir):
        for pkg_id_str in os.listdir(setup_pkg_dir):
            print("Installing setup package: {}".format(pkg_id_str))
            if not PackageId.is_id(pkg_id_str):
                raise ValidationError(
                    "Invalid package id in setup package: {}".format(
                        pkg_id_str))
            pkg_id = PackageId(pkg_id_str)
            if pkg_id.version != "setup":
                raise ValidationError(
                    "Setup packages (those in `{0}`) must have the version setup. "
                    "Bad package: {1}".format(setup_pkg_dir, pkg_id_str))

            # Make sure there is no existing package
            if repository.has_package(pkg_id_str):
                print("WARNING: Ignoring already installed package {}".format(
                    pkg_id_str))

            repository.add(copy_fetcher, pkg_id_str)
            setup_packages_to_activate.append(pkg_id_str)

    # If active.json is set on the host, use that as the set of packages to
    # activate. Otherwise just use the set of currently active packages (those
    # active in the bootstrap tarball)
    to_activate = None
    active_path = install.get_config_filename("setup-flags/active.json")
    if os.path.exists(active_path):
        print("Loaded active packages from", active_path)
        to_activate = load_json(active_path)

        # Ensure all packages are local
        print("Ensuring all packages in active set {} are local".format(
            ",".join(to_activate)))
        for package in to_activate:
            repository.add(fetcher, package)
    else:
        print("Calculated active packages from bootstrap tarball")
        to_activate = list(install.get_active())

        # Fetch and activate all requested additional packages to accompany the bootstrap packages.
        cluster_packages_filename = install.get_config_filename(
            "setup-flags/cluster-packages.json")
        cluster_packages = if_exists(load_json, cluster_packages_filename)
        print("Checking for cluster packages in:", cluster_packages_filename)
        if cluster_packages:
            if not isinstance(cluster_packages, list):
                print(
                    'ERROR: {} should contain a JSON list of packages. Got a {}'
                    .format(cluster_packages_filename, type(cluster_packages)))
            print("Loading cluster-packages: {}".format(cluster_packages))

            for package_id_str in cluster_packages:
                # Validate the package ids
                pkg_id = PackageId(package_id_str)

                # Fetch the packages if not local
                if not repository.has_package(package_id_str):
                    repository.add(fetcher, package_id_str)

                # Add the package to the set to activate
                setup_packages_to_activate.append(package_id_str)
        else:
            print("No cluster-packages specified")

    # Calculate the full set of final packages (Explicit activations + setup packages).
    # De-duplicate using a set.
    to_activate = list(set(to_activate + setup_packages_to_activate))

    print("Activating packages")
    install.activate(repository.load_packages(to_activate))
Пример #5
0
def _do_bootstrap(install, repository):
    # These files should be set by the environment which initially builds
    # the host (cloud-init).
    repository_url = if_exists(load_string, install.get_config_filename("setup-flags/repository-url"))

    # TODO(cmaloney): If there is 1+ master, grab the active config from a master.
    # If the config can't be grabbed from any of them, fail.
    def fetcher(id, target):
        if repository_url is None:
            raise ValidationError("ERROR: Non-local package {} but no repository url given.".format(repository_url))
        return requests_fetcher(repository_url, id, target, os.getcwd())

    # Copy host/cluster-specific packages written to the filesystem manually
    # from the setup-packages folder into the repository. Do not overwrite or
    # merge existing packages, hard fail instead.
    setup_packages_to_activate = []
    setup_pkg_dir = install.get_config_filename("setup-packages")
    copy_fetcher = partial(_copy_fetcher, setup_pkg_dir)
    if os.path.exists(setup_pkg_dir):
        for pkg_id_str in os.listdir(setup_pkg_dir):
            print("Installing setup package: {}".format(pkg_id_str))
            if not PackageId.is_id(pkg_id_str):
                raise ValidationError("Invalid package id in setup package: {}".format(pkg_id_str))
            pkg_id = PackageId(pkg_id_str)
            if pkg_id.version != "setup":
                raise ValidationError(
                    "Setup packages (those in `{0}`) must have the version setup. "
                    "Bad package: {1}".format(setup_pkg_dir, pkg_id_str))

            # Make sure there is no existing package
            if repository.has_package(pkg_id_str):
                print("WARNING: Ignoring already installed package {}".format(pkg_id_str))

            repository.add(copy_fetcher, pkg_id_str)
            setup_packages_to_activate.append(pkg_id_str)

    # If active.json is set on the host, use that as the set of packages to
    # activate. Otherwise just use the set of currently active packages (those
    # active in the bootstrap tarball)
    to_activate = None
    active_path = install.get_config_filename("setup-flags/active.json")
    if os.path.exists(active_path):
        print("Loaded active packages from", active_path)
        to_activate = load_json(active_path)

        # Ensure all packages are local
        print("Ensuring all packages in active set {} are local".format(",".join(to_activate)))
        for package in to_activate:
            repository.add(fetcher, package)
    else:
        print("Calculated active packages from bootstrap tarball")
        to_activate = list(install.get_active())

        # Fetch and activate all requested additional packages to accompany the bootstrap packages.
        cluster_packages_filename = install.get_config_filename("setup-flags/cluster-packages.json")
        cluster_packages = if_exists(load_json, cluster_packages_filename)
        print("Checking for cluster packages in:", cluster_packages_filename)
        if cluster_packages:
            if not isinstance(cluster_packages, list):
                print('ERROR: {} should contain a JSON list of packages. Got a {}'.format(cluster_packages_filename,
                                                                                          type(cluster_packages)))
            print("Loading cluster-packages: {}".format(cluster_packages))

            for package_id_str in cluster_packages:
                # Validate the package ids
                pkg_id = PackageId(package_id_str)

                # Fetch the packages if not local
                if not repository.has_package(package_id_str):
                    repository.add(fetcher, package_id_str)

                # Add the package to the set to activate
                setup_packages_to_activate.append(package_id_str)
        else:
            print("No cluster-packages specified")

    # Calculate the full set of final packages (Explicit activations + setup packages).
    # De-duplicate using a set.
    to_activate = list(set(to_activate + setup_packages_to_activate))

    print("Activating packages")
    install.activate(repository.load_packages(to_activate))