def __init__(self,
                 root_dir,
                 salt_root_path='vendor',
                 clone_path='formula-repos',
                 salt_root='_root'):
        """
        Initialise application paths and collect together the
        metadata

        Args:
            root_dir(string): The root directory to use
            salt_root_dir(string): The directory to use for the salt
                root
            clone_path(string): The directory to put formula into
            salt_root(string): The directory to link formula into
        """
        # Run sanity checks on pygit2
        pygit2_utils.pygit2_check()

        self.roots_dir = os.path.join(root_dir, salt_root_path, salt_root)
        self.repos_dir = os.path.join(root_dir, salt_root_path, clone_path)

        self._root_dir = root_dir
        self._shaker_metadata = ShakerMetadata(root_dir)
示例#2
0
    def __init__(self, root_dir, salt_root_path='vendor',
                 clone_path='formula-repos', salt_root='_root'):
        """
        Initialise application paths and collect together the
        metadata

        Args:
            root_dir(string): The root directory to use
            salt_root_dir(string): The directory to use for the salt
                root
            clone_path(string): The directory to put formula into
            salt_root(string): The directory to link formula into
        """
        # Run sanity checks on pygit2
        pygit2_utils.pygit2_check()

        self.roots_dir = os.path.join(root_dir, salt_root_path, salt_root)
        self.repos_dir = os.path.join(root_dir, salt_root_path, clone_path)

        self._root_dir = root_dir
        self._shaker_metadata = ShakerMetadata(root_dir)
示例#3
0
class Shaker(object):
    """
    Shaker takes in a metadata yaml file and uses this to resolve a set
    of dependencies into a pinned and versioned set in a
    formula-requirements.txt file. This can then be used to synchronise
    a set of salt-formulas with remote versions pinned to the specified
    versions.

    Starting from a root formula and calculate all necessary dependencies,
    based on metadata stored in each formula.

       -

    Salt Shaker works by creating an extra file root that must be copied up to
    your salt server and added to the master config.


    The formula-requirements.txt file
    ---------------------------------

    The format of the file is simply a list of git-cloneable urls with an
    optional revision specified on the end. At the moment the only form a
    version comparison accepted is `==`. The version can be a tag, a branch
    name or anything that ``git rev-parse`` understands (i.e. a plain sha or
    the output of ``git describe`` such as ``v0.2.2-1-g1b520c5``).

    Example::

        [email protected]:ministryofjustice/ntp-formula.git==v1.2.3
        [email protected]:ministryofjustice/repos-formula.git==my_branch
        [email protected]:ministryofjustice/php-fpm-formula.git
        [email protected]:ministryofjustice/utils-formula.git
        [email protected]:ministryofjustice/java-formula.git
        [email protected]:ministryofjustice/redis-formula.git==v0.2.2-1-g1b520c5
        [email protected]:ministryofjustice/logstash-formula.git
        [email protected]:ministryofjustice/sensu-formula.git
        [email protected]:ministryofjustice/rabbitmq-formula.git
        [email protected]:saltstack-formulas/users-formula.git


    """
    def __init__(self, root_dir, salt_root_path='vendor',
                 clone_path='formula-repos', salt_root='_root'):
        """
        Initialise application paths and collect together the
        metadata

        Args:
            root_dir(string): The root directory to use
            salt_root_dir(string): The directory to use for the salt
                root
            clone_path(string): The directory to put formula into
            salt_root(string): The directory to link formula into
        """
        # Run sanity checks on pygit2
        pygit2_utils.pygit2_check()

        self.roots_dir = os.path.join(root_dir, salt_root_path, salt_root)
        self.repos_dir = os.path.join(root_dir, salt_root_path, clone_path)

        self._root_dir = root_dir
        self._shaker_metadata = ShakerMetadata(root_dir)

    def install_requirements(self,
                             simulate=False,
                             enable_remote_check=False):
        """
        simulate(bool): True to only simulate the run,
            false to carry it through for real
        enable_remote_check(bool): True to enable remote
            checks when installing pinned versions
        """
        logger.Logger().info("Shaker::install_requirements: "
                             "Installing pinned requirements..."
                             "dependencies will be installed "
                             "from the stored formula requirements")
        self._load_local_requirements()
        self._install_versioned_requirements(overwrite=False,
                                             simulate=simulate,
                                             enable_remote_check=enable_remote_check)

    def update_requirements(self,
                            simulate=False):
        """
        Update the formula-requirements from the metadata,
        then install them

        Args:
            simulate(bool): True to only simulate the run,
                false to carry it through for real
        """
        logger.Logger().info("Shaker::install_requirements: "
                             "Updating and Installing requirements..."
                             "all dependencies will be "
                             "re-calculated from the metadata")
        self._update_local_requirements()
        self._install_versioned_requirements(overwrite=True,
                                             simulate=simulate,
                                             enable_remote_check=True)

    def check_requirements(self):
        """
        Check the current formula-requirements against those that
        would be generated from the metadata,
        """
        logger.Logger().info("Shaker::check_requirements: "
                             "Checking the current requirements "
                             "against an update")
        self._load_local_requirements(enable_remote_check=True)
        current_requirements = self._shaker_remote.get_requirements()
        self._update_local_requirements()
        new_requirements = self._shaker_remote.get_requirements()

        requirements_diff = metadata.compare_requirements(current_requirements,
                                                          new_requirements)

        if len(requirements_diff) == 0:
            logger.Logger().info("Shaker::check_requirements: "
                                 "No formula requirements changes found")
        else:
            for requirement_pair in requirements_diff:
                first_entry = requirement_pair[0]
                second_entry = requirement_pair[1]
                if len(first_entry) == 0:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "New entry %s"
                                         % (second_entry))
                elif len(second_entry) == 0:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "Deprecated entry %s"
                                         % (first_entry))
                else:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "Unequal entries %s != %s"
                                         % (first_entry,
                                            second_entry))
        return requirements_diff

    def _load_local_requirements(self,
                                 enable_remote_check=False):
        """
        Load the requirements file and update the remote dependencies

        Args:
            enable_remote_check(bool): False to use current formula without checking
                remotely for updates. True to use remote repository API to
                recalculate shas
        """
        logger.Logger().info("Shaker: Loading the current formula requirements...")
        self._shaker_remote = ShakerRemote(self._shaker_metadata.local_requirements)
        if enable_remote_check:
            logger.Logger().info("Shaker: Updating the current formula requirements "
                                 "dependencies...")
            self._shaker_remote.update_dependencies()

    def _update_local_requirements(self):
        """
        Update the requirements from metadata entries, overriding the
        current formula requirements
        """
        logger.Logger().info("Shaker: Updating the formula requirements...")

        self._shaker_metadata.update_dependencies(ignore_local_requirements=True)
        self._shaker_remote = ShakerRemote(self._shaker_metadata.dependencies)
        self._shaker_remote.update_dependencies()

    def _install_versioned_requirements(self,
                                        overwrite=False,
                                        simulate=False,
                                        enable_remote_check=False
                                        ):
        """
        Install all of the versioned requirements found

        Args:
            overwrite(bool): True to overwrite dependencies,
                false otherwise
            simulate(bool): True to only simulate the run,
                false to carry it through for real
            enable_remote_check(bool): False to use current formula without checking
                remotely for updates. True to use remote repository API to
                recalculate shas
        """
        if not simulate:
            if enable_remote_check:
                logger.Logger().info("Shaker::install_requirements: Updating requirements tag target shas")
                self._shaker_remote.update_dependencies()
            else:
                logger.Logger().info("Shaker::install_requirements: No remote check, not updating tag target shas")
            logger.Logger().info("Shaker::install_requirements: Installing requirements...")
            successful, unsuccessful = self._shaker_remote.install_dependencies(overwrite=overwrite,
                                                                                enable_remote_check=enable_remote_check)

            # If we have unsuccessful updates, then we should fail before writing the requirements file
            if unsuccessful > 0:
                msg = ("Shaker::install_requirements: %s successful, %s failed"
                       % (successful, unsuccessful))
                raise ShakerRequirementsUpdateException(msg)

            if enable_remote_check:
                logger.Logger().info("Shaker: Writing requirements file...")
                self._shaker_remote.write_requirements(overwrite=True, backup=False)
        else:
            requirements = '\n'.join(self._shaker_remote.get_requirements())
            logger.Logger().warning("Shaker: Simulation mode enabled, "
                                    "no changes will be made...\n%s\n\n"
                                    % (requirements))
class Shaker(object):
    """
    Shaker takes in a metadata yaml file and uses this to resolve a set
    of dependencies into a pinned and versioned set in a
    formula-requirements.txt file. This can then be used to synchronise
    a set of salt-formulas with remote versions pinned to the specified
    versions.

    Starting from a root formula and calculate all necessary dependencies,
    based on metadata stored in each formula.

       -

    Salt Shaker works by creating an extra file root that must be copied up to
    your salt server and added to the master config.


    The formula-requirements.txt file
    ---------------------------------

    The format of the file is simply a list of git-cloneable urls with an
    optional revision specified on the end. At the moment the only form a
    version comparison accepted is `==`. The version can be a tag, a branch
    name or anything that ``git rev-parse`` understands (i.e. a plain sha or
    the output of ``git describe`` such as ``v0.2.2-1-g1b520c5``).

    Example::

        [email protected]:ministryofjustice/ntp-formula.git==v1.2.3
        [email protected]:ministryofjustice/repos-formula.git==my_branch
        [email protected]:ministryofjustice/php-fpm-formula.git
        [email protected]:ministryofjustice/utils-formula.git
        [email protected]:ministryofjustice/java-formula.git
        [email protected]:ministryofjustice/redis-formula.git==v0.2.2-1-g1b520c5
        [email protected]:ministryofjustice/logstash-formula.git
        [email protected]:ministryofjustice/sensu-formula.git
        [email protected]:ministryofjustice/rabbitmq-formula.git
        [email protected]:saltstack-formulas/users-formula.git


    """
    def __init__(self,
                 root_dir,
                 salt_root_path='vendor',
                 clone_path='formula-repos',
                 salt_root='_root'):
        """
        Initialise application paths and collect together the
        metadata

        Args:
            root_dir(string): The root directory to use
            salt_root_dir(string): The directory to use for the salt
                root
            clone_path(string): The directory to put formula into
            salt_root(string): The directory to link formula into
        """
        # Run sanity checks on pygit2
        pygit2_utils.pygit2_check()

        self.roots_dir = os.path.join(root_dir, salt_root_path, salt_root)
        self.repos_dir = os.path.join(root_dir, salt_root_path, clone_path)

        self._root_dir = root_dir
        self._shaker_metadata = ShakerMetadata(root_dir)

    def install_requirements(self, simulate=False, enable_remote_check=False):
        """
        simulate(bool): True to only simulate the run,
            false to carry it through for real
        enable_remote_check(bool): True to enable remote
            checks when installing pinned versions
        """
        logger.Logger().info("Shaker::install_requirements: "
                             "Installing pinned requirements..."
                             "dependencies will be installed "
                             "from the stored formula requirements")
        self._load_local_requirements()
        self._install_versioned_requirements(
            overwrite=False,
            simulate=simulate,
            enable_remote_check=enable_remote_check)

    def update_requirements(self, simulate=False):
        """
        Update the formula-requirements from the metadata,
        then install them

        Args:
            simulate(bool): True to only simulate the run,
                false to carry it through for real
        """
        logger.Logger().info("Shaker::install_requirements: "
                             "Updating and Installing requirements..."
                             "all dependencies will be "
                             "re-calculated from the metadata")
        self._update_local_requirements()
        self._install_versioned_requirements(overwrite=True,
                                             simulate=simulate,
                                             enable_remote_check=True)

    def check_requirements(self):
        """
        Check the current formula-requirements against those that
        would be generated from the metadata,
        """
        logger.Logger().info("Shaker::check_requirements: "
                             "Checking the current requirements "
                             "against an update")
        self._load_local_requirements(enable_remote_check=True)
        current_requirements = self._shaker_remote.get_requirements()
        self._update_local_requirements()
        new_requirements = self._shaker_remote.get_requirements()

        requirements_diff = metadata.compare_requirements(
            current_requirements, new_requirements)

        if len(requirements_diff) == 0:
            logger.Logger().info("Shaker::check_requirements: "
                                 "No formula requirements changes found")
        else:
            for requirement_pair in requirements_diff:
                first_entry = requirement_pair[0]
                second_entry = requirement_pair[1]
                if len(first_entry) == 0:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "New entry %s" % (second_entry))
                elif len(second_entry) == 0:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "Deprecated entry %s" % (first_entry))
                else:
                    logger.Logger().info("Shaker::check_requirements: "
                                         "Unequal entries %s != %s" %
                                         (first_entry, second_entry))
        return requirements_diff

    def _load_local_requirements(self, enable_remote_check=False):
        """
        Load the requirements file and update the remote dependencies

        Args:
            enable_remote_check(bool): False to use current formula without checking
                remotely for updates. True to use remote repository API to
                recalculate shas
        """
        logger.Logger().info(
            "Shaker: Loading the current formula requirements...")
        self._shaker_remote = ShakerRemote(
            self._shaker_metadata.local_requirements)
        if enable_remote_check:
            logger.Logger().info(
                "Shaker: Updating the current formula requirements "
                "dependencies...")
            self._shaker_remote.update_dependencies()

    def _update_local_requirements(self):
        """
        Update the requirements from metadata entries, overriding the
        current formula requirements
        """
        logger.Logger().info("Shaker: Updating the formula requirements...")

        self._shaker_metadata.update_dependencies(
            ignore_local_requirements=True)
        self._shaker_remote = ShakerRemote(self._shaker_metadata.dependencies)
        self._shaker_remote.update_dependencies()

    def _install_versioned_requirements(self,
                                        overwrite=False,
                                        simulate=False,
                                        enable_remote_check=False):
        """
        Install all of the versioned requirements found

        Args:
            overwrite(bool): True to overwrite dependencies,
                false otherwise
            simulate(bool): True to only simulate the run,
                false to carry it through for real
            enable_remote_check(bool): False to use current formula without checking
                remotely for updates. True to use remote repository API to
                recalculate shas
        """
        if not simulate:
            if enable_remote_check:
                logger.Logger().info(
                    "Shaker::install_requirements: Updating requirements tag target shas"
                )
                self._shaker_remote.update_dependencies()
            else:
                logger.Logger().info(
                    "Shaker::install_requirements: No remote check, not updating tag target shas"
                )
            logger.Logger().info(
                "Shaker::install_requirements: Installing requirements...")
            successful, unsuccessful = self._shaker_remote.install_dependencies(
                overwrite=overwrite, enable_remote_check=enable_remote_check)

            # If we have unsuccessful updates, then we should fail before writing the requirements file
            if unsuccessful > 0:
                msg = (
                    "Shaker::install_requirements: %s successful, %s failed" %
                    (successful, unsuccessful))
                raise ShakerRequirementsUpdateException(msg)

            if enable_remote_check:
                logger.Logger().info("Shaker: Writing requirements file...")
                self._shaker_remote.write_requirements(overwrite=True,
                                                       backup=False)
        else:
            requirements = '\n'.join(self._shaker_remote.get_requirements())
            logger.Logger().warning("Shaker: Simulation mode enabled, "
                                    "no changes will be made...\n%s\n\n" %
                                    (requirements))