Example #1
0
    def process_hosts(self):

        if not self.args.tmp_folder:
            Logger.errorout("tmp folder must be defined")

        # Deleting the tmp folder to keep installs clean
        Helpers.delete_path(self.tmp_folder)

        try:
            if self.args.template_files:
                template_paths = self.args.template_files

                # Incase one long string is entered
                if isinstance(self.args.template_files, basestring):
                    template_paths = self.args.template_files.split(' ')

                self.template_values = Helpers.load_templating(template_paths)
        except Exception as exception:
            Logger.errorout("No templating problem: %s" % exception.message)

        if self.args.template_json:
            try:
                self.template_values.update(
                    json.loads(self.args.template_json.replace('\\"', '"')))
            except Exception as err:
                Logger.errorout("Error parsing --template-json",
                                error=err.message)

        if self.args.template_filtering:
            self.template_values = Helpers.filter_object(
                self.template_values, self.args.template_filtering)

        ConnManager.set_globals(self.args.ssh_user, self.args.ssh_keyfile,
                                self.args.ssh_port, self.args.app_folder,
                                self.args.app_binary, self.args.dryrun)

        # Load any files reference to appetite scripts folder before this
        # Working directories change with repo management
        repo_status = self.repo_manager.pull_repo(self.args.clean_repo)

        if not self.args.dryrun and repo_status < 0:
            Logger.errorout('Repo Error, Look at logs for details')

        repo_check_status = self.repo_manager.check_for_update()

        Logger.add_track_info(self.repo_manager.track)

        triggered = repo_check_status['triggered'] or repo_status == 1

        Logger.info('Repo pull',
                    output=repo_check_status['output'],
                    triggered=triggered)

        if not self.args.dryrun and not self.args.skip_repo_sync and not triggered:
            Logger.info('No repo update found', complete=False)
            self.print_track_info(False)
            sys.exit(0)

        self.repo_manager.set_commit_id()

        # Load in deploymentmethods.conf
        self.deployment_manager = DeploymentMethodsManager(
            self.repo_name, "", self.scratch_location,
            self.args.deployment_methods_file)

        # Generate hosts
        if self.args.hosts:
            # Incase one long string is entered
            if len(self.args.hosts) == 1:
                self.args.hosts = self.args.hosts[0].split(' ')

            for host in self.args.hosts:
                split_hostname = host.strip("'\"").split(':')
                clean_hostname = split_hostname[0].split('.')[0].strip("'\"")

                # With user name, the ssh hostname can be defined.  This allows
                # for IP addresses to be defined incase there is no DNS.
                host_data = Helpers.pull_class_from_host(
                    self.name_formatting, clean_hostname, self.host_classes)

                if host_data:
                    # Can use a specified hostname/IP.
                    # Default is the given hostname
                    ssh_host = split_hostname[len(split_hostname) - 1]
                    self.appetite_hosts.add_host(self, clean_hostname,
                                                 host_data, ssh_host)
        else:
            # Create hosts based on classes
            for host_class in self.host_classes:
                self.appetite_hosts.add_host(
                    self,
                    Helpers.build_hostname(
                        self.name_formatting,  # pylint: disable=no-value-for-parameter
                        host_class,
                        1))

        if self.appetite_hosts.is_empty():
            Logger.errorout("No hosts found after filtering")

        if self.args.clean_metas:
            Helpers.delete_path(self.meta_folder)

        # Only update if a manifest file is not found
        self.update_manifests(check_if_exists=True)

        Logger.info("appetite started",
                    use_templating=self.args.templating,
                    firstrun=self.args.firstrun)

        self.populate_apps_to_hosts()

        self.ssh_app_commands = ConnManager.SshAppCommands(
            self.app_commands_file, self.template_values)

        changes_found = self.create_host_directories_and_tar()

        if changes_found:
            Logger.info("Start host updates")

            self.update_hosts()

            Logger.info("End host updates")

        self.print_track_info(changes_found)
        Logger.info("Appetite complete", complete=True, changes=changes_found)
Example #2
0
    def populate_apps_to_hosts(self):
        """Parses the manifest and adds apps to hosts

        :return: None
        """
        Helpers.check_file(self.manifest_path)

        with open(self.manifest_path, 'rU') as csvfile:
            mreader = csv.reader(csvfile, delimiter=',', quotechar='"')
            first_row = True

            # Go though each app
            for row in mreader:
                # Remove header if it exists
                if first_row:
                    # Defines column headers in manifest
                    column_headers = {
                        col_name: -1
                        for col_name in Consts.DEFAULT_COLUMN_HEADER
                    }

                    # Get indexes for headers from the first row
                    num_columns = len(row)
                    for k in column_headers:
                        value_index = next((index
                                            for index in range(0, num_columns)
                                            if row[index].lower() == k), -1)
                        if value_index < 0:
                            Logger.errorout("Manifest header is missing",
                                            header=k)
                        column_headers[k] = value_index

                    first_row = False
                    continue

                if len(row) > 1:
                    row_values = Helpers.create_obj({
                        "commit_id":
                        row[column_headers['commitid']],
                        "app_clean":
                        self.deployment_manager.name_filter.sub(
                            "", row[column_headers['application']]),
                        "app":
                        row[column_headers['application']],
                        "deployment":
                        row[column_headers['deploymentmethod']],
                        "white_list":
                        row[column_headers['whitelist']].split(','),
                        "black_list":
                        row[column_headers['blacklist']].split(',')
                    })

                    app_folder = os.path.join(self.apps_folder, row_values.app)

                    if self.args.build_test_apps:
                        # for testing - create test folders for apps
                        if not os.path.exists(app_folder):
                            Helpers.create_path(
                                os.path.join(app_folder, "folder"), True)
                            app_test_file = "%s/%s.txt" % (
                                app_folder, row_values.app_clean)
                            with open(app_test_file, 'wb') as touch:
                                touch.write("")

                    # validate commit id
                    if len(row_values.commit_id
                           ) > 0 and not Helpers.validate_commit_id(
                               row_values.commit_id):
                        Logger.critical("Invalid commit ID",
                                        commit_id=row_values.commit_id)

                    # Go through each host and see
                    # if the app is needed for the host
                    for host in self.appetite_hosts:
                        self.add_to_host(host, row_values)
                        self.bootstrap_firstrun_hosts(host, row_values)

            if self.args.new_host_brakes and next(
                (True
                 for host in self.appetite_hosts if host.bootstrap), False):
                self.args.num_connections = 1

            if self.appetite_hosts.is_empty():
                Logger.errorout("Manifest misconfiguration, "
                                "no apps for any hosts")
Example #3
0
    def __init__(self, is_already_running):
        self.args = parse_args()

        # default path for configs
        self.app_config_dir = os.path.abspath("../config")

        # Update args from config file
        if self.args.config_file:
            abs_config_file = os.path.abspath(
                os.path.dirname(self.args.config_file))
            self.args.__dict__.update(
                AppetiteArgs.load_args(self.args.config_file))
            self.app_config_dir = abs_config_file

        self.app_commands_file = os.path.join(self.app_config_dir,
                                              "commands.conf")

        if self.args.command_conf:
            self.app_commands_file = os.path.join(
                os.path.abspath(os.path.expandvars(self.args.command_conf)))

        AppetiteArgs.args_check(self.args)

        # Set up logging after args are set
        Logger.setup_logging('appetite_%s' % self.args.refname,
                             self.args.refname, True,
                             self.args.disable_logging, self.args.silent,
                             self.args.logging_path)

        if is_already_running:
            Logger.info("Appetite is already processing")
            self.print_track_info(False, Helpers.get_track())
            return

        # Get host classes for filtering
        self.host_classes = self.args.host_classes.split(" ") \
            if isinstance(self.args.host_classes, basestring) \
            else self.args.host_classes

        # Reverse sorting needed for name filtering
        self.host_classes.sort(reverse=True)

        # Get boot ordering
        self.boot_ordering = Helpers.get_enchanced_boot_order(
            self.args.boot_order, self.host_classes)

        self.repo_name = self.args.repo_url.split('/')[-1].split('.')[0]
        self.scratch_location = os.path.join(
            os.path.abspath(os.path.expandvars(self.args.scratch_dir)),
            self.args.refname)
        self.repo_path = os.path.join(self.scratch_location, self.repo_name)
        self.apps_folder = os.path.join(self.repo_path, self.args.apps_folder)
        self.manifest_path = os.path.join(self.repo_path,
                                          Consts.CONFIG_PATH_NAME,
                                          self.args.apps_manifest)
        self.tmp_folder = os.path.join(self.scratch_location,
                                       self.args.tmp_folder)
        self.meta_folder = os.path.join(self.scratch_location, 'meta')
        self.tars_folder = os.path.join(self.tmp_folder, 'tars')
        self.hosts_folder = os.path.join(self.tmp_folder, 'hosts')
        self.remote_apps_path = os.path.normpath(self.args.app_folder)
        self.base_location, self.base_name = os.path.split(
            self.remote_apps_path)
        self.name_formatting = self.args.name_formatting.strip('"\'')
        self.meta_name = "%s%s" % (Consts.APPS_METADATE_FILENAME,
                                   self.args.refname)
        self.meta_remote_folder = os.path.join(self.remote_apps_path,
                                               Consts.META_DIR)
        self.meta_remote_logs_folder = os.path.join(
            self.meta_remote_folder, Consts.HOST_LOGS_FOLDER_NAME)
        self.meta_remote_file = "%s.json" % os.path.join(
            self.meta_remote_folder, self.meta_name)

        if self.args.clean:
            Helpers.delete_path(self.scratch_location)

        self.repo_manager = RepoManager(self.repo_name, self.args.repo_url,
                                        self.args.repo_branch, "",
                                        self.scratch_location,
                                        self.args.apps_manifest,
                                        self.args.dryrun)

        Logger.debug_on(self.args.debug)

        if not self.args.tmp_folder:
            Logger.errorout("tmp folder must be defined")

        # Deleting the tmp folder to keep installs clean
        Helpers.delete_path(self.tmp_folder)

        self.template_values = {}

        try:
            if self.args.template_files:
                template_paths = self.args.template_files

                # Incase one long string is entered
                if isinstance(self.args.template_files, basestring):
                    template_paths = self.args.template_files.split(' ')

                self.template_values = Helpers.load_templating(template_paths)
        except Exception as exception:
            Logger.errorout("No templating problem: %s" % exception.message)

        if self.args.template_json:
            try:
                self.template_values.update(
                    json.loads(self.args.template_json.replace('\\"', '"')))
            except Exception as err:
                Logger.errorout("Error parsing --template-json",
                                error=err.message)

        if self.args.template_filtering:
            self.template_values = Helpers.filter_object(
                self.template_values, self.args.template_filtering)

        ConnManager.set_globals(self.args.ssh_user, self.args.ssh_keyfile,
                                self.args.ssh_port, self.args.app_folder,
                                self.args.app_binary, self.args.dryrun)

        self.ssh_app_commands = ConnManager.SshAppCommands(
            self.app_commands_file, self.template_values)

        # Load any files reference to appetite scripts folder before this
        # Working directories change with repo management
        repo_status = self.repo_manager.pull_repo(self.args.clean_repo)

        if not self.args.dryrun and repo_status < 0:
            Logger.errorout('Repo Error, Look at logs for details')

        repo_check_status = self.repo_manager.check_for_update()

        Logger.add_track_info(self.repo_manager.track)

        triggered = repo_check_status['triggered'] or repo_status == 1

        Logger.info('Repo pull',
                    output=repo_check_status['output'],
                    triggered=triggered)

        if not self.args.dryrun and not self.args.skip_repo_sync and not triggered:
            Logger.info('No repo update found', complete=False)
            self.print_track_info(False)
            sys.exit(0)

        self.repo_manager.set_commit_id()

        # Load in deploymentmethods.conf
        self.deployment_manager = DeploymentMethodsManager(
            self.repo_name, "", self.scratch_location)

        # Generate hosts
        if self.args.hosts:
            # Incase one long string is entered
            if len(self.args.hosts) == 1:
                self.args.hosts = self.args.hosts[0].split(' ')

            for host in self.args.hosts:
                split_hostname = host.strip("'\"").split(':')
                clean_hostname = split_hostname[0].split('.')[0].strip("'\"")

                # With user name, the ssh hostname can be defined.  This allows
                # for IP addresses to be defined incase there is no DNS.
                host_data = Helpers.pull_class_from_host(
                    self.name_formatting, clean_hostname, self.host_classes)

                if host_data:
                    # Can use a specified hostname/IP.
                    # Default is the given hostname
                    ssh_host = split_hostname[len(split_hostname) - 1]
                    self.appetite_hosts.add_host(self, clean_hostname,
                                                 host_data, ssh_host)
        else:
            # Create hosts based on classes
            for host_class in self.host_classes:
                self.appetite_hosts.add_host(
                    self,
                    Helpers.build_hostname(
                        self.name_formatting,  # pylint: disable=no-value-for-parameter
                        host_class,
                        1))

        if self.appetite_hosts.is_empty():
            Logger.errorout("No hosts found after filtering")

        if self.args.clean_metas:
            Helpers.delete_path(self.meta_folder)

        # Only update if a manifest file is not found
        self.update_manifests(check_if_exists=True)

        Logger.info("appetite started",
                    use_templating=self.args.templating,
                    firstrun=self.args.firstrun)

        self.populate_apps_to_hosts()

        changes_found = self.create_host_directories_and_tar()

        if changes_found:
            Logger.info("Start host updates")

            self.update_hosts()

            Logger.info("End host updates")

        self.print_track_info(changes_found)
        Logger.info("Appetite complete", complete=True, changes=changes_found)