示例#1
0
    def check_partial_install(self):
        """
    If an installation did not complete successfully, check if installation was partially complete and
    log the partially completed version to REPO_VERSION_HISTORY_FILE.
    :return:
    """
        Logger.info(
            "Installation of packages failed. Checking if installation was partially complete"
        )
        Logger.info("Old versions: {0}".format(self.old_versions))

        new_versions = get_hdp_versions(self.stack_root_folder)
        Logger.info("New versions: {0}".format(new_versions))

        deltas = set(new_versions) - set(self.old_versions)
        Logger.info("Deltas: {0}".format(deltas))

        # Get version without build number
        normalized_repo_version = self.repository_version.split('-')[0]

        if 1 == len(deltas):
            # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
            partial_install_version = next(iter(deltas)).strip()
            write_actual_version_to_history_file(normalized_repo_version,
                                                 partial_install_version)
            Logger.info("Version {0} was partially installed. ".format(
                partial_install_version))
示例#2
0
  def compute_actual_version(self):
    """
    After packages are installed, determine what the new actual version is.
    """

    # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
    # to correct value if it is not
    self.actual_version = None
    self.repo_version_with_build_number = None
    if self.repository_version:
      m = re.search("[\d\.]+-\d+", self.repository_version)
      if m:
        # Contains a build number
        self.repo_version_with_build_number = self.repository_version
        self.structured_output['actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
        self.put_structured_out(self.structured_output)

    Logger.info("Attempting to determine actual version with build number.")
    Logger.info("Old versions: {0}".format(self.old_versions))

    new_versions = get_hdp_versions(self.stack_root_folder)
    Logger.info("New versions: {0}".format(new_versions))

    deltas = set(new_versions) - set(self.old_versions)
    Logger.info("Deltas: {0}".format(deltas))

    # Get version without build number
    normalized_repo_version = self.repository_version.split('-')[0]

    if 1 == len(deltas):
      self.actual_version = next(iter(deltas)).strip()
      self.structured_output['actual_version'] = self.actual_version
      self.put_structured_out(self.structured_output)
      write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
      Logger.info(
        "Found actual version {0} by checking the delta between versions before and after installing packages".format(
          self.actual_version))
    else:
      # If the first install attempt does a partial install and is unable to report this to the server,
      # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
      Logger.info("Cannot determine actual version installed by checking the delta between versions "
                  "before and after installing package")
      Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
      self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
      if self.actual_version is not None:
        self.actual_version = self.actual_version.strip()
        self.structured_output['actual_version'] = self.actual_version
        self.put_structured_out(self.structured_output)
        Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
      else:
        msg = "Could not determine actual version installed. Try reinstalling packages again."
        raise Fail(msg)
示例#3
0
    def compute_actual_version(self):
        """
    After packages are installed, determine what the new actual version is, in order to save it.
    """
        Logger.info(
            "Attempting to determine actual version with build number.")
        Logger.info("Old versions: {0}".format(self.old_versions))

        new_versions = get_hdp_versions()
        Logger.info("New versions: {0}".format(new_versions))

        deltas = set(new_versions) - set(self.old_versions)
        Logger.info("Deltas: {0}".format(deltas))

        # Get HDP version without build number
        normalized_repo_version = self.repository_version.split('-')[0]

        if 1 == len(deltas):
            self.actual_version = next(iter(deltas)).strip()
            self.structured_output['actual_version'] = self.actual_version
            self.put_structured_out(self.structured_output)
            write_actual_version_to_history_file(normalized_repo_version,
                                                 self.actual_version)
        else:
            Logger.info(
                "Cannot determine a new actual version installed by using the delta method."
            )
            # If the first install attempt does a partial install and is unable to report this to the server,
            # then a subsequent attempt will report an empty delta. For this reason, it is important to search the
            # repo version history file to determine if we previously did write an actual_version.
            self.actual_version = read_actual_version_from_history_file(
                normalized_repo_version)
            if self.actual_version is not None:
                self.actual_version = self.actual_version.strip()
                self.structured_output['actual_version'] = self.actual_version
                self.put_structured_out(self.structured_output)
                Logger.info(
                    "Found actual version {0} by parsing file {1}".format(
                        self.actual_version, REPO_VERSION_HISTORY_FILE))
            elif self.repo_version_with_build_number is None:
                # It's likely that this host does not have any Stack Components installed, so only contains AMS.
                # So just use repo version value provided by server (we already put it to structured output)
                if not os.path.exists(self.stack_root_folder):
                    # Special case when this host does not contain any HDP components, but still contains other components like AMS.
                    msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
                          "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
                    Logger.info(msg)
                else:
                    msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
                          "Try reinstalling packages again.".format(self.stack_root_folder)
                    raise Fail(msg)
示例#4
0
    def compute_actual_version(self):
        """
    After packages are installed, determine what the new actual version is, in order to save it.
    """
        Logger.info(
            "Attempting to determine actual version with build number.")
        Logger.info("Old versions: {0}".format(self.old_versions))

        new_versions = get_hdp_versions()
        Logger.info("New versions: {0}".format(new_versions))

        deltas = set(new_versions) - set(self.old_versions)
        Logger.info("Deltas: {0}".format(deltas))

        # Get HDP version without build number
        normalized_repo_version = self.repository_version.split('-')[0]

        if 1 == len(deltas):
            self.actual_version = next(iter(deltas)).strip()
            self.structured_output['actual_version'] = self.actual_version
            self.put_structured_out(self.structured_output)
            write_actual_version_to_history_file(normalized_repo_version,
                                                 self.actual_version)
        else:
            Logger.info(
                "Cannot determine a new actual version installed by using the delta method."
            )
            # If the first install attempt does a partial install and is unable to report this to the server,
            # then a subsequent attempt will report an empty delta. For this reason, it is important to search the
            # repo version history file to determine if we previously did write an actual_version.
            self.actual_version = read_actual_version_from_history_file(
                normalized_repo_version)
            if self.actual_version is not None:
                self.actual_version = self.actual_version.strip()
                self.structured_output['actual_version'] = self.actual_version
                self.put_structured_out(self.structured_output)
                Logger.info(
                    "Found actual version {0} by parsing file {1}".format(
                        self.actual_version, REPO_VERSION_HISTORY_FILE))
            elif self.repo_version_with_build_number is None:
                msg = "Could not determine actual version installed. Try reinstalling packages again."
                raise Fail(msg)
示例#5
0
  def check_partial_install(self):
    """
    If an installation did not complete successfully, check if installation was partially complete and
    log the partially completed version to REPO_VERSION_HISTORY_FILE.
    :return:
    """
    Logger.info("Installation of packages failed. Checking if installation was partially complete")
    Logger.info("Old versions: {0}".format(self.old_versions))

    new_versions = get_hdp_versions(self.stack_root_folder)
    Logger.info("New versions: {0}".format(new_versions))

    deltas = set(new_versions) - set(self.old_versions)
    Logger.info("Deltas: {0}".format(deltas))

    # Get version without build number
    normalized_repo_version = self.repository_version.split('-')[0]

    if 1 == len(deltas):
      # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE
      partial_install_version = next(iter(deltas)).strip()
      write_actual_version_to_history_file(normalized_repo_version, partial_install_version)
      Logger.info("Version {0} was partially installed. ".format(partial_install_version))
示例#6
0
class InstallPackages(Script):
    """
  This script is a part of Rolling Upgrade workflow and is described at
  appropriate design doc.
  It installs repositories to the node and then installs packages.
  For now, repositories are installed into individual files.
  """

    UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
    REPO_FILE_NAME_PREFIX = 'HDP-'
    STACK_TO_ROOT_FOLDER = {"HDP": "/usr/hdp"}

    def actionexecute(self, env):
        num_errors = 0

        # Parse parameters
        config = Script.get_config()

        repo_rhel_suse = config['configurations']['cluster-env'][
            'repo_suse_rhel_template']
        repo_ubuntu = config['configurations']['cluster-env'][
            'repo_ubuntu_template']
        template = repo_rhel_suse if OSCheck.is_redhat_family(
        ) or OSCheck.is_suse_family() else repo_ubuntu

        # Handle a SIGTERM and SIGINT gracefully
        signal.signal(signal.SIGTERM, self.abort_handler)
        signal.signal(signal.SIGINT, self.abort_handler)

        # Select dict that contains parameters
        try:
            self.repository_version = config['roleParams'][
                'repository_version']
            base_urls = json.loads(config['roleParams']['base_urls'])
            package_list = json.loads(config['roleParams']['package_list'])
            stack_id = config['roleParams']['stack_id']
        except KeyError:
            # Last try
            self.repository_version = config['commandParams'][
                'repository_version']
            base_urls = json.loads(config['commandParams']['base_urls'])
            package_list = json.loads(config['commandParams']['package_list'])
            stack_id = config['commandParams']['stack_id']

        # current stack information
        self.current_hdp_stack_version = None
        if 'stack_version' in config['hostLevelParams']:
            current_stack_version_unformatted = str(
                config['hostLevelParams']['stack_version'])
            self.current_hdp_stack_version = format_hdp_stack_version(
                current_stack_version_unformatted)

        stack_name = None
        self.stack_root_folder = None
        if stack_id and "-" in stack_id:
            stack_split = stack_id.split("-")
            if len(stack_split) == 2:
                stack_name = stack_split[0].upper()
                if stack_name in self.STACK_TO_ROOT_FOLDER:
                    self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[
                        stack_name]
        if self.stack_root_folder is None:
            raise Fail(
                "Cannot determine the stack's root directory by parsing the stack_id property, {0}"
                .format(str(stack_id)))
        if self.repository_version is None:
            raise Fail("Cannot determine the repository version to install")

        self.repository_version = self.repository_version.strip()

        # Install/update repositories
        installed_repositories = []
        self.current_repositories = []
        self.current_repo_files = set()

        # Enable base system repositories
        # We don't need that for RHEL family, because we leave all repos enabled
        # except disabled HDP* ones
        if OSCheck.is_suse_family():
            self.current_repositories.append('base')
        elif OSCheck.is_ubuntu_family():
            self.current_repo_files.add('base')

        Logger.info("Will install packages for repository version {0}".format(
            self.repository_version))
        try:
            append_to_file = False
            for url_info in base_urls:
                repo_name, repo_file = self.install_repository(
                    url_info, append_to_file, template)
                self.current_repositories.append(repo_name)
                self.current_repo_files.add(repo_file)
                append_to_file = True

            installed_repositories = list_ambari_managed_repos()
        except Exception, err:
            Logger.logger.exception(
                "Cannot distribute repositories. Error: {0}".format(str(err)))
            num_errors += 1

        # Build structured output with initial values
        self.structured_output = {
            'ambari_repositories': installed_repositories,
            'installed_repository_version': self.repository_version,
            'stack_id': stack_id,
            'package_installation_result': 'FAIL'
        }
        self.put_structured_out(self.structured_output)

        if num_errors > 0:
            raise Fail("Failed to distribute repositories/install packages")

        # Initial list of versions, used to compute the new version installed
        self.old_versions = get_hdp_versions(self.stack_root_folder)

        try:
            is_package_install_successful = False
            ret_code = self.install_packages(package_list)
            if ret_code == 0:
                self.structured_output[
                    'package_installation_result'] = 'SUCCESS'
                self.put_structured_out(self.structured_output)
                is_package_install_successful = True
            else:
                num_errors += 1
        except Exception, err:
            num_errors += 1
            Logger.logger.exception(
                "Could not install packages. Error: {0}".format(str(err)))
示例#7
0
    def compute_actual_version(self):
        """
    After packages are installed, determine what the new actual version is.
    """

        # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
        # to correct value if it is not
        self.actual_version = None
        self.repo_version_with_build_number = None
        if self.repository_version:
            m = re.search("[\d\.]+-\d+", self.repository_version)
            if m:
                # Contains a build number
                self.repo_version_with_build_number = self.repository_version
                self.structured_output[
                    'actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
                self.put_structured_out(self.structured_output)

        Logger.info(
            "Attempting to determine actual version with build number.")
        Logger.info("Old versions: {0}".format(self.old_versions))

        new_versions = get_hdp_versions(self.stack_root_folder)
        Logger.info("New versions: {0}".format(new_versions))

        deltas = set(new_versions) - set(self.old_versions)
        Logger.info("Deltas: {0}".format(deltas))

        # Get version without build number
        normalized_repo_version = self.repository_version.split('-')[0]

        if 1 == len(deltas):
            self.actual_version = next(iter(deltas)).strip()
            self.structured_output['actual_version'] = self.actual_version
            self.put_structured_out(self.structured_output)
            write_actual_version_to_history_file(normalized_repo_version,
                                                 self.actual_version)
            Logger.info(
                "Found actual version {0} by checking the delta between versions before and after installing packages"
                .format(self.actual_version))
        else:
            # If the first install attempt does a partial install and is unable to report this to the server,
            # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
            Logger.info(
                "Cannot determine actual version installed by checking the delta between versions "
                "before and after installing package")
            Logger.info(
                "Will try to find for the actual version by searching for best possible match in the list of versions installed"
            )
            self.actual_version = self.find_best_fit_version(
                new_versions, self.repository_version)
            if self.actual_version is not None:
                self.actual_version = self.actual_version.strip()
                self.structured_output['actual_version'] = self.actual_version
                self.put_structured_out(self.structured_output)
                Logger.info(
                    "Found actual version {0} by searching for best possible match"
                    .format(self.actual_version))
            else:
                # It's likely that this host does not have any Stack Components installed, so only contains AMS.
                # So just use repo version value provided by server (we already put it to structured output)
                if not os.path.exists(self.stack_root_folder):
                    # Special case when this host does not contain any HDP components, but still contains other components like AMS.
                    msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
                          "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
                    Logger.info(msg)
                else:
                    msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
                          "Try reinstalling packages again.".format(self.stack_root_folder)
                    raise Fail(msg)
示例#8
0
    def compute_actual_version(self):
        """
    After packages are installed, determine what the new actual version is.
    """

        # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
        # to correct value if it is not
        self.actual_version = None
        self.repo_version_with_build_number = None
        if self.repository_version:
            m = re.search("[\d\.]+-\d+", self.repository_version)
            if m:
                # Contains a build number
                self.repo_version_with_build_number = self.repository_version
                self.structured_output[
                    'actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
                self.put_structured_out(self.structured_output)

        Logger.info(
            "Attempting to determine actual version with build number.")
        Logger.info("Old versions: {0}".format(self.old_versions))

        new_versions = get_hdp_versions(self.stack_root_folder)
        Logger.info("New versions: {0}".format(new_versions))

        deltas = set(new_versions) - set(self.old_versions)
        Logger.info("Deltas: {0}".format(deltas))

        # Get version without build number
        normalized_repo_version = self.repository_version.split('-')[0]

        if 1 == len(deltas):
            self.actual_version = next(iter(deltas)).strip()
            self.structured_output['actual_version'] = self.actual_version
            self.put_structured_out(self.structured_output)
            write_actual_version_to_history_file(normalized_repo_version,
                                                 self.actual_version)
            Logger.info(
                "Found actual version {0} by checking the delta between versions before and after installing packages"
                .format(self.actual_version))
        else:
            # If the first install attempt does a partial install and is unable to report this to the server,
            # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
            Logger.info(
                "Cannot determine actual version installed by checking the delta between versions "
                "before and after installing package")
            Logger.info(
                "Will try to find for the actual version by searching for best possible match in the list of versions installed"
            )
            self.actual_version = self.find_best_fit_version(
                new_versions, self.repository_version)
            if self.actual_version is not None:
                self.actual_version = self.actual_version.strip()
                self.structured_output['actual_version'] = self.actual_version
                self.put_structured_out(self.structured_output)
                Logger.info(
                    "Found actual version {0} by searching for best possible match"
                    .format(self.actual_version))
            else:
                msg = "Could not determine actual version installed. Try reinstalling packages again."
                raise Fail(msg)
  def compute_actual_version(self):
    """
    After packages are installed, determine what the new actual version is.
    """

    # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
    # to correct value if it is not
    self.actual_version = None
    self.repo_version_with_build_number = None
    if self.repository_version:
      m = re.search("[\d\.]+-\d+", self.repository_version)
      if m:
        # Contains a build number
        self.repo_version_with_build_number = self.repository_version
        self.structured_output['actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
        self.put_structured_out(self.structured_output)

    Logger.info("Attempting to determine actual version with build number.")
    Logger.info("Old versions: {0}".format(self.old_versions))

    new_versions = get_hdp_versions(self.stack_root_folder)
    Logger.info("New versions: {0}".format(new_versions))

    deltas = set(new_versions) - set(self.old_versions)
    Logger.info("Deltas: {0}".format(deltas))

    # Get version without build number
    normalized_repo_version = self.repository_version.split('-')[0]

    if 1 == len(deltas):
      self.actual_version = next(iter(deltas)).strip()
      self.structured_output['actual_version'] = self.actual_version
      self.put_structured_out(self.structured_output)
      write_actual_version_to_history_file(normalized_repo_version, self.actual_version)
      Logger.info(
        "Found actual version {0} by checking the delta between versions before and after installing packages".format(
          self.actual_version))
    else:
      # If the first install attempt does a partial install and is unable to report this to the server,
      # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version
      Logger.info("Cannot determine actual version installed by checking the delta between versions "
                  "before and after installing package")
      Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed")
      self.actual_version = self.find_best_fit_version(new_versions, self.repository_version)
      if self.actual_version is not None:
        self.actual_version = self.actual_version.strip()
        self.structured_output['actual_version'] = self.actual_version
        self.put_structured_out(self.structured_output)
        Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version))
      else:
        # It's likely that this host does not have any Stack Components installed, so only contains AMS.
        # So just use repo version value provided by server (we already put it to structured output)
        if not os.path.exists(self.stack_root_folder):
          # Special case when this host does not contain any HDP components, but still contains other components like AMS.
          msg = "Could not determine actual version. This stack's root directory ({0}) is not present on this host, so this host does not contain any versionable components. " \
                "Therefore, ignore this host and allow other hosts to report the correct repository version.".format(self.stack_root_folder)
          Logger.info(msg)
        else:
          msg = "Could not determine actual version. This stack's root directory ({0}) exists but was not able to determine the actual repository version installed. " \
                "Try reinstalling packages again.".format(self.stack_root_folder)
          raise Fail(msg)
示例#10
0
class InstallPackages(Script):
    """
  This script is a part of Rolling Upgrade workflow and is described at
  appropriate design doc.
  It installs repositories to the node and then installs packages.
  For now, repositories are installed into individual files.
  """

    UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"]
    REPO_FILE_NAME_PREFIX = 'HDP-'
    STACK_TO_ROOT_FOLDER = {"HDP": "/usr/hdp"}

    def actionexecute(self, env):
        num_errors = 0

        # Parse parameters
        config = Script.get_config()

        repo_rhel_suse = config['configurations']['cluster-env'][
            'repo_suse_rhel_template']
        repo_ubuntu = config['configurations']['cluster-env'][
            'repo_ubuntu_template']
        template = repo_rhel_suse if OSCheck.is_redhat_family(
        ) or OSCheck.is_suse_family() else repo_ubuntu

        # Handle a SIGTERM and SIGINT gracefully
        signal.signal(signal.SIGTERM, self.abort_handler)
        signal.signal(signal.SIGINT, self.abort_handler)

        # Select dict that contains parameters
        try:
            self.repository_version = config['roleParams'][
                'repository_version']
            base_urls = json.loads(config['roleParams']['base_urls'])
            package_list = json.loads(config['roleParams']['package_list'])
            stack_id = config['roleParams']['stack_id']
        except KeyError:
            # Last try
            self.repository_version = config['commandParams'][
                'repository_version']
            base_urls = json.loads(config['commandParams']['base_urls'])
            package_list = json.loads(config['commandParams']['package_list'])
            stack_id = config['commandParams']['stack_id']

        stack_name = None
        self.stack_root_folder = None
        if stack_id and "-" in stack_id:
            stack_split = stack_id.split("-")
            if len(stack_split) == 2:
                stack_name = stack_split[0].upper()
                if stack_name in self.STACK_TO_ROOT_FOLDER:
                    self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[
                        stack_name]
        if self.stack_root_folder is None:
            raise Fail(
                "Cannot determine the stack's root directory by parsing the stack_id property, {0}"
                .format(str(stack_id)))

        self.repository_version = self.repository_version.strip()

        # Install/update repositories
        installed_repositories = []
        self.current_repositories = []
        self.current_repo_files = set()

        Logger.info("Will install packages for repository version {0}".format(
            self.repository_version))
        try:
            append_to_file = False
            for url_info in base_urls:
                repo_name, repo_file = self.install_repository(
                    url_info, append_to_file, template)
                self.current_repositories.append(repo_name)
                self.current_repo_files.add(repo_file)
                append_to_file = True

            installed_repositories = list_ambari_managed_repos()
        except Exception, err:
            Logger.logger.exception(
                "Cannot distribute repositories. Error: {0}".format(str(err)))
            num_errors += 1

        # Build structured output with initial values
        self.structured_output = {
            'ambari_repositories': installed_repositories,
            'installed_repository_version': self.repository_version,
            'stack_id': stack_id,
            'package_installation_result': 'FAIL'
        }
        self.put_structured_out(self.structured_output)

        if num_errors > 0:
            raise Fail("Failed to distribute repositories/install packages")

        # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed
        # to correct value if it is not
        self.actual_version = None
        if self.repository_version:
            m = re.search("[\d\.]+-\d+", self.repository_version)
            if m:
                # Contains a build number
                self.repo_version_with_build_number = self.repository_version
                self.structured_output[
                    'actual_version'] = self.repo_version_with_build_number  # This is the best value known so far.
                self.put_structured_out(self.structured_output)
            else:
                self.repo_version_with_build_number = None

        # Initial list of versions, used to compute the new version installed
        self.old_versions = get_hdp_versions()

        try:
            # It's possible for the process to receive a SIGTERM while installing the packages
            ret_code = self.install_packages(package_list)
            if ret_code == 0:
                self.structured_output[
                    'package_installation_result'] = 'SUCCESS'
                self.put_structured_out(self.structured_output)
            else:
                num_errors += 1
        except Exception, err:
            Logger.logger.exception(
                "Could not install packages. Error: {0}".format(str(err)))