def check_partial_install(self): """ If an installation did not complete successfully, check if installation was partially complete and log the partially completed version to REPO_VERSION_HISTORY_FILE. :return: """ Logger.info( "Installation of packages failed. Checking if installation was partially complete" ) Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_stack_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): # Some packages were installed successfully. Log this version to REPO_VERSION_HISTORY_FILE partial_install_version = next(iter(deltas)).strip() write_actual_version_to_history_file(normalized_repo_version, partial_install_version) Logger.info("Version {0} was partially installed. ".format( partial_install_version))
def get_lower_versions(self, current_version): versions = get_stack_versions(stack_root) Logger.info("available versions: {0}".format(str(versions))) lover_versions = [] for version in versions: if self.compare(version, current_version) < 0: lover_versions.append(version) Logger.info("version %s added to remove" % (version)) return lover_versions
def compute_actual_version(self): """ After packages are installed, determine what the new actual version is. """ # If the repo contains a build number, optimistically assume it to be the actual_version. It will get changed # to correct value if it is not self.actual_version = None self.repo_version_with_build_number = None if self.repository_version: m = re.search("[\d\.]+-\d+", self.repository_version) if m: # Contains a build number self.repo_version_with_build_number = self.repository_version self.structured_output['actual_version'] = self.repo_version_with_build_number # This is the best value known so far. self.put_structured_out(self.structured_output) Logger.info("Attempting to determine actual version with build number.") Logger.info("Old versions: {0}".format(self.old_versions)) new_versions = get_stack_versions(self.stack_root_folder) Logger.info("New versions: {0}".format(new_versions)) deltas = set(new_versions) - set(self.old_versions) Logger.info("Deltas: {0}".format(deltas)) # Get version without build number normalized_repo_version = self.repository_version.split('-')[0] if 1 == len(deltas): self.actual_version = next(iter(deltas)).strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) write_actual_version_to_history_file(normalized_repo_version, self.actual_version) Logger.info( "Found actual version {0} by checking the delta between versions before and after installing packages".format( self.actual_version)) else: # If the first install attempt does a partial install and is unable to report this to the server, # then a subsequent attempt will report an empty delta. For this reason, we search for a best fit version for the repo version Logger.info("Cannot determine actual version installed by checking the delta between versions " "before and after installing package") Logger.info("Will try to find for the actual version by searching for best possible match in the list of versions installed") self.actual_version = self.find_best_fit_version(new_versions, self.repository_version) if self.actual_version is not None: self.actual_version = self.actual_version.strip() self.structured_output['actual_version'] = self.actual_version self.put_structured_out(self.structured_output) Logger.info("Found actual version {0} by searching for best possible match".format(self.actual_version)) else: msg = "Could not determine actual version installed. Try reinstalling packages again." raise Fail(msg)
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if not command_repository.items: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) new_repo_files = Script.repository_util.create_repo_files() self.repo_files.update(new_repo_files) except Exception as err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { 'package_installation_result': 'FAIL', 'repository_version_id': command_repository.version_id } self.put_structured_out(self.structured_output) try: # check package manager non-completed transactions if self.repo_mgr.check_uncompleted_transactions(): self.repo_mgr.print_uncompleted_transaction_hint() num_errors += 1 except Exception as e: # we need to ignore any exception Logger.warning( "Failed to check for uncompleted package manager transactions: " + str(e)) if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") # Initial list of versions, used to compute the new version installed self.old_versions = get_stack_versions(self.stack_root_folder) try: is_package_install_successful = False ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output[ 'package_installation_result'] = 'SUCCESS' self.put_structured_out(self.structured_output) is_package_install_successful = True else: num_errors += 1 except Exception as err: num_errors += 1 Logger.logger.exception( "Could not install packages. Error: {0}".format(str(err))) # Provide correct exit code if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") self._fix_default_links_for_current() # if installing a version of HDP that needs some symlink love, then create them if is_package_install_successful and 'actual_version' in self.structured_output: self._relink_configurations_with_conf_select( stack_id, self.structured_output['actual_version'])
class InstallPackages(Script): """ This script is a part of Rolling Upgrade workflow and is described at appropriate design doc. It installs repositories to the node and then installs packages. For now, repositories are installed into individual files. """ UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if 0 == len(command_repository.repositories): Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) create_repo_files(template, command_repository) except Exception, err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { 'package_installation_result': 'FAIL', 'repository_version_id': command_repository.version_id } self.put_structured_out(self.structured_output) if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") # Initial list of versions, used to compute the new version installed self.old_versions = get_stack_versions(self.stack_root_folder) try: is_package_install_successful = False ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output[ 'package_installation_result'] = 'SUCCESS' self.put_structured_out(self.structured_output) is_package_install_successful = True else: num_errors += 1 except Exception, err: num_errors += 1 Logger.logger.exception( "Could not install packages. Error: {0}".format(str(err)))
class InstallPackages(Script): """ This script is a part of Rolling Upgrade workflow and is described at appropriate design doc. It installs repositories to the node and then installs packages. For now, repositories are installed into individual files. """ UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version_id = None # Select dict that contains parameters try: self.repository_version = config['roleParams']['repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] if 'repository_version_id' in config['roleParams']: self.repository_version_id = config['roleParams']['repository_version_id'] except KeyError: # Last try self.repository_version = config['commandParams']['repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] if 'repository_version_id' in config['commandParams']: self.repository_version_id = config['commandParams']['repository_version_id'] # current stack information self.current_stack_version_formatted = None if 'stack_version' in config['hostLevelParams']: current_stack_version_unformatted = str(config['hostLevelParams']['stack_version']) self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted) self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories installed_repositories = [] self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format(self.repository_version)) if 0 == len(base_urls): Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version)) try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository(url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos(self.stack_name) except Exception, err: Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { 'ambari_repositories': installed_repositories, 'installed_repository_version': self.repository_version, 'stack_id': stack_id, 'package_installation_result': 'FAIL' } if self.repository_version_id is not None: self.structured_output['repository_version_id'] = self.repository_version_id self.put_structured_out(self.structured_output) if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") # Initial list of versions, used to compute the new version installed self.old_versions = get_stack_versions(self.stack_root_folder) try: is_package_install_successful = False ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output['package_installation_result'] = 'SUCCESS' self.put_structured_out(self.structured_output) is_package_install_successful = True else: num_errors += 1 except Exception, err: num_errors += 1 Logger.logger.exception("Could not install packages. Error: {0}".format(str(err)))