def load_available_packages(self): from ambari_commons.repo_manager import ManagerFactory if self.available_packages_in_repos: return self.available_packages_in_repos config = self.get_config() service_name = config['serviceName'] if 'serviceName' in config else None repos = CommandRepository(config['repositoryFile']) from resource_management.libraries.functions import lzo_utils # remove repos with 'GPL' tag when GPL license is not approved repo_tags_to_skip = set() if not lzo_utils.is_gpl_license_accepted(): repo_tags_to_skip.add("GPL") repos.items = [r for r in repos.items if not (repo_tags_to_skip & r.tags)] repo_ids = [repo.repo_id for repo in repos.items] Logger.info("Command repositories: {0}".format(", ".join(repo_ids))) repos.items = [x for x in repos.items if (not x.applicable_services or service_name in x.applicable_services) ] applicable_repo_ids = [repo.repo_id for repo in repos.items] Logger.info("Applicable repositories: {0}".format(", ".join(applicable_repo_ids))) pkg_provider = ManagerFactory.get() try: self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(repos) except Exception as err: Logger.exception("Unable to load available packages") self.available_packages_in_repos = [] return self.available_packages_in_repos
def load_available_packages(self): from ambari_commons.repo_manager import ManagerFactory if self.available_packages_in_repos: return self.available_packages_in_repos config = self.get_config() service_name = config['serviceName'] if 'serviceName' in config else None repos = CommandRepository(config['repositoryFile']) repo_ids = [repo.repo_id for repo in repos.items] Logger.info("Command repositories: {0}".format(", ".join(repo_ids))) repos.items = [x for x in repos.items if (not x.applicable_services or service_name in x.applicable_services) ] applicable_repo_ids = [repo.repo_id for repo in repos.items] Logger.info("Applicable repositories: {0}".format(", ".join(applicable_repo_ids))) pkg_provider = ManagerFactory.get() try: self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(repos) except Exception as err: Logger.exception("Unable to load available packages") self.available_packages_in_repos = [] return self.available_packages_in_repos
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if 0 == len(command_repository.repositories): Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) create_repo_files(template, command_repository) except Exception, err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1
def get_package_from_available(self, name, available_packages_in_repos=None): """ This function matches package names with ${stack_version} placeholder to actual package names from Ambari-managed repository. Package names without ${stack_version} placeholder are returned as is. """ if STACK_VERSION_PLACEHOLDER not in name: return name if not available_packages_in_repos: available_packages_in_repos = self.load_available_packages() from resource_management.libraries.functions.default import default package_delimiter = '-' if OSCheck.is_ubuntu_family() else '_' package_regex = name.replace(STACK_VERSION_PLACEHOLDER, '(\d|{0})+'.format(package_delimiter)) + "$" repo = default('/repositoryFile', None) name_with_version = None if repo: command_repo = CommandRepository(repo) version_str = command_repo.version_string.replace('.', package_delimiter).replace("-", package_delimiter) name_with_version = name.replace(STACK_VERSION_PLACEHOLDER, version_str) for package in available_packages_in_repos: if re.match(package_regex, package): return package if name_with_version: raise Fail("No package found for {0}(expected name: {1})".format(name, name_with_version)) else: raise Fail("Cannot match package for regexp name {0}. Available packages: {1}".format(name, self.available_packages_in_repos))
def install_repos(): import params if params.host_sys_prepped: return template = params.repo_rhel_suse if OSCheck.is_suse_family( ) or OSCheck.is_redhat_family() else params.repo_ubuntu # use this newer way of specifying repositories, if available if params.repo_file is not None: create_repo_files(template, CommandRepository(params.repo_file)) return _alter_repo("create", params.repo_info, template) if params.service_repo_info: _alter_repo("create", params.service_repo_info, template)
def get_current_component_version(): """ Returns best available version for the component at different stages (install, start, stop) :return best matched version or None :rtype str|None """ from resource_management.core.exceptions import Fail from resource_management.libraries.functions.default import default from resource_management.libraries.functions.stack_select import get_role_component_current_stack_version from resource_management.libraries.functions.repository_util import CommandRepository version = default("/commandParams/version", None) if not version: repository = CommandRepository(default("/repositoryFile", {})) if not repository.resolved: try: version = get_role_component_current_stack_version() except (Fail, TypeError): pass else: version = repository.version_string return version
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if not command_repository.items: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) new_repo_files = Script.repository_util.create_repo_files() self.repo_files.update(new_repo_files) except Exception as err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { 'package_installation_result': 'FAIL', 'repository_version_id': command_repository.version_id } self.put_structured_out(self.structured_output) try: # check package manager non-completed transactions if self.repo_mgr.check_uncompleted_transactions(): self.repo_mgr.print_uncompleted_transaction_hint() num_errors += 1 except Exception as e: # we need to ignore any exception Logger.warning( "Failed to check for uncompleted package manager transactions: " + str(e)) if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") # Initial list of versions, used to compute the new version installed self.old_versions = get_stack_versions(self.stack_root_folder) try: is_package_install_successful = False ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output[ 'package_installation_result'] = 'SUCCESS' self.put_structured_out(self.structured_output) is_package_install_successful = True else: num_errors += 1 except Exception as err: num_errors += 1 Logger.logger.exception( "Could not install packages. Error: {0}".format(str(err))) # Provide correct exit code if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") self._fix_default_links_for_current() # if installing a version of HDP that needs some symlink love, then create them if is_package_install_successful and 'actual_version' in self.structured_output: self._relink_configurations_with_conf_select( stack_id, self.structured_output['actual_version'])
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool( config['ambariLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint( config['ambariLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False packages_installed_before = [] stack_selector_package = stack_tools.get_stack_tool_package( stack_tools.STACK_SELECTOR_NAME) try: # install the stack-selector; we need to supply the action as "upgrade" here since the normal # install command will skip if the package is already installed in the system. # This is required for non-versioned components, like stack-select, since each version of # the stack comes with one. Also, scope the install by repository since we need to pick a # specific repo that the stack-select tools are coming out of in case there are multiple # patches installed repositories = config['repositoryFile']['repositories'] command_repos = CommandRepository(config['repositoryFile']) repository_ids = [ repository['repoId'] for repository in repositories ] repos_to_use = {} for repo_id in repository_ids: if repo_id in self.repo_files: repos_to_use[repo_id] = self.repo_files[repo_id] self.repo_mgr.upgrade_package( stack_selector_package, RepoCallContext(ignore_errors=False, use_repos=repos_to_use, retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count)) packages_installed_before = self.repo_mgr.installed_packages() packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: available_packages_in_repos = self.repo_mgr.get_available_packages_in_repos( command_repos) except Exception: available_packages_in_repos = [] installation_context = RepoCallContext( ignore_errors=False, retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) for package in filtered_package_list: name = self.get_package_from_available( package['name'], available_packages_in_repos) # This enables upgrading non-versioned packages, despite the fact they exist. # Needed by 'mahout' which is non-version but have to be updated self.repo_mgr.upgrade_package(name, installation_context) except Exception as err: ret_code = 1 Logger.logger.error( "Package Manager failed to install packages: {0}".format( str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = self.repo_mgr.installed_packages() packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): self.repo_mgr.remove_package(package, RepoCallContext()) if not self.repo_mgr.verify_dependencies(): ret_code = 1 Logger.logger.error("Failure while verifying dependencies") Logger.logger.error( "*******************************************************************************" ) Logger.logger.error( "Manually verify and fix package dependencies and then re-run install_packages" ) Logger.logger.error( "*******************************************************************************" ) # Compute the actual version in order to save it in structured out try: if ret_code == 0: self.compute_actual_version() else: self.check_partial_install() except Fail as err: ret_code = 1 Logger.logger.exception( "Failure while computing actual version. Error: {0}".format( str(err))) return ret_code
def load_available_packages(self): if self.available_packages_in_repos: return self.available_packages_in_repos pkg_provider = get_provider("Package") try: self.available_packages_in_repos = pkg_provider.get_available_packages_in_repos(CommandRepository(self.get_config()['repositoryFile'])) except Exception as err: Logger.exception("Unable to load available packages") self.available_packages_in_repos = []
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version_id = None base_urls = [] # Select dict that contains parameters try: if 'base_urls' in config['roleParams']: base_urls = json.loads(config['roleParams']['base_urls']) self.repository_version = config['roleParams']['repository_version'] package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] if 'repository_version_id' in config['roleParams']: self.repository_version_id = config['roleParams']['repository_version_id'] except KeyError: pass # current stack information self.current_stack_version_formatted = None if 'stack_version' in config['hostLevelParams']: current_stack_version_unformatted = str(config['hostLevelParams']['stack_version']) self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted) self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format(self.repository_version)) if 0 == len(base_urls): Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version)) try: if 'repositoryFile' in config: create_repo_files(template, CommandRepository(config['repositoryFile'])) else: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository(url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True except Exception, err: Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1