def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Install packages packages_were_checked = False try: Package( "hdp-select", action="upgrade", ) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name']) Package( name, action= "upgrade" # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated ) except Exception, err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def test_analyze_zypper_out(self, spwt_mock, get_os_family_mock): get_os_family_mock.return_value = 'suse' stringToRead = """Refreshing service 'susecloud'. Loading repository data... Reading installed packages... S | Name | Type | Version | Arch | Repository --+-----------------------------------+---------+------------------------+--------+---------------------- i | ConsoleKit | package | 0.2.10-64.65.1 | x86_64 | SLES11-SP1-Updates i | gweb | package | 2.2.0-99 | noarch | Hortonworks Data Platform Utils Version - HDP-UTILS-1.1.0.15 i | hadoop | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP i | hadoop-libhdfs | package | 1.2.0.1.3.0.0-107 | x86_64 | HDP i | ambari-server | package | 1.2.4.9-1 | noarch | Ambari 1.x i | hdp_mon_ganglia_addons | package | 1.2.4.9-1 | noarch | Ambari 1.x i | Minimal | pattern | 11-38.13.9 | x86_64 | SLES11-SP1""" result = {} result['out'] = stringToRead result['err'] = "" result['retCode'] = 0 spwt_mock.return_value = result installedPackages = [] packages_analyzer.allInstalledPackages(installedPackages) self.assertEqual(7, len(installedPackages)) self.assertTrue(installedPackages[1][0], "gweb") self.assertTrue(installedPackages[3][2], "HDP") self.assertTrue(installedPackages[6][1], "11-38.13.9")
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Install packages packages_were_checked = False try: Package(self.get_base_packages_to_install()) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], self.repository_version) Package( name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) except Exception, err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint(config['hostLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME) try: Package(stack_selector_package, action="upgrade", retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count ) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name']) Package(name, action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count ) except Exception, err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace('.', '-') else: package_version_string = self.repository_version.replace('-', '_') package_version_string = package_version_string.replace('.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def test_analyze_yum_output_err(self, subprocessWithTimeout_mock, get_os_family_mock): get_os_family_mock.return_value = OSConst.REDHAT_FAMILY result = {} result['out'] = "" result['err'] = "" result['retCode'] = 1 subprocessWithTimeout_mock.return_value = result installedPackages = [] packages_analyzer.allInstalledPackages(installedPackages) self.assertEqual(installedPackages, [])
def test_analyze_yum_output(self, subprocessWithTimeout_mock, get_os_family_mock): get_os_family_mock.return_value = 'redhat' stringToRead = """Loaded plugins: amazon-id, product-id, rhui-lb, security, subscription-manager Updating certificate-based repositories. Installed Packages AMBARI.dev.noarch 1.x-1.el6 installed PyXML.x86_64 0.8.4-19.el6 @koji-override-0 Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch 3-7.el6 @koji-override-0 hcatalog.noarch 0.11.0.1.3.0.0-107.el6 @HDP-1.3.0 hesiod.x86_64 3.1.0-19.el6 @koji-override-0/$releasever hive.noarch 0.11.0.1.3.0.0-107.el6 @HDP-1.3.0 oracle-server-db.x86 1.3.17-2 @Oracle-11g ambari-log4j.noarch 1.2.5.9-1 @AMBARI.dev-1.x libconfuse.x86_64 2.7-4.el6 @HDP-epel""" result = {} result['out'] = stringToRead result['err'] = "" result['retCode'] = 0 subprocessWithTimeout_mock.return_value = result installedPackages = [] packages_analyzer.allInstalledPackages(installedPackages) self.assertEqual(9, len(installedPackages)) for package in installedPackages: self.assertTrue(package[0] in ["AMBARI.dev.noarch", "PyXML.x86_64", "oracle-server-db.x86", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch", "hcatalog.noarch", "hesiod.x86_64", "hive.noarch", "ambari-log4j.noarch", "libconfuse.x86_64"]) self.assertTrue(package[1] in ["1.x-1.el6", "0.8.4-19.el6", "3-7.el6", "3.1.0-19.el6", "0.11.0.1.3.0.0-107.el6", "1.2.5.9-1", "1.3.17-2", "1.2.5.9-1", "2.7-4.el6"]) self.assertTrue(package[2] in ["installed", "koji-override-0", "HDP-1.3.0", "koji-override-0/$releasever", "AMBARI.dev-1.x", "Oracle-11g", "HDP-epel"]) packages = packages_analyzer.getInstalledPkgsByNames(["AMBARI", "Red_Hat_Enterprise", "hesiod", "hive"], installedPackages) self.assertEqual(4, len(packages)) expected = ["AMBARI.dev.noarch", "Red_Hat_Enterprise_Linux-Release_Notes-6-en-US.noarch", "hesiod.x86_64", "hive.noarch"] for package in expected: self.assertTrue(package in packages) detailedPackages = packages_analyzer.getPackageDetails(installedPackages, packages) self.assertEqual(4, len(detailedPackages)) for package in detailedPackages: self.assertTrue(package['version'] in ["1.x-1.el6", "3-7.el6", "3.1.0-19.el6", "0.11.0.1.3.0.0-107.el6"]) self.assertTrue(package['repoName'] in ["installed", "koji-override-0", "HDP-1.3.0", "koji-override-0/$releasever"]) self.assertFalse(package['repoName'] in ["AMBARI.dev-1.x"])
def get_packages_to_remove(self, version): packages = [] formated_version = version.replace('.', '_').replace('-', '_') all_installed_packages = [] allInstalledPackages(all_installed_packages) all_installed_packages = [ package[0] for package in all_installed_packages ] for package in all_installed_packages: if formated_version in package and self.stack_tool_package not in package: packages.append(package) Logger.info("%s added to remove" % (package)) return packages
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Clear cache of package manager right before installation of the packages self._clear_package_manager_cache() # Install packages packages_were_checked = False try: Package(self.get_base_packages_to_install()) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], self.repository_version) Package(name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) except Exception, err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace('.', '-') else: package_version_string = self.repository_version.replace('-', '_') package_version_string = package_version_string.replace('.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def execute_existing_repos_and_installed_packages_check(self, config): installedPackages = [] availablePackages = [] packages_analyzer.allInstalledPackages(installedPackages) packages_analyzer.allAvailablePackages(availablePackages) repos = [] packages_analyzer.getInstalledRepos(self.PACKAGES, installedPackages + availablePackages, self.IGNORE_PACKAGES_FROM_REPOS, repos) packagesInstalled = packages_analyzer.getInstalledPkgsByRepo(repos, self.IGNORE_PACKAGES, installedPackages) additionalPkgsInstalled = packages_analyzer.getInstalledPkgsByNames( self.ADDITIONAL_PACKAGES, installedPackages) allPackages = list(set(packagesInstalled + additionalPkgsInstalled)) installedPackages = packages_analyzer.getPackageDetails(installedPackages, allPackages) repos = packages_analyzer.getReposToRemove(repos, self.IGNORE_REPOS) return installedPackages, repos
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool( config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint( config['hostLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False stack_selector_package = stack_tools.get_stack_tool_package( stack_tools.STACK_SELECTOR_NAME) try: Package(stack_selector_package, action="upgrade", retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: available_packages_in_repos = packages_analyzer.get_available_packages_in_repos( config['repositoryFile']['repositories']) except Exception: available_packages_in_repos = [] for package in filtered_package_list: name = self.get_package_from_available( package['name'], available_packages_in_repos) Package( name, action= "upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except Exception as err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove") if not verifyDependencies(): ret_code = 1 Logger.logger.error("Failure while verifying dependencies") Logger.logger.error( "*******************************************************************************" ) Logger.logger.error( "Manually verify and fix package dependencies and then re-run install_packages" ) Logger.logger.error( "*******************************************************************************" ) # Compute the actual version in order to save it in structured out try: if ret_code == 0: self.compute_actual_version() else: self.check_partial_install() except Fail as err: ret_code = 1 Logger.logger.exception( "Failure while computing actual version. Error: {0}".format( str(err))) return ret_code
class InstallPackages(Script): """ This script is a part of Rolling Upgrade workflow and is described at appropriate design doc. It installs repositories to the node and then installs packages. For now, repositories are installed into individual files. """ UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] REPO_FILE_NAME_PREFIX = 'HDP-' def actionexecute(self, env): delayed_fail = False package_install_result = False # Parse parameters config = Script.get_config() # Select dict that contains parameters try: repository_version = config['roleParams']['repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: # Last try repository_version = config['commandParams']['repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] # Install/update repositories installed_repositories = [] current_repositories = [ 'base' ] # Some our packages are installed from the base repo current_repo_files = set(['base']) old_versions = self.hdp_versions() try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository( url_info, repository_version, append_to_file) current_repositories.append(repo_name) current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos() except Exception, err: print "Can not distribute repositories." print traceback.format_exc() delayed_fail = True # Install packages if not delayed_fail: packages_were_checked = False try: packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], repository_version) Package( name, use_repos=list(current_repo_files) if OSCheck.is_ubuntu_family() else current_repositories) package_install_result = True except Exception, err: print "Can not install packages." print traceback.format_exc() delayed_fail = True # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = repository_version.replace( '.', '-') else: package_version_string = repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")