def list_ambari_managed_repos(stack_name): """ Lists all repositories that are present at host """ stack_name = stack_name.upper() # TODO : get it dynamically from the server repository_names = [stack_name, stack_name + "-UTILS" ] if OSCheck.is_ubuntu_family(): repo_dir = '/etc/apt/sources.list.d/' elif OSCheck.is_redhat_family(): # Centos/RHEL 5/6 repo_dir = '/etc/yum.repos.d/' elif OSCheck.is_suse_family(): repo_dir = '/etc/zypp/repos.d/' else: raise Fail('Can not dermine repo dir') repos = [] for name in repository_names: # List all files that match pattern files = glob.glob(os.path.join(repo_dir, name) + '*') for f in files: filename = os.path.basename(f) # leave out extension reponame = os.path.splitext(filename)[0] repos.append(reponame) # get uniq strings seen = set() uniq = [s for s in repos if not (s in seen or seen.add(s))] return uniq
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) min_ver = format_stack_version("2.2") real_ver = format_stack_version(version) if stack_name == "HDP": if compare_versions(real_ver, min_ver) >= 0: stack_selector_path = stack_tools.get_stack_tool_path( stack_tools.STACK_SELECTOR_NAME) cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if compare_versions(real_ver, format_stack_version("2.3")) >= 0: # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info_json = config['hostLevelParams']['repo_info'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]: base_url = item["base_url"] repo_name = item["repo_name"] repo_id = item["repo_id"] repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [repo_name] + self.UBUNTU_REPO_COMPONENTS_POSTFIX Repository(repo_id, action = "create", base_url = base_url, mirror_list = None, repo_file_name = repo_name, repo_template = template, components = ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")} except Exception, exception: Logger.logger.exception("ERROR: There was an unexpected error while updating repositories") raise Fail("Failed to update repo files!")
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info = config['repositoryFile'] for item in repo_info["repositories"]: base_url = item["baseUrl"] repo_name = item["repoName"] repo_id = item["repoId"] distribution = item["distribution"] if "distribution" in item else None components = item["components"] if "components" in item else None repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [distribution if distribution else repo_name] + \ [components.replace(",", " ") if components else self.UBUNTU_REPO_COMPONENTS_POSTFIX] Repository(repo_id, action = "create", base_url = base_url, mirror_list = None, repo_file_name = repo_name, repo_template = template, components = ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")} except Exception, exception: Logger.logger.exception("ERROR: There was an unexpected error while updating repositories") raise Fail("Failed to update repo files!")
def install_repos(): import params template = "repo_suse_rhel.j2" if OSCheck.is_suse_family( ) or OSCheck.is_redhat_family() else "repo_ubuntu.j2" _alter_repo("create", params.repo_info, template) if params.service_repo_info: _alter_repo("create", params.service_repo_info, template)
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if 0 == len(command_repository.repositories): Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) create_repo_files(template, command_repository) except Exception, err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1
def actionexecute(self, env): version = default('/commandParams/version', None) if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) formatted_version = format_stack_version(version) if not formatted_version: raise Fail("Unable to determine a properly formatted stack version from {0}".format(version)) stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) # this script runs on all hosts; if this host doesn't have stack components, # then don't invoke the stack tool # (no need to log that it's skipped - the function will do that) if is_host_skippable(stack_selector_path, formatted_version): return # invoke "set all" cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if code != 0: raise Exception("Command '{0}' exit code is nonzero".format(cmd)) if check_stack_feature(StackFeature.CONFIG_VERSIONING, formatted_version): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) real_ver = format_stack_version(version) if real_ver and check_stack_feature(StackFeature.ROLLING_UPGRADE, real_ver): stack_selector_path = stack_tools.get_stack_tool_path(stack_tools.STACK_SELECTOR_NAME) cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if code != 0: raise Exception("Command '{0}' exit code is nonzero".format(cmd)) if real_ver and check_stack_feature(StackFeature.CONFIG_VERSIONING, real_ver): # backup the old and symlink /etc/[component]/conf to <stack-root>/current/[component] for k, v in conf_select.get_package_dirs().iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def get_lzo_packages(stack_version_unformatted): lzo_packages = [] script_instance = Script.get_instance() if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] if stack_version_unformatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_unformatted): if OSCheck.is_ubuntu_family(): lzo_packages += [ script_instance.format_package_name( "hadooplzo-${stack_version}"), script_instance.format_package_name( "hadooplzo-${stack_version}-native") ] else: lzo_packages += [ script_instance.format_package_name( "hadooplzo_${stack_version}"), script_instance.format_package_name( "hadooplzo_${stack_version}-native") ] else: lzo_packages += ["hadoop-lzo"] return lzo_packages
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) min_ver = format_hdp_stack_version("2.2") real_ver = format_hdp_stack_version(version) if stack_name == "HDP": if compare_versions(real_ver, min_ver) >= 0: cmd = ('hdp-select', 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if compare_versions(real_ver, format_hdp_stack_version("2.3")) >= 0: # backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component] for k, v in conf_select.PACKAGE_DIRS.iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def list_ambari_managed_repos(stack_name): """ Lists all repositories that are present at host """ stack_name = stack_name.upper() # TODO : get it dynamically from the server repository_names = [stack_name, stack_name + "-UTILS"] if OSCheck.is_ubuntu_family(): repo_dir = '/etc/apt/sources.list.d/' elif OSCheck.is_redhat_family(): # Centos/RHEL 5/6 repo_dir = '/etc/yum.repos.d/' elif OSCheck.is_suse_family(): repo_dir = '/etc/zypp/repos.d/' else: raise Fail('Can not dermine repo dir') repos = [] for name in repository_names: # List all files that match pattern files = glob.glob(os.path.join(repo_dir, name) + '*') for f in files: filename = os.path.basename(f) # leave out extension reponame = os.path.splitext(filename)[0] repos.append(reponame) # get uniq strings seen = set() uniq = [s for s in repos if not (s in seen or seen.add(s))] return uniq
def get_lzo_packages(): lzo_packages = [] script_instance = Script.get_instance() if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: lzo_packages += ["liblzo2-2"] elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] stack_version_unformatted = stack_features.get_stack_feature_version( Script.get_config() ) # only used to check stack_feature, NOT as package version! if stack_version_unformatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_unformatted): if OSCheck.is_ubuntu_family(): lzo_packages += [ script_instance.format_package_name( "hadooplzo-${stack_version}"), script_instance.format_package_name( "hadooplzo-${stack_version}-native") ] else: lzo_packages += [ script_instance.format_package_name( "hadooplzo_${stack_version}"), script_instance.format_package_name( "hadooplzo_${stack_version}-native") ] return lzo_packages
def execute_transparent_huge_page_check(self, config): Logger.info("Transparent huge page check started.") thp_regex = "\[(.+)\]" file_name = None if OSCheck.is_ubuntu_family(): file_name = THP_FILE_UBUNTU elif OSCheck.is_redhat_family(): file_name = THP_FILE_REDHAT if file_name and os.path.isfile(file_name): with open(file_name) as f: file_content = f.read() transparent_huge_page_check_structured_output = { "exit_code": 0, "message": str(re.search(thp_regex, file_content).groups()[0]) } else: transparent_huge_page_check_structured_output = { "exit_code": 0, "message": "" } Logger.info("Transparent huge page check completed.") return transparent_huge_page_check_structured_output
def install_repository(self, url_info, repository_version, append_to_file): template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else "repo_ubuntu.j2" repo = { 'repoName': "{0}-{1}".format(url_info['name'], repository_version) } if not 'baseUrl' in url_info: repo['baseurl'] = None else: repo['baseurl'] = url_info['baseUrl'] if not 'mirrorsList' in url_info: repo['mirrorsList'] = None else: repo['mirrorsList'] = url_info['mirrorsList'] ubuntu_components = [url_info['name'] ] + self.UBUNTU_REPO_COMPONENTS_POSTFIX file_name = self.REPO_FILE_NAME_PREFIX + repository_version Repository( repo['repoName'], action="create", base_url=repo['baseurl'], mirror_list=repo['mirrorsList'], repo_file_name=file_name, repo_template=template, append_to_file=append_to_file, components=ubuntu_components, # ubuntu specific ) return repo['repoName'], file_name
def install_repository(self, url_info, repository_version, append_to_file): template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else "repo_ubuntu.j2" repo = { 'repoName': "{0}-{1}".format(url_info['name'], repository_version) } if not 'baseUrl' in url_info: repo['baseurl'] = None else: repo['baseurl'] = url_info['baseUrl'] if not 'mirrorsList' in url_info: repo['mirrorsList'] = None else: repo['mirrorsList'] = url_info['mirrorsList'] ubuntu_components = [url_info['name']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX file_name = self.REPO_FILE_NAME_PREFIX + repository_version Repository(repo['repoName'], action = "create", base_url = repo['baseurl'], mirror_list = repo['mirrorsList'], repo_file_name = file_name, repo_template = template, append_to_file = append_to_file, components = ubuntu_components, # ubuntu specific ) return repo['repoName'], file_name
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Install packages packages_were_checked = False try: Package(self.get_base_packages_to_install()) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], self.repository_version) Package( name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) except Exception, err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def install_repos(): import params if params.host_sys_prepped: return template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu _alter_repo("create", params.repo_info, template) if params.service_repo_info: _alter_repo("create", params.service_repo_info, template)
def getRepoDir(self): if OSCheck.is_redhat_family(): return "/etc/yum.repos.d" elif OSCheck.is_suse_family(): return "/etc/zypp/repos.d" elif OSCheck.is_ubuntu_family(): return "/etc/apt/sources.list.d" else: raise Exception("Unsupported OS family '{0}'".format( OSCheck.get_os_family()))
def get_lzo_packages(stack_version_unformatted): lzo_packages = ["hadoop-lzo"] if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.2') >= 0: if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += [format("hadooplzo_{underscored_version}_*")] elif OSCheck.is_ubuntu_family(): lzo_packages += [format("hadooplzo_{dashed_version}_*")] return lzo_packages
def install_repos(): import params_repo #if OSCheck.is_ubuntu_family(): #cassandra_repo_file = params_repo.repo_ubuntu_path #with open(path, "a") as f: #f.write(params_repo.repo_ubuntu_content) if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): with open(params_repo.repo_rhel_suse_path, "w") as f: f.writelines(params_repo.repo_rhel_suse_content) else: raise Exception('Cassandra repo file path not set for current OS.')
def get_clearcache_cmd(self): if OSCheck.is_redhat_family(): Logger.info("Clear repository cache for the RedHat OS family"); return ("/usr/bin/yum", "clean", "all") elif OSCheck.is_suse_family(): Logger.info("Clear repository cache for the SUSE OS family"); return ('/usr/bin/zypper', 'refresh') elif OSCheck.is_ubuntu_family(): Logger.info("Clear repository cache for the Ubuntu OS family"); return ('/usr/bin/apt-get', 'update') else: raise Exception("Unsupported OS family: '{0}' ".format(OSCheck.get_os_family()))
def __init__(self, config, tags_to_skip): self.tags_to_skip = tags_to_skip # repo templates repo_file = config['repositoryFile'] repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] if is_empty(repo_file): return self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu self.command_repository = CommandRepository(repo_file)
def _clear_package_manager_cache(self): package_manager_cmd = "" if OSCheck.is_redhat_family(): package_manager_cmd = ("/usr/bin/yum", "clean", "metadata") if OSCheck.is_suse_family(): package_manager_cmd = ("/usr/bin/zypper", "-q", "-n", "clean") if OSCheck.is_ubuntu_family(): return Logger.debug("Clearing repo manager metadata") Execute(package_manager_cmd, logoutput=False, sudo=True)
def install_repos(): import params if params.host_sys_prepped: return template = params.repo_rhel_suse if OSCheck.is_suse_family( ) or OSCheck.is_redhat_family() else params.repo_ubuntu # use this newer way of specifying repositories, if available if params.repo_file is not None: create_repo_files(template, CommandRepository(params.repo_file)) return _alter_repo("create", params.repo_info, template) if params.service_repo_info: _alter_repo("create", params.service_repo_info, template)
def get_elastic_config_path(default="/etc/default/elasticsearch"): """ Defines the path to the Elasticsearch environment file. This path will differ based on the OS family. :param default: The path used if the OS family is not recognized. """ path = default if OSCheck.is_redhat_family(): path = "/etc/sysconfig/elasticsearch" elif OSCheck.is_ubuntu_family(): path = "/etc/default/elasticsearch" else: Logger.error("Unexpected OS family; using default path={0}".format(path)) return path
def get_lzo_packages(stack_version_unformatted): lzo_packages = [] if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] if stack_version_unformatted and check_stack_feature( StackFeature.ROLLING_UPGRADE, stack_version_unformatted): lzo_packages += ["hadooplzo_*"] else: lzo_packages += ["hadoop-lzo"] return lzo_packages
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Clear cache of package manager right before installation of the packages self._clear_package_manager_cache() # Install packages packages_were_checked = False try: Package(self.get_base_packages_to_install()) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], self.repository_version) Package(name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) except Exception, err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace('.', '-') else: package_version_string = self.repository_version.replace('-', '_') package_version_string = package_version_string.replace('.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def __init__(self, config): """ Constructor for RepositoryUtil :type config dict """ # repo templates repo_file = config['repositoryFile'] repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] if is_empty(repo_file): return self.template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu self.command_repository = CommandRepository(repo_file)
def get_lzo_packages(stack_version_unformatted): lzo_packages = [] if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') stack_version_formatted = format_stack_version(stack_version_unformatted) if stack_version_formatted != "" and compare_versions(stack_version_formatted, '2.2') >= 0: lzo_packages += ["hadooplzo_*"] else: lzo_packages += ["hadoop-lzo"] return lzo_packages
def actionexecute(self, env): summary = upgrade_summary.get_upgrade_summary() if summary is None: Logger.warning("There is no upgrade in progress") return if summary.associated_version is None: Logger.warning( "There is no version associated with the upgrade in progress") return if summary.orchestration != "STANDARD": Logger.warning( "The 'stack-select set all' command can only be invoked during STANDARD upgrades" ) return if summary.direction.lower( ) != Direction.UPGRADE or summary.is_downgrade_allowed or summary.is_revert: Logger.warning( "The 'stack-select set all' command can only be invoked during an UPGRADE which cannot be downgraded" ) return # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) stack_selector_path = stack_tools.get_stack_tool_path( stack_tools.STACK_SELECTOR_NAME) # this script runs on all hosts; if this host doesn't have stack components, # then don't invoke the stack tool # (no need to log that it's skipped - the function will do that) if is_host_skippable(stack_selector_path, summary.associated_version): return # invoke "set all" cmd = ('ambari-python-wrap', stack_selector_path, 'set', 'all', summary.associated_version) code, out = shell.call(cmd, sudo=True) if code != 0: raise Exception("Command '{0}' exit code is nonzero".format(cmd))
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info_json = config['hostLevelParams']['repo_info'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]: base_url = item["base_url"] repo_name = item["repo_name"] repo_id = item["repo_id"] repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family( ) or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [repo_name ] + self.UBUNTU_REPO_COMPONENTS_POSTFIX Repository( repo_id, action="create", base_url=base_url, mirror_list=None, repo_file_name=repo_name, repo_template=template, components=ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = { "exit_code": 0, "message": format("Repository files successfully updated!") } except Exception, exception: Logger.logger.exception( "ERROR: There was an unexpected error while updating repositories" ) raise Fail("Failed to update repo files!")
def get_lzo_packages(stack_version_unformatted): lzo_packages = [] script_instance = Script.get_instance() if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: lzo_packages += ["liblzo2-2", "hadoop-lzo-native"] elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted): if OSCheck.is_ubuntu_family(): lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") , script_instance.format_package_name("hadooplzo-${stack_version}-native")] else: lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"), script_instance.format_package_name("hadooplzo_${stack_version}-native")] else: lzo_packages += ["hadoop-lzo"] return lzo_packages
def get_lzo_packages(): lzo_packages = [] script_instance = Script.get_instance() if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: lzo_packages += ["liblzo2-2"] elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] if OSCheck.is_ubuntu_family(): lzo_packages += [ script_instance.format_package_name("hadooplzo-${stack_version}"), script_instance.format_package_name( "hadooplzo-${stack_version}-native") ] else: lzo_packages += [ script_instance.format_package_name("hadooplzo_${stack_version}"), script_instance.format_package_name( "hadooplzo_${stack_version}-native") ] return lzo_packages
master_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_master_heapsize']) regionserver_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_regionserver_heapsize']) regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] regionserver_xmn_percent = expect("/configurations/hbase-env/hbase_regionserver_xmn_ratio", float) regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) hbase_regionserver_shutdown_timeout = expect('/configurations/hbase-env/hbase_regionserver_shutdown_timeout', int, 30) phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): phoenix_package = format("phoenix_{underscored_version}_*") elif OSCheck.is_ubuntu_family(): phoenix_package = format("phoenix-{dashed_version}-.*") pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] local_dir = config['configurations']['hbase-site']['hbase.local.dir'] ioengine_param = default('/configurations/hbase-site/hbase.bucketcache.ioengine', None) client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf") ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) # Select dict that contains parameters try: self.repository_version = config['roleParams']['repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: # Last try self.repository_version = config['commandParams']['repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] stack_name = None self.stack_root_folder = None if stack_id and "-" in stack_id: stack_split = stack_id.split("-") if len(stack_split) == 2: stack_name = stack_split[0].upper() if stack_name in self.STACK_TO_ROOT_FOLDER: self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[stack_name] if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory by parsing the stack_id property, {0}".format(str(stack_id))) if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories installed_repositories = [] self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format(self.repository_version)) try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository(url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos() except Exception, err: Logger.logger.exception("Cannot distribute repositories. Error: {0}".format(str(err))) num_errors += 1
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) # Select dict that contains parameters try: self.repository_version = config['roleParams'][ 'repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: # Last try self.repository_version = config['commandParams'][ 'repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] # current stack information self.current_hdp_stack_version = None if 'stack_version' in config['hostLevelParams']: current_stack_version_unformatted = str( config['hostLevelParams']['stack_version']) self.current_hdp_stack_version = format_hdp_stack_version( current_stack_version_unformatted) stack_name = None self.stack_root_folder = None if stack_id and "-" in stack_id: stack_split = stack_id.split("-") if len(stack_split) == 2: stack_name = stack_split[0].upper() if stack_name in self.STACK_TO_ROOT_FOLDER: self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[ stack_name] if self.stack_root_folder is None: raise Fail( "Cannot determine the stack's root directory by parsing the stack_id property, {0}" .format(str(stack_id))) if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories installed_repositories = [] self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format( self.repository_version)) try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository( url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos() except Exception, err: Logger.logger.exception( "Cannot distribute repositories. Error: {0}".format(str(err))) num_errors += 1
installed_packages, repos = self.execute_existing_repos_and_installed_packages_check(config) structured_output[CHECK_INSTALLED_PACKAGES] = installed_packages structured_output[CHECK_EXISTING_REPOS] = repos except Exception, exception : print "There was an unknown error while checking installed packages and existing repositories: " + str(exception) structured_output[CHECK_INSTALLED_PACKAGES] = {"exit_code" : 1, "message": str(exception)} structured_output[CHECK_EXISTING_REPOS] = {"exit_code" : 1, "message": str(exception)} # Here we are checking transparent huge page if CHECK_TRANSPARENT_HUGE_PAGE is in check_execute_list if CHECK_TRANSPARENT_HUGE_PAGE in check_execute_list: try : thp_regex = "\[(.+)\]" file_name = None if OSCheck.is_ubuntu_family(): file_name = THP_FILE_UBUNTU elif OSCheck.is_redhat_family(): file_name = THP_FILE_REDHAT if file_name and os.path.isfile(file_name): with open(file_name) as f: file_content = f.read() structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 0, "message": str(re.search(thp_regex, file_content).groups()[0])} else: structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 0, "message": ""} except Exception, exception : print "There was an unknown error while getting transparent huge page data: " + str(exception) structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {"exit_code" : 1, "message": str(exception)} # this is necessary for HostCleanup to know later what were the results. self.reportFileHandler.writeHostChecksCustomActionsFile(structured_output)
regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 if not has_phoenix and not phoenix_enabled: exclude_packages = ['phoenix*'] else: exclude_packages = [] underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): phoenix_package = format("phoenix_{underscored_version}_*") elif OSCheck.is_ubuntu_family(): phoenix_package = format("phoenix-{dashed_version}-.*") pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] local_dir = config['configurations']['hbase-site']['hbase.local.dir'] client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf") ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]