def server_files(): import params rrd_py_path = params.rrd_py_path Directory(rrd_py_path, recursive=True ) rrd_py_file_path = path.join(rrd_py_path, "rrd.py") TemplateConfig(rrd_py_file_path, owner="root", group="root", mode=0755 ) rrd_file_owner = params.gmetad_user Directory(params.rrdcached_base_dir, owner=rrd_file_owner, group=rrd_file_owner, mode=0755, recursive=True ) if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): File( params.ganglia_apache_config_file, content = Template("ganglia.conf.j2"), mode = 0644 )
def actionexecute(self, env): config = Script.get_config() structured_output = {} try: repo_info_json = config['hostLevelParams']['repo_info'] repo_info_dict = json.loads(repo_info_json) for item in repo_info_dict["repositories"]: base_url = item["base_url"] repo_name = item["repo_name"] repo_id = item["repo_id"] repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else repo_ubuntu ubuntu_components = [repo_name] + self.UBUNTU_REPO_COMPONENTS_POSTFIX Repository(repo_id, action = "create", base_url = base_url, mirror_list = None, repo_file_name = repo_name, repo_template = template, components = ubuntu_components, # ubuntu specific ) structured_output["repo_update"] = {"exit_code" : 0, "message": format("Repository files successfully updated!")} except Exception, exception: Logger.logger.exception("ERROR: There was an unexpected error while updating repositories") raise Fail("Failed to update repo files!")
def install_packages(self, env, exclude_packages=[]): """ List of packages that are required< by service is received from the server as a command parameter. The method installs all packages from this list """ config = self.get_config() try: package_list_str = config['hostLevelParams']['package_list'] if isinstance(package_list_str, basestring) and len(package_list_str) > 0: package_list = json.loads(package_list_str) for package in package_list: if not package['name'] in exclude_packages: name = package['name'] if OSCheck.is_windows_family(): if name[-4:] == ".msi": #TODO all msis must be located in resource folder of server, change it to repo later Msi(name, http_source=os.path.join(config['hostLevelParams']['jdk_location'])) else: Package(name) except KeyError: pass # No reason to worry if OSCheck.is_windows_family(): #TODO hacky install of windows msi, remove it or move to old(2.1) stack definition when component based install will be implemented install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"), config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"), str(config['hostLevelParams']['stack_version'])) reload_windows_env() pass
def list_ambari_managed_repos(stack_name): """ Lists all repositories that are present at host """ stack_name = stack_name.upper() # TODO : get it dynamically from the server repository_names = [stack_name, stack_name + "-UTILS" ] if OSCheck.is_ubuntu_family(): repo_dir = '/etc/apt/sources.list.d/' elif OSCheck.is_redhat_family(): # Centos/RHEL 5/6 repo_dir = '/etc/yum.repos.d/' elif OSCheck.is_suse_family(): repo_dir = '/etc/zypp/repos.d/' else: raise Fail('Can not dermine repo dir') repos = [] for name in repository_names: # List all files that match pattern files = glob.glob(os.path.join(repo_dir, name) + '*') for f in files: filename = os.path.basename(f) # leave out extension reponame = os.path.splitext(filename)[0] repos.append(reponame) # get uniq strings seen = set() uniq = [s for s in repos if not (s in seen or seen.add(s))] return uniq
def install_repository(self, url_info, repository_version, append_to_file): template = "repo_suse_rhel.j2" if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else "repo_ubuntu.j2" repo = { 'repoName': "{0}-{1}".format(url_info['name'], repository_version) } if not 'baseUrl' in url_info: repo['baseurl'] = None else: repo['baseurl'] = url_info['baseUrl'] if not 'mirrorsList' in url_info: repo['mirrorsList'] = None else: repo['mirrorsList'] = url_info['mirrorsList'] ubuntu_components = [url_info['name']] + self.UBUNTU_REPO_COMPONENTS_POSTFIX file_name = self.REPO_FILE_NAME_PREFIX + repository_version Repository(repo['repoName'], action = "create", base_url = repo['baseurl'], mirror_list = repo['mirrorsList'], repo_file_name = file_name, repo_template = template, append_to_file = append_to_file, components = ubuntu_components, # ubuntu specific ) return repo['repoName'], file_name
def get_serivice_params(self): self.system = System.get_instance() if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): self.service_name = "apache2" self.httpd_conf_dir = '/etc/apache2' else: self.service_name = "httpd" self.httpd_conf_dir = '/etc/httpd/conf'
def install_repos(): import params if params.host_sys_prepped: return template = params.repo_rhel_suse if OSCheck.is_suse_family() or OSCheck.is_redhat_family() else params.repo_ubuntu _alter_repo("create", params.repo_info, template) if params.service_repo_info: _alter_repo("create", params.service_repo_info, template)
def stop(self, env): if OSCheck.is_suse_family(): Execute('rckadmind stop') Execute('rckrb5kdc stop') elif OSCheck.is_ubuntu_family(): Execute('service krb5-kdc stop') Execute('service krb5-admin-server stop') else: Execute('service krb5kdc stop') Execute('service kadmin stop')
def get_base_packages_to_install(self): """ HACK: list packages which should be installed without disabling any repos. (This is planned to fix in Ambari-2.2) """ base_packages_to_install = ['fuse'] if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): base_packages_to_install.append('libfuse2') else: base_packages_to_install.append('fuse-libs') return base_packages_to_install
def _clear_package_manager_cache(self): package_manager_cmd = "" if OSCheck.is_redhat_family(): package_manager_cmd = ("/usr/bin/yum", "clean", "metadata") if OSCheck.is_suse_family(): package_manager_cmd = ("/usr/bin/zypper", "-q", "-n", "clean") if OSCheck.is_ubuntu_family(): return Logger.debug("Clearing repo manager metadata") Execute(package_manager_cmd, logoutput=False, sudo=True)
def get_elastic_config_path(default="/etc/default/elasticsearch"): """ Defines the path to the Elasticsearch environment file. This path will differ based on the OS family. :param default: The path used if the OS family is not recognized. """ path = default if OSCheck.is_redhat_family(): path = "/etc/sysconfig/elasticsearch" elif OSCheck.is_ubuntu_family(): path = "/etc/default/elasticsearch" else: Logger.error("Unexpected OS family; using default path={0}".format(path)) return path
def launch_python_subprocess(self, command, tmpout, tmperr): """ Creates subprocess with given parameters. This functionality was moved to separate method to make possible unit testing """ close_fds = None if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else True command_env = dict(os.environ) if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: command_env["PYTHONPATH"] = os.pathsep.join(sys.path) for k, v in command_env.iteritems(): command_env[k] = str(v) return subprocess.Popen(command, stdout=tmpout, stderr=tmperr, close_fds=close_fds, env=command_env, preexec_fn=self.preexec_fn)
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ if configurations is None: return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) if not OOZIE_URL_KEY in configurations: return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all # interfaces, which doesn't work on Windows localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0' oozie_url = configurations[OOZIE_URL_KEY] oozie_url = oozie_url.replace(urlparse(oozie_url).hostname,localhost_address) try: command, env = get_check_command(oozie_url, host_name, configurations) # execute the command Execute(command, environment=env) return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) except KerberosPropertiesNotFound, ex: return (RESULT_CODE_UNKNOWN, [str(ex)])
def actionexecute(self, env): config = Script.get_config() version = default('/commandParams/version', None) stack_name = default('/hostLevelParams/stack_name', "") if not version: raise Fail("Value is required for '/commandParams/version'") # other os? if OSCheck.is_redhat_family(): cmd = ('/usr/bin/yum', 'clean', 'all') code, out = shell.call(cmd, sudo=True) min_ver = format_hdp_stack_version("2.2") real_ver = format_hdp_stack_version(version) if stack_name == "HDP": if compare_versions(real_ver, min_ver) >= 0: cmd = ('hdp-select', 'set', 'all', version) code, out = shell.call(cmd, sudo=True) if compare_versions(real_ver, format_hdp_stack_version("2.3")) >= 0: # backup the old and symlink /etc/[component]/conf to /usr/hdp/current/[component] for k, v in conf_select.PACKAGE_DIRS.iteritems(): for dir_def in v: link_config(dir_def['conf_dir'], dir_def['current_dir'])
def start(self, env): # Attempt to reconfigure the service before starting self.configure(env) # Create or update the administrator account KerberosScript.create_or_update_administrator_identity() if OSCheck.is_suse_family(): Execute('rckadmind start') Execute('rckrb5kdc start') elif OSCheck.is_ubuntu_family(): Execute('service krb5-kdc start') Execute('service krb5-admin-server start') else: Execute('service krb5kdc start') Execute('service kadmin start')
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 # Clear cache of package manager right before installation of the packages self._clear_package_manager_cache() # Install packages packages_were_checked = False try: Package(self.get_base_packages_to_install()) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], self.repository_version) Package(name, use_repos=list(self.current_repo_files) if OSCheck.is_ubuntu_family() else self.current_repositories, skip_repos=[self.REPO_FILE_NAME_PREFIX + "*"] if OSCheck.is_redhat_family() else []) except Exception, err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace('.', '-') else: package_version_string = self.repository_version.replace('-', '_') package_version_string = package_version_string.replace('.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
def status(self, env): import params if OSCheck.is_suse_family(): try: Execute('checkproc `which krb5kdc`') Execute('checkproc `which kadmind`') except Fail as ex: raise ComponentIsNotRunning() elif OSCheck.is_ubuntu_family(): check_process_status(params.kdamin_pid_path) check_process_status(params.krb5kdc_pid_path) else: check_process_status(params.kdamin_pid_path) check_process_status(params.krb5kdc_pid_path)
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ if configurations is None: return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) if not OOZIE_URL_KEY in configurations: return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all # interfaces, which doesn't work on Windows localhost_address = 'localhost' if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY else '0.0.0.0' https_port = None # try to get https port form oozie-env content if OOZIE_ENV_CONTENT in configurations: for line in configurations[OOZIE_ENV_CONTENT].splitlines(): result = re.match(OOZIE_ENV_HTTPS_RE, line) if result is not None: https_port = result.group(1) # or from oozie-site.xml if https_port is None and OOZIE_HTTPS_PORT in configurations: https_port = configurations[OOZIE_HTTPS_PORT] oozie_url = configurations[OOZIE_URL_KEY] # construct proper url for https if https_port is not None: parsed_url = urlparse(oozie_url) oozie_url = oozie_url.replace(parsed_url.scheme, "https") if parsed_url.port is None: oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) else: oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port)) # https will not work with localhost address, we need put fqdn if https_port is None: oozie_url = oozie_url.replace(urlparse(oozie_url).hostname, localhost_address) try: command, env, oozie_user = get_check_command(oozie_url, host_name, configurations) # execute the command Execute(command, environment=env, user=oozie_user, ) return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) except KerberosPropertiesNotFound, ex: return (RESULT_CODE_UNKNOWN, [str(ex)])
def bind_signal_handlers(agentPid): global _handler if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY: if os.getpid() == agentPid: signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGUSR1, debug) _handler = HeartbeatStopHandlersLinux() else: _handler = HeartbeatStopHandlersWindows() return _handler
def format_package_name(self, package_name, repo_id): """ This method overcomes problems at SLES SP3. Zypper here behaves differently than at SP1, and refuses to install packages by mask if there is any installed package that matches this mask. So we preppend concrete HDP version to mask under Suse """ if OSCheck.is_suse_family() and '*' in package_name: mask_version = re.search(r'((_\d+)*(_)?\*)', package_name).group(0) formatted_version = '_' + repo_id.replace('.', '_').replace('-', '_') + '*' return package_name.replace(mask_version, formatted_version) else: return package_name
def on_failure(self, pythonCommand, result): """ Log some useful information after task failure. """ logger.info("Command " + pprint.pformat(pythonCommand) + " failed with exitcode=" + str(result['exitcode'])) if OSCheck.is_windows_family(): cmd_list = ["WMIC path win32_process get Caption,Processid,Commandline", "netstat -an"] else: cmd_list = ["ps faux", "netstat -tulpn"] shell_runner = shellRunner() for cmd in cmd_list: ret = shell_runner.run(cmd) logger.info("Command '{0}' returned {1}. {2}{3}".format(cmd, ret["exitCode"], ret["error"], ret["output"]))
def execute_java_home_available_check(self, config): print "Java home check started." java_home = config['commandParams']['java_home'] print "Java home to check: " + java_home java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" if not os.path.isfile(os.path.join(java_home, "bin", java_bin)): print "Java home doesn't exist!" java_home_check_structured_output = {"exit_code" : 1, "message": "Java home doesn't exist!"} else: print "Java home exists!" java_home_check_structured_output = {"exit_code" : 0, "message": "Java home exists!"} return java_home_check_structured_output
def turn_off_autostart(service): if OSCheck.is_ubuntu_family(): Execute(('update-rc.d', service, 'disable'), path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', sudo = True ) Execute(('service', service, 'stop'), sudo = True, ignore_failures=True, ) File(format('/etc/init/{service}.override'), # disable upstart job content = 'manual', ) else: Execute(('chkconfig', service, 'off'), path='/usr/sbin:/sbin:/usr/local/bin:/bin:/usr/bin', sudo = True, )
def bind_signal_handlers(agentPid): global _handler if OSCheck.get_os_family() != OSConst.WINSRV_FAMILY: if os.getpid() == agentPid: signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGUSR2, remote_debug) # Interrupt running process, and provide a python prompt for it try: import faulthandler # This is not default module, has to be installed separately faulthandler.enable(file=sys.stderr, all_threads=True) faulthandler.register(signal.SIGUSR1, file=sys.stderr, all_threads=True, chain=False) sys.stderr.write("Registered faulthandler\n") except ImportError: pass # Module is not included into python distribution _handler = HeartbeatStopHandlersLinux() else: _handler = HeartbeatStopHandlersWindows() return _handler
def get_lzo_packages(stack_version_unformatted): lzo_packages = [] script_instance = Script.get_instance() if OSCheck.is_suse_family() and int(OSCheck.get_os_major_version()) >= 12: lzo_packages += ["liblzo2-2", "hadoop-lzo-native"] elif OSCheck.is_redhat_family() or OSCheck.is_suse_family(): lzo_packages += ["lzo", "hadoop-lzo-native"] elif OSCheck.is_ubuntu_family(): lzo_packages += ["liblzo2-2"] if stack_version_unformatted and check_stack_feature(StackFeature.ROLLING_UPGRADE, stack_version_unformatted): if OSCheck.is_ubuntu_family(): lzo_packages += [script_instance.format_package_name("hadooplzo-${stack_version}") , script_instance.format_package_name("hadooplzo-${stack_version}-native")] else: lzo_packages += [script_instance.format_package_name("hadooplzo_${stack_version}"), script_instance.format_package_name("hadooplzo_${stack_version}-native")] else: lzo_packages += ["hadoop-lzo"] return lzo_packages
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if not command_repository.items: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) new_repo_files = create_repo_files(template, command_repository) self.repo_files.update(new_repo_files) except Exception as err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1 # Build structured output with initial values self.structured_output = { 'package_installation_result': 'FAIL', 'repository_version_id': command_repository.version_id } self.put_structured_out(self.structured_output) try: # check package manager non-completed transactions if self.pkg_provider.check_uncompleted_transactions(): self.pkg_provider.print_uncompleted_transaction_hint() num_errors += 1 except Exception as e: # we need to ignore any exception Logger.warning( "Failed to check for uncompleted package manager transactions: " + str(e)) if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") # Initial list of versions, used to compute the new version installed self.old_versions = get_stack_versions(self.stack_root_folder) try: is_package_install_successful = False ret_code = self.install_packages(package_list) if ret_code == 0: self.structured_output[ 'package_installation_result'] = 'SUCCESS' self.put_structured_out(self.structured_output) is_package_install_successful = True else: num_errors += 1 except Exception as err: num_errors += 1 Logger.logger.exception( "Could not install packages. Error: {0}".format(str(err))) # Provide correct exit code if num_errors > 0: raise Fail("Failed to distribute repositories/install packages") self._fix_default_links_for_current() # if installing a version of HDP that needs some symlink love, then create them if is_package_install_successful and 'actual_version' in self.structured_output: self._relink_configurations_with_conf_select( stack_id, self.structured_output['actual_version'])
"/configurations/hbase-env/hbase_regionserver_xmn_ratio", float) regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) hbase_regionserver_shutdown_timeout = expect( '/configurations/hbase-env/hbase_regionserver_shutdown_timeout', int, 30) phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): phoenix_package = format("phoenix_{underscored_version}_*") elif OSCheck.is_ubuntu_family(): phoenix_package = format("phoenix-{dashed_version}-.*") pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] local_dir = config['configurations']['hbase-site']['hbase.local.dir'] ioengine_param = default( '/configurations/hbase-site/hbase.bucketcache.ioengine', None) client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format( "{hbase_conf_dir}/hbase_regionserver_jaas.conf") queryserver_jaas_config_file = format(
security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] #hadoop params hdfs_log_dir_prefix = config['configurations']['hadoop-env'][ 'hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env'][ 'hadoop_pid_dir_prefix'] hadoop_root_logger = config['configurations']['hadoop-env'][ 'hadoop_root_logger'] if Script.is_hdp_stack_greater_or_equal( "2.0") and Script.is_hdp_stack_less_than( "2.1") and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env'][ 'namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env'][ 'namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option( "/configurations/hadoop-env/namenode_opt_permsize", "128m") namenode_opt_maxpermsize = format_jvm_option( "/configurations/hadoop-env/namenode_opt_maxpermsize", "256m")
#for create_hdfs_directory hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_principal_name = default('/configurations/hadoop-env/hdfs_principal_name', 'missing_principal').replace("_HOST", hostname) # Tez-related properties tez_user = config['configurations']['tez-env']['tez_user'] # Tez jars tez_local_api_jars = '/usr/lib/tez/tez*.jar' tez_local_lib_jars = '/usr/lib/tez/lib/*.jar' # Tez libraries tez_lib_uris = default("/configurations/tez-site/tez.lib.uris", None) if OSCheck.is_ubuntu_family(): mysql_configname = '/etc/mysql/my.cnf' else: mysql_configname = '/etc/my.cnf' mysql_user = '******' # Hive security hive_authorization_enabled = config['configurations']['hive-site'][ 'hive.security.authorization.enabled'] mysql_jdbc_driver_jar = "/usr/share/java/mysql-connector-java.jar" hive_site_config = dict(config['configurations']['hive-site']) ########################################################
if has_namenode and 'rca_enabled' in config['configurations']['hadoop-env']: rca_enabled = config['configurations']['hadoop-env']['rca_enabled'] else: rca_enabled = False rca_disabled_prefix = "###" if rca_enabled == True: rca_prefix = "" else: rca_prefix = rca_disabled_prefix #hadoop-env.sh java_home = config['hostLevelParams']['java_home'] if hdp_stack_version != "" and compare_versions( hdp_stack_version, '2.0') >= 0 and compare_versions( hdp_stack_version, '2.1') < 0 and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env'][ 'namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env'][ 'namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option( "/configurations/hadoop-env/namenode_opt_permsize", "128m") namenode_opt_maxpermsize = format_jvm_option( "/configurations/hadoop-env/namenode_opt_maxpermsize", "256m")
import stat import string import sys import tempfile from ambari_commons.exceptions import FatalException from ambari_commons.os_check import OSCheck, OSConst from ambari_commons.os_family_impl import OsFamilyImpl from ambari_commons.os_utils import run_os_command, search_file, set_file_permissions from ambari_commons.logging_utils import get_debug_mode, print_info_msg, print_warning_msg, print_error_msg, \ set_debug_mode from ambari_server.properties import Properties from ambari_server.userInput import get_validated_string_input from ambari_server.utils import compare_versions, locate_file OS_VERSION = OSCheck().get_os_major_version() OS_TYPE = OSCheck.get_os_type() OS_FAMILY = OSCheck.get_os_family() PID_NAME = "ambari-server.pid" # Non-root user setup commands NR_USER_PROPERTY = "ambari-server.user" BLIND_PASSWORD = "******" # Common messages PRESS_ENTER_MSG = "Press <enter> to continue." OS_FAMILY_PROPERTY = "server.os_family" OS_TYPE_PROPERTY = "server.os_type"
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool( config['ambariLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint( config['ambariLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False packages_installed_before = [] stack_selector_package = stack_tools.get_stack_tool_package( stack_tools.STACK_SELECTOR_NAME) try: # install the stack-selector; we need to supply the action as "upgrade" here since the normal # install command will skip if the package is already installed in the system. # This is required for non-versioned components, like stack-select, since each version of # the stack comes with one. Also, scope the install by repository since we need to pick a # specific repo that the stack-select tools are coming out of in case there are multiple # patches installed repositories = config['repositoryFile']['repositories'] command_repos = CommandRepository(config['repositoryFile']) repository_ids = [ repository['repoId'] for repository in repositories ] repos_to_use = {} for repo_id in repository_ids: if repo_id in self.repo_files: repos_to_use[repo_id] = self.repo_files[repo_id] self.repo_mgr.upgrade_package( stack_selector_package, RepoCallContext(use_repos=repos_to_use, retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count)) packages_installed_before = self.repo_mgr.installed_packages() packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: available_packages_in_repos = self.repo_mgr.get_available_packages_in_repos( command_repos) except Exception: available_packages_in_repos = [] for package in filtered_package_list: name = self.get_package_from_available( package['name'], available_packages_in_repos) # This enables upgrading non-versioned packages, despite the fact they exist. # Needed by 'mahout' which is non-version but have to be updated self.repo_mgr.upgrade_package( name, RepoCallContext(retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count)) except Exception as err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = self.repo_mgr.installed_packages() packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): self.repo_mgr.remove_package(package, RepoCallContext()) if not self.repo_mgr.verify_dependencies(): ret_code = 1 Logger.logger.error("Failure while verifying dependencies") Logger.logger.error( "*******************************************************************************" ) Logger.logger.error( "Manually verify and fix package dependencies and then re-run install_packages" ) Logger.logger.error( "*******************************************************************************" ) # Compute the actual version in order to save it in structured out try: if ret_code == 0: self.compute_actual_version() else: self.check_partial_install() except Fail as err: ret_code = 1 Logger.logger.exception( "Failure while computing actual version. Error: {0}".format( str(err))) return ret_code
from resource_management.core.resources import Execute from resource_management.core.shell import call from resource_management.libraries.functions import format from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions import get_klist_path from ambari_commons.os_check import OSConst, OSCheck from ambari_commons.os_family_impl import OsFamilyFuncImpl, OsFamilyImpl from urlparse import urlparse import os import re RESULT_CODE_OK = 'OK' RESULT_CODE_CRITICAL = 'CRITICAL' RESULT_CODE_UNKNOWN = 'UNKNOWN' if OSCheck.is_windows_family(): OOZIE_ENV_HTTPS_RE = r"set\s+OOZIE_HTTPS_PORT=(\d+)" else: OOZIE_ENV_HTTPS_RE = r"export\s+OOZIE_HTTPS_PORT=(\d+)" # The configured Kerberos executable search paths, if any KERBEROS_EXECUTABLE_SEARCH_PATHS_KEY = '{{kerberos-env/executable_search_paths}}' OOZIE_URL_KEY = '{{oozie-site/oozie.base.url}}' SECURITY_ENABLED = '{{cluster-env/security_enabled}}' OOZIE_USER = '******' OOZIE_CONF_DIR = '/usr/hdp/current/oozie-server/conf' OOZIE_CONF_DIR_LEGACY = '/etc/oozie/conf' OOZIE_HTTPS_PORT = '{{oozie-site/oozie.https.port}}' OOZIE_ENV_CONTENT = '{{oozie-env/content}}'
lzo_packages = get_lzo_packages(stack_version_unformatted) exclude_packages = [] if not lzo_enabled: exclude_packages += lzo_packages name_node_params = default("/commandParams/namenode", None) #hadoop params hadoop_env_sh_template = config['configurations']['hadoop-env']['content'] #hadoop-env.sh java_home = config['hostLevelParams']['java_home'] java_version = int(config['hostLevelParams']['java_version']) if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.0') >= 0 and compare_versions(hdp_stack_version, '2.1') < 0 and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") jtnode_opt_newsize = "200m" jtnode_opt_maxnewsize = "200m" jtnode_heapsize = "1024m"
else: # params.dfs_http_policy == "HTTP_ONLY" or not defined: secure_dn_ports_are_in_use = utils.is_secure_port(dfs_dn_port) or utils.is_secure_port(dfs_dn_http_port) if secure_dn_ports_are_in_use: hadoop_secure_dn_user = hdfs_user else: hadoop_secure_dn_user = '******' ambari_libs_dir = "/var/lib/ambari-agent/lib" limits_conf_dir = "/etc/security/limits.d" hdfs_user_nofile_limit = default("/configurations/hadoop-env/hdfs_user_nofile_limit", "128000") hdfs_user_nproc_limit = default("/configurations/hadoop-env/hdfs_user_nproc_limit", "65536") create_lib_snappy_symlinks = not Script.is_hdp_stack_greater_or_equal("2.2") if Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.1") and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" execute_path = os.environ['PATH'] + os.pathsep + hadoop_bin_dir ulimit_cmd = "ulimit -c unlimited ; " snappy_so = "libsnappy.so" so_target_dir_x86 = format("{hadoop_lib_home}/native/Linux-i386-32") so_target_dir_x64 = format("{hadoop_lib_home}/native/Linux-amd64-64") so_target_x86 = format("{so_target_dir_x86}/{snappy_so}") so_target_x64 = format("{so_target_dir_x64}/{snappy_so}") so_src_dir_x86 = format("{hadoop_home}/lib") so_src_dir_x64 = format("{hadoop_home}/lib64")
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool( config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint( config['hostLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False stack_selector_package = stack_tools.get_stack_tool_package( stack_tools.STACK_SELECTOR_NAME) try: Package(stack_selector_package, action="upgrade", retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name']) Package( name, action= "upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated retry_on_repo_unavailability= agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count) except Exception, err: ret_code = 1 Logger.logger.exception( "Package Manager failed to install packages. Error: {0}". format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace( '.', '-') else: package_version_string = self.repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
log_dir = config['configurations']['hbase-env']['hbase_log_dir'] java_io_tmpdir = config['configurations']['hbase-env']['hbase_java_io_tmpdir'] master_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_master_heapsize']) regionserver_heapsize = ensure_unit_for_memory(config['configurations']['hbase-env']['hbase_regionserver_heapsize']) regionserver_xmn_max = config['configurations']['hbase-env']['hbase_regionserver_xmn_max'] regionserver_xmn_percent = expect("/configurations/hbase-env/hbase_regionserver_xmn_ratio", float) regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) hbase_regionserver_shutdown_timeout = expect('/configurations/hbase-env/hbase_regionserver_shutdown_timeout', int, 30) phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): phoenix_package = format("phoenix-*") pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] local_dir = config['configurations']['hbase-site']['hbase.local.dir'] ioengine_param = default('/configurations/hbase-site/hbase.bucketcache.ioengine', None) client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf") ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present has_ganglia_server = not len(ganglia_server_hosts) == 0 if has_ganglia_server:
def osdisks(): if OSCheck.get_os_family() == OSConst.WINSRV_FAMILY: return Hardware._osdisks_win() else: return Hardware._osdisks_linux()
exception) structured_output[CHECK_INSTALLED_PACKAGES] = { "exit_code": 1, "message": str(exception) } structured_output[CHECK_EXISTING_REPOS] = { "exit_code": 1, "message": str(exception) } # Here we are checking transparent huge page if CHECK_TRANSPARENT_HUGE_PAGE is in check_execute_list if CHECK_TRANSPARENT_HUGE_PAGE in check_execute_list: try: thp_regex = "\[(.+)\]" file_name = None if OSCheck.is_ubuntu_family(): file_name = THP_FILE_UBUNTU elif OSCheck.is_redhat_family(): file_name = THP_FILE_REDHAT if file_name and os.path.isfile(file_name): with open(file_name) as f: file_content = f.read() structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = { "exit_code": 0, "message": str( re.search(thp_regex, file_content).groups()[0]) } else: structured_output[CHECK_TRANSPARENT_HUGE_PAGE] = {
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ from resource_management import * from utils import get_property_value, get_unstructured_data from ambari_commons.os_check import OSCheck krb5_conf_dir = '/etc' krb5_conf_file = 'krb5.conf' krb5_conf_path = krb5_conf_dir + '/' + krb5_conf_file if OSCheck.is_suse_family(): kdc_conf_dir = '/var/lib/kerberos/krb5kdc' elif OSCheck.is_ubuntu_family(): kdc_conf_dir = '/etc/krb5kdc' else: kdc_conf_dir = '/var/kerberos/krb5kdc' kdc_conf_file = 'kdc.conf' kdc_conf_path = kdc_conf_dir + '/' + kdc_conf_file kadm5_acl_dir = kdc_conf_dir # Typically kadm5.acl and kdc.conf exist in the same directory kadm5_acl_file = 'kadm5.acl' kadm5_acl_path = kadm5_acl_dir + '/' + kadm5_acl_file config = Script.get_config() tmp_dir = Script.get_tmp_dir() host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False)
hadoop_conf_empty_dir = None versioned_hdp_root = '/usr/hdp/current' #security params security_enabled = config['configurations']['cluster-env']['security_enabled'] #java params java_home = config['hostLevelParams']['java_home'] #hadoop params hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] if Script.is_hdp_stack_greater_or_equal("2.0") and Script.is_hdp_stack_less_than("2.1") and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") jtnode_opt_newsize = "200m" jtnode_opt_maxnewsize = "200m" jtnode_heapsize = "1024m"
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) # Select dict that contains parameters try: self.repository_version = config['roleParams'][ 'repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: # Last try self.repository_version = config['commandParams'][ 'repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] # current stack information self.current_hdp_stack_version = None if 'stack_version' in config['hostLevelParams']: current_stack_version_unformatted = str( config['hostLevelParams']['stack_version']) self.current_hdp_stack_version = format_hdp_stack_version( current_stack_version_unformatted) stack_name = None self.stack_root_folder = None if stack_id and "-" in stack_id: stack_split = stack_id.split("-") if len(stack_split) == 2: stack_name = stack_split[0].upper() if stack_name in self.STACK_TO_ROOT_FOLDER: self.stack_root_folder = self.STACK_TO_ROOT_FOLDER[ stack_name] if self.stack_root_folder is None: raise Fail( "Cannot determine the stack's root directory by parsing the stack_id property, {0}" .format(str(stack_id))) if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories installed_repositories = [] self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format( self.repository_version)) try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository( url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos() except Exception, err: Logger.logger.exception( "Cannot distribute repositories. Error: {0}".format(str(err))) num_errors += 1
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() repo_rhel_suse = config['configurations']['cluster-env']['repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env']['repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family() or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version_id = None base_urls = [] # Select dict that contains parameters try: if 'base_urls' in config['roleParams']: base_urls = json.loads(config['roleParams']['base_urls']) self.repository_version = config['roleParams']['repository_version'] package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] if 'repository_version_id' in config['roleParams']: self.repository_version_id = config['roleParams']['repository_version_id'] except KeyError: pass # current stack information self.current_stack_version_formatted = None if 'stack_version' in config['hostLevelParams']: current_stack_version_unformatted = str(config['hostLevelParams']['stack_version']) self.current_stack_version_formatted = format_stack_version(current_stack_version_unformatted) self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() # Install/update repositories self.current_repositories = [] self.current_repo_files = set() # Enable base system repositories # We don't need that for RHEL family, because we leave all repos enabled # except disabled HDP* ones if OSCheck.is_suse_family(): self.current_repositories.append('base') elif OSCheck.is_ubuntu_family(): self.current_repo_files.add('base') Logger.info("Will install packages for repository version {0}".format(self.repository_version)) if 0 == len(base_urls): Logger.warning("Repository list is empty. Ambari may not be managing the repositories for {0}.".format(self.repository_version)) try: if 'repositoryFile' in config: create_repo_files(template, CommandRepository(config['repositoryFile'])) else: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository(url_info, append_to_file, template) self.current_repositories.append(repo_name) self.current_repo_files.add(repo_file) append_to_file = True except Exception, err: Logger.logger.exception("Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1
def install_packages(self, package_list): """ Actually install the packages using the package manager. :param package_list: List of package names to install :return: Returns 0 if no errors were found, and 1 otherwise. """ ret_code = 0 config = self.get_config() agent_stack_retry_on_unavailability = cbool(config['hostLevelParams']['agent_stack_retry_on_unavailability']) agent_stack_retry_count = cint(config['hostLevelParams']['agent_stack_retry_count']) # Install packages packages_were_checked = False stack_selector_package = stack_tools.get_stack_tool_package(stack_tools.STACK_SELECTOR_NAME) try: Package(stack_selector_package, action="upgrade", retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count ) packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [package[0] for package in packages_installed_before] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) try: available_packages_in_repos = packages_analyzer.get_available_packages_in_repos(config['repositoryFile']['repositories']) except Exception: available_packages_in_repos = [] for package in filtered_package_list: name = self.get_package_from_available(package['name'], available_packages_in_repos) Package(name, action="upgrade", # this enables upgrading non-versioned packages, despite the fact they exist. Needed by 'mahout' which is non-version but have to be updated retry_on_repo_unavailability=agent_stack_retry_on_unavailability, retry_count=agent_stack_retry_count ) except Exception as err: ret_code = 1 Logger.logger.exception("Package Manager failed to install packages. Error: {0}".format(str(err))) # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [package[0] for package in packages_installed_after] packages_installed_before = set(packages_installed_before) new_packages_installed = [package for package in packages_installed_after if package not in packages_installed_before] if OSCheck.is_ubuntu_family(): package_version_string = self.repository_version.replace('.', '-') else: package_version_string = self.repository_version.replace('-', '_') package_version_string = package_version_string.replace('.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove") if not verifyDependencies(): ret_code = 1 Logger.logger.error("Failure while verifying dependencies") Logger.logger.error("*******************************************************************************") Logger.logger.error("Manually verify and fix package dependencies and then re-run install_packages") Logger.logger.error("*******************************************************************************") # Compute the actual version in order to save it in structured out try: if ret_code == 0: self.compute_actual_version() else: self.check_partial_install() except Fail as err: ret_code = 1 Logger.logger.exception("Failure while computing actual version. Error: {0}".format(str(err))) return ret_code
def actionexecute(self, env): num_errors = 0 # Parse parameters config = Script.get_config() try: command_repository = CommandRepository(config['repositoryFile']) except KeyError: raise Fail( "The command repository indicated by 'repositoryFile' was not found" ) repo_rhel_suse = config['configurations']['cluster-env'][ 'repo_suse_rhel_template'] repo_ubuntu = config['configurations']['cluster-env'][ 'repo_ubuntu_template'] template = repo_rhel_suse if OSCheck.is_redhat_family( ) or OSCheck.is_suse_family() else repo_ubuntu # Handle a SIGTERM and SIGINT gracefully signal.signal(signal.SIGTERM, self.abort_handler) signal.signal(signal.SIGINT, self.abort_handler) self.repository_version = command_repository.version_string # Select dict that contains parameters try: package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: pass self.stack_name = Script.get_stack_name() if self.stack_name is None: raise Fail("Cannot determine the stack name") self.stack_root_folder = Script.get_stack_root() if self.stack_root_folder is None: raise Fail("Cannot determine the stack's root directory") if self.repository_version is None: raise Fail("Cannot determine the repository version to install") self.repository_version = self.repository_version.strip() try: if not command_repository.repositories: Logger.warning( "Repository list is empty. Ambari may not be managing the repositories for {0}." .format(self.repository_version)) else: Logger.info( "Will install packages for repository version {0}".format( self.repository_version)) new_repo_files = create_repo_files(template, command_repository) self.repo_files.update(new_repo_files) except Exception, err: Logger.logger.exception( "Cannot install repository files. Error: {0}".format(str(err))) num_errors += 1
Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ from ambari_commons.os_check import OSCheck from resource_management.libraries.functions import format from resource_management.libraries.functions import conf_select from resource_management.libraries.functions.version import format_hdp_stack_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script if OSCheck.is_windows_family(): from params_windows import * else: from params_linux import * host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) # server configurations config = Script.get_config() stack_name = default("/hostLevelParams/stack_name", None) # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
class InstallPackages(Script): """ This script is a part of Rolling Upgrade workflow and is described at appropriate design doc. It installs repositories to the node and then installs packages. For now, repositories are installed into individual files. """ UBUNTU_REPO_COMPONENTS_POSTFIX = ["main"] REPO_FILE_NAME_PREFIX = 'HDP-' def actionexecute(self, env): delayed_fail = False package_install_result = False # Parse parameters config = Script.get_config() # Select dict that contains parameters try: repository_version = config['roleParams']['repository_version'] base_urls = json.loads(config['roleParams']['base_urls']) package_list = json.loads(config['roleParams']['package_list']) stack_id = config['roleParams']['stack_id'] except KeyError: # Last try repository_version = config['commandParams']['repository_version'] base_urls = json.loads(config['commandParams']['base_urls']) package_list = json.loads(config['commandParams']['package_list']) stack_id = config['commandParams']['stack_id'] # Install/update repositories installed_repositories = [] current_repositories = [ 'base' ] # Some our packages are installed from the base repo current_repo_files = set(['base']) old_versions = self.hdp_versions() try: append_to_file = False for url_info in base_urls: repo_name, repo_file = self.install_repository( url_info, repository_version, append_to_file) current_repositories.append(repo_name) current_repo_files.add(repo_file) append_to_file = True installed_repositories = list_ambari_managed_repos() except Exception, err: print "Can not distribute repositories." print traceback.format_exc() delayed_fail = True # Install packages if not delayed_fail: packages_were_checked = False try: packages_installed_before = [] allInstalledPackages(packages_installed_before) packages_installed_before = [ package[0] for package in packages_installed_before ] packages_were_checked = True filtered_package_list = self.filter_package_list(package_list) for package in filtered_package_list: name = self.format_package_name(package['name'], repository_version) Package( name, use_repos=list(current_repo_files) if OSCheck.is_ubuntu_family() else current_repositories) package_install_result = True except Exception, err: print "Can not install packages." print traceback.format_exc() delayed_fail = True # Remove already installed packages in case of fail if packages_were_checked and packages_installed_before: packages_installed_after = [] allInstalledPackages(packages_installed_after) packages_installed_after = [ package[0] for package in packages_installed_after ] packages_installed_before = set(packages_installed_before) new_packages_installed = [ package for package in packages_installed_after if package not in packages_installed_before ] if OSCheck.is_ubuntu_family(): package_version_string = repository_version.replace( '.', '-') else: package_version_string = repository_version.replace( '-', '_') package_version_string = package_version_string.replace( '.', '_') for package in new_packages_installed: if package_version_string and (package_version_string in package): Package(package, action="remove")
regionserver_xmn_percent = config['configurations']['hbase-env']['hbase_regionserver_xmn_ratio'] regionserver_xmn_size = calc_xmn_from_xms(regionserver_heapsize, regionserver_xmn_percent, regionserver_xmn_max) phoenix_hosts = default('/clusterHostInfo/phoenix_query_server_hosts', []) phoenix_enabled = default('/configurations/hbase-env/phoenix_sql_enabled', False) has_phoenix = len(phoenix_hosts) > 0 if not has_phoenix and not phoenix_enabled: exclude_packages = ['phoenix*'] else: exclude_packages = [] underscored_version = stack_version_unformatted.replace('.', '_') dashed_version = stack_version_unformatted.replace('.', '-') if OSCheck.is_redhat_family() or OSCheck.is_suse_family(): phoenix_package = format("phoenix_{underscored_version}_*") elif OSCheck.is_ubuntu_family(): phoenix_package = format("phoenix-{dashed_version}-.*") pid_dir = status_params.pid_dir tmp_dir = config['configurations']['hbase-site']['hbase.tmp.dir'] local_dir = config['configurations']['hbase-site']['hbase.local.dir'] client_jaas_config_file = format("{hbase_conf_dir}/hbase_client_jaas.conf") master_jaas_config_file = format("{hbase_conf_dir}/hbase_master_jaas.conf") regionserver_jaas_config_file = format("{hbase_conf_dir}/hbase_regionserver_jaas.conf") queryserver_jaas_config_file = format("{hbase_conf_dir}/hbase_queryserver_jaas.conf") ganglia_server_hosts = default('/clusterHostInfo/ganglia_server_host', []) # is not passed when ganglia is not present ganglia_server_host = '' if len(ganglia_server_hosts) == 0 else ganglia_server_hosts[0]
from resource_management import * from ambari_commons.os_check import OSCheck config = Script.get_config() hive_pid_dir = config['configurations']['hive-env']['hive_pid_dir'] hive_pid = 'hive-server.pid' hive_metastore_pid = 'hive.pid' hcat_pid_dir = config['configurations']['hive-env'][ 'hcat_pid_dir'] #hcat_pid_dir webhcat_pid_file = format('{hcat_pid_dir}/webhcat.pid') process_name = 'mysqld' if OSCheck.is_suse_family() or OSCheck.is_ubuntu_family(): daemon_name = 'mysql' else: daemon_name = 'mysqld' # Security related/required params hostname = config['hostname'] security_enabled = config['configurations']['cluster-env']['security_enabled'] hadoop_conf_dir = "/etc/hadoop/conf" kinit_path_local = functions.get_kinit_path() tmp_dir = Script.get_tmp_dir() hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hive_user = config['configurations']['hive-env']['hive_user'] hive_conf_dir = "/etc/hive/conf" webhcat_user = config['configurations']['hive-env']['webhcat_user'] webhcat_conf_dir = '/etc/hive-webhcat/conf'
# for create_hdfs_directory security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] smokeuser_principal = config['configurations']['cluster-env'][ 'smokeuser_principal_name'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) hdfs_site = config['configurations']['hdfs-site'] default_fs = config['configurations']['core-site']['fs.defaultFS'] dfs_type = default("/commandParams/dfs_type", "") # MYSQL if OSCheck.is_ubuntu_family(): mysql_configname = '/etc/mysql/my.cnf' else: mysql_configname = '/etc/my.cnf' daemon_name = status_params.daemon_name # There will always be exactly one mysql_host mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][ 0] mysql_port = config['configurations']['metron-env'][ 'metron_enrichment_db_port'] mysql_adduser_path = tmp_dir + "/addMysqlUser.sh" mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh" mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
def execute(parameters=None, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: parameters (dictionary): a mapping of parameter key to value host_name (string): the name of this host where the alert is running """ if parameters is None: return (RESULT_CODE_UNKNOWN, ['There were no parameters supplied to the script.']) if not OOZIE_URL_KEY in parameters: return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all # interfaces, which doesn't work on Windows localhost_address = 'localhost' if OSCheck.get_os_family( ) == OSConst.WINSRV_FAMILY else '0.0.0.0' oozie_url = parameters[OOZIE_URL_KEY] oozie_url = oozie_url.replace( urlparse(oozie_url).hostname, localhost_address) security_enabled = False if SECURITY_ENABLED in parameters: security_enabled = str(parameters[SECURITY_ENABLED]).upper() == 'TRUE' command = format( "source /etc/oozie/conf/oozie-env.sh ; oozie admin -oozie {oozie_url} -status" ) try: # kinit if security is enabled so that oozie-env.sh can make the web request kerberos_env = None if security_enabled: if OOZIE_KEYTAB in parameters and OOZIE_PRINCIPAL in parameters: oozie_keytab = parameters[OOZIE_KEYTAB] oozie_principal = parameters[OOZIE_PRINCIPAL] # substitute _HOST in kerberos principal with actual fqdn oozie_principal = oozie_principal.replace('_HOST', host_name) else: return (RESULT_CODE_UNKNOWN, [ 'The Oozie keytab and principal are required parameters when security is enabled.' ]) # Create the kerberos credentials cache (ccache) file and set it in the environment to use # when executing curl env = Environment.get_instance() ccache_file = "{0}{1}oozie_alert_cc_{2}".format( env.tmp_dir, sep, getpid()) kerberos_env = {'KRB5CCNAME': ccache_file} klist_path_local = get_klist_path() klist_command = format("{klist_path_local} -s {ccache_file}") # Determine if we need to kinit by testing to see if the relevant cache exists and has # non-expired tickets. Tickets are marked to expire after 5 minutes to help reduce the number # it kinits we do but recover quickly when keytabs are regenerated return_code, _ = call(klist_command) if return_code != 0: kinit_path_local = get_kinit_path() kinit_command = format( "{kinit_path_local} -l 5m -kt {oozie_keytab} {oozie_principal}; " ) # kinit Execute(kinit_command, environment=kerberos_env) # execute the command Execute(command, environment=kerberos_env) return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) except Exception, ex: return (RESULT_CODE_CRITICAL, [str(ex)])
download_file(check_db_connection_url, check_db_connection_path) except Exception, e: message = "Error downloading DBConnectionVerification.jar from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output # download jdbc driver from ambari-server resources try: download_file(jdbc_url, jdbc_path) if db_name == DB_MSSQL and OSCheck.is_windows_family(): jdbc_auth_path = os.path.join(agent_cache_dir, JDBC_AUTH_SYMLINK_MSSQL) jdbc_auth_url = jdk_location + JDBC_AUTH_SYMLINK_MSSQL download_file(jdbc_auth_url, jdbc_auth_path) except Exception, e: message = format("Error: Ambari Server cannot download the database JDBC driver and is unable to test the " \ "database connection. You must run ambari-server setup --jdbc-db={db_name} " \ "--jdbc-driver=/path/to/your/{db_name}/driver.jar on the Ambari Server host to make the JDBC " \ "driver available for download and to enable testing the database connection.\n") + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir): print "DB connection check started." # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] if db_name == DB_MYSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MYSQL jdbc_driver = JDBC_DRIVER_MYSQL jdbc_name = JDBC_DRIVER_SYMLINK_MYSQL elif db_name == DB_ORACLE: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_ORACLE jdbc_driver = JDBC_DRIVER_ORACLE jdbc_name = JDBC_DRIVER_SYMLINK_ORACLE elif db_name == DB_POSTGRESQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_POSTGRESQL jdbc_driver = JDBC_DRIVER_POSTGRESQL jdbc_name = JDBC_DRIVER_SYMLINK_POSTGRESQL elif db_name == DB_MSSQL: jdbc_url = jdk_location + JDBC_DRIVER_SYMLINK_MSSQL jdbc_driver = JDBC_DRIVER_MSSQL jdbc_name = JDBC_DRIVER_SYMLINK_MSSQL db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath( config["hostLevelParams"]["agentCacheDir"]) check_db_connection_url = jdk_location + check_db_connection_jar_name jdbc_path = os.path.join(agent_cache_dir, jdbc_name) check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" class_path_delimiter = ":" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin", java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output environment = {"no_proxy": format("{ambari_server_hostname}")} # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = "{0}/{1}".format(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path=install_path) except Exception, e: message = "Error installing java.\n" + str(e) print message db_connection_check_structured_output = { "exit_code": 1, "message": message } return db_connection_check_structured_output
def execute_db_connection_check(self, config, tmp_dir): Logger.info("DB connection check started.") # initialize needed data ambari_server_hostname = config['commandParams']['ambari_server_host'] check_db_connection_jar_name = "DBConnectionVerification.jar" jdk_location = config['commandParams']['jdk_location'] java_home = config['commandParams']['java_home'] db_name = config['commandParams']['db_name'] no_jdbc_error_message = None if db_name == DB_MYSQL: jdbc_driver_mysql_name = default("/ambariLevelParams/custom_mysql_jdbc_name", None) if not jdbc_driver_mysql_name: no_jdbc_error_message = "The MySQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mysql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = CheckHost.build_url(jdk_location, jdbc_driver_mysql_name) jdbc_driver_class = JDBC_DRIVER_CLASS_MYSQL jdbc_name = jdbc_driver_mysql_name elif db_name == DB_ORACLE: jdbc_driver_oracle_name = default("/ambariLevelParams/custom_oracle_jdbc_name", None) if not jdbc_driver_oracle_name: no_jdbc_error_message = "The Oracle JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=oracle --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = CheckHost.build_url(jdk_location, jdbc_driver_oracle_name) jdbc_driver_class = JDBC_DRIVER_CLASS_ORACLE jdbc_name = jdbc_driver_oracle_name elif db_name == DB_POSTGRESQL: jdbc_driver_postgres_name = default("/ambariLevelParams/custom_postgres_jdbc_name", None) if not jdbc_driver_postgres_name: no_jdbc_error_message = "The Postgres JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=postgres --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = CheckHost.build_url(jdk_location, jdbc_driver_postgres_name) jdbc_driver_class = JDBC_DRIVER_CLASS_POSTGRESQL jdbc_name = jdbc_driver_postgres_name elif db_name == DB_MSSQL: jdbc_driver_mssql_name = default("/ambariLevelParams/custom_mssql_jdbc_name", None) if not jdbc_driver_mssql_name: no_jdbc_error_message = "The MSSQL JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=mssql --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = CheckHost.build_url(jdk_location, jdbc_driver_mssql_name) jdbc_driver_class = JDBC_DRIVER_CLASS_MSSQL jdbc_name = jdbc_driver_mssql_name elif db_name == DB_SQLA: jdbc_driver_sqla_name = default("/ambariLevelParams/custom_sqlanywhere_jdbc_name", None) if not jdbc_driver_sqla_name: no_jdbc_error_message = "The SQLAnywhere JDBC driver has not been set. Please ensure that you have executed 'ambari-server setup --jdbc-db=sqlanywhere --jdbc-driver=/path/to/jdbc_driver'." else: jdbc_url = CheckHost.build_url(jdk_location, jdbc_driver_sqla_name) jdbc_driver_class = JDBC_DRIVER_CLASS_SQLA jdbc_name = jdbc_driver_sqla_name else: no_jdbc_error_message = format("'{db_name}' database type not supported.") if no_jdbc_error_message: Logger.warning(no_jdbc_error_message) db_connection_check_structured_output = {"exit_code" : 1, "message": no_jdbc_error_message} return db_connection_check_structured_output db_connection_url = config['commandParams']['db_connection_url'] user_name = config['commandParams']['user_name'] user_passwd = config['commandParams']['user_passwd'] agent_cache_dir = os.path.abspath(config["agentLevelParams"]["agentCacheDir"]) check_db_connection_url = CheckHost.build_url(jdk_location, check_db_connection_jar_name) jdbc_path = os.path.join(agent_cache_dir, jdbc_name) class_path_delimiter = ":" if db_name == DB_SQLA: jdbc_jar_path = agent_cache_dir + JDBC_DRIVER_SQLA_JAR_PATH_IN_ARCHIVE java_library_path = agent_cache_dir + JARS_PATH_IN_ARCHIVE_SQLA + class_path_delimiter + agent_cache_dir + \ LIBS_PATH_IN_ARCHIVE_SQLA else: jdbc_jar_path = jdbc_path java_library_path = agent_cache_dir check_db_connection_path = os.path.join(agent_cache_dir, check_db_connection_jar_name) java_bin = "java" if OSCheck.is_windows_family(): java_bin = "java.exe" class_path_delimiter = ";" java_exec = os.path.join(java_home, "bin",java_bin) if ('jdk_name' not in config['commandParams'] or config['commandParams']['jdk_name'] == None \ or config['commandParams']['jdk_name'] == '') and not os.path.isfile(java_exec): message = "Custom java is not available on host. Please install it. Java home should be the same as on server. " \ "\n" Logger.warning(message) db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output environment = { "no_proxy": format("{ambari_server_hostname}") } # download and install java if it doesn't exists if not os.path.isfile(java_exec): jdk_name = config['commandParams']['jdk_name'] jdk_url = CheckHost.build_url(jdk_location, jdk_name) jdk_download_target = os.path.join(agent_cache_dir, jdk_name) java_dir = os.path.dirname(java_home) try: download_file(jdk_url, jdk_download_target) except Exception, e: message = "Error downloading JDK from Ambari Server resources. Check network access to " \ "Ambari Server.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output if jdk_name.endswith(".exe"): install_cmd = "{0} /s INSTALLDIR={1} STATIC=1 WEB_JAVA=0 /L \\var\\log\\ambari-agent".format( os_utils.quote_path(jdk_download_target), os_utils.quote_path(java_home), ) install_path = [java_dir] try: Execute(install_cmd, path = install_path) except Exception, e: message = "Error installing java.\n" + str(e) Logger.exception(message) db_connection_check_structured_output = {"exit_code" : 1, "message": message} return db_connection_check_structured_output
def execute(self): """ Sets up logging; Parses command parameters and executes method relevant to command type """ logger, chout, cherr = Logger.initialize_logger() # parse arguments if len(sys.argv) < 7: logger.error("Script expects at least 6 arguments") print USAGE.format(os.path.basename( sys.argv[0])) # print to stdout sys.exit(1) command_name = str.lower(sys.argv[1]) self.command_data_file = sys.argv[2] self.basedir = sys.argv[3] self.stroutfile = sys.argv[4] self.load_structured_out() self.logging_level = sys.argv[5] Script.tmp_dir = sys.argv[6] logging_level_str = logging._levelNames[self.logging_level] chout.setLevel(logging_level_str) logger.setLevel(logging_level_str) # on windows we need to reload some of env variables manually because there is no default paths for configs(like # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated # in agent, so other Script executions will not be able to access to new env variables if OSCheck.is_windows_family(): reload_windows_env() try: with open(self.command_data_file) as f: pass Script.config = ConfigDictionary(json.load(f)) # load passwords here(used on windows to impersonate different users) Script.passwords = {} for k, v in _PASSWORD_MAP.iteritems(): if get_path_from_configuration( k, Script.config) and get_path_from_configuration( v, Script.config): Script.passwords[get_path_from_configuration( k, Script.config)] = get_path_from_configuration( v, Script.config) except IOError: logger.exception( "Can not read json file with command parameters: ") sys.exit(1) # Run class method depending on a command type try: method = self.choose_method_to_execute(command_name) with Environment(self.basedir, tmp_dir=Script.tmp_dir) as env: env.config.download_path = Script.tmp_dir method(env) if command_name == "install": self.set_version() except ClientComponentHasNoStatus or ComponentIsNotRunning: # Support of component status checks. # Non-zero exit code is interpreted as an INSTALLED status of a component sys.exit(1) except Fail: logger.exception( "Error while executing command '{0}':".format(command_name)) sys.exit(1) finally: if self.should_expose_component_version(command_name): self.save_component_version_to_structured_out()
secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_https_port) elif dfs_http_policy == "HTTP_AND_HTTPS": secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) or is_secure_port(dfs_dn_https_port) else: # params.dfs_http_policy == "HTTP_ONLY" or not defined: secure_dn_ports_are_in_use = is_secure_port(dfs_dn_port) or is_secure_port(dfs_dn_http_port) if secure_dn_ports_are_in_use: hadoop_secure_dn_user = hdfs_user else: hadoop_secure_dn_user = '******' #hadoop params hdfs_log_dir_prefix = config['configurations']['hadoop-env']['hdfs_log_dir_prefix'] hadoop_pid_dir_prefix = config['configurations']['hadoop-env']['hadoop_pid_dir_prefix'] hadoop_root_logger = config['configurations']['hadoop-env']['hadoop_root_logger'] if hdp_stack_version != "" and compare_versions(hdp_stack_version, '2.0') >= 0 and compare_versions(hdp_stack_version, '2.1') < 0 and not OSCheck.is_suse_family(): # deprecated rhel jsvc_path jsvc_path = "/usr/libexec/bigtop-utils" else: jsvc_path = "/usr/lib/bigtop-utils" hadoop_heapsize = config['configurations']['hadoop-env']['hadoop_heapsize'] namenode_heapsize = config['configurations']['hadoop-env']['namenode_heapsize'] namenode_opt_newsize = config['configurations']['hadoop-env']['namenode_opt_newsize'] namenode_opt_maxnewsize = config['configurations']['hadoop-env']['namenode_opt_maxnewsize'] namenode_opt_permsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_permsize","128m") namenode_opt_maxpermsize = format_jvm_option("/configurations/hadoop-env/namenode_opt_maxpermsize","256m") jtnode_opt_newsize = "200m" jtnode_opt_maxnewsize = "200m" jtnode_heapsize = "1024m"
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ if configurations is None: return (RESULT_CODE_UNKNOWN, ['There were no configurations supplied to the script.']) if not OOZIE_URL_KEY in configurations: return (RESULT_CODE_UNKNOWN, ['The Oozie URL is a required parameter.']) # use localhost on Windows, 0.0.0.0 on others; 0.0.0.0 means bind to all # interfaces, which doesn't work on Windows localhost_address = 'localhost' if OSCheck.get_os_family( ) == OSConst.WINSRV_FAMILY else '0.0.0.0' https_port = None # try to get https port form oozie-env content if OOZIE_ENV_CONTENT in configurations: for line in configurations[OOZIE_ENV_CONTENT].splitlines(): result = re.match(OOZIE_ENV_HTTPS_RE, line) if result is not None: https_port = result.group(1) # or from oozie-site.xml if https_port is None and OOZIE_HTTPS_PORT in configurations: https_port = configurations[OOZIE_HTTPS_PORT] oozie_url = configurations[OOZIE_URL_KEY] # construct proper url for https if https_port is not None: parsed_url = urlparse(oozie_url) oozie_url = oozie_url.replace(parsed_url.scheme, "https") if parsed_url.port is None: oozie_url.replace(parsed_url.hostname, ":".join([parsed_url.hostname, str(https_port)])) else: oozie_url = oozie_url.replace(str(parsed_url.port), str(https_port)) # https will not work with localhost address, we need put fqdn if https_port is None: oozie_url = oozie_url.replace( urlparse(oozie_url).hostname, localhost_address) try: command, env, oozie_user = get_check_command(oozie_url, host_name, configurations) # execute the command Execute( command, environment=env, user=oozie_user, ) return (RESULT_CODE_OK, ["Successful connection to {0}".format(oozie_url)]) except KerberosPropertiesNotFound, ex: return (RESULT_CODE_UNKNOWN, [str(ex)])