def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if Script.is_stack_greater_or_equal("4.2"): # phoenix uses hbase configs stack_select.select_packages(params.version)
def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if Script.is_stack_greater_or_equal('4.1.0.0'): conf_select.select(params.stack_name, "hadoop", params.version) stack_select.select("hadoop-hdfs-nfs3", params.version)
def check_stack_feature(stack_feature, stack_version): """ Given a stack_feature and a specific stack_version, it validates that the feature is supported by the stack_version. :param stack_feature: Feature name to check if it is supported by the stack. For example: "rolling_upgrade" :param stack_version: Version of the stack :return: Will return True if successful, otherwise, False. """ stack_features_config = default( "/configurations/cluster-env/stack_features", None) data = _DEFAULT_STACK_FEATURES if stack_features_config: data = json.loads(stack_features_config) for feature in data["stack_features"]: if feature["name"] == stack_feature: if "min_version" in feature: min_version = feature["min_version"] if Script.is_stack_less_than(min_version): return False if "max_version" in feature: max_version = feature["max_version"] if Script.is_stack_greater_or_equal(max_version): return False return True return False
def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if Script.is_stack_greater_or_equal("4.2"): # phoenix uses hbase configs conf_select.select(params.stack_name, "hbase", params.version) stack_select.select("phoenix-server", params.version)
def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Metastore Rolling Upgrade pre-restart") import params env.set_params(params) if Script.is_stack_greater_or_equal("4.1.0.0"): self.upgrade_schema(env) if params.version and compare_versions( format_stack_version(params.version), '4.0.0.0') >= 0: stack_select.select_packages(params.version)
def pre_upgrade_restart(self, env, upgrade_type=None): Logger.info("Executing Metastore Stack Upgrade pre-restart") import params env.set_params(params) is_stack_hdp_23 = Script.is_stack_greater_or_equal("4.2.0.0") is_upgrade = params.upgrade_direction == Direction.UPGRADE if is_stack_hdp_23 and is_upgrade: self.upgrade_schema(env) if params.version and compare_versions( format_stack_version(params.version), '4.0.0.0') >= 0: stack_select.select_packages(params.version)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of BigInsights-4.1 and higher """ if not Script.is_stack_greater_or_equal("4.1"): Logger.info("Can only link configs for BigInsights-4.1 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDP-2.3 and higher """ if not Script.is_stack_greater_or_equal("2.3"): Logger.info("Can only link configs for HDP-2.3 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info( "Could not load 'version' from {0}".format(struct_out_file)) return for k, v in conf_select.PACKAGE_DIRS.iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
def link_configs(struct_out_file): """ Links configs, only on a fresh install of HDF-0.3 and higher """ import params if not Script.is_stack_greater_or_equal("0.3"): Logger.info("Can only link configs for HDF-0.3 and higher.") return json_version = load_version(struct_out_file) if not json_version: Logger.info("Could not load 'version' from {0}".format(struct_out_file)) return # On parallel command execution this should be executed by a single process at a time. with FcntlBasedProcessLock(params.link_configs_lock_file, enabled = params.is_parallel_execution_enabled, skip_fcntl_failures = True): for k, v in conf_select.get_package_dirs().iteritems(): conf_select.convert_conf_directories_to_symlinks(k, json_version, v)
# server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() sudo = AMBARI_SUDO_BINARY stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", Direction.UPGRADE) version = default("/commandParams/version", None) storm_component_home_dir = status_params.storm_component_home_dir conf_dir = status_params.conf_dir stack_version_unformatted = str(config['hostLevelParams']['stack_version']) stack_version_formatted = format_stack_version(stack_version_unformatted) stack_is_hdp22_or_further = Script.is_stack_greater_or_equal("2.2") # default hadoop params rest_lib_dir = "/usr/lib/storm/contrib/storm-rest" storm_bin_dir = "/usr/bin" storm_lib_dir = "/usr/lib/storm/lib/" # hadoop parameters for 2.2+ if stack_is_hdp22_or_further: rest_lib_dir = format("{storm_component_home_dir}/contrib/storm-rest") storm_bin_dir = format("{storm_component_home_dir}/bin") storm_lib_dir = format("{storm_component_home_dir}/lib") log4j_dir = format("{storm_component_home_dir}/log4j2") storm_user = config['configurations']['storm-env']['storm_user'] log_dir = config['configurations']['storm-env']['storm_log_dir']
hadoop_home = stack_select.get_hadoop_dir("home", force_latest_on_upgrade=True) hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec", force_latest_on_upgrade=True) hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_stack_root = '/usr/hdp/current' hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") datanode_max_locked_memory = config['configurations']['hdfs-site'][ 'dfs.datanode.max.locked.memory'] is_datanode_max_locked_memory_set = not is_empty( config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) # HDP 2.2+ params if Script.is_stack_greater_or_equal("2.2"): mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*" # not supported in HDP 2.2+ hadoop_conf_empty_dir = None if not security_enabled: hadoop_secure_dn_user = '******' else: dfs_dn_port = get_port(dfs_dn_addr) dfs_dn_http_port = get_port(dfs_dn_http_addr) dfs_dn_https_port = get_port(dfs_dn_https_addr) # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports if dfs_http_policy == "HTTPS_ONLY": secure_dn_ports_are_in_use = is_secure_port( dfs_dn_port) or is_secure_port(dfs_dn_https_port)
def pre_upgrade_restart(self, env, upgrade_type=None): import params env.set_params(params) if Script.is_stack_greater_or_equal('4.1.0.0'): stack_select.select_packages(params.version)
# force the use of "current" in the hook hadoop_home = stack_select.get_hadoop_dir("home") hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec") hadoop_conf_empty_dir = "/etc/hadoop/conf.empty" hadoop_secure_dn_user = hdfs_user hadoop_dir = "/etc/hadoop" versioned_stack_root = '/usr/iop/current' hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir") datanode_max_locked_memory = config['configurations']['hdfs-site'][ 'dfs.datanode.max.locked.memory'] is_datanode_max_locked_memory_set = not is_empty( config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory']) # IOP 4.0+ params if Script.is_stack_greater_or_equal("4.0"): mapreduce_libs_path = "/usr/iop/current/hadoop-mapreduce-client/*" # not supported in IOP 4.0+ hadoop_conf_empty_dir = None if not security_enabled: hadoop_secure_dn_user = '******' else: dfs_dn_port = get_port(dfs_dn_addr) dfs_dn_http_port = get_port(dfs_dn_http_addr) dfs_dn_https_port = get_port(dfs_dn_https_addr) # We try to avoid inability to start datanode as a plain user due to usage of root-owned ports if dfs_http_policy == "HTTPS_ONLY": secure_dn_ports_are_in_use = is_secure_port( dfs_dn_port) or is_secure_port(dfs_dn_https_port)
def execute(configurations={}, parameters={}, host_name=None): """ Returns a tuple containing the result code and a pre-formatted result label Keyword arguments: configurations (dictionary): a mapping of configuration key to value parameters (dictionary): a mapping of script parameter key to value host_name (string): the name of this host where the alert is running """ if configurations is None: return (('UNKNOWN', ['There were no configurations supplied to the script.'])) ranger_link = None ranger_auth_link = None ranger_get_user = None admin_username = None admin_password = None ranger_admin_username = None ranger_admin_password = None security_enabled = False stack_is_hdp25_or_further = Script.is_stack_greater_or_equal("2.5") if RANGER_ADMIN_URL in configurations: ranger_link = configurations[RANGER_ADMIN_URL] if ranger_link.endswith('/'): ranger_link = ranger_link[:-1] ranger_auth_link = '{0}/{1}'.format( ranger_link, 'service/public/api/repository/count') ranger_get_user = '******'.format(ranger_link, 'service/xusers/users') if ADMIN_USERNAME in configurations: admin_username = configurations[ADMIN_USERNAME] if ADMIN_PASSWORD in configurations: admin_password = configurations[ADMIN_PASSWORD] if RANGER_ADMIN_USERNAME in configurations: ranger_admin_username = configurations[RANGER_ADMIN_USERNAME] if RANGER_ADMIN_PASSWORD in configurations: ranger_admin_password = configurations[RANGER_ADMIN_PASSWORD] if SECURITY_ENABLED in configurations: security_enabled = str( configurations[SECURITY_ENABLED]).upper() == 'TRUE' label = None result_code = 'OK' try: if security_enabled and stack_is_hdp25_or_further: result_code = 'UNKNOWN' label = 'This alert will get skipped for Ranger Admin on kerberos env' else: admin_http_code = check_ranger_login(ranger_auth_link, admin_username, admin_password) if admin_http_code == 200: get_user_code = get_ranger_user(ranger_get_user, admin_username, admin_password, ranger_admin_username) if get_user_code: user_http_code = check_ranger_login( ranger_auth_link, ranger_admin_username, ranger_admin_password) if user_http_code == 200: result_code = 'OK' label = 'Login Successful for users {0} and {1}'.format( admin_username, ranger_admin_username) elif user_http_code == 401: result_code = 'CRITICAL' label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format( ranger_admin_username) else: result_code = 'WARNING' label = 'Ranger Admin service is not reachable, please restart the service' else: result_code = 'OK' label = 'Login Successful for user: {0}. User:{1} user not yet synced with Ranger'.format( admin_username, ranger_admin_username) elif admin_http_code == 401: result_code = 'CRITICAL' label = 'User:{0} credentials on Ambari UI are not in sync with Ranger'.format( admin_username) else: result_code = 'WARNING' label = 'Ranger Admin service is not reachable, please restart the service' except Exception, e: label = str(e) result_code = 'UNKNOWN' logger.exception(label)