def all_jmx_namenode_addresses(hdfs_site, name_service): nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = is_https_enabled_in_hdfs( hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service, nn_unique_id) if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) if INADDR_ANY in value and rpc_key in hdfs_site: rpc_value = str(hdfs_site[rpc_key]) if INADDR_ANY not in rpc_value: rpc_host = rpc_value.split(":")[0] value = value.replace(INADDR_ANY, rpc_host) jmx_uri = JMX_URI_DEFAULT.format(protocol, value) yield nn_unique_id, value, jmx_uri
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = get_nameservice(hdfs_site) nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id) if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) if INADDR_ANY in value and rpc_key in hdfs_site: rpc_value = str(hdfs_site[rpc_key]) if INADDR_ANY not in rpc_value: rpc_host = rpc_value.split(":")[0] value = value.replace(INADDR_ANY, rpc_host) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user, last_retry=True): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = get_nameservice(hdfs_site) nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id) if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) if INADDR_ANY in value and rpc_key in hdfs_site: rpc_value = str(hdfs_site[rpc_key]) if INADDR_ANY not in rpc_value: rpc_host = rpc_value.split(":")[0] value = value.replace(INADDR_ANY, rpc_host) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled, last_retry) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def __init__(self, hdfs_site, nameservice, run_user, security_enabled, logoutput=None): self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) address_property = 'dfs.namenode.https-address' if self.is_https_enabled else 'dfs.namenode.http-address' address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, nameservice, address_property, security_enabled, run_user) protocol = "https" if self.is_https_enabled else "http" self.address = format("{protocol}://{address}") self.run_user = run_user self.security_enabled = security_enabled self.logoutput = logoutput
def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None): https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address', security_enabled, run_user) http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address', security_enabled, run_user) self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) address = https_nn_address if self.is_https_enabled else http_nn_address protocol = "https" if self.is_https_enabled else "http" self.address = format("{protocol}://{address}") self.run_user = run_user self.security_enabled = security_enabled self.logoutput = logoutput
def get_hdfs_cluster_id_from_jmx(hdfs_site, security_enabled, run_user): name_services = get_nameservices(hdfs_site) for name_service in name_services: for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses( hdfs_site, name_service): jmx_uri = jmx_uri.format(JMX_BEAN_NN_INFO) is_https_enabled = is_https_enabled_in_hdfs( hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) state = get_value_from_jmx(jmx_uri, 'ClusterId', security_enabled, run_user, is_https_enabled) if state: return state Logger.info("Cannot get clusterId from {0}".format(jmx_uri)) raise Fail( "Cannot get clsuterId from jmx, since none of the namenodes is running/accessible via jmx." )
def _get_namenode_states_noretries_single_ns(hdfs_site, name_service, security_enabled, run_user, last_retry=True): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses( hdfs_site, name_service): is_https_enabled = is_https_enabled_in_hdfs( hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) jmx_uri = jmx_uri.format(JMX_BEAN_FS) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled, last_retry) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format( name_service, nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, address)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, address)) else: unknown_namenodes.append((nn_unique_id, address)) return active_namenodes, standby_namenodes, unknown_namenodes
fs_default_name = config['configurations']['core-site']['fs.defaultFS'] hadoop_security_auth_to_local = config['configurations']['core-site'][ 'hadoop.security.auth_to_local'] if security_enabled: sn_principal_name = default( "/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]") sn_principal_name = sn_principal_name.replace('_HOST', hostname.lower()) # for curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' is_https_enabled = is_https_enabled_in_hdfs( config['configurations']['hdfs-site']['dfs.http.policy'], config['configurations']['hdfs-site']['dfs.https.enable']) # ranger hdfs plugin section start # ranger host ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", []) has_ranger_admin = not len(ranger_admin_hosts) == 0 # ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature xml_configurations_supported = check_stack_feature( StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks) # ambari-server hostname ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]