def all_jmx_namenode_addresses(hdfs_site, name_service):
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = is_https_enabled_in_hdfs(
hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service, nn_unique_id)
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
if INADDR_ANY in value and rpc_key in hdfs_site:
rpc_value = str(hdfs_site[rpc_key])
if INADDR_ANY not in rpc_value:
rpc_host = rpc_value.split(":")[0]
value = value.replace(INADDR_ANY, rpc_host)
jmx_uri = JMX_URI_DEFAULT.format(protocol, value)
yield nn_unique_id, value, jmx_uri
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = get_nameservice(hdfs_site)
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id)
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
if INADDR_ANY in value and rpc_key in hdfs_site:
rpc_value = str(hdfs_site[rpc_key])
if INADDR_ANY not in rpc_value:
rpc_host = rpc_value.split(":")[0]
value = value.replace(INADDR_ANY, rpc_host)
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id)
code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user, last_retry=True):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = get_nameservice(hdfs_site)
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id)
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
if INADDR_ANY in value and rpc_key in hdfs_site:
rpc_value = str(hdfs_site[rpc_key])
if INADDR_ANY not in rpc_value:
rpc_host = rpc_value.split(":")[0]
value = value.replace(INADDR_ANY, rpc_host)
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled, last_retry)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id)
code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def __init__(self, hdfs_site, nameservice, run_user, security_enabled, logoutput=None):
self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
address_property = 'dfs.namenode.https-address' if self.is_https_enabled else 'dfs.namenode.http-address'
address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, nameservice, address_property,
security_enabled, run_user)
protocol = "https" if self.is_https_enabled else "http"
self.address = format("{protocol}://{address}")
self.run_user = run_user
self.security_enabled = security_enabled
self.logoutput = logoutput
def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None):
https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address',
security_enabled, run_user)
http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address',
security_enabled, run_user)
self.is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
address = https_nn_address if self.is_https_enabled else http_nn_address
protocol = "https" if self.is_https_enabled else "http"
self.address = format("{protocol}://{address}")
self.run_user = run_user
self.security_enabled = security_enabled
self.logoutput = logoutput
def get_hdfs_cluster_id_from_jmx(hdfs_site, security_enabled, run_user):
name_services = get_nameservices(hdfs_site)
for name_service in name_services:
for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses(
hdfs_site, name_service):
jmx_uri = jmx_uri.format(JMX_BEAN_NN_INFO)
is_https_enabled = is_https_enabled_in_hdfs(
hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
state = get_value_from_jmx(jmx_uri, 'ClusterId', security_enabled,
run_user, is_https_enabled)
if state:
return state
Logger.info("Cannot get clusterId from {0}".format(jmx_uri))
raise Fail(
"Cannot get clsuterId from jmx, since none of the namenodes is running/accessible via jmx."
)
def _get_namenode_states_noretries_single_ns(hdfs_site,
name_service,
security_enabled,
run_user,
last_retry=True):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses(
hdfs_site, name_service):
is_https_enabled = is_https_enabled_in_hdfs(
hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
jmx_uri = jmx_uri.format(JMX_BEAN_FS)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled,
run_user, is_https_enabled, last_retry)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(
name_service, nn_unique_id)
code, out = shell.call(check_service_cmd,
logoutput=True,
user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, address))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, address))
else:
unknown_namenodes.append((nn_unique_id, address))
return active_namenodes, standby_namenodes, unknown_namenodes
fs_default_name = config['configurations']['core-site']['fs.defaultFS']
hadoop_security_auth_to_local = config['configurations']['core-site'][
'hadoop.security.auth_to_local']
if security_enabled:
sn_principal_name = default(
"/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal",
"nn/[email protected]")
sn_principal_name = sn_principal_name.replace('_HOST', hostname.lower())
# for curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
is_https_enabled = is_https_enabled_in_hdfs(
config['configurations']['hdfs-site']['dfs.http.policy'],
config['configurations']['hdfs-site']['dfs.https.enable'])
# ranger hdfs plugin section start
# ranger host
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
# ranger support xml_configuration flag, instead of depending on ranger xml_configurations_supported/ranger-env, using stack feature
xml_configurations_supported = check_stack_feature(
StackFeature.RANGER_XML_CONFIGURATION, version_for_stack_feature_checks)
# ambari-server hostname
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
