def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = get_nameservice(hdfs_site)
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id)
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
if INADDR_ANY in value and rpc_key in hdfs_site:
rpc_value = str(hdfs_site[rpc_key])
if INADDR_ANY not in rpc_value:
rpc_host = rpc_value.split(":")[0]
value = value.replace(INADDR_ANY, rpc_host)
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id)
code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = hdfs_site['dfs.nameservices']
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(
hdfs_site['dfs.https.enable']) else False
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState',
security_enabled, run_user,
is_https_enabled)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -getServiceState {0}".format(
nn_unique_id)
code, out = shell.call(check_service_cmd,
logoutput=True,
user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def get_hdfs_cluster_id_from_jmx(hdfs_site, security_enabled, run_user):
name_services = get_nameservices(hdfs_site)
for name_service in name_services:
for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses(
hdfs_site, name_service):
jmx_uri = jmx_uri.format(JMX_BEAN_NN_INFO)
is_https_enabled = is_https_enabled_in_hdfs(
hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
state = get_value_from_jmx(jmx_uri, 'ClusterId', security_enabled,
run_user, is_https_enabled)
if state:
return state
Logger.info("Cannot get clusterId from {0}".format(jmx_uri))
raise Fail(
"Cannot get clsuterId from jmx, since none of the namenodes is running/accessible via jmx."
)
def _get_namenode_states_noretries_single_ns(hdfs_site,
name_service,
security_enabled,
run_user,
last_retry=True):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses(
hdfs_site, name_service):
is_https_enabled = is_https_enabled_in_hdfs(
hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable'])
jmx_uri = jmx_uri.format(JMX_BEAN_FS)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled,
run_user, is_https_enabled, last_retry)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(
name_service, nn_unique_id)
code, out = shell.call(check_service_cmd,
logoutput=True,
user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, address))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, address))
else:
unknown_namenodes.append((nn_unique_id, address))
return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states(hdfs_site):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = hdfs_site['dfs.nameservices']
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = hdfs_site['dfs.https.enabled'] if not is_empty(
hdfs_site['dfs.https.enabled']) else False
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id)
else:
key = "https://" + NAMENODE_HTTPS_FRAGMENT.format(
name_service, nn_unique_id)
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
jmx_uri = JMX_URI_FRAGMENT.format(value)
state = get_value_from_jmx(jmx_uri, 'State')
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def __init__(self):
"""
Initializes all fields by querying the Namenode state.
Raises a ValueError if unable to construct the object.
"""
import params
self.name_service = default(
'/configurations/hdfs-site/dfs.internal.nameservices', None)
if self.name_service is None:
self.name_service = default(
'/configurations/hdfs-site/dfs.nameservices', None)
if not self.name_service:
raise ValueError(
"Could not retrieve property dfs.nameservices or dfs.internal.nameservices"
)
nn_unique_ids_key = "dfs.ha.namenodes." + str(self.name_service)
# List of the nn unique ids
self.nn_unique_ids = default(
"/configurations/hdfs-site/" + nn_unique_ids_key, None)
if not self.nn_unique_ids:
raise ValueError("Could not retrieve property " +
nn_unique_ids_key)
self.nn_unique_ids = self.nn_unique_ids.split(",")
self.nn_unique_ids = [x.strip() for x in self.nn_unique_ids]
policy = default("/configurations/hdfs-site/dfs.http.policy",
"HTTP_ONLY")
self.encrypted = policy.upper() == "HTTPS_ONLY"
jmx_uri_fragment = (
"https" if self.encrypted else "http"
) + "://{0}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem"
namenode_http_fragment = "dfs.namenode.http-address.{0}.{1}"
namenode_https_fragment = "dfs.namenode.https-address.{0}.{1}"
# Dictionary where the key is the Namenode State (e.g., ACTIVE), and the value is a set of hostnames
self.namenode_state_to_hostnames = {}
# Dictionary from nn unique id name to a tuple of (http address, https address)
self.nn_unique_id_to_addresses = {}
for nn_unique_id in self.nn_unique_ids:
http_key = namenode_http_fragment.format(self.name_service,
nn_unique_id)
https_key = namenode_https_fragment.format(self.name_service,
nn_unique_id)
http_value = default("/configurations/hdfs-site/" + http_key, None)
https_value = default("/configurations/hdfs-site/" + https_key,
None)
actual_value = https_value if self.encrypted else http_value
hostname = actual_value.split(":")[0].strip(
) if actual_value and ":" in actual_value else None
self.nn_unique_id_to_addresses[nn_unique_id] = (http_value,
https_value)
try:
if not hostname:
raise Exception(
"Could not retrieve hostname from address " +
actual_value)
jmx_uri = jmx_uri_fragment.format(actual_value)
state = get_value_from_jmx(jmx_uri, "tag.HAState",
params.security_enabled,
params.hdfs_user,
params.is_https_enabled)
# If JMX parsing failed
if not state:
run_user = default("/configurations/hadoop-env/hdfs_user",
"hdfs")
check_service_cmd = "hdfs haadmin -getServiceState {0}".format(
nn_unique_id)
code, out = shell.call(check_service_cmd,
logoutput=True,
user=run_user)
if code == 0 and out:
if NAMENODE_STATE.STANDBY in out:
state = NAMENODE_STATE.STANDBY
elif NAMENODE_STATE.ACTIVE in out:
state = NAMENODE_STATE.ACTIVE
if not state:
raise Exception(
"Could not retrieve Namenode state from URL " +
jmx_uri)
state = state.lower()
if state not in [
NAMENODE_STATE.ACTIVE, NAMENODE_STATE.STANDBY
]:
state = NAMENODE_STATE.UNKNOWN
if state in self.namenode_state_to_hostnames:
self.namenode_state_to_hostnames[state].add(hostname)
else:
hostnames = set([
hostname,
])
self.namenode_state_to_hostnames[state] = hostnames
except:
Logger.error("Could not get namenode state for " +
nn_unique_id)
def __init__(self):
"""
Initializes all fields by querying the Namenode state.
Raises a ValueError if unable to construct the object.
"""
import params
self.name_service = default("/configurations/hdfs-site/dfs.nameservices", None)
if not self.name_service:
raise ValueError("Could not retrieve property dfs.nameservices")
nn_unique_ids_key = "dfs.ha.namenodes." + str(self.name_service)
# List of the nn unique ids
self.nn_unique_ids = default("/configurations/hdfs-site/" + nn_unique_ids_key, None)
if not self.nn_unique_ids:
raise ValueError("Could not retrieve property " + nn_unique_ids_key)
self.nn_unique_ids = self.nn_unique_ids.split(",")
self.nn_unique_ids = [x.strip() for x in self.nn_unique_ids]
policy = default("/configurations/hdfs-site/dfs.http.policy", "HTTP_ONLY")
self.encrypted = policy.upper() == "HTTPS_ONLY"
jmx_uri_fragment = ("https" if self.encrypted else "http") + "://{0}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem"
namenode_http_fragment = "dfs.namenode.http-address.{0}.{1}"
namenode_https_fragment = "dfs.namenode.https-address.{0}.{1}"
# Dictionary where the key is the Namenode State (e.g., ACTIVE), and the value is a set of hostnames
self.namenode_state_to_hostnames = {}
# Dictionary from nn unique id name to a tuple of (http address, https address)
self.nn_unique_id_to_addresses = {}
for nn_unique_id in self.nn_unique_ids:
http_key = namenode_http_fragment.format(self.name_service, nn_unique_id)
https_key = namenode_https_fragment.format(self.name_service, nn_unique_id)
http_value = default("/configurations/hdfs-site/" + http_key, None)
https_value = default("/configurations/hdfs-site/" + https_key, None)
actual_value = https_value if self.encrypted else http_value
hostname = actual_value.split(":")[0].strip() if actual_value and ":" in actual_value else None
self.nn_unique_id_to_addresses[nn_unique_id] = (http_value, https_value)
try:
if not hostname:
raise Exception("Could not retrieve hostname from address " + actual_value)
jmx_uri = jmx_uri_fragment.format(actual_value)
state = get_value_from_jmx(jmx_uri, "tag.HAState", params.security_enabled, params.hdfs_user, params.is_https_enabled)
# If JMX parsing failed
if not state:
run_user = default("/configurations/hadoop-env/hdfs_user", "hdfs")
check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id)
code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
if code == 0 and out:
if NAMENODE_STATE.STANDBY in out:
state = NAMENODE_STATE.STANDBY
elif NAMENODE_STATE.ACTIVE in out:
state = NAMENODE_STATE.ACTIVE
if not state:
raise Exception("Could not retrieve Namenode state from URL " + jmx_uri)
state = state.lower()
if state not in [NAMENODE_STATE.ACTIVE, NAMENODE_STATE.STANDBY]:
state = NAMENODE_STATE.UNKNOWN
if state in self.namenode_state_to_hostnames:
self.namenode_state_to_hostnames[state].add(hostname)
else:
hostnames = set([hostname, ])
self.namenode_state_to_hostnames[state] = hostnames
except:
Logger.error("Could not get namenode state for " + nn_unique_id)
