def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = get_nameservice(hdfs_site) nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = is_https_enabled_in_hdfs(hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service,nn_unique_id) if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) if INADDR_ANY in value and rpc_key in hdfs_site: rpc_value = str(hdfs_site[rpc_key]) if INADDR_ANY not in rpc_value: rpc_host = rpc_value.split(":")[0] value = value.replace(INADDR_ANY, rpc_host) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format(get_nameservice(hdfs_site), nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = hdfs_site['dfs.nameservices'] nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty( hdfs_site['dfs.https.enable']) else False if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id) protocol = "http" else: key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id) protocol = "https" if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) jmx_uri = JMX_URI_FRAGMENT.format(protocol, value) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -getServiceState {0}".format( nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def get_hdfs_cluster_id_from_jmx(hdfs_site, security_enabled, run_user): name_services = get_nameservices(hdfs_site) for name_service in name_services: for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses( hdfs_site, name_service): jmx_uri = jmx_uri.format(JMX_BEAN_NN_INFO) is_https_enabled = is_https_enabled_in_hdfs( hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) state = get_value_from_jmx(jmx_uri, 'ClusterId', security_enabled, run_user, is_https_enabled) if state: return state Logger.info("Cannot get clusterId from {0}".format(jmx_uri)) raise Fail( "Cannot get clsuterId from jmx, since none of the namenodes is running/accessible via jmx." )
def _get_namenode_states_noretries_single_ns(hdfs_site, name_service, security_enabled, run_user, last_retry=True): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] for nn_unique_id, address, jmx_uri in all_jmx_namenode_addresses( hdfs_site, name_service): is_https_enabled = is_https_enabled_in_hdfs( hdfs_site['dfs.http.policy'], hdfs_site['dfs.https.enable']) jmx_uri = jmx_uri.format(JMX_BEAN_FS) state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled, last_retry) # If JMX parsing failed if not state: check_service_cmd = "hdfs haadmin -ns {0} -getServiceState {1}".format( name_service, nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if HDFS_NN_STATE_STANDBY in out: state = HDFS_NN_STATE_STANDBY elif HDFS_NN_STATE_ACTIVE in out: state = HDFS_NN_STATE_ACTIVE if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, address)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, address)) else: unknown_namenodes.append((nn_unique_id, address)) return active_namenodes, standby_namenodes, unknown_namenodes
def get_namenode_states(hdfs_site): """ return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....] """ active_namenodes = [] standby_namenodes = [] unknown_namenodes = [] name_service = hdfs_site['dfs.nameservices'] nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service # now we have something like 'nn1,nn2,nn3,nn4' # turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id] # ie dfs.namenode.http-address.hacluster.nn1 nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',') for nn_unique_id in nn_unique_ids: is_https_enabled = hdfs_site['dfs.https.enabled'] if not is_empty( hdfs_site['dfs.https.enabled']) else False if not is_https_enabled: key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id) else: key = "https://" + NAMENODE_HTTPS_FRAGMENT.format( name_service, nn_unique_id) if key in hdfs_site: # use str() to ensure that unicode strings do not have the u' in them value = str(hdfs_site[key]) jmx_uri = JMX_URI_FRAGMENT.format(value) state = get_value_from_jmx(jmx_uri, 'State') if state == HDFS_NN_STATE_ACTIVE: active_namenodes.append((nn_unique_id, value)) elif state == HDFS_NN_STATE_STANDBY: standby_namenodes.append((nn_unique_id, value)) else: unknown_namenodes.append((nn_unique_id, value)) return active_namenodes, standby_namenodes, unknown_namenodes
def __init__(self): """ Initializes all fields by querying the Namenode state. Raises a ValueError if unable to construct the object. """ import params self.name_service = default( '/configurations/hdfs-site/dfs.internal.nameservices', None) if self.name_service is None: self.name_service = default( '/configurations/hdfs-site/dfs.nameservices', None) if not self.name_service: raise ValueError( "Could not retrieve property dfs.nameservices or dfs.internal.nameservices" ) nn_unique_ids_key = "dfs.ha.namenodes." + str(self.name_service) # List of the nn unique ids self.nn_unique_ids = default( "/configurations/hdfs-site/" + nn_unique_ids_key, None) if not self.nn_unique_ids: raise ValueError("Could not retrieve property " + nn_unique_ids_key) self.nn_unique_ids = self.nn_unique_ids.split(",") self.nn_unique_ids = [x.strip() for x in self.nn_unique_ids] policy = default("/configurations/hdfs-site/dfs.http.policy", "HTTP_ONLY") self.encrypted = policy.upper() == "HTTPS_ONLY" jmx_uri_fragment = ( "https" if self.encrypted else "http" ) + "://{0}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem" namenode_http_fragment = "dfs.namenode.http-address.{0}.{1}" namenode_https_fragment = "dfs.namenode.https-address.{0}.{1}" # Dictionary where the key is the Namenode State (e.g., ACTIVE), and the value is a set of hostnames self.namenode_state_to_hostnames = {} # Dictionary from nn unique id name to a tuple of (http address, https address) self.nn_unique_id_to_addresses = {} for nn_unique_id in self.nn_unique_ids: http_key = namenode_http_fragment.format(self.name_service, nn_unique_id) https_key = namenode_https_fragment.format(self.name_service, nn_unique_id) http_value = default("/configurations/hdfs-site/" + http_key, None) https_value = default("/configurations/hdfs-site/" + https_key, None) actual_value = https_value if self.encrypted else http_value hostname = actual_value.split(":")[0].strip( ) if actual_value and ":" in actual_value else None self.nn_unique_id_to_addresses[nn_unique_id] = (http_value, https_value) try: if not hostname: raise Exception( "Could not retrieve hostname from address " + actual_value) jmx_uri = jmx_uri_fragment.format(actual_value) state = get_value_from_jmx(jmx_uri, "tag.HAState", params.security_enabled, params.hdfs_user, params.is_https_enabled) # If JMX parsing failed if not state: run_user = default("/configurations/hadoop-env/hdfs_user", "hdfs") check_service_cmd = "hdfs haadmin -getServiceState {0}".format( nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if NAMENODE_STATE.STANDBY in out: state = NAMENODE_STATE.STANDBY elif NAMENODE_STATE.ACTIVE in out: state = NAMENODE_STATE.ACTIVE if not state: raise Exception( "Could not retrieve Namenode state from URL " + jmx_uri) state = state.lower() if state not in [ NAMENODE_STATE.ACTIVE, NAMENODE_STATE.STANDBY ]: state = NAMENODE_STATE.UNKNOWN if state in self.namenode_state_to_hostnames: self.namenode_state_to_hostnames[state].add(hostname) else: hostnames = set([ hostname, ]) self.namenode_state_to_hostnames[state] = hostnames except: Logger.error("Could not get namenode state for " + nn_unique_id)
def __init__(self): """ Initializes all fields by querying the Namenode state. Raises a ValueError if unable to construct the object. """ import params self.name_service = default("/configurations/hdfs-site/dfs.nameservices", None) if not self.name_service: raise ValueError("Could not retrieve property dfs.nameservices") nn_unique_ids_key = "dfs.ha.namenodes." + str(self.name_service) # List of the nn unique ids self.nn_unique_ids = default("/configurations/hdfs-site/" + nn_unique_ids_key, None) if not self.nn_unique_ids: raise ValueError("Could not retrieve property " + nn_unique_ids_key) self.nn_unique_ids = self.nn_unique_ids.split(",") self.nn_unique_ids = [x.strip() for x in self.nn_unique_ids] policy = default("/configurations/hdfs-site/dfs.http.policy", "HTTP_ONLY") self.encrypted = policy.upper() == "HTTPS_ONLY" jmx_uri_fragment = ("https" if self.encrypted else "http") + "://{0}/jmx?qry=Hadoop:service=NameNode,name=FSNamesystem" namenode_http_fragment = "dfs.namenode.http-address.{0}.{1}" namenode_https_fragment = "dfs.namenode.https-address.{0}.{1}" # Dictionary where the key is the Namenode State (e.g., ACTIVE), and the value is a set of hostnames self.namenode_state_to_hostnames = {} # Dictionary from nn unique id name to a tuple of (http address, https address) self.nn_unique_id_to_addresses = {} for nn_unique_id in self.nn_unique_ids: http_key = namenode_http_fragment.format(self.name_service, nn_unique_id) https_key = namenode_https_fragment.format(self.name_service, nn_unique_id) http_value = default("/configurations/hdfs-site/" + http_key, None) https_value = default("/configurations/hdfs-site/" + https_key, None) actual_value = https_value if self.encrypted else http_value hostname = actual_value.split(":")[0].strip() if actual_value and ":" in actual_value else None self.nn_unique_id_to_addresses[nn_unique_id] = (http_value, https_value) try: if not hostname: raise Exception("Could not retrieve hostname from address " + actual_value) jmx_uri = jmx_uri_fragment.format(actual_value) state = get_value_from_jmx(jmx_uri, "tag.HAState", params.security_enabled, params.hdfs_user, params.is_https_enabled) # If JMX parsing failed if not state: run_user = default("/configurations/hadoop-env/hdfs_user", "hdfs") check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id) code, out = shell.call(check_service_cmd, logoutput=True, user=run_user) if code == 0 and out: if NAMENODE_STATE.STANDBY in out: state = NAMENODE_STATE.STANDBY elif NAMENODE_STATE.ACTIVE in out: state = NAMENODE_STATE.ACTIVE if not state: raise Exception("Could not retrieve Namenode state from URL " + jmx_uri) state = state.lower() if state not in [NAMENODE_STATE.ACTIVE, NAMENODE_STATE.STANDBY]: state = NAMENODE_STATE.UNKNOWN if state in self.namenode_state_to_hostnames: self.namenode_state_to_hostnames[state].add(hostname) else: hostnames = set([hostname, ]) self.namenode_state_to_hostnames[state] = hostnames except: Logger.error("Could not get namenode state for " + nn_unique_id)