def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None): https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address', security_enabled, run_user) http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address', security_enabled, run_user) # check for dfs.http.policy and after that for deprecated(for newer stacks) dfs.https.enable self.is_https_enabled = False if not is_empty(hdfs_site['dfs.http.policy']): self.is_https_enabled = hdfs_site['dfs.http.policy'].lower() == "https_only" elif not is_empty(hdfs_site['dfs.https.enable']): self.is_https_enabled = hdfs_site['dfs.https.enable'] address = https_nn_address if self.is_https_enabled else http_nn_address protocol = "https" if self.is_https_enabled else "http" self.address = format("{protocol}://{address}") self.run_user = run_user self.security_enabled = security_enabled self.logoutput = logoutput
def get_port_from_url(address): """ Return port from URL. If address is UnknownConfiguration, UnknownConfiguration will be returned. If no port was found, Fail will be raised. """ if not is_empty(address): port = re.findall(":([\d]{1,5})(?=/|$)", address) if port: return port[0] raise Fail("No port in URL:{0}".format(address)) else: return address
enable_ranger_yarn = default( "/configurations/ranger-yarn-plugin-properties/ranger-yarn-plugin-enabled", "No") enable_ranger_yarn = True if enable_ranger_yarn.lower() == 'yes' else False # ranger yarn-plugin supported flag, instead of using is_supported_yarn_ranger/yarn-env, using stack feature is_supported_yarn_ranger = check_stack_feature( StackFeature.YARN_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks) # get ranger yarn properties if enable_ranger_yarn is True if enable_ranger_yarn and is_supported_yarn_ranger: # get ranger policy url policymgr_mgr_url = config['configurations']['ranger-yarn-security'][ 'ranger.plugin.yarn.policy.rest.url'] if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') # ranger audit db user xa_audit_db_user = default( '/configurations/admin-properties/audit_db_user', 'rangerlogger') xa_audit_db_password = '' if not is_empty( config['configurations']['admin-properties']['audit_db_password'] ) and stack_supports_ranger_audit_db and has_ranger_admin: xa_audit_db_password = config['configurations']['admin-properties'][ 'audit_db_password'] # ranger yarn service/repository name repo_name = str(config['clusterName']) + '_yarn'
policymgr_mgr_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] if 'admin-properties' in config[ 'configurations'] and 'policymgr_external_url' in config[ 'configurations'][ 'admin-properties'] and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') xa_audit_db_name = default('/configurations/admin-properties/audit_db_name', 'ranger_audits') xa_audit_db_user = default('/configurations/admin-properties/audit_db_user', 'rangerlogger') xa_db_host = config['configurations']['admin-properties']['db_host'] repo_name = str(config['clusterName']) + '_knox' repo_name_value = config['configurations']['ranger-knox-security'][ 'ranger.plugin.knox.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value knox_home = config['configurations']['ranger-knox-plugin-properties'][ 'KNOX_HOME'] common_name_for_certificate = config['configurations'][ 'ranger-knox-plugin-properties']['common.name.for.certificate'] repo_config_username = config['configurations'][ 'ranger-knox-plugin-properties']['REPOSITORY_CONFIG_USERNAME'] ranger_env = config['configurations']['ranger-env'] ranger_plugin_properties = config['configurations'][ 'ranger-knox-plugin-properties'] policy_user = config['configurations']['ranger-knox-plugin-properties'][ 'policy_user']
) == 'yes' else False xa_audit_db_is_enabled = False xa_audit_db_password = '' # ranger elasticsearch properties if enable_ranger_elasticsearch: # get ranger policy url policymgr_mgr_url = config['configurations']['admin-properties'][ 'policymgr_external_url'] if xml_configurations_supported: policymgr_mgr_url = config['configurations'][ 'ranger-elasticsearch-security'][ 'ranger.plugin.elasticsearch.policy.rest.url'] if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') # ranger elasticsearch service name repo_name = str(config['clusterName']) + '_elasticsearch' repo_name_value = config['configurations'][ 'ranger-elasticsearch-security'][ 'ranger.plugin.elasticsearch.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value common_name_for_certificate = config['configurations'][ 'ranger-elasticsearch-plugin-properties'][ 'common.name.for.certificate'] repo_config_username = config['configurations'][ 'ranger-elasticsearch-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
storm_worker_log4j_content = config['configurations']['storm-worker-log4j'][ 'content'] # some commands may need to supply the JAAS location when running as storm storm_jaas_file = format("{conf_dir}/storm_jaas.conf") # For curl command in ranger plugin to get db connector jdk_location = config['hostLevelParams']['jdk_location'] java_share_dir = '/usr/share/java' if has_ranger_admin: enable_ranger_storm = ( config['configurations']['ranger-storm-plugin-properties'] ['ranger-storm-plugin-enabled'].lower() == 'yes') xa_audit_db_password = '' if not is_empty(config['configurations']['admin-properties'] ['audit_db_password']) and stack_supports_ranger_audit_db: xa_audit_db_password = unicode( config['configurations']['admin-properties']['audit_db_password']) repo_config_password = unicode( config['configurations']['ranger-storm-plugin-properties'] ['REPOSITORY_CONFIG_PASSWORD']) xa_audit_db_flavor = ( config['configurations']['admin-properties']['DB_FLAVOR']).lower() previous_jdbc_jar_name = None if stack_supports_ranger_audit_db: if xa_audit_db_flavor == 'mysql': jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name", None) previous_jdbc_jar_name = default( "/hostLevelParams/previous_custom_mysql_jdbc_name", None)
rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves else: rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts') smoke_test_user = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name'] smokeuser_permissions = "RWXCA" service_check_data = get_unique_id_and_date() user_group = config['configurations']['cluster-env']["user_group"] if security_enabled: _hostname_lowercase = config['hostname'].lower() master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase) regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase) _queryserver_jaas_princ = config['configurations']['hbase-site']['phoenix.queryserver.kerberos.principal'] if not is_empty(_queryserver_jaas_princ): queryserver_jaas_princ =_queryserver_jaas_princ.replace('_HOST',_hostname_lowercase) master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file'] regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file'] queryserver_keytab_path = config['configurations']['hbase-site']['phoenix.queryserver.keytab.file'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) if security_enabled: kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_principal_name};") kinit_cmd_master = format("{kinit_path_local} -kt {master_keytab_path} {master_jaas_princ};") else: kinit_cmd = "" kinit_cmd_master = ""
'smokeuser_principal_name'] smokeuser_permissions = "RWXCA" service_check_data = get_unique_id_and_date() user_group = config['configurations']['cluster-env']["user_group"] if security_enabled: _hostname_lowercase = config['hostname'].lower() master_jaas_princ = config['configurations']['hbase-site'][ 'hbase.master.kerberos.principal'].replace('_HOST', _hostname_lowercase) regionserver_jaas_princ = config['configurations']['hbase-site'][ 'hbase.regionserver.kerberos.principal'].replace( '_HOST', _hostname_lowercase) _queryserver_jaas_princ = config['configurations']['hbase-site'][ 'phoenix.queryserver.kerberos.principal'] if not is_empty(_queryserver_jaas_princ): queryserver_jaas_princ = _queryserver_jaas_princ.replace( '_HOST', _hostname_lowercase) master_keytab_path = config['configurations']['hbase-site'][ 'hbase.master.keytab.file'] regionserver_keytab_path = config['configurations']['hbase-site'][ 'hbase.regionserver.keytab.file'] queryserver_keytab_path = config['configurations']['hbase-site'][ 'phoenix.queryserver.keytab.file'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) if security_enabled: kinit_cmd = format(
_hostname_lowercase = config['hostname'].lower() master_jaas_princ = config['configurations']['hbase-site'][ 'hbase.master.kerberos.principal'].replace('_HOST', _hostname_lowercase) master_keytab_path = config['configurations']['hbase-site'][ 'hbase.master.keytab.file'] regionserver_jaas_princ = config['configurations']['hbase-site'][ 'hbase.regionserver.kerberos.principal'].replace( '_HOST', _hostname_lowercase) _rest_server_jaas_princ = config['configurations']['hbase-site'][ 'hbase.rest.kerberos.principal'] _rest_server_spnego_jaas_princ = config['configurations']['hbase-site'][ 'hbase.rest.authentication.kerberos.principal'] _queryserver_jaas_princ = config['configurations']['hbase-site'][ 'phoenix.queryserver.kerberos.principal'] if not is_empty(_queryserver_jaas_princ): queryserver_jaas_princ = _queryserver_jaas_princ.replace( '_HOST', _hostname_lowercase) if not is_empty(_rest_server_jaas_princ): rest_server_jaas_princ = _rest_server_jaas_princ.replace( '_HOST', _hostname_lowercase) if not is_empty(_rest_server_spnego_jaas_princ): rest_server_spnego_jaas_princ = _rest_server_spnego_jaas_princ.replace( '_HOST', _hostname_lowercase) regionserver_keytab_path = config['configurations']['hbase-site'][ 'hbase.regionserver.keytab.file'] rest_server_keytab_path = config['configurations']['hbase-site'][ 'hbase.rest.keytab.file'] rest_server_spnego_keytab_path = config['configurations']['hbase-site'][ 'hbase.rest.authentication.kerberos.keytab']
service_check_data = get_unique_id_and_date() user_group = config['configurations']['cluster-env']["user_group"] if security_enabled: _hostname_lowercase = config['agentLevelParams']['hostname'].lower() master_jaas_princ = config['configurations']['hbase-site'][ 'hbase.master.kerberos.principal'].replace('_HOST', _hostname_lowercase) master_keytab_path = config['configurations']['hbase-site'][ 'hbase.master.keytab.file'] regionserver_jaas_princ = config['configurations']['hbase-site'][ 'hbase.regionserver.kerberos.principal'].replace( '_HOST', _hostname_lowercase) _queryserver_jaas_princ = config['configurations']['hbase-site'][ 'phoenix.queryserver.kerberos.principal'] if not is_empty(_queryserver_jaas_princ): queryserver_jaas_princ = _queryserver_jaas_princ.replace( '_HOST', _hostname_lowercase) regionserver_keytab_path = config['configurations']['hbase-site'][ 'hbase.regionserver.keytab.file'] queryserver_keytab_path = config['configurations']['hbase-site'][ 'phoenix.queryserver.keytab.file'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab'] hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab'] kinit_path_local = get_kinit_path( default('/configurations/kerberos-env/executable_search_paths', None)) if security_enabled: kinit_cmd = format( "{kinit_path_local} -kt {hbase_user_keytab} {hbase_principal_name};") kinit_cmd_master = format(
# server configurations config = Script.get_config() hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"] yarn_user = hadoop_user hdfs_user = hadoop_user smokeuser = hadoop_user config_dir = os.environ["HADOOP_CONF_DIR"] hadoop_home = os.environ["HADOOP_HOME"] yarn_home = os.environ["HADOOP_YARN_HOME"] hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False) _authentication = config['configurations']['core-site']['hadoop.security.authentication'] security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos') smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab'] kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None)) rm_host = config['clusterHostInfo']['resourcemanager_hosts'][0] rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1] rm_https_port = "8090" rm_webui_address = format("{rm_host}:{rm_port}") rm_webui_https_address = format("{rm_host}:{rm_https_port}") hs_host = config['clusterHostInfo']['historyserver_hosts'][0] hs_port = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'].split(':')[-1] hs_webui_address = format("{hs_host}:{hs_port}") hadoop_mapred2_jar_location = os.path.join(os.environ["HADOOP_COMMON_HOME"], "share", "hadoop", "mapreduce") hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
tickTime = config['configurations']['zoo.cfg']['tickTime'] initLimit = config['configurations']['zoo.cfg']['initLimit'] syncLimit = config['configurations']['zoo.cfg']['syncLimit'] clientPort = config['configurations']['zoo.cfg']['clientPort'] if 'zoo.cfg' in config['configurations']: zoo_cfg_properties_map = config['configurations']['zoo.cfg'].copy() # Fix the data dir - ZK won't start unless the backslashes are doubled zoo_cfg_properties_map['dataDir'] = zk_data_dir else: zoo_cfg_properties_map = {} zoo_cfg_properties_map_length = len(zoo_cfg_properties_map) zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts'] zookeeper_hosts.sort() hostname = config['hostname'] _authentication = config['configurations']['core-site'][ 'hadoop.security.authentication'] security_enabled = (not is_empty(_authentication) and _authentication == 'kerberos') user_group = None zookeeper_win_service_name = status_params.zookeeper_win_service_name #log4j.properties if (('zookeeper-log4j' in config['configurations']) and ('content' in config['configurations']['zookeeper-log4j'])): log4j_props = config['configurations']['zookeeper-log4j']['content'] else: log4j_props = None
# ranger yarn plugin enabled property enable_ranger_yarn = default( "/configurations/ranger-yarn-plugin-properties/ranger-yarn-plugin-enabled", "No") enable_ranger_yarn = True if enable_ranger_yarn.lower() == 'yes' else False # ranger yarn-plugin supported flag, instead of using is_supported_yarn_ranger/yarn-env, using stack feature is_supported_yarn_ranger = True # get ranger yarn properties if enable_ranger_yarn is True if enable_ranger_yarn and is_supported_yarn_ranger: # get ranger policy url policymgr_mgr_url = config['configurations']['ranger-yarn-security'][ 'ranger.plugin.yarn.policy.rest.url'] if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') # ranger yarn service/repository name repo_name = str(config['clusterName']) + '_yarn' repo_name_value = config['configurations']['ranger-yarn-security'][ 'ranger.plugin.yarn.service.name'] if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}": repo_name = repo_name_value # ranger-env config ranger_env = config['configurations']['ranger-env'] # create ranger-env config having external ranger credential properties if not has_ranger_admin and enable_ranger_yarn: external_admin_username = default(