def __init__(self, hdfs_site, run_user, security_enabled, logoutput=None):
https_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.https-address',
security_enabled, run_user)
http_nn_address = namenode_ha_utils.get_property_for_active_namenode(hdfs_site, 'dfs.namenode.http-address',
security_enabled, run_user)
# check for dfs.http.policy and after that for deprecated(for newer stacks) dfs.https.enable
self.is_https_enabled = False
if not is_empty(hdfs_site['dfs.http.policy']):
self.is_https_enabled = hdfs_site['dfs.http.policy'].lower() == "https_only"
elif not is_empty(hdfs_site['dfs.https.enable']):
self.is_https_enabled = hdfs_site['dfs.https.enable']
address = https_nn_address if self.is_https_enabled else http_nn_address
protocol = "https" if self.is_https_enabled else "http"
self.address = format("{protocol}://{address}")
self.run_user = run_user
self.security_enabled = security_enabled
self.logoutput = logoutput
def get_port_from_url(address):
"""
Return port from URL. If address is UnknownConfiguration,
UnknownConfiguration will be returned. If no port was found, Fail will be
raised.
"""
if not is_empty(address):
port = re.findall(":([\d]{1,5})(?=/|$)", address)
if port:
return port[0]
raise Fail("No port in URL:{0}".format(address))
else:
return address
enable_ranger_yarn = default(
"/configurations/ranger-yarn-plugin-properties/ranger-yarn-plugin-enabled",
"No")
enable_ranger_yarn = True if enable_ranger_yarn.lower() == 'yes' else False
# ranger yarn-plugin supported flag, instead of using is_supported_yarn_ranger/yarn-env, using stack feature
is_supported_yarn_ranger = check_stack_feature(
StackFeature.YARN_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks)
# get ranger yarn properties if enable_ranger_yarn is True
if enable_ranger_yarn and is_supported_yarn_ranger:
# get ranger policy url
policymgr_mgr_url = config['configurations']['ranger-yarn-security'][
'ranger.plugin.yarn.policy.rest.url']
if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
# ranger audit db user
xa_audit_db_user = default(
'/configurations/admin-properties/audit_db_user', 'rangerlogger')
xa_audit_db_password = ''
if not is_empty(
config['configurations']['admin-properties']['audit_db_password']
) and stack_supports_ranger_audit_db and has_ranger_admin:
xa_audit_db_password = config['configurations']['admin-properties'][
'audit_db_password']
# ranger yarn service/repository name
repo_name = str(config['clusterName']) + '_yarn'
policymgr_mgr_url = config['configurations']['admin-properties'][
'policymgr_external_url']
if 'admin-properties' in config[
'configurations'] and 'policymgr_external_url' in config[
'configurations'][
'admin-properties'] and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
xa_audit_db_name = default('/configurations/admin-properties/audit_db_name',
'ranger_audits')
xa_audit_db_user = default('/configurations/admin-properties/audit_db_user',
'rangerlogger')
xa_db_host = config['configurations']['admin-properties']['db_host']
repo_name = str(config['clusterName']) + '_knox'
repo_name_value = config['configurations']['ranger-knox-security'][
'ranger.plugin.knox.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
knox_home = config['configurations']['ranger-knox-plugin-properties'][
'KNOX_HOME']
common_name_for_certificate = config['configurations'][
'ranger-knox-plugin-properties']['common.name.for.certificate']
repo_config_username = config['configurations'][
'ranger-knox-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations'][
'ranger-knox-plugin-properties']
policy_user = config['configurations']['ranger-knox-plugin-properties'][
'policy_user']
) == 'yes' else False
xa_audit_db_is_enabled = False
xa_audit_db_password = ''
# ranger elasticsearch properties
if enable_ranger_elasticsearch:
# get ranger policy url
policymgr_mgr_url = config['configurations']['admin-properties'][
'policymgr_external_url']
if xml_configurations_supported:
policymgr_mgr_url = config['configurations'][
'ranger-elasticsearch-security'][
'ranger.plugin.elasticsearch.policy.rest.url']
if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
# ranger elasticsearch service name
repo_name = str(config['clusterName']) + '_elasticsearch'
repo_name_value = config['configurations'][
'ranger-elasticsearch-security'][
'ranger.plugin.elasticsearch.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
common_name_for_certificate = config['configurations'][
'ranger-elasticsearch-plugin-properties'][
'common.name.for.certificate']
repo_config_username = config['configurations'][
'ranger-elasticsearch-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
storm_worker_log4j_content = config['configurations']['storm-worker-log4j'][
'content']
# some commands may need to supply the JAAS location when running as storm
storm_jaas_file = format("{conf_dir}/storm_jaas.conf")
# For curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
if has_ranger_admin:
enable_ranger_storm = (
config['configurations']['ranger-storm-plugin-properties']
['ranger-storm-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = ''
if not is_empty(config['configurations']['admin-properties']
['audit_db_password']) and stack_supports_ranger_audit_db:
xa_audit_db_password = unicode(
config['configurations']['admin-properties']['audit_db_password'])
repo_config_password = unicode(
config['configurations']['ranger-storm-plugin-properties']
['REPOSITORY_CONFIG_PASSWORD'])
xa_audit_db_flavor = (
config['configurations']['admin-properties']['DB_FLAVOR']).lower()
previous_jdbc_jar_name = None
if stack_supports_ranger_audit_db:
if xa_audit_db_flavor == 'mysql':
jdbc_jar_name = default("/hostLevelParams/custom_mysql_jdbc_name",
None)
previous_jdbc_jar_name = default(
"/hostLevelParams/previous_custom_mysql_jdbc_name", None)
rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/slave_hosts') #if hbase_rs_hosts not given it is assumed that region servers on same nodes as slaves
else:
rs_hosts = default('/clusterHostInfo/hbase_rs_hosts', '/clusterHostInfo/all_hosts')
smoke_test_user = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
smokeuser_permissions = "RWXCA"
service_check_data = get_unique_id_and_date()
user_group = config['configurations']['cluster-env']["user_group"]
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
master_jaas_princ = config['configurations']['hbase-site']['hbase.master.kerberos.principal'].replace('_HOST',_hostname_lowercase)
regionserver_jaas_princ = config['configurations']['hbase-site']['hbase.regionserver.kerberos.principal'].replace('_HOST',_hostname_lowercase)
_queryserver_jaas_princ = config['configurations']['hbase-site']['phoenix.queryserver.kerberos.principal']
if not is_empty(_queryserver_jaas_princ):
queryserver_jaas_princ =_queryserver_jaas_princ.replace('_HOST',_hostname_lowercase)
master_keytab_path = config['configurations']['hbase-site']['hbase.master.keytab.file']
regionserver_keytab_path = config['configurations']['hbase-site']['hbase.regionserver.keytab.file']
queryserver_keytab_path = config['configurations']['hbase-site']['phoenix.queryserver.keytab.file']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab']
kinit_path_local = get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
if security_enabled:
kinit_cmd = format("{kinit_path_local} -kt {hbase_user_keytab} {hbase_principal_name};")
kinit_cmd_master = format("{kinit_path_local} -kt {master_keytab_path} {master_jaas_princ};")
else:
kinit_cmd = ""
kinit_cmd_master = ""
'smokeuser_principal_name']
smokeuser_permissions = "RWXCA"
service_check_data = get_unique_id_and_date()
user_group = config['configurations']['cluster-env']["user_group"]
if security_enabled:
_hostname_lowercase = config['hostname'].lower()
master_jaas_princ = config['configurations']['hbase-site'][
'hbase.master.kerberos.principal'].replace('_HOST',
_hostname_lowercase)
regionserver_jaas_princ = config['configurations']['hbase-site'][
'hbase.regionserver.kerberos.principal'].replace(
'_HOST', _hostname_lowercase)
_queryserver_jaas_princ = config['configurations']['hbase-site'][
'phoenix.queryserver.kerberos.principal']
if not is_empty(_queryserver_jaas_princ):
queryserver_jaas_princ = _queryserver_jaas_princ.replace(
'_HOST', _hostname_lowercase)
master_keytab_path = config['configurations']['hbase-site'][
'hbase.master.keytab.file']
regionserver_keytab_path = config['configurations']['hbase-site'][
'hbase.regionserver.keytab.file']
queryserver_keytab_path = config['configurations']['hbase-site'][
'phoenix.queryserver.keytab.file']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
if security_enabled:
kinit_cmd = format(
_hostname_lowercase = config['hostname'].lower()
master_jaas_princ = config['configurations']['hbase-site'][
'hbase.master.kerberos.principal'].replace('_HOST',
_hostname_lowercase)
master_keytab_path = config['configurations']['hbase-site'][
'hbase.master.keytab.file']
regionserver_jaas_princ = config['configurations']['hbase-site'][
'hbase.regionserver.kerberos.principal'].replace(
'_HOST', _hostname_lowercase)
_rest_server_jaas_princ = config['configurations']['hbase-site'][
'hbase.rest.kerberos.principal']
_rest_server_spnego_jaas_princ = config['configurations']['hbase-site'][
'hbase.rest.authentication.kerberos.principal']
_queryserver_jaas_princ = config['configurations']['hbase-site'][
'phoenix.queryserver.kerberos.principal']
if not is_empty(_queryserver_jaas_princ):
queryserver_jaas_princ = _queryserver_jaas_princ.replace(
'_HOST', _hostname_lowercase)
if not is_empty(_rest_server_jaas_princ):
rest_server_jaas_princ = _rest_server_jaas_princ.replace(
'_HOST', _hostname_lowercase)
if not is_empty(_rest_server_spnego_jaas_princ):
rest_server_spnego_jaas_princ = _rest_server_spnego_jaas_princ.replace(
'_HOST', _hostname_lowercase)
regionserver_keytab_path = config['configurations']['hbase-site'][
'hbase.regionserver.keytab.file']
rest_server_keytab_path = config['configurations']['hbase-site'][
'hbase.rest.keytab.file']
rest_server_spnego_keytab_path = config['configurations']['hbase-site'][
'hbase.rest.authentication.kerberos.keytab']
service_check_data = get_unique_id_and_date()
user_group = config['configurations']['cluster-env']["user_group"]
if security_enabled:
_hostname_lowercase = config['agentLevelParams']['hostname'].lower()
master_jaas_princ = config['configurations']['hbase-site'][
'hbase.master.kerberos.principal'].replace('_HOST',
_hostname_lowercase)
master_keytab_path = config['configurations']['hbase-site'][
'hbase.master.keytab.file']
regionserver_jaas_princ = config['configurations']['hbase-site'][
'hbase.regionserver.kerberos.principal'].replace(
'_HOST', _hostname_lowercase)
_queryserver_jaas_princ = config['configurations']['hbase-site'][
'phoenix.queryserver.kerberos.principal']
if not is_empty(_queryserver_jaas_princ):
queryserver_jaas_princ = _queryserver_jaas_princ.replace(
'_HOST', _hostname_lowercase)
regionserver_keytab_path = config['configurations']['hbase-site'][
'hbase.regionserver.keytab.file']
queryserver_keytab_path = config['configurations']['hbase-site'][
'phoenix.queryserver.keytab.file']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
hbase_user_keytab = config['configurations']['hbase-env']['hbase_user_keytab']
kinit_path_local = get_kinit_path(
default('/configurations/kerberos-env/executable_search_paths', None))
if security_enabled:
kinit_cmd = format(
"{kinit_path_local} -kt {hbase_user_keytab} {hbase_principal_name};")
kinit_cmd_master = format(
# server configurations
config = Script.get_config()
hadoop_user = config["configurations"]["cluster-env"]["hadoop.user.name"]
yarn_user = hadoop_user
hdfs_user = hadoop_user
smokeuser = hadoop_user
config_dir = os.environ["HADOOP_CONF_DIR"]
hadoop_home = os.environ["HADOOP_HOME"]
yarn_home = os.environ["HADOOP_YARN_HOME"]
hadoop_ssl_enabled = default("/configurations/core-site/hadoop.ssl.enabled", False)
_authentication = config['configurations']['core-site']['hadoop.security.authentication']
security_enabled = ( not is_empty(_authentication) and _authentication == 'kerberos')
smoke_user_keytab = config['configurations']['hadoop-env']['smokeuser_keytab']
kinit_path_local = functions.get_kinit_path(default('/configurations/kerberos-env/executable_search_paths', None))
rm_host = config['clusterHostInfo']['resourcemanager_hosts'][0]
rm_port = config['configurations']['yarn-site']['yarn.resourcemanager.webapp.address'].split(':')[-1]
rm_https_port = "8090"
rm_webui_address = format("{rm_host}:{rm_port}")
rm_webui_https_address = format("{rm_host}:{rm_https_port}")
hs_host = config['clusterHostInfo']['historyserver_hosts'][0]
hs_port = config['configurations']['mapred-site']['mapreduce.jobhistory.webapp.address'].split(':')[-1]
hs_webui_address = format("{hs_host}:{hs_port}")
hadoop_mapred2_jar_location = os.path.join(os.environ["HADOOP_COMMON_HOME"], "share", "hadoop", "mapreduce")
hadoopMapredExamplesJarName = "hadoop-mapreduce-examples-2.*.jar"
tickTime = config['configurations']['zoo.cfg']['tickTime']
initLimit = config['configurations']['zoo.cfg']['initLimit']
syncLimit = config['configurations']['zoo.cfg']['syncLimit']
clientPort = config['configurations']['zoo.cfg']['clientPort']
if 'zoo.cfg' in config['configurations']:
zoo_cfg_properties_map = config['configurations']['zoo.cfg'].copy()
# Fix the data dir - ZK won't start unless the backslashes are doubled
zoo_cfg_properties_map['dataDir'] = zk_data_dir
else:
zoo_cfg_properties_map = {}
zoo_cfg_properties_map_length = len(zoo_cfg_properties_map)
zookeeper_hosts = config['clusterHostInfo']['zookeeper_hosts']
zookeeper_hosts.sort()
hostname = config['hostname']
_authentication = config['configurations']['core-site'][
'hadoop.security.authentication']
security_enabled = (not is_empty(_authentication)
and _authentication == 'kerberos')
user_group = None
zookeeper_win_service_name = status_params.zookeeper_win_service_name
#log4j.properties
if (('zookeeper-log4j' in config['configurations'])
and ('content' in config['configurations']['zookeeper-log4j'])):
log4j_props = config['configurations']['zookeeper-log4j']['content']
else:
log4j_props = None
# ranger yarn plugin enabled property
enable_ranger_yarn = default(
"/configurations/ranger-yarn-plugin-properties/ranger-yarn-plugin-enabled",
"No")
enable_ranger_yarn = True if enable_ranger_yarn.lower() == 'yes' else False
# ranger yarn-plugin supported flag, instead of using is_supported_yarn_ranger/yarn-env, using stack feature
is_supported_yarn_ranger = True
# get ranger yarn properties if enable_ranger_yarn is True
if enable_ranger_yarn and is_supported_yarn_ranger:
# get ranger policy url
policymgr_mgr_url = config['configurations']['ranger-yarn-security'][
'ranger.plugin.yarn.policy.rest.url']
if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
# ranger yarn service/repository name
repo_name = str(config['clusterName']) + '_yarn'
repo_name_value = config['configurations']['ranger-yarn-security'][
'ranger.plugin.yarn.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
# ranger-env config
ranger_env = config['configurations']['ranger-env']
# create ranger-env config having external ranger credential properties
if not has_ranger_admin and enable_ranger_yarn:
external_admin_username = default(
