def is_https_enabled_in_hdfs(dfs_http_policy, dfs_https_enable):
https_enabled = False
if not is_empty(dfs_http_policy):
https_enabled = dfs_http_policy.lower() == "https_only"
elif not is_empty(dfs_https_enable):
https_enabled = dfs_https_enable
return https_enabled
def is_ha_enabled(hdfs_site):
dfs_ha_nameservices = get_nameservice(hdfs_site)
if not dfs_ha_nameservices or is_empty(dfs_ha_nameservices):
return False
dfs_ha_namenode_ids = hdfs_site[format("dfs.ha.namenodes.{dfs_ha_nameservices}")]
if not is_empty(dfs_ha_namenode_ids):
dfs_ha_namemodes_ids_list = dfs_ha_namenode_ids.split(",")
dfs_ha_namenode_ids_array_len = len(dfs_ha_namemodes_ids_list)
if dfs_ha_namenode_ids_array_len > 1:
return True
return False
def do_keystore_setup(rolling_upgrade=False):
import params
ranger_home = params.ranger_home
cred_lib_path = params.cred_lib_path
cred_setup_prefix = params.cred_setup_prefix
if rolling_upgrade:
ranger_home = format("/usr/hdp/{version}/ranger-admin")
cred_lib_path = os.path.join(ranger_home,"cred","lib","*")
cred_setup_prefix = (format('{ranger_home}/ranger_credential_helper.py'), '-l', cred_lib_path)
if not is_empty(params.ranger_credential_provider_path):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True
)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password):
jceks_path = params.ranger_credential_provider_path
cred_setup = cred_setup_prefix + ('-f', jceks_path, '-k', params.ranger_jpa_audit_jdbc_credential_alias, '-v', PasswordString(params.ranger_ambari_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'RANGER_ADMIN_HOME':ranger_home, 'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True
)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
def get_path_from_url(address):
"""
Return port from URL. If the address is numeric, the address is assumed to be a port and is returned.
If address is UnknownConfiguration, UnknownConfiguration will be returned.
"""
if is_empty(address):
return address
result = re.findall("^((.+)://)?(([a-zA-Z0-9]|\.|-)*)(:([\d]{2,}))?/(.*)$", address)
if result:
return result[0][6]
return None
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = hdfs_site['dfs.nameservices']
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(hdfs_site['dfs.https.enable']) else False
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service,nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service,nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState', security_enabled, run_user, is_https_enabled)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -getServiceState {0}".format(nn_unique_id)
code, out = shell.call(check_service_cmd, logoutput=True, user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def get_port_from_url(address):
"""
Return port from URL. If the address is numeric, the address is assumed to be a port and is returned.
If address is UnknownConfiguration, UnknownConfiguration will be returned.
If no port was found, Fail will be raised.
"""
if is_empty(address):
return address
if isinstance(address, (int, long)):
return address
port = re.findall(":([\d]{1,5})(?=/|$)", address)
if port:
return port[0]
elif address.isdigit():
return address
raise Fail("No port in URL:{0}".format(address))
'xml_configurations_supported']
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
ranger_admin_log_dir = default(
"/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
#need to set the defaut to false to satisfy downgrade from 4.2,5 to 4.2 or 4.1
is_supported_solr_ranger = default(
'/configurations/solr-env/is_supported_solr_ranger', False)
#ranger solr properties
if has_ranger_admin and is_supported_solr_ranger:
enable_ranger_solr = config['configurations'][
'ranger-solr-plugin-properties']['ranger-solr-plugin-enabled']
enable_ranger_solr = not is_empty(
enable_ranger_solr) and enable_ranger_solr.lower() == 'yes'
policymgr_mgr_url = config['configurations']['admin-properties'][
'policymgr_external_url']
if 'admin-properties' in config[
'configurations'] and 'policymgr_external_url' in config[
'configurations'][
'admin-properties'] and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
xa_audit_db_flavor = config['configurations']['admin-properties'][
'DB_FLAVOR']
xa_audit_db_flavor = xa_audit_db_flavor.lower(
) if xa_audit_db_flavor else None
xa_audit_db_name = default(
'/configurations/admin-properties/audit_db_name', 'ranger_audits')
xa_audit_db_user = default(
'/configurations/admin-properties/audit_db_user', 'rangerlogger')
repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
if security_enabled:
sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]")
sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower())
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties']
policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user']
#For curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enable'] if \
not is_empty(config['configurations']['hdfs-site']['dfs.https.enable']) else False
if has_ranger_admin:
enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
if xa_audit_db_flavor == 'mysql':
jdbc_symlink_name = "mysql-jdbc-driver.jar"
jdbc_jar_name = "mysql-connector-java.jar"
audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
jdbc_driver = "com.mysql.jdbc.Driver"
elif xa_audit_db_flavor == 'oracle':
jdbc_jar_name = "ojdbc6.jar"
jdbc_symlink_name = "oracle-jdbc-driver.jar"
usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml')
if stack_supports_ranger_log4j:
usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.properties')
cred_validator_file = format('{usersync_home}/native/credValidator.uexe')
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
usersync_exturl = config['configurations']['admin-properties'][
'policymgr_external_url']
if usersync_exturl.endswith('/'):
usersync_exturl = usersync_exturl.rstrip('/')
ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0]
ugsync_host = 'localhost'
usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts']
if not is_empty(usersync_host_info) and len(usersync_host_info) > 0:
ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0]
ranger_external_url = config['configurations']['admin-properties'][
'policymgr_external_url']
if ranger_external_url.endswith('/'):
ranger_external_url = ranger_external_url.rstrip('/')
ranger_db_name = config['configurations']['admin-properties']['db_name']
ranger_auditdb_name = default('/configurations/admin-properties/audit_db_name',
'ranger_audits')
sql_command_invoker = config['configurations']['admin-properties'][
'SQL_COMMAND_INVOKER']
db_root_user = config['configurations']['admin-properties']['db_root_user']
db_root_password = unicode(
config['configurations']['admin-properties']['db_root_password'])
db_host = config['configurations']['admin-properties']['db_host']
def do_keystore_setup(upgrade_type=None):
import params
ranger_home = params.ranger_home
cred_lib_path = params.cred_lib_path
if not is_empty(params.ranger_credential_provider_path):
ranger_credential_helper(cred_lib_path, params.ranger_jpa_jdbc_credential_alias, params.ranger_ambari_db_password, params.ranger_credential_provider_path)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if not is_empty(params.ranger_credential_provider_path) and (params.ranger_audit_source_type).lower() == 'db' and not is_empty(params.ranger_ambari_audit_db_password):
ranger_credential_helper(cred_lib_path, params.ranger_jpa_audit_jdbc_credential_alias, params.ranger_ambari_audit_db_password, params.ranger_credential_provider_path)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if params.ranger_auth_method.upper() == "LDAP":
ranger_ldap_auth_password = params.ranger_usersync_ldap_ldapbindpassword
if params.ranger_ldap_bind_auth_password != "{{ranger_usersync_ldap_ldapbindpassword}}":
ranger_ldap_auth_password = params.ranger_ldap_bind_auth_password
ranger_credential_helper(params.cred_lib_path, params.ranger_ldap_password_alias, ranger_ldap_auth_password, params.ranger_credential_provider_path)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if params.ranger_auth_method.upper() == "ACTIVE_DIRECTORY":
ranger_ad_auth_password = params.ranger_usersync_ldap_ldapbindpassword
if params.ranger_ad_bind_auth_password != "{{ranger_usersync_ldap_ldapbindpassword}}":
ranger_ad_auth_password = params.ranger_ad_bind_auth_password
ranger_credential_helper(params.cred_lib_path, params.ranger_ad_password_alias, ranger_ad_auth_password, params.ranger_credential_provider_path)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
if params.stack_supports_secure_ssl_password:
ranger_credential_helper(params.cred_lib_path, params.ranger_truststore_alias, params.truststore_password, params.ranger_credential_provider_path)
if params.https_enabled and not params.http_enabled:
ranger_credential_helper(params.cred_lib_path, params.ranger_https_keystore_alias, params.https_keystore_password, params.ranger_credential_provider_path)
File(params.ranger_credential_provider_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0750,
owner = params.unix_user,
group = params.unix_group
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group,
cd_access = 'a',
create_parents=True,
mode=0755
)
File(format('{ranger_ugsync_conf}/ranger-usersync-env-logdir.sh'),
content = format("export logdir={usersync_log_dir}"),
owner = params.unix_user,
group = params.unix_group,
mode=0755
)
Directory(format("{ranger_ugsync_conf}/"),
owner = params.unix_user
)
if upgrade_type is not None:
src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if params.stack_supports_ranger_log4j:
File(format('{usersync_home}/conf/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=params.usersync_log4j,
mode=0644
)
elif upgrade_type is not None and not params.stack_supports_ranger_log4j:
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
XmlConfig("ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_lib = os.path.join(usersync_home,"lib","*")
ranger_credential_helper(cred_lib, 'usersync.ssl.key.password', params.ranger_usersync_keystore_password, params.ugsync_jceks_path)
if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
ranger_credential_helper(cred_lib, 'ranger.usersync.ldap.bindalias', params.ranger_usersync_ldap_ldapbindpassword, params.ugsync_jceks_path)
ranger_credential_helper(cred_lib, 'usersync.ssl.truststore.password', params.ranger_usersync_truststore_password, params.ugsync_jceks_path)
File(params.ugsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user,
group = params.unix_group
)
File(params.usersync_services_file,
mode = 0755,
)
Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'")
Execute(cmd, logoutput=True, user = params.unix_user)
File(params.ranger_usersync_keystore_file,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
create_core_site_xml(ranger_ugsync_conf)
mysql_host = config['clusterHostInfo']['metron_enrichment_mysql_server_hosts'][
0]
mysql_port = config['configurations']['metron-env'][
'metron_enrichment_db_port']
mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
enrichment_metron_user = config['configurations']['metron-env'][
'metron_enrichment_db_user']
enrichment_metron_user_passwd = config['configurations']['metron-env'][
'metron_enrichment_db_password']
enrichment_metron_user_passwd = unicode(
enrichment_metron_user_passwd) if not is_empty(
enrichment_metron_user_passwd) else enrichment_metron_user_passwd
mysql_process_name = status_params.mysql_process_name
# create partial functions with common arguments for every HdfsResource call
# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
HdfsResource = functools.partial(
HdfsResource,
user=hdfs_user,
hdfs_resource_ignore_file=
"/var/lib/ambari-agent/data/.hdfs_resource_ignore",
security_enabled=security_enabled,
keytab=hdfs_user_keytab,
kinit_path_local=kinit_path_local,
hadoop_bin_dir=hadoop_bin_dir,
hadoop_conf_dir=hadoop_conf_dir,
principal_name=hdfs_principal_name,
enable_ranger_kafka = default(
"configurations/ranger-kafka-plugin-properties/ranger-kafka-plugin-enabled",
"No")
enable_ranger_kafka = True if enable_ranger_kafka.lower() == 'yes' else False
# ranger kafka-plugin supported flag, instead of dependending on is_supported_kafka_ranger/kafka-env.xml, using stack feature
is_supported_kafka_ranger = check_stack_feature(
StackFeature.KAFKA_RANGER_PLUGIN_SUPPORT, version_for_stack_feature_checks)
# ranger kafka properties
if enable_ranger_kafka and is_supported_kafka_ranger:
# get ranger policy url
policymgr_mgr_url = config['configurations']['ranger-kafka-security'][
'ranger.plugin.kafka.policy.rest.url']
if not is_empty(policymgr_mgr_url) and policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
# ranger audit db user
xa_audit_db_user = default(
'/configurations/admin-properties/audit_db_user', 'rangerlogger')
xa_audit_db_password = ''
if not is_empty(
config['configurations']['admin-properties']['audit_db_password']
) and stack_supports_ranger_audit_db and has_ranger_admin:
xa_audit_db_password = config['configurations']['admin-properties'][
'audit_db_password']
# ranger kafka service/repository name
repo_name = str(config['clusterName']) + '_kafka'
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations'][
'ranger-hbase-plugin-properties']
policy_user = config['configurations']['ranger-hbase-plugin-properties'][
'policy_user']
#For curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
enable_ranger_hbase = False
if has_ranger_admin:
enable_ranger_hbase = (
config['configurations']['ranger-hbase-plugin-properties']
['ranger-hbase-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = ''
if not is_empty(config['configurations']['admin-properties']
['audit_db_password']) and stack_supports_ranger_audit_db:
xa_audit_db_password = unicode(
config['configurations']['admin-properties']['audit_db_password'])
repo_config_password = unicode(
config['configurations']['ranger-hbase-plugin-properties']
['REPOSITORY_CONFIG_PASSWORD'])
xa_audit_db_flavor = (
config['configurations']['admin-properties']['DB_FLAVOR']).lower()
previous_jdbc_jar_name = None
if xa_audit_db_flavor == 'mysql':
jdbc_symlink_name = "mysql-jdbc-driver.jar"
jdbc_jar_name = "mysql-connector-java.jar"
previous_jdbc_jar_name = default(
"/hostLevelParams/previous_custom_mysql_jdbc_name", None)
audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
has_ranger_admin = not len(ranger_admin_hosts) == 0
xml_configurations_supported = config['configurations']['ranger-env'][
'xml_configurations_supported']
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
ranger_admin_log_dir = default(
"/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
is_supported_kafka_ranger = config['configurations']['kafka-env'][
'is_supported_kafka_ranger']
#ranger kafka properties
if has_ranger_admin and is_supported_kafka_ranger:
enable_ranger_kafka = config['configurations'][
'ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled']
enable_ranger_kafka = not is_empty(
enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes'
policymgr_mgr_url = config['configurations']['admin-properties'][
'policymgr_external_url']
sql_connector_jar = config['configurations']['admin-properties'][
'SQL_CONNECTOR_JAR']
xa_audit_db_flavor = config['configurations']['admin-properties'][
'DB_FLAVOR']
xa_audit_db_flavor = xa_audit_db_flavor.lower(
) if xa_audit_db_flavor else None
xa_audit_db_name = config['configurations']['admin-properties'][
'audit_db_name']
xa_audit_db_user = config['configurations']['admin-properties'][
'audit_db_user']
xa_audit_db_password = unicode(
config['configurations']['admin-properties']['audit_db_password'])
xa_db_host = config['configurations']['admin-properties']['db_host']
usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync")
admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml')
security_app_context_file = format('{ranger_conf}/security-applicationContext.xml')
ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml')
usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml')
cred_validator_file = format('{usersync_home}/native/credValidator.uexe')
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
usersync_exturl = config['configurations']['admin-properties']['policymgr_external_url']
ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0]
ugsync_host = 'localhost'
usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts']
if not is_empty(usersync_host_info) and len(usersync_host_info) > 0:
ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0]
ranger_external_url = config['configurations']['admin-properties']['policymgr_external_url']
ranger_db_name = config['configurations']['admin-properties']['db_name']
ranger_auditdb_name = config['configurations']['admin-properties']['audit_db_name']
sql_command_invoker = config['configurations']['admin-properties']['SQL_COMMAND_INVOKER']
db_root_user = config['configurations']['admin-properties']['db_root_user']
db_root_password = unicode(config['configurations']['admin-properties']['db_root_password'])
db_host = config['configurations']['admin-properties']['db_host']
ranger_db_user = config['configurations']['admin-properties']['db_user']
ranger_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
ranger_db_password = unicode(config['configurations']['admin-properties']['db_password'])
#ranger-env properties
oracle_home = default("/configurations/ranger-env/oracle_home", "-")
def get_namenode_states_noretries(hdfs_site, security_enabled, run_user):
"""
return format [('nn1', 'hdfs://hostname1:port1'), ('nn2', 'hdfs://hostname2:port2')] , [....], [....]
"""
active_namenodes = []
standby_namenodes = []
unknown_namenodes = []
name_service = hdfs_site['dfs.nameservices']
nn_unique_ids_key = 'dfs.ha.namenodes.' + name_service
# now we have something like 'nn1,nn2,nn3,nn4'
# turn it into dfs.namenode.[property].[dfs.nameservices].[nn_unique_id]
# ie dfs.namenode.http-address.hacluster.nn1
nn_unique_ids = hdfs_site[nn_unique_ids_key].split(',')
for nn_unique_id in nn_unique_ids:
is_https_enabled = hdfs_site['dfs.https.enable'] if not is_empty(
hdfs_site['dfs.https.enable']) else False
rpc_key = NAMENODE_RPC_FRAGMENT.format(name_service, nn_unique_id)
if not is_https_enabled:
key = NAMENODE_HTTP_FRAGMENT.format(name_service, nn_unique_id)
protocol = "http"
else:
key = NAMENODE_HTTPS_FRAGMENT.format(name_service, nn_unique_id)
protocol = "https"
if key in hdfs_site:
# use str() to ensure that unicode strings do not have the u' in them
value = str(hdfs_site[key])
if INADDR_ANY in value and rpc_key in hdfs_site:
rpc_value = str(hdfs_site[rpc_key])
if INADDR_ANY not in rpc_value:
rpc_host = rpc_value.split(":")[0]
value = value.replace(INADDR_ANY, rpc_host)
jmx_uri = JMX_URI_FRAGMENT.format(protocol, value)
state = get_value_from_jmx(jmx_uri, 'tag.HAState',
security_enabled, run_user,
is_https_enabled)
# If JMX parsing failed
if not state:
check_service_cmd = "hdfs haadmin -getServiceState {0}".format(
nn_unique_id)
code, out = shell.call(check_service_cmd,
logoutput=True,
user=run_user)
if code == 0 and out:
if HDFS_NN_STATE_STANDBY in out:
state = HDFS_NN_STATE_STANDBY
elif HDFS_NN_STATE_ACTIVE in out:
state = HDFS_NN_STATE_ACTIVE
if state == HDFS_NN_STATE_ACTIVE:
active_namenodes.append((nn_unique_id, value))
elif state == HDFS_NN_STATE_STANDBY:
standby_namenodes.append((nn_unique_id, value))
else:
unknown_namenodes.append((nn_unique_id, value))
return active_namenodes, standby_namenodes, unknown_namenodes
def hive(name=None):
import params
if name == 'hiveserver2':
# copy tarball to HDFS feature not supported
if not (params.stack_version_formatted_major
and check_stack_feature(StackFeature.COPY_TARBALL_TO_HDFS,
params.stack_version_formatted_major)):
params.HdfsResource(params.webhcat_apps_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=0755)
# Create webhcat dirs.
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.hcat_hdfs_user_mode)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode)
# ****** Begin Copy Tarballs ******
# *********************************
# if copy tarball to HDFS feature supported copy mapreduce.tar.gz and tez.tar.gz to HDFS
if params.stack_version_formatted_major and check_stack_feature(
StackFeature.COPY_TARBALL_TO_HDFS,
params.stack_version_formatted_major):
copy_to_hdfs("mapreduce",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("tez",
params.user_group,
params.hdfs_user,
host_sys_prepped=params.host_sys_prepped)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name +
"_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(
source_file_pattern) if "*" in source_file_pattern else [
source_file_pattern
]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file,
host_sys_prepped=params.host_sys_prepped)
# ******* End Copy Tarballs *******
# *********************************
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(
params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(
params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777
) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
Directory(params.hive_etc_dir_prefix, mode=0755)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']
['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
# Generate atlas-application.properties.xml file
if has_atlas_in_cluster():
atlas_hook_filepath = os.path.join(params.hive_config_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
if name == 'hiveserver2':
XmlConfig(
"hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']
['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
if params.hive_metastore_site_supported and name == 'metastore':
XmlConfig(
"hivemetastore-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']
['hivemetastore-site'],
configuration_attributes=params.config['configuration_attributes']
['hivemetastore-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if name == 'metastore' or name == 'hiveserver2':
if params.hive_jdbc_target is not None and not os.path.exists(
params.hive_jdbc_target):
jdbc_connector(params.hive_jdbc_target,
params.hive_previous_jdbc_jar)
if params.hive2_jdbc_target is not None and not os.path.exists(
params.hive2_jdbc_target):
jdbc_connector(params.hive2_jdbc_target,
params.hive2_previous_jdbc_jar)
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
if name == 'metastore':
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hivemetastore.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hivemetastore.properties.j2"))
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.init_metastore_schema:
create_schema_cmd = format(
"export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_schematool_bin}/schematool -initSchema "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p} -verbose")
check_schema_created_cmd = as_user(
format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_schematool_bin}/schematool -info "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p} -verbose"),
params.hive_user)
# HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
# Fixing it with the hack below:
quoted_hive_metastore_user_passwd = quote_bash_args(
quote_bash_args(params.hive_metastore_user_passwd))
if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[
1:-1]
Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(
check_schema_created_cmd.replace(
format("-passWord {quoted_hive_metastore_user_passwd}"),
"-passWord " + utils.PASSWORDS_HIDE_STRING))
Execute(create_schema_cmd,
not_if=check_schema_created_cmd,
user=params.hive_user)
elif name == 'hiveserver2':
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}')))
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
if name != "client":
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
def enable_kms_plugin():
import params
if params.has_ranger_admin:
ranger_flag = False
if params.stack_supports_ranger_kerberos and params.security_enabled:
if not is_empty(params.rangerkms_principal
) and params.rangerkms_principal != '':
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.rangerkms_keytab,
params.rangerkms_principal)
else:
ranger_flag = check_ranger_service_support_kerberos(
params.kms_user, params.spengo_keytab,
params.spnego_principal)
else:
ranger_flag = check_ranger_service()
if not ranger_flag:
Logger.error('Error in Get/Create service for Ranger Kms.')
current_datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
File(format('{kms_conf_dir}/ranger-security.xml'),
owner=params.kms_user,
group=params.kms_group,
mode=0644,
content=format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
Directory([
os.path.join('/etc', 'ranger', params.repo_name),
os.path.join('/etc', 'ranger', params.repo_name, 'policycache')
],
owner=params.kms_user,
group=params.kms_group,
mode=0775,
create_parents=True)
File(os.path.join('/etc', 'ranger', params.repo_name, 'policycache',
format('kms_{repo_name}.json')),
owner=params.kms_user,
group=params.kms_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(
params.config['configurations']['ranger-kms-audit'])
if params.plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
params.plugin_audit_password_property] = "crypted"
XmlConfig(
"ranger-kms-audit.xml",
conf_dir=params.kms_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-audit'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
XmlConfig(
"ranger-kms-security.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']
['ranger-kms-security'],
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-security'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
# remove plain-text password from xml configs
ranger_kms_policymgr_ssl_copy = {}
ranger_kms_policymgr_ssl_copy.update(
params.config['configurations']['ranger-kms-policymgr-ssl'])
for prop in params.kms_plugin_password_properties:
if prop in ranger_kms_policymgr_ssl_copy:
ranger_kms_policymgr_ssl_copy[prop] = "crypted"
XmlConfig(
"ranger-policymgr-ssl.xml",
conf_dir=params.kms_conf_dir,
configurations=ranger_kms_policymgr_ssl_copy,
configuration_attributes=params.config['configuration_attributes']
['ranger-kms-policymgr-ssl'],
owner=params.kms_user,
group=params.kms_group,
mode=0744)
if params.xa_audit_db_is_enabled:
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'auditDBCred', '-v',
PasswordString(params.xa_audit_db_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslKeyStore', '-v',
PasswordString(params.ssl_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = params.cred_setup_prefix + (
'-f', params.credential_file, '-k', 'sslTrustStore', '-v',
PasswordString(params.ssl_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.credential_file,
owner=params.kms_user,
group=params.kms_group,
mode=0640)
# create ranger kms audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kms",
type="directory",
action="create_on_execute",
owner=params.kms_user,
group=params.kms_group,
mode=0750,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.xa_audit_hdfs_is_enabled and len(params.namenode_host) > 1:
Logger.info(
'Audit to Hdfs enabled in NameNode HA environment, creating hdfs-site.xml'
)
XmlConfig(
"hdfs-site.xml",
conf_dir=params.kms_conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.kms_user,
group=params.kms_group,
mode=0644)
else:
File(format('{kms_conf_dir}/hdfs-site.xml'), action="delete")
usersync_log_dir = default("/configurations/ranger-env/ranger_usersync_log_dir", "/var/log/ranger/usersync")
admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir", "/var/log/ranger/admin")
ranger_admin_default_file = format('{ranger_conf}/ranger-admin-default-site.xml')
security_app_context_file = format('{ranger_conf}/security-applicationContext.xml')
ranger_ugsync_default_file = format('{ranger_ugsync_conf}/ranger-ugsync-default.xml')
usgsync_log4j_file = format('{ranger_ugsync_conf}/log4j.xml')
cred_validator_file = format('{usersync_home}/native/credValidator.uexe')
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
usersync_exturl = config['configurations']['admin-properties']['policymgr_external_url']
ranger_host = config['clusterHostInfo']['ranger_admin_hosts'][0]
ugsync_host = 'localhost'
usersync_host_info = config['clusterHostInfo']['ranger_usersync_hosts']
if not is_empty(usersync_host_info) and len(usersync_host_info) > 0:
ugsync_host = config['clusterHostInfo']['ranger_usersync_hosts'][0]
ranger_external_url = config['configurations']['admin-properties']['policymgr_external_url']
ranger_db_name = config['configurations']['admin-properties']['db_name']
ranger_auditdb_name = config['configurations']['admin-properties']['audit_db_name']
sql_command_invoker = config['configurations']['admin-properties']['SQL_COMMAND_INVOKER']
db_root_user = config['configurations']['admin-properties']['db_root_user']
db_root_password = unicode(config['configurations']['admin-properties']['db_root_password'])
db_host = config['configurations']['admin-properties']['db_host']
ranger_db_user = config['configurations']['admin-properties']['db_user']
ranger_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
ranger_db_password = unicode(config['configurations']['admin-properties']['db_password'])
#ranger-env properties
oracle_home = default("/configurations/ranger-env/oracle_home", "-")
repo_config_username = config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_USERNAME']
if security_enabled:
sn_principal_name = default("/configurations/hdfs-site/dfs.secondary.namenode.kerberos.principal", "nn/[email protected]")
sn_principal_name = sn_principal_name.replace('_HOST',hostname.lower())
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations']['ranger-hdfs-plugin-properties']
policy_user = config['configurations']['ranger-hdfs-plugin-properties']['policy_user']
#For curl command in ranger plugin to get db connector
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
is_https_enabled = config['configurations']['hdfs-site']['dfs.https.enable'] if \
not is_empty(config['configurations']['hdfs-site']['dfs.https.enable']) else False
if has_ranger_admin:
enable_ranger_hdfs = (config['configurations']['ranger-hdfs-plugin-properties']['ranger-hdfs-plugin-enabled'].lower() == 'yes')
xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
repo_config_password = unicode(config['configurations']['ranger-hdfs-plugin-properties']['REPOSITORY_CONFIG_PASSWORD'])
xa_audit_db_flavor = (config['configurations']['admin-properties']['DB_FLAVOR']).lower()
if xa_audit_db_flavor == 'mysql':
jdbc_symlink_name = "mysql-jdbc-driver.jar"
jdbc_jar_name = "mysql-connector-java.jar"
audit_jdbc_url = format('jdbc:mysql://{xa_db_host}/{xa_audit_db_name}')
jdbc_driver = "com.mysql.jdbc.Driver"
elif xa_audit_db_flavor == 'oracle':
jdbc_jar_name = "ojdbc6.jar"
jdbc_symlink_name = "oracle-jdbc-driver.jar"
def hive_interactive(name=None):
import params
MB_TO_BYTES = 1048576
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_metastore_warehouse_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0700)
# create directories for llap package
pkg_dir = '/user/' + params.hive_user + '/.yarn'
for dir in [pkg_dir, pkg_dir + '/package', pkg_dir + '/package/LLAP']:
# hdfsresouces handles parent creation badly
params.HdfsResource(dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.user_group,
mode=0755)
if not is_empty(params.tez_hook_proto_base_directory):
params.HdfsResource(params.tez_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01755)
if not is_empty(params.hive_hook_proto_base_directory):
params.HdfsResource(params.hive_hook_proto_base_directory,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_meta = params.tez_hook_proto_base_directory + "dag_meta"
params.HdfsResource(dag_meta,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
dag_data = params.tez_hook_proto_base_directory + "dag_data"
params.HdfsResource(dag_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
app_data = params.tez_hook_proto_base_directory + "app_data"
params.HdfsResource(app_data,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=01777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_metastore_warehouse_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
# list of properties that should be excluded from the config
# this approach is a compromise against adding a dedicated config
# type for hive_server_interactive or needed config groups on a
# per component basis
exclude_list = ['hive.enforce.bucketing', 'hive.enforce.sorting']
# List of configs to be excluded from hive2 client, but present in Hive2 server.
exclude_list_for_hive2_client = [
'javax.jdo.option.ConnectionPassword',
'hadoop.security.credential.provider.path'
]
Logger.info("Directories to fill with configs: %s" %
str(params.hive_conf_dirs_list))
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
'''
As hive2/hive-site.xml only contains the new + the changed props compared to hive/hive-site.xml,
we need to merge hive/hive-site.xml and hive2/hive-site.xml and store it in hive2/hive-site.xml.
'''
merged_hive_interactive_site = {}
merged_hive_interactive_site.update(params.hive_site_config)
merged_hive_interactive_site.update(
params.config['configurations']['hive-interactive-site'])
for item in exclude_list:
if item in merged_hive_interactive_site.keys():
del merged_hive_interactive_site[item]
merged_hive_interactive_site[
'hive.llap.daemon.vcpus.per.instance'] = format(
merged_hive_interactive_site['hive.llap.daemon.vcpus.per.instance']
)
merged_hive_interactive_site[
'hive.server2.active.passive.ha.enable'] = str(
params.hive_server_interactive_ha).lower()
'''
Config 'hive.llap.io.memory.size' calculated value in stack_advisor is in MB as of now. We need to
convert it to bytes before we write it down to config file.
'''
if 'hive.llap.io.memory.size' in merged_hive_interactive_site.keys():
hive_llap_io_mem_size_in_mb = merged_hive_interactive_site.get(
"hive.llap.io.memory.size")
hive_llap_io_mem_size_in_bytes = long(
hive_llap_io_mem_size_in_mb) * MB_TO_BYTES
merged_hive_interactive_site[
'hive.llap.io.memory.size'] = hive_llap_io_mem_size_in_bytes
Logger.info(
"Converted 'hive.llap.io.memory.size' value from '{0} MB' to '{1} Bytes' before writing "
"it to config file.".format(hive_llap_io_mem_size_in_mb,
hive_llap_io_mem_size_in_bytes))
'''
Hive2 doesn't have support for Atlas, we need to remove the Hook 'org.apache.atlas.hive.hook.HiveHook',
which would have come in config 'hive.exec.post.hooks' during the site merge logic, if Atlas is installed.
'''
# Generate atlas-application.properties.xml file
if params.enable_atlas_hook and params.stack_supports_atlas_hook_for_hive_interactive:
Logger.info("Setup for Atlas Hive2 Hook started.")
atlas_hook_filepath = os.path.join(
params.hive_server_interactive_conf_dir,
params.atlas_hook_filename)
setup_atlas_hook(SERVICE.HIVE,
params.hive_atlas_application_properties,
atlas_hook_filepath, params.hive_user,
params.user_group)
Logger.info("Setup for Atlas Hive2 Hook done.")
else:
# Required for HDP 2.5 stacks
Logger.info(
"Skipping setup for Atlas Hook, as it is disabled/ not supported.")
remove_atlas_hook_if_exists(merged_hive_interactive_site)
'''
As tez_hive2/tez-site.xml only contains the new + the changed props compared to tez/tez-site.xml,
we need to merge tez/tez-site.xml and tez_hive2/tez-site.xml and store it in tez_hive2/tez-site.xml.
'''
merged_tez_interactive_site = {}
if 'tez-site' in params.config['configurations']:
merged_tez_interactive_site.update(
params.config['configurations']['tez-site'])
Logger.info(
"Retrieved 'tez/tez-site' for merging with 'tez_hive2/tez-interactive-site'."
)
else:
Logger.error(
"Tez's 'tez-site' couldn't be retrieved from passed-in configurations."
)
merged_tez_interactive_site.update(
params.config['configurations']['tez-interactive-site'])
XmlConfig("tez-site.xml",
conf_dir=params.tez_interactive_conf_dir,
configurations=merged_tez_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['tez-interactive-site'],
owner=params.tez_interactive_user,
group=params.user_group,
mode=0664)
'''
Merge properties from hiveserver2-interactive-site into hiveserver2-site
'''
merged_hiveserver2_interactive_site = {}
if 'hiveserver2-site' in params.config['configurations']:
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-site'])
Logger.info(
"Retrieved 'hiveserver2-site' for merging with 'hiveserver2-interactive-site'."
)
else:
Logger.error(
"'hiveserver2-site' couldn't be retrieved from passed-in configurations."
)
merged_hiveserver2_interactive_site.update(
params.config['configurations']['hiveserver2-interactive-site'])
# Create config files under hive_server_interactive_conf_dir:
# hive-site.xml
# hive-env.sh
# llap-daemon-log4j2.properties
# llap-cli-log4j2.properties
# hive-log4j2.properties
# hive-exec-log4j2.properties
# beeline-log4j2.properties
hive_server_interactive_conf_dir = params.hive_server_interactive_conf_dir
mode_identified = 0600
merged_hive_interactive_site = update_credential_provider_path(
merged_hive_interactive_site, 'hive-site',
os.path.join(conf_dir, 'hive-site.jceks'), params.hive_user,
params.user_group)
XmlConfig("hive-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hive_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hive-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
XmlConfig("hiveserver2-site.xml",
conf_dir=hive_server_interactive_conf_dir,
configurations=merged_hiveserver2_interactive_site,
configuration_attributes=params.config['configurationAttributes']
['hiveserver2-interactive-site'],
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(format("{hive_server_interactive_conf_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.hive_interactive_env_sh_template))
llap_daemon_log4j_filename = 'llap-daemon-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_daemon_log4j_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_daemon_log4j))
llap_cli_log4j2_filename = 'llap-cli-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{llap_cli_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.llap_cli_log4j2))
hive_log4j2_filename = 'hive-log4j2.properties'
File(format("{hive_server_interactive_conf_dir}/{hive_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_log4j2))
hive_exec_log4j2_filename = 'hive-exec-log4j2.properties'
File(format(
"{hive_server_interactive_conf_dir}/{hive_exec_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.hive_exec_log4j2))
beeline_log4j2_filename = 'beeline-log4j2.properties'
File(
format("{hive_server_interactive_conf_dir}/{beeline_log4j2_filename}"),
mode=mode_identified,
group=params.user_group,
owner=params.hive_user,
content=InlineTemplate(params.beeline_log4j2))
XmlConfig("beeline-site.xml",
conf_dir=conf_dir,
configurations=params.beeline_site_config,
owner=params.hive_user,
group=params.user_group,
mode=mode_identified)
File(os.path.join(hive_server_interactive_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llapdaemon.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llapdaemon.j2"))
File(format(
"{hive_server_interactive_conf_dir}/hadoop-metrics2-llaptaskscheduler.properties"
),
owner=params.hive_user,
group=params.user_group,
mode=mode_identified,
content=Template("hadoop-metrics2-llaptaskscheduler.j2"))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
create_parents=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if not os.path.exists(params.target_hive_interactive):
jdbc_connector(params.target_hive_interactive,
params.hive_intaractive_previous_jdbc_jar)
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}/{check_db_connection_jar_name}")),
mode=0644)
File(params.start_hiveserver2_interactive_path,
mode=0755,
content=Template(format('{start_hiveserver2_interactive_script}')))
Directory(params.hive_pid_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_log_dir,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
Directory(params.hive_interactive_var_lib,
create_parents=True,
cd_access='a',
owner=params.hive_user,
group=params.user_group,
mode=0755)
generate_logfeeder_input_config(
'hive', Template("input.config-hive.json.j2", extra_imports=[default]))
def yarn(name=None, config_dir=None):
"""
:param name: Component name, apptimelineserver, nodemanager, resourcemanager, or None (defaults for client)
:param config_dir: Which config directory to write configs to, which could be different during rolling upgrade.
"""
import params
if config_dir is None:
config_dir = params.hadoop_conf_dir
if name == "historyserver":
if params.yarn_log_aggregation_enabled:
params.HdfsResource(params.yarn_nm_app_log_dir,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group,
mode=01777,
recursive_chmod=True)
# create the /tmp folder with proper permissions if it doesn't exist yet
if params.entity_file_history_directory.startswith('/tmp'):
params.HdfsResource(
params.hdfs_tmp_dir,
action="create_on_execute",
type="directory",
owner=params.hdfs_user,
mode=0777,
)
params.HdfsResource(params.entity_file_history_directory,
action="create_on_execute",
type="directory",
owner=params.yarn_user,
group=params.user_group)
params.HdfsResource("/mapred",
type="directory",
action="create_on_execute",
owner=params.mapred_user)
params.HdfsResource("/mapred/system",
type="directory",
action="create_on_execute",
owner=params.hdfs_user)
params.HdfsResource(params.mapreduce_jobhistory_done_dir,
type="directory",
action="create_on_execute",
owner=params.mapred_user,
group=params.user_group,
change_permissions_for_parents=True,
mode=0777)
params.HdfsResource(None, action="execute")
Directory(
params.jhs_leveldb_state_store_dir,
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access="a",
recursive_ownership=True,
)
#<editor-fold desc="Node Manager Section">
if name == "nodemanager":
# First start after enabling/disabling security
if params.toggle_nm_security:
Directory(params.nm_local_dirs_list + params.nm_log_dirs_list,
action='delete')
# If yarn.nodemanager.recovery.dir exists, remove this dir
if params.yarn_nodemanager_recovery_dir:
Directory(InlineTemplate(
params.yarn_nodemanager_recovery_dir).get_content(),
action='delete')
# Setting NM marker file
if params.security_enabled:
Directory(params.nm_security_marker_dir)
File(
params.nm_security_marker,
content=
"Marker file to track first start after enabling/disabling security. "
"During first start yarn local, log dirs are removed and recreated"
)
elif not params.security_enabled:
File(params.nm_security_marker, action="delete")
if not params.security_enabled or params.toggle_nm_security:
# handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable (unmounted), and intended to reside on a different mount.
nm_log_dir_to_mount_file_content = handle_mounted_dirs(
create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
# create a history file used by handle_mounted_dirs
File(params.nm_log_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_log_dir_to_mount_file_content)
nm_local_dir_to_mount_file_content = handle_mounted_dirs(
create_local_dir, params.nm_local_dirs,
params.nm_local_dir_to_mount_file, params)
File(params.nm_local_dir_to_mount_file,
owner=params.hdfs_user,
group=params.user_group,
mode=0644,
content=nm_local_dir_to_mount_file_content)
#</editor-fold>
if params.yarn_nodemanager_recovery_dir:
Directory(
InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access='a',
)
Directory(
[params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[
params.mapred_pid_dir_prefix, params.mapred_pid_dir,
params.mapred_log_dir_prefix, params.mapred_log_dir
],
owner=params.mapred_user,
group=params.user_group,
create_parents=True,
cd_access='a',
)
Directory(
[params.yarn_log_dir_prefix],
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
ignore_failures=True,
cd_access='a',
)
XmlConfig(
"core-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['core-site'],
configuration_attributes=params.config['configuration_attributes']
['core-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
# During RU, Core Masters and Slaves need hdfs-site.xml
# TODO, instead of specifying individual configs, which is susceptible to breaking when new configs are added,
# RU should rely on all available in <stack-root>/<version>/hadoop/conf
if 'hdfs-site' in params.config['configurations']:
XmlConfig(
"hdfs-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.hdfs_user,
group=params.user_group,
mode=0644)
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"yarn-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['yarn-site'],
configuration_attributes=params.config['configuration_attributes']
['yarn-site'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.yarn_user,
group=params.user_group,
mode=0644)
if name == 'resourcemanager':
Directory(
params.rm_nodes_exclude_dir,
mode=0755,
create_parents=True,
cd_access='a',
)
File(params.rm_nodes_exclude_path,
owner=params.yarn_user,
group=params.user_group)
File(params.yarn_job_summary_log,
owner=params.yarn_user,
group=params.user_group)
if not is_empty(
params.node_label_enable
) and params.node_label_enable or is_empty(
params.node_label_enable) and params.node_labels_dir:
params.HdfsResource(params.node_labels_dir,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0700)
params.HdfsResource(None, action="execute")
elif name == 'apptimelineserver':
Directory(
params.ats_leveldb_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# if stack support application timeline-service state store property (timeline_state_store stack feature)
if params.stack_supports_timeline_state_store:
Directory(
params.ats_leveldb_state_store_dir,
owner=params.yarn_user,
group=params.user_group,
create_parents=True,
cd_access="a",
)
# app timeline server 1.5 directories
if not is_empty(params.entity_groupfs_store_dir):
parent_path = os.path.dirname(params.entity_groupfs_store_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_store_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_store_dir_mode)
if not is_empty(params.entity_groupfs_active_dir):
parent_path = os.path.dirname(params.entity_groupfs_active_dir)
params.HdfsResource(parent_path,
type="directory",
action="create_on_execute",
change_permissions_for_parents=True,
owner=params.yarn_user,
group=params.user_group,
mode=0755)
params.HdfsResource(params.entity_groupfs_active_dir,
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.user_group,
mode=params.entity_groupfs_active_dir_mode)
params.HdfsResource(None, action="execute")
File(format("{limits_conf_dir}/yarn.conf"),
mode=0644,
content=Template('yarn.conf.j2'))
File(format("{limits_conf_dir}/mapreduce.conf"),
mode=0644,
content=Template('mapreduce.conf.j2'))
File(os.path.join(config_dir, "yarn-env.sh"),
owner=params.yarn_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.yarn_env_sh_template))
container_executor = format("{yarn_container_bin}/container-executor")
File(container_executor,
group=params.yarn_executor_container_group,
mode=params.container_executor_mode)
File(os.path.join(config_dir, "container-executor.cfg"),
group=params.user_group,
mode=0644,
content=Template('container-executor.cfg.j2'))
Directory(params.cgroups_dir,
group=params.user_group,
create_parents=True,
mode=0755,
cd_access="a")
if params.security_enabled:
tc_mode = 0644
tc_owner = "root"
else:
tc_mode = None
tc_owner = params.hdfs_user
File(os.path.join(config_dir, "mapred-env.sh"),
owner=tc_owner,
mode=0755,
content=InlineTemplate(params.mapred_env_sh_template))
if params.security_enabled:
File(os.path.join(params.hadoop_bin, "task-controller"),
owner="root",
group=params.mapred_tt_group,
mode=06050)
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
mode=tc_mode,
group=params.mapred_tt_group,
content=Template("taskcontroller.cfg.j2"))
else:
File(os.path.join(config_dir, 'taskcontroller.cfg'),
owner=tc_owner,
content=Template("taskcontroller.cfg.j2"))
if "mapred-site" in params.config['configurations']:
XmlConfig(
"mapred-site.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['mapred-site'],
configuration_attributes=params.config['configuration_attributes']
['mapred-site'],
owner=params.mapred_user,
group=params.user_group)
if "capacity-scheduler" in params.config['configurations']:
XmlConfig(
"capacity-scheduler.xml",
conf_dir=config_dir,
configurations=params.config['configurations']
['capacity-scheduler'],
configuration_attributes=params.config['configuration_attributes']
['capacity-scheduler'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-client" in params.config['configurations']:
XmlConfig(
"ssl-client.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
Directory(
params.hadoop_conf_secure_dir,
create_parents=True,
owner='root',
group=params.user_group,
cd_access='a',
)
XmlConfig(
"ssl-client.xml",
conf_dir=params.hadoop_conf_secure_dir,
configurations=params.config['configurations']['ssl-client'],
configuration_attributes=params.config['configuration_attributes']
['ssl-client'],
owner=params.hdfs_user,
group=params.user_group)
if "ssl-server" in params.config['configurations']:
XmlConfig(
"ssl-server.xml",
conf_dir=config_dir,
configurations=params.config['configurations']['ssl-server'],
configuration_attributes=params.config['configuration_attributes']
['ssl-server'],
owner=params.hdfs_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'fair-scheduler.xml')):
File(os.path.join(config_dir, 'fair-scheduler.xml'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-client.xml.example')):
File(os.path.join(config_dir, 'ssl-client.xml.example'),
owner=params.mapred_user,
group=params.user_group)
if os.path.exists(os.path.join(config_dir, 'ssl-server.xml.example')):
File(os.path.join(config_dir, 'ssl-server.xml.example'),
owner=params.mapred_user,
group=params.user_group)
def setup_ranger_hive_metastore_service():
"""
Creates ranger hive service in ranger admin installed in same cluster for cluster depolyed in cloud env.
"""
import params
if params.has_ranger_admin and params.ranger_hive_metastore_lookup:
repo_name = str(params.config['clusterName']) + '_hive'
repo_name_value = params.config['configurations'][
'ranger-hive-security']['ranger.plugin.hive.service.name']
if not is_empty(
repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
hive_ranger_plugin_config = {
'username':
params.config['configurations']['ranger-hive-plugin-properties']
['REPOSITORY_CONFIG_USERNAME'],
'password':
params.config['configurations']['ranger-hive-plugin-properties']
['REPOSITORY_CONFIG_PASSWORD'],
'jdbc.driverClassName':
params.config['configurations']['ranger-hive-plugin-properties']
['jdbc.driverClassName'],
'jdbc.url':
'none',
'commonNameForCertificate':
params.config['configurations']['ranger-hive-plugin-properties']
['common.name.for.certificate'],
'ambari.service.check.user':
params.config['configurations']['ranger-hive-plugin-properties']
['policy_user']
}
if params.security_enabled:
hive_ranger_plugin_config[
'policy.download.auth.users'] = params.hive_user
hive_ranger_plugin_config[
'tag.download.auth.users'] = params.hive_user
hive_ranger_plugin_config[
'policy.grantrevoke.auth.users'] = params.hive_user
custom_ranger_service_config = generate_ranger_service_config(
params.config['configurations']['ranger-hive-plugin-properties'])
if len(custom_ranger_service_config) > 0:
hive_ranger_plugin_config.update(custom_ranger_service_config)
hive_ranger_plugin_repo = {
'isEnabled': 'true',
'configs': hive_ranger_plugin_config,
'description': 'Hive service',
'name': repo_name,
'type': 'hive'
}
ranger_admin_obj = RangeradminV2(
url=params.config['configurations']['ranger-hive-security']
['ranger.plugin.hive.policy.rest.url'],
skip_if_rangeradmin_down=not params.retryAble)
ranger_admin_obj.create_ranger_repository(
component='hive',
repo_name=repo_name,
repo_properties=hive_ranger_plugin_repo,
ambari_ranger_admin=params.config['configurations']['ranger-env']
['ranger_admin_username'],
ambari_ranger_password=params.config['configurations']
['ranger-env']['ranger_admin_password'],
admin_uname=params.config['configurations']['ranger-env']
['admin_username'],
admin_password=params.config['configurations']['ranger-env']
['admin_password'],
policy_user=params.config['configurations']
['ranger-hive-plugin-properties']['policy_user'],
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user=params.hive_user,
component_user_principal=params.hive_metastore_principal_with_host
if params.security_enabled else None,
component_user_keytab=params.hive_metastore_keytab_path
if params.security_enabled else None)
def hive(name=None):
import params
if name == 'hiveserver2':
# HDP 2.1.* or lower
if params.hdp_stack_version_major != "" and compare_versions(
params.hdp_stack_version_major, "2.2.0.0") < 0:
params.HdfsResource(params.webhcat_apps_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=0755)
# Create webhcat dirs.
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode)
# ****** Begin Copy Tarballs ******
# *********************************
# HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS
if params.hdp_stack_version_major != "" and compare_versions(
params.hdp_stack_version_major, '2.2') >= 0:
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user)
copy_to_hdfs("tez", params.user_group, params.hdfs_user)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account for both HDP 2.1 and 2.2
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name +
"_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(
source_file_pattern) if "*" in source_file_pattern else [
source_file_pattern
]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file)
# ******* End Copy Tarballs *******
# *********************************
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777)
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(
params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(
params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777
) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
Directory(params.hive_etc_dir_prefix, mode=0755)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig(
"hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']
['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
setup_atlas_hive()
if params.hive_specific_configs_supported and name == 'hiveserver2':
XmlConfig(
"hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']
['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template))
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
recursive=True,
owner='root',
group='root')
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2"))
if name == 'metastore' or name == 'hiveserver2':
jdbc_connector()
File(
format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content=DownloadSource(
format("{jdk_location}{check_db_connection_jar_name}")),
mode=0644,
)
if name == 'metastore':
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh'))
if params.init_metastore_schema:
create_schema_cmd = format(
"export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -initSchema "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}")
check_schema_created_cmd = as_user(
format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -info "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}"),
params.hive_user)
Execute(create_schema_cmd,
not_if=check_schema_created_cmd,
user=params.hive_user)
elif name == 'hiveserver2':
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}')))
if name != "client":
crt_directory(params.hive_pid_dir)
crt_directory(params.hive_log_dir)
crt_directory(params.hive_var_lib)
# upgrades would cause these directories to have a version instead of "current"
# which would cause a lot of problems when writing out hadoop-env.sh; instead
# force the use of "current" in the hook
hdfs_user_nofile_limit = default(
"/configurations/hadoop-env/hdfs_user_nofile_limit", "128000")
hadoop_home = stack_select.get_hadoop_dir("home")
hadoop_libexec_dir = stack_select.get_hadoop_dir("libexec")
hadoop_lib_home = stack_select.get_hadoop_dir("lib")
hadoop_dir = "/etc/hadoop"
hadoop_java_io_tmpdir = os.path.join(tmp_dir, "hadoop_java_io_tmpdir")
datanode_max_locked_memory = config['configurations']['hdfs-site'][
'dfs.datanode.max.locked.memory']
is_datanode_max_locked_memory_set = not is_empty(
config['configurations']['hdfs-site']['dfs.datanode.max.locked.memory'])
mapreduce_libs_path = "/usr/hdp/current/hadoop-mapreduce-client/*"
if not security_enabled:
hadoop_secure_dn_user = '******'
else:
dfs_dn_port = get_port(dfs_dn_addr)
dfs_dn_http_port = get_port(dfs_dn_http_addr)
dfs_dn_https_port = get_port(dfs_dn_https_addr)
# We try to avoid inability to start datanode as a plain user due to usage of root-owned ports
if dfs_http_policy == "HTTPS_ONLY":
secure_dn_ports_are_in_use = is_secure_port(
dfs_dn_port) or is_secure_port(dfs_dn_https_port)
elif dfs_http_policy == "HTTP_AND_HTTPS":
secure_dn_ports_are_in_use = is_secure_port(
def setup_hiveserver2():
import params
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}')))
File(os.path.join(params.hive_server_conf_dir,
"hadoop-metrics2-hiveserver2.properties"),
owner=params.hive_user,
group=params.user_group,
content=Template("hadoop-metrics2-hiveserver2.properties.j2"),
mode=0600)
XmlConfig(
"hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']
['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0600)
# copy tarball to HDFS feature not supported
params.HdfsResource(params.webhcat_apps_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=0755)
# Create webhcat dirs.
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.hcat_hdfs_user_mode)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode)
# if warehouse directory is in DFS
if not params.whs_dir_protocol or params.whs_dir_protocol == urlparse(
params.default_fs).scheme:
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777)
else:
Logger.info(
format(
"Not creating warehouse directory '{hive_apps_whs_dir}', as the location is not in DFS."
))
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(
params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(
params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777
) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
def setup_tagsync(upgrade_type=None):
import params
ranger_tagsync_home = params.ranger_tagsync_home
ranger_home = params.ranger_home
ranger_tagsync_conf = params.ranger_tagsync_conf
tagsync_log4j_file = format('{ranger_tagsync_conf}/log4j.xml')
Directory(format("{ranger_tagsync_conf}"),
owner = params.unix_user,
group = params.unix_group,
create_parents = True
)
Directory(params.ranger_pid_dir,
mode=0750,
create_parents=True,
owner = params.unix_user,
group = params.unix_group,
cd_access = "a",
)
Directory(params.tagsync_log_dir,
create_parents = True,
owner = params.unix_user,
group = params.unix_group,
cd_access = "a",
mode=0755
)
File(format('{ranger_tagsync_conf}/ranger-tagsync-env-logdir.sh'),
content = format("export RANGER_TAGSYNC_LOG_DIR={tagsync_log_dir}"),
owner = params.unix_user,
group = params.unix_group,
mode=0755
)
XmlConfig("ranger-tagsync-site.xml",
conf_dir=ranger_tagsync_conf,
configurations=params.config['configurations']['ranger-tagsync-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-tagsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
PropertiesFile(format('{ranger_tagsync_conf}/application.properties'),
properties = params.tagsync_application_properties,
mode=0755,
owner=params.unix_user,
group=params.unix_group
)
if params.stack_supports_ranger_log4j:
File(format('{ranger_tagsync_conf}/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=params.tagsync_log4j,
mode=0644
)
src_file = format('{ranger_tagsync_home}/conf.dist/log4j.xml')
dst_file = format('{tagsync_log4j_file}')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if os.path.isfile(tagsync_log4j_file):
File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group)
else:
Logger.warning('Required file {0} does not exist, copying the file to {1} path'.format(tagsync_log4j_file, ranger_tagsync_conf))
src_file = format('{ranger_tagsync_home}/conf.dist/log4j.xml')
dst_file = format('{tagsync_log4j_file}')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
File(tagsync_log4j_file, owner=params.unix_user, group=params.unix_group)
cred_lib = os.path.join(ranger_tagsync_home,"lib","*")
if not is_empty(params.tagsync_jceks_path) and not is_empty(params.ranger_tagsync_tagadmin_password) and params.tagsync_enabled:
ranger_credential_helper(cred_lib, 'tagadmin.user.password', params.ranger_tagsync_tagadmin_password, params.tagsync_jceks_path)
File(params.tagsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File(params.tagsync_services_file,
mode = 0755,
)
Execute(('ln','-sf', format('{tagsync_services_file}'),'/usr/bin/ranger-tagsync'),
not_if=format("ls /usr/bin/ranger-tagsync"),
only_if=format("ls {tagsync_services_file}"),
sudo=True)
create_core_site_xml(ranger_tagsync_conf)
if check_stack_feature(StackFeature.HIVE_WEBHCAT_SPECIFIC_CONFIGS, version_for_stack_feature_checks):
# this is NOT a typo. Configs for hcatalog/webhcat point to a
# specific directory which is NOT called 'conf'
hcat_conf_dir = format('{stack_root}/current/hive-webhcat/etc/hcatalog')
config_dir = format('{stack_root}/current/hive-webhcat/etc/webhcat')
if check_stack_feature(StackFeature.HIVE_METASTORE_SITE_SUPPORT, version_for_stack_feature_checks):
hive_metastore_site_supported = True
execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir
hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
#JDBC driver jar name
hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
hive_database_name = config['configurations']['hive-env']['hive_database_name']
hive_database = config['configurations']['hive-env']['hive_database']
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0755,
owner = params.unix_user,
group = params.user_group,
cd_access = "a",
create_parents=True
)
if params.stack_supports_pid:
File(format('{ranger_ugsync_conf}/ranger-usersync-env-piddir.sh'),
content = format("export USERSYNC_PID_DIR_PATH={ranger_pid_dir}\nexport UNIX_USERSYNC_USER={unix_user}"),
owner = params.unix_user,
group = params.unix_group,
mode=0755
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group,
cd_access = 'a',
create_parents=True,
mode=0755,
recursive_ownership = True
)
File(format('{ranger_ugsync_conf}/ranger-usersync-env-logdir.sh'),
content = format("export logdir={usersync_log_dir}"),
owner = params.unix_user,
group = params.unix_group,
mode=0755
)
Directory(format("{ranger_ugsync_conf}/"),
owner = params.unix_user
)
if upgrade_type is not None:
src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
if params.stack_supports_ranger_log4j:
File(format('{usersync_home}/conf/log4j.properties'),
owner=params.unix_user,
group=params.unix_group,
content=InlineTemplate(params.usersync_log4j),
mode=0644
)
elif upgrade_type is not None and not params.stack_supports_ranger_log4j:
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# remove plain-text password from xml configs
ranger_ugsync_site_copy = {}
ranger_ugsync_site_copy.update(params.config['configurations']['ranger-ugsync-site'])
for prop in params.ranger_usersync_password_properties:
if prop in ranger_ugsync_site_copy:
ranger_ugsync_site_copy[prop] = "_"
XmlConfig("ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=ranger_ugsync_site_copy,
configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.key.password', params.ranger_usersync_keystore_password, params.ugsync_jceks_path)
if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
ranger_credential_helper(params.ugsync_cred_lib, 'ranger.usersync.ldap.bindalias', params.ranger_usersync_ldap_ldapbindpassword, params.ugsync_jceks_path)
ranger_credential_helper(params.ugsync_cred_lib, 'usersync.ssl.truststore.password', params.ranger_usersync_truststore_password, params.ugsync_jceks_path)
File(params.ugsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user,
group = params.unix_group
)
File(params.usersync_services_file,
mode = 0755,
)
if upgrade_type is not None and params.stack_supports_config_versioning:
if os.path.islink('/usr/bin/ranger-usersync'):
Link('/usr/bin/ranger-usersync', action="delete")
Link('/usr/bin/ranger-usersync', to=params.usersync_services_file)
Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'")
Execute(cmd, logoutput=True, user = params.unix_user)
File(params.ranger_usersync_keystore_file,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
create_core_site_xml(ranger_ugsync_conf)
dfs_type=dfs_type)
repo_name = str(config['clusterName']) + '_atlas'
ssl_keystore_password = unicode(
config['configurations']['ranger-atlas-policymgr-ssl']
['xasecure.policymgr.clientssl.keystore.password'])
ssl_truststore_password = unicode(
config['configurations']['ranger-atlas-policymgr-ssl']
['xasecure.policymgr.clientssl.truststore.password'])
credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
xa_audit_hdfs_is_enabled = default(
'/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs',
False)
enable_ranger_atlas = config['configurations'][
'ranger-atlas-plugin-properties']['ranger-atlas-plugin-enabled']
enable_ranger_atlas = not is_empty(
enable_ranger_atlas) and enable_ranger_atlas.lower() == 'yes'
enable_ranger_hbase = config['configurations'][
'ranger-hbase-plugin-properties']['ranger-hbase-plugin-enabled']
enable_ranger_hbase = not is_empty(
enable_ranger_hbase) and enable_ranger_hbase.lower() == 'yes'
policymgr_mgr_url = config['configurations']['admin-properties'][
'policymgr_external_url']
downloaded_custom_connector = None
driver_curl_source = None
driver_curl_target = None
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations'][
'ranger-atlas-plugin-properties']
pig_tar_dest_file = webhcat_apps_dir + "/pig.tar.gz"
hadoop_streaming_tar_source = hadoop_streaming_jars # this contains *
sqoop_tar_source = sqoop_tar_file # this contains *
hadoop_streaming_tar_dest_dir = webhcat_apps_dir
sqoop_tar_dest_dir = webhcat_apps_dir
tarballs_mode = 0755
execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir
hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName']
hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL']
hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
#JDBC driver jar name
hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName']
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
hive_database_name = config['configurations']['hive-env']['hive_database_name']
hive_database = config['configurations']['hive-env']['hive_database']
hive_use_existing_db = hive_database.startswith('Existing')
# NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE.
mysql_port = config['configurations']['metron-env'][
'metron_enrichment_db_port']
mysql_adduser_path = tmp_dir + "/addMysqlUser.sh"
mysql_deluser_path = tmp_dir + "/removeMysqlUser.sh"
mysql_create_geoip_path = tmp_dir + "/createMysqlGeoIp.sh"
enrichment_hosts = default("/clusterHostInfo/enrichment_host", [])
enrichment_host = enrichment_hosts[0] if len(enrichment_hosts) > 0 else None
enrichment_metron_user = config['configurations']['metron-env'][
'metron_enrichment_db_user']
enrichment_metron_user_passwd = config['configurations']['metron-env'][
'metron_enrichment_db_password']
enrichment_metron_user_passwd = unicode(
enrichment_metron_user_passwd) if not is_empty(
enrichment_metron_user_passwd) else enrichment_metron_user_passwd
mysql_process_name = status_params.mysql_process_name
# create partial functions with common arguments for every HdfsResource call
# to create/delete hdfs directory/file/copyfromlocal we need to call params.HdfsResource in code
HdfsResource = functools.partial(
HdfsResource,
user=hdfs_user,
hdfs_resource_ignore_file=
"/var/lib/ambari-agent/data/.hdfs_resource_ignore",
security_enabled=security_enabled,
keytab=hdfs_user_keytab,
kinit_path_local=kinit_path_local,
hadoop_bin_dir=hadoop_bin_dir,
hadoop_conf_dir=hadoop_conf_dir,
principal_name=hdfs_principal_name,
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(
params.ranger_usersync_ldap_ldapbindpassword
) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
Directory(params.ranger_pid_dir,
mode=0750,
owner=params.unix_user,
group=params.unix_group)
Directory(params.usersync_log_dir,
owner=params.unix_user,
group=params.unix_group)
Directory(ranger_ugsync_conf,
owner=params.unix_user,
group=params.unix_group,
recursive=True)
if upgrade_type is not None:
src_file = format(
'{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_conf}/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{usersync_home}/conf.dist/log4j.properties')
dst_file = format('{usersync_conf}/log4j.properties')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
# Copies /etc/hadoop/conf.empty/core-site.xml to /etc/ranger/admin/conf/core-site.xml
src_file = format('/etc/hadoop/conf.empty/core-site.xml')
dst_file = format('/etc/ranger/admin/conf/core-site.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
# Added by RAI
#makes 1) ln - sf / etc / ranger / admin / conf / usr / lib / ranger - admin / ews / webapp / WEB - INF / classes
# 2) ln - sf / var / log / ranger // usr / lib / ranger - usersync / logs#
Execute(('ln', '-sf', format('{ranger_conf}'),
format('{ranger_home}/ews/webapp/WEB-INF/classes')),
not_if=format("ls {ranger_home}/ews/webapp/WEB-INF/classes"),
only_if=format("ls {ranger_conf}"),
sudo=True)
Execute(('ln', '-sf', format('{usersync_log_dir}'),
format('{usersync_home}/logs')),
not_if=format("ls {usersync_home}/logs"),
only_if=format("ls {usersync_log_dir}"),
sudo=True)
XmlConfig(
"ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']
['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file,
owner=params.unix_user,
group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_file = format('{ranger_home}/ranger_credential_helper.py')
if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')):
cred_file = format('{usersync_home}/ranger_credential_helper.py')
cred_lib = os.path.join(usersync_home, "lib", "*")
cred_setup_prefix = (cred_file, '-l', cred_lib)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password',
'-v', PasswordString(
params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias',
'-v', PasswordString(
params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
cred_setup = cred_setup_prefix + (
'-f', params.ugsync_jceks_path, '-k',
'usersync.ssl.truststore.password', '-v',
PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup,
environment={'JAVA_HOME': params.java_home},
logoutput=True,
sudo=True)
File(params.ugsync_jceks_path,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
File([params.usersync_start, params.usersync_stop],
owner=params.unix_user,
group=params.unix_group)
File(
params.usersync_services_file,
mode=0755,
)
Execute(('ln', '-sf', format('{usersync_services_file}'),
'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format(
"{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'"
)
Execute(cmd, logoutput=True, user=params.unix_user)
File(params.ranger_usersync_keystore_file,
owner=params.unix_user,
group=params.unix_group,
mode=0640)
def hive(name=None):
import params
if name == 'hiveserver2':
# HDP 2.1.* or lower
if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, "2.2.0.0") < 0:
params.HdfsResource(params.webhcat_apps_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=0755
)
# Create webhcat dirs.
if params.hcat_hdfs_user_dir != params.webhcat_hdfs_user_dir:
params.HdfsResource(params.hcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hcat_user,
mode=params.hcat_hdfs_user_mode
)
params.HdfsResource(params.webhcat_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.webhcat_user,
mode=params.webhcat_hdfs_user_mode
)
# ****** Begin Copy Tarballs ******
# *********************************
# HDP 2.2 or higher, copy mapreduce.tar.gz to HDFS
if params.hdp_stack_version_major != "" and compare_versions(params.hdp_stack_version_major, '2.2') >= 0:
copy_to_hdfs("mapreduce", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("tez", params.user_group, params.hdfs_user, host_sys_prepped=params.host_sys_prepped)
# Always copy pig.tar.gz and hive.tar.gz using the appropriate mode.
# This can use a different source and dest location to account for both HDP 2.1 and 2.2
copy_to_hdfs("pig",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.pig_tar_source,
custom_dest_file=params.pig_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
copy_to_hdfs("hive",
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=params.hive_tar_source,
custom_dest_file=params.hive_tar_dest_file,
host_sys_prepped=params.host_sys_prepped)
wildcard_tarballs = ["sqoop", "hadoop_streaming"]
for tarball_name in wildcard_tarballs:
source_file_pattern = eval("params." + tarball_name + "_tar_source")
dest_dir = eval("params." + tarball_name + "_tar_dest_dir")
if source_file_pattern is None or dest_dir is None:
continue
source_files = glob.glob(source_file_pattern) if "*" in source_file_pattern else [source_file_pattern]
for source_file in source_files:
src_filename = os.path.basename(source_file)
dest_file = os.path.join(dest_dir, src_filename)
copy_to_hdfs(tarball_name,
params.user_group,
params.hdfs_user,
file_mode=params.tarballs_mode,
custom_source_file=source_file,
custom_dest_file=dest_file,
host_sys_prepped=params.host_sys_prepped)
# ******* End Copy Tarballs *******
# *********************************
# Create Hive Metastore Warehouse Dir
params.HdfsResource(params.hive_apps_whs_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=0777
)
# Create Hive User Dir
params.HdfsResource(params.hive_hdfs_user_dir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
mode=params.hive_hdfs_user_mode
)
if not is_empty(params.hive_exec_scratchdir) and not urlparse(params.hive_exec_scratchdir).path.startswith("/tmp"):
params.HdfsResource(params.hive_exec_scratchdir,
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hdfs_user,
mode=0777) # Hive expects this dir to be writeable by everyone as it is used as a temp dir
params.HdfsResource(None, action="execute")
Directory(params.hive_etc_dir_prefix,
mode=0755
)
# We should change configurations for client as well as for server.
# The reason is that stale-configs are service-level, not component.
for conf_dir in params.hive_conf_dirs_list:
fill_conf_dir(conf_dir)
XmlConfig("hive-site.xml",
conf_dir=params.hive_config_dir,
configurations=params.hive_site_config,
configuration_attributes=params.config['configuration_attributes']['hive-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
setup_atlas_hive()
if params.hive_specific_configs_supported and name == 'hiveserver2':
XmlConfig("hiveserver2-site.xml",
conf_dir=params.hive_server_conf_dir,
configurations=params.config['configurations']['hiveserver2-site'],
configuration_attributes=params.config['configuration_attributes']['hiveserver2-site'],
owner=params.hive_user,
group=params.user_group,
mode=0644)
File(format("{hive_config_dir}/hive-env.sh"),
owner=params.hive_user,
group=params.user_group,
content=InlineTemplate(params.hive_env_sh_template)
)
# On some OS this folder could be not exists, so we will create it before pushing there files
Directory(params.limits_conf_dir,
recursive=True,
owner='root',
group='root'
)
File(os.path.join(params.limits_conf_dir, 'hive.conf'),
owner='root',
group='root',
mode=0644,
content=Template("hive.conf.j2")
)
if (name == 'metastore' or name == 'hiveserver2') and not os.path.exists(params.target):
jdbc_connector()
File(format("/usr/lib/ambari-agent/{check_db_connection_jar_name}"),
content = DownloadSource(format("{jdk_location}{check_db_connection_jar_name}")),
mode = 0644,
)
if name == 'metastore':
File(params.start_metastore_path,
mode=0755,
content=StaticFile('startMetastore.sh')
)
if params.init_metastore_schema:
create_schema_cmd = format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -initSchema "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}")
check_schema_created_cmd = as_user(format("export HIVE_CONF_DIR={hive_server_conf_dir} ; "
"{hive_bin}/schematool -info "
"-dbType {hive_metastore_db_type} "
"-userName {hive_metastore_user_name} "
"-passWord {hive_metastore_user_passwd!p}"), params.hive_user)
# HACK: in cases with quoted passwords and as_user (which does the quoting as well) !p won't work for hiding passwords.
# Fixing it with the hack below:
quoted_hive_metastore_user_passwd = quote_bash_args(quote_bash_args(params.hive_metastore_user_passwd))
if quoted_hive_metastore_user_passwd[0] == "'" and quoted_hive_metastore_user_passwd[-1] == "'" \
or quoted_hive_metastore_user_passwd[0] == '"' and quoted_hive_metastore_user_passwd[-1] == '"':
quoted_hive_metastore_user_passwd = quoted_hive_metastore_user_passwd[1:-1]
Logger.sensitive_strings[repr(check_schema_created_cmd)] = repr(check_schema_created_cmd.replace(
format("-passWord {quoted_hive_metastore_user_passwd}"), "-passWord " + utils.PASSWORDS_HIDE_STRING))
Execute(create_schema_cmd,
not_if = check_schema_created_cmd,
user = params.hive_user
)
elif name == 'hiveserver2':
File(params.start_hiveserver2_path,
mode=0755,
content=Template(format('{start_hiveserver2_script}'))
)
if name != "client":
crt_directory(params.hive_pid_dir)
crt_directory(params.hive_log_dir)
crt_directory(params.hive_var_lib)
def get_port_from_url(address):
if not is_empty(address):
return address.split(':')[-1]
else:
return address
hive_tar_dest_file = webhcat_apps_dir + "/hive.tar.gz" pig_tar_dest_file = webhcat_apps_dir + "/pig.tar.gz" hadoop_streaming_tar_source = hadoop_streaming_jars # this contains * sqoop_tar_source = sqoop_tar_file # this contains * hadoop_streaming_tar_dest_dir = webhcat_apps_dir sqoop_tar_dest_dir = webhcat_apps_dir tarballs_mode = 0755 execute_path = os.environ['PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir hive_metastore_user_name = config['configurations']['hive-site']['javax.jdo.option.ConnectionUserName'] hive_jdbc_connection_url = config['configurations']['hive-site']['javax.jdo.option.ConnectionURL'] hive_metastore_user_passwd = config['configurations']['hive-site']['javax.jdo.option.ConnectionPassword'] hive_metastore_user_passwd = unicode(hive_metastore_user_passwd) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd hive_metastore_db_type = config['configurations']['hive-env']['hive_database_type'] #HACK Temporarily use dbType=azuredb while invoking schematool if hive_metastore_db_type == "mssql": hive_metastore_db_type = "azuredb" #users hive_user = config['configurations']['hive-env']['hive_user'] #JDBC driver jar name hive_jdbc_driver = config['configurations']['hive-site']['javax.jdo.option.ConnectionDriverName'] # NOT SURE THAT IT'S A GOOD IDEA TO USE PATH TO CLASS IN DRIVER, MAYBE IT WILL BE BETTER TO USE DB TYPE. # BECAUSE PATH TO CLASSES COULD BE CHANGED sqla_db_used = False if hive_jdbc_driver == "com.microsoft.sqlserver.jdbc.SQLServerDriver": jdbc_jar_name = "sqljdbc4.jar" jdbc_symlink_name = "mssql-jdbc-driver.jar"
def metadata(type='server'):
import params
# Needed by both Server and Client
Directory(params.conf_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
if type == "server":
Directory([params.pid_dir],
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(format('{conf_dir}/solr'),
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True,
recursive_ownership=True)
Directory(params.log_dir,
mode=0755,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.data_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
Directory(params.expanded_war_dir,
mode=0644,
cd_access='a',
owner=params.metadata_user,
group=params.user_group,
create_parents=True)
File(format("{expanded_war_dir}/atlas.war"),
content=StaticFile(
format('{metadata_home}/server/webapp/atlas.war')))
File(format("{conf_dir}/atlas-log4j.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_log4j_content))
File(format("{conf_dir}/atlas-env.sh"),
owner=params.metadata_user,
group=params.user_group,
mode=0755,
content=InlineTemplate(params.metadata_env_content))
if not is_empty(params.atlas_admin_username) and not is_empty(
params.atlas_admin_password):
psswd_output = hashlib.sha256(
params.atlas_admin_password).hexdigest()
ModifyPropertiesFile(
format("{conf_dir}/users-credentials.properties"),
properties={
format('{atlas_admin_username}'):
format('ROLE_ADMIN::{psswd_output}')
},
owner=params.metadata_user)
files_to_chown = [
format("{conf_dir}/policy-store.txt"),
format("{conf_dir}/users-credentials.properties")
]
for file in files_to_chown:
if os.path.exists(file):
Execute(
('chown', format('{metadata_user}:{user_group}'), file),
sudo=True)
Execute(('chmod', '644', file), sudo=True)
if params.metadata_solrconfig_content:
File(format("{conf_dir}/solr/solrconfig.xml"),
mode=0644,
owner=params.metadata_user,
group=params.user_group,
content=InlineTemplate(params.metadata_solrconfig_content))
# Needed by both Server and Client
PropertiesFile(format('{conf_dir}/{conf_file}'),
properties=params.application_properties,
mode=0644,
owner=params.metadata_user,
group=params.user_group)
if params.security_enabled:
TemplateConfig(format(params.atlas_jaas_file),
owner=params.metadata_user)
if type == 'server' and params.search_backend_solr and params.has_infra_solr:
solr_cloud_util.setup_solr_client(params.config)
check_znode()
jaasFile = params.atlas_jaas_file if params.security_enabled else None
upload_conf_set('atlas_configs', jaasFile)
if params.security_enabled: # update permissions before creating the collections
solr_cloud_util.add_solr_roles(
params.config,
roles=[
params.infra_solr_role_atlas,
params.infra_solr_role_ranger_audit,
params.infra_solr_role_dev
],
new_service_principals=[params.atlas_jaas_principal])
create_collection('vertex_index', 'atlas_configs', jaasFile)
create_collection('edge_index', 'atlas_configs', jaasFile)
create_collection('fulltext_index', 'atlas_configs', jaasFile)
if params.security_enabled:
secure_znode(format('{infra_solr_znode}/configs/atlas_configs'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/vertex_index'),
jaasFile)
secure_znode(format('{infra_solr_znode}/collections/edge_index'),
jaasFile)
secure_znode(
format('{infra_solr_znode}/collections/fulltext_index'),
jaasFile)
File(params.atlas_hbase_setup,
group=params.user_group,
owner=params.hbase_user,
content=Template("atlas_hbase_setup.rb.j2"))
is_atlas_upgrade_support = check_stack_feature(
StackFeature.ATLAS_UPGRADE_SUPPORT,
get_stack_feature_version(params.config))
if is_atlas_upgrade_support and params.security_enabled:
File(params.atlas_kafka_setup,
group=params.user_group,
owner=params.kafka_user,
content=Template("atlas_kafka_acl.sh.j2"))
# files required only in case if kafka broker is not present on the host as configured component
if not params.host_with_kafka:
File(format("{kafka_conf_dir}/kafka-env.sh"),
owner=params.kafka_user,
content=InlineTemplate(params.kafka_env_sh_template))
File(format("{kafka_conf_dir}/kafka_jaas.conf"),
group=params.user_group,
owner=params.kafka_user,
content=Template("kafka_jaas.conf.j2"))
if params.stack_supports_atlas_hdfs_site_on_namenode_ha and len(
params.namenode_host) > 1:
XmlConfig(
"hdfs-site.xml",
conf_dir=params.conf_dir,
configurations=params.config['configurations']['hdfs-site'],
configuration_attributes=params.config['configuration_attributes']
['hdfs-site'],
owner=params.metadata_user,
group=params.user_group,
mode=0644)
else:
File(format('{conf_dir}/hdfs-site.xml'), action="delete")
version_for_stack_feature_checks):
hive_metastore_site_supported = True
execute_path = os.environ[
'PATH'] + os.pathsep + hive_bin + os.pathsep + hadoop_bin_dir
hive_metastore_user_name = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionUserName']
hive_jdbc_connection_url = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionURL']
hive_metastore_user_passwd = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionPassword']
hive_metastore_user_passwd = unicode(
hive_metastore_user_passwd
) if not is_empty(hive_metastore_user_passwd) else hive_metastore_user_passwd
hive_metastore_db_type = config['configurations']['hive-env'][
'hive_database_type']
#HACK Temporarily use dbType=azuredb while invoking schematool
if hive_metastore_db_type == "mssql":
hive_metastore_db_type = "azuredb"
#users
hive_user = config['configurations']['hive-env']['hive_user']
#JDBC driver jar name
hive_jdbc_driver = config['configurations']['hive-site'][
'javax.jdo.option.ConnectionDriverName']
jdk_location = config['hostLevelParams']['jdk_location']
java_share_dir = '/usr/share/java'
# *********************** RANGER PLUGIN CHANGES ***********************
# ranger host
# **********************************************************************
ranger_admin_hosts = default("/clusterHostInfo/ranger_admin_hosts", [])
has_ranger_admin = not len(ranger_admin_hosts) == 0
xml_configurations_supported = config['configurations']['ranger-env']['xml_configurations_supported']
ambari_server_hostname = config['clusterHostInfo']['ambari_server_host'][0]
ranger_admin_log_dir = default("/configurations/ranger-env/ranger_admin_log_dir","/var/log/ranger/admin")
is_supported_kafka_ranger = config['configurations']['kafka-env']['is_supported_kafka_ranger']
#ranger kafka properties
if has_ranger_admin and is_supported_kafka_ranger:
enable_ranger_kafka = config['configurations']['ranger-kafka-plugin-properties']['ranger-kafka-plugin-enabled']
enable_ranger_kafka = not is_empty(enable_ranger_kafka) and enable_ranger_kafka.lower() == 'yes'
policymgr_mgr_url = config['configurations']['admin-properties']['policymgr_external_url']
sql_connector_jar = config['configurations']['admin-properties']['SQL_CONNECTOR_JAR']
xa_audit_db_flavor = config['configurations']['admin-properties']['DB_FLAVOR']
xa_audit_db_flavor = xa_audit_db_flavor.lower() if xa_audit_db_flavor else None
xa_audit_db_name = config['configurations']['admin-properties']['audit_db_name']
xa_audit_db_user = config['configurations']['admin-properties']['audit_db_user']
xa_audit_db_password = unicode(config['configurations']['admin-properties']['audit_db_password'])
xa_db_host = config['configurations']['admin-properties']['db_host']
repo_name = str(config['clusterName']) + '_kafka'
ranger_env = config['configurations']['ranger-env']
ranger_plugin_properties = config['configurations']['ranger-kafka-plugin-properties']
ranger_kafka_audit = config['configurations']['ranger-kafka-audit']
ranger_kafka_audit_attrs = config['configuration_attributes']['ranger-kafka-audit']
security_enabled=security_enabled,
keytab=hdfs_user_keytab,
kinit_path_local=kinit_path_local,
hadoop_bin_dir=hadoop_bin_dir,
hadoop_conf_dir=hadoop_conf_dir,
principal_name=hdfs_principal_name,
hdfs_site=hdfs_site,
default_fs=default_fs,
immutable_paths=get_not_managed_resources(),
dfs_type=dfs_type)
# ranger atlas service/repository name
repo_name = str(config['clusterName']) + '_atlas'
repo_name_value = config['configurations']['ranger-atlas-security'][
'ranger.plugin.atlas.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
ssl_keystore_password = config['configurations'][
'ranger-atlas-policymgr-ssl'][
'xasecure.policymgr.clientssl.keystore.password']
ssl_truststore_password = config['configurations'][
'ranger-atlas-policymgr-ssl'][
'xasecure.policymgr.clientssl.truststore.password']
credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
xa_audit_hdfs_is_enabled = default(
'/configurations/ranger-atlas-audit/xasecure.audit.destination.hdfs',
False)
# get ranger policy url
policymgr_mgr_url = config['configurations']['ranger-atlas-security'][
db_name = config['configurations']['kms-properties']['db_name']
db_user = config['configurations']['kms-properties']['db_user']
db_password = unicode(
config['configurations']['kms-properties']['db_password'])
kms_master_key_password = unicode(
config['configurations']['kms-properties']['KMS_MASTER_KEY_PASSWD'])
credential_provider_path = config['configurations']['dbks-site'][
'ranger.ks.jpa.jdbc.credential.provider.path']
jdbc_alias = config['configurations']['dbks-site'][
'ranger.ks.jpa.jdbc.credential.alias']
masterkey_alias = config['configurations']['dbks-site'][
'ranger.ks.masterkey.credential.alias']
repo_name = str(config['clusterName']) + '_kms'
repo_name_value = config['configurations']['ranger-kms-security'][
'ranger.plugin.kms.service.name']
if not is_empty(repo_name_value) and repo_name_value != "{{repo_name}}":
repo_name = repo_name_value
cred_lib_path = os.path.join(kms_home, "cred", "lib", "*")
cred_setup_prefix = (format('{kms_home}/ranger_credential_helper.py'), '-l',
cred_lib_path)
credential_file = format('/etc/ranger/{repo_name}/cred.jceks')
# ranger kms ssl enabled config
ranger_kms_ssl_enabled = config['configurations']['ranger-kms-site'][
'ranger.service.https.attrib.ssl.enabled']
url_scheme = "http"
if ranger_kms_ssl_enabled:
url_scheme = "https"
if has_ranger_admin:
policymgr_mgr_url = config['configurations']['admin-properties'][
def setup_usersync(upgrade_type=None):
import params
usersync_home = params.usersync_home
ranger_home = params.ranger_home
ranger_ugsync_conf = params.ranger_ugsync_conf
if not is_empty(params.ranger_usersync_ldap_ldapbindpassword) and params.ug_sync_source == 'org.apache.ranger.ldapusersync.process.LdapUserGroupBuilder':
password_validation(params.ranger_usersync_ldap_ldapbindpassword)
if upgrade_type is not None:
usersync_home = format("/usr/hdp/{version}/ranger-usersync")
ranger_home = format("/usr/hdp/{version}/ranger-admin")
ranger_ugsync_conf = format("/usr/hdp/{version}/ranger-usersync/conf")
Directory(params.ranger_pid_dir,
mode=0750,
owner = params.unix_user,
group = params.unix_group
)
Directory(params.usersync_log_dir,
owner = params.unix_user,
group = params.unix_group
)
Directory(format("{ranger_ugsync_conf}/"),
owner = params.unix_user
)
if upgrade_type is not None:
src_file = format('{usersync_home}/conf.dist/ranger-ugsync-default.xml')
dst_file = format('{usersync_home}/conf/ranger-ugsync-default.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
src_file = format('{usersync_home}/conf.dist/log4j.xml')
dst_file = format('{usersync_home}/conf/log4j.xml')
Execute(('cp', '-f', src_file, dst_file), sudo=True)
XmlConfig("ranger-ugsync-site.xml",
conf_dir=ranger_ugsync_conf,
configurations=params.config['configurations']['ranger-ugsync-site'],
configuration_attributes=params.config['configuration_attributes']['ranger-ugsync-site'],
owner=params.unix_user,
group=params.unix_group,
mode=0644)
if os.path.isfile(params.ranger_ugsync_default_file):
File(params.ranger_ugsync_default_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.usgsync_log4j_file):
File(params.usgsync_log4j_file, owner=params.unix_user, group=params.unix_group)
if os.path.isfile(params.cred_validator_file):
File(params.cred_validator_file, group=params.unix_group, mode=04555)
cred_file = format('{ranger_home}/ranger_credential_helper.py')
if os.path.isfile(format('{usersync_home}/ranger_credential_helper.py')):
cred_file = format('{usersync_home}/ranger_credential_helper.py')
cred_lib = os.path.join(usersync_home,"lib","*")
cred_setup_prefix = (cred_file, '-l', cred_lib)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.key.password', '-v', PasswordString(params.ranger_usersync_keystore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'ranger.usersync.ldap.bindalias', '-v', PasswordString(params.ranger_usersync_ldap_ldapbindpassword), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
cred_setup = cred_setup_prefix + ('-f', params.ugsync_jceks_path, '-k', 'usersync.ssl.truststore.password', '-v', PasswordString(params.ranger_usersync_truststore_password), '-c', '1')
Execute(cred_setup, environment={'JAVA_HOME': params.java_home}, logoutput=True, sudo=True)
File(params.ugsync_jceks_path,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
File([params.usersync_start, params.usersync_stop],
owner = params.unix_user,
group = params.unix_group
)
File(params.usersync_services_file,
mode = 0755,
)
Execute(('ln','-sf', format('{usersync_services_file}'),'/usr/bin/ranger-usersync'),
not_if=format("ls /usr/bin/ranger-usersync"),
only_if=format("ls {usersync_services_file}"),
sudo=True)
if not os.path.isfile(params.ranger_usersync_keystore_file):
cmd = format("{java_home}/bin/keytool -genkeypair -keyalg RSA -alias selfsigned -keystore '{ranger_usersync_keystore_file}' -keypass {ranger_usersync_keystore_password!p} -storepass {ranger_usersync_keystore_password!p} -validity 3600 -keysize 2048 -dname '{default_dn_name}'")
Execute(cmd, logoutput=True, user = params.unix_user)
File(params.ranger_usersync_keystore_file,
owner = params.unix_user,
group = params.unix_group,
mode = 0640
)
