def setup_ranger_hive(rolling_upgrade = False):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if rolling_upgrade:
hdp_version = params.version
setup_ranger_plugin('hive-server2', 'hive',
params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source,
params.ranger_driver_curl_target, params.java64_home,
params.repo_name, params.hive_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_hive, conf_dict=params.hive_server_conf_dir,
component_user=params.hive_user, component_group=params.user_group, cache_service_list=['hiveServer2'],
plugin_audit_properties=params.config['configurations']['ranger-hive-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hive-audit'],
plugin_security_properties=params.config['configurations']['ranger-hive-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hive-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hive-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hive-policymgr-ssl'],
component_list=['hive-client', 'hive-metastore', 'hive-server2'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override = hdp_version)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_hive():
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin('hive-server2', 'hive',
params.ranger_downloaded_custom_connector, params.ranger_driver_curl_source,
params.ranger_driver_curl_target, params.java64_home,
params.repo_name, params.hive_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_hive, conf_dict=params.hive_server_conf_dir,
component_user=params.hive_user, component_group=params.user_group, cache_service_list=['hiveServer2'],
plugin_audit_properties=params.config['configurations']['ranger-hive-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hive-audit'],
plugin_security_properties=params.config['configurations']['ranger-hive-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hive-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hive-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hive-policymgr-ssl'],
component_list=['hive-client', 'hive-metastore', 'hive-server2'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password
)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_kafka():
import params
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info("Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_kafka and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('kafka-broker', 'kafka',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.kafka_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_kafka, conf_dict=params.conf_dir,
component_user=params.kafka_user, component_group=params.user_group, cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit, plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security, plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl, plugin_policymgr_ssl_attributes=params.ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination', params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True
)
File(params.setup_ranger_env_sh_target,
owner = params.kafka_user,
group = params.user_group,
mode = 0755
)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.has_ranger_admin and params.security_enabled:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if upgrade_type is not None:
hdp_version = params.version
if params.retryAble:
Logger.info("Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_storm and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('storm-nimbus', 'storm',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.storm_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_storm, conf_dict=params.conf_dir,
component_user=params.storm_user, component_group=params.user_group, cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']['ranger-storm-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']['ranger-storm-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-storm-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.has_ranger_admin and params.security_enabled:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info("Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_storm and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('storm-nimbus', 'storm',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.storm_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_storm, conf_dict=params.conf_dir,
component_user=params.storm_user, component_group=params.user_group, cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']['ranger-storm-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']['ranger-storm-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-storm-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_yarn():
import params
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
setup_ranger_plugin(
'hadoop-yarn-resourcemanager',
'yarn',
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.yarn_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_yarn,
conf_dict=params.hadoop_conf_dir,
component_user=params.yarn_user,
component_group=params.user_group,
cache_service_list=['yarn'],
plugin_audit_properties=params.config['configurations']
['ranger-yarn-audit'],
plugin_audit_attributes=params.config['configuration_attributes']
['ranger-yarn-audit'],
plugin_security_properties=params.config['configurations']
['ranger-yarn-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-yarn-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-yarn-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configuration_attributes']['ranger-yarn-policymgr-ssl'],
component_list=['hadoop-yarn-resourcemanager'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_yarn():
import params
if params.enable_ranger_yarn:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info("YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/yarn",
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.yarn_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception("Audit directory creation in HDFS for YARN Ranger plugin failed with error:\n{0}".format(err))
setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn', params.previous_jdbc_jar,
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.yarn_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_yarn, conf_dict=params.hadoop_conf_dir,
component_user=params.yarn_user, component_group=params.user_group, cache_service_list=['yarn'],
plugin_audit_properties=params.config['configurations']['ranger-yarn-audit'], plugin_audit_attributes=params.config['configurationAttributes']['ranger-yarn-audit'],
plugin_security_properties=params.config['configurations']['ranger-yarn-security'], plugin_security_attributes=params.config['configurationAttributes']['ranger-yarn-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-yarn-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configurationAttributes']['ranger-yarn-policymgr-ssl'],
component_list=['hadoop-yarn-resourcemanager'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble,
is_security_enabled = params.security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
component_user_principal=params.rm_principal_name if params.security_enabled else None,
component_user_keytab=params.rm_keytab if params.security_enabled else None
)
def setup_ranger_hdfs(rolling_upgrade=False):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if rolling_upgrade:
hdp_version = params.version
setup_ranger_plugin(
'hadoop-client',
'hdfs',
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.hdfs_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hdfs,
conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user,
component_group=params.user_group,
cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']
['ranger-hdfs-audit'],
plugin_audit_attributes=params.config['configuration_attributes']
['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hdfs-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hdfs-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configuration_attributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override=hdp_version)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_atlas(upgrade_type=None):
import params
if params.enable_ranger_atlas:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retry_enabled:
Logger.info("ATLAS: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("ATLAS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.enable_ranger_atlas and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.metadata_user,
group=params.user_group,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/atlas",
type="directory",
action="create_on_execute",
owner=params.metadata_user,
group=params.user_group,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('atlas-server', 'atlas',None,
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.atlas_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_atlas, conf_dict=params.conf_dir,
component_user=params.metadata_user, component_group=params.user_group, cache_service_list=['atlas'],
plugin_audit_properties=params.config['configurations']['ranger-atlas-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-atlas-audit'],
plugin_security_properties=params.config['configurations']['ranger-atlas-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-atlas-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-atlas-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-atlas-policymgr-ssl'],
component_list=['atlas-server'], audit_db_is_enabled=False,
credential_file=params.credential_file, xa_audit_db_password=None,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2', skip_if_rangeradmin_down = not params.retry_enabled, is_security_enabled = params.security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
component_user_principal=params.atlas_jaas_principal if params.security_enabled else None,
component_user_keytab=params.atlas_keytab_path if params.security_enabled else None)
else:
Logger.info('Ranger Atlas plugin is not enabled')
def setup_ranger_kafka():
import params
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'kafka-broker',
'kafka',
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2')
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination',
params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_yarn():
import params
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info("YARN: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("YARN: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_yarn and params.xa_audit_hdfs_is_enabled:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/yarn",
type="directory",
action="create_on_execute",
owner=params.yarn_user,
group=params.yarn_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('hadoop-yarn-resourcemanager', 'yarn',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.yarn_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_yarn, conf_dict=params.hadoop_conf_dir,
component_user=params.yarn_user, component_group=params.user_group, cache_service_list=['yarn'],
plugin_audit_properties=params.config['configurations']['ranger-yarn-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-yarn-audit'],
plugin_security_properties=params.config['configurations']['ranger-yarn-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-yarn-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-yarn-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-yarn-policymgr-ssl'],
component_list=['hadoop-yarn-resourcemanager'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2', skip_if_rangeradmin_down= not params.retryAble
)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_kafka():
import params
if params.has_ranger_admin:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
"kafka-broker",
"kafka",
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=["kafka"],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.ranger_kafka_policymgr_ssl_attrs,
component_list=["kafka-broker"],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version="v2",
)
if params.enable_ranger_kafka:
Execute(
("cp", "--remove-destination", params.setup_ranger_env_sh_source, params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True,
)
File(params.setup_ranger_env_sh_target, owner=params.kafka_user, group=params.user_group, mode=0755)
else:
Logger.info("Ranger admin not installed")
def setup_ranger_nifi(upgrade_type=None):
import params, os
if params.has_ranger_admin and params.enable_ranger_nifi:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info("nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
api_version=None
if params.stack_supports_ranger_kerberos:
api_version='v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin('nifi', 'nifi', params.previous_jdbc_jar,
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.nifi_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_nifi, conf_dict=params.nifi_config_dir,
component_user=params.nifi_user, component_group=params.nifi_group, cache_service_list=['nifi'],
plugin_audit_properties=params.config['configurations']['ranger-nifi-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-nifi-audit'],
plugin_security_properties=params.config['configurations']['ranger-nifi-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-nifi-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-nifi-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-nifi-policymgr-ssl'],
component_list=[], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
is_security_enabled = params.security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
component_user_principal=params.ranger_nifi_principal if params.security_enabled else None,
component_user_keytab=params.ranger_nifi_keytab if params.security_enabled else None)
#change permissions of ranger xml that were written to 0400
File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'), owner=params.nifi_user, group=params.nifi_group, mode=0400)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_hdfs(upgrade_type=None):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
setup_ranger_plugin('hadoop-client', 'hdfs',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.hdfs_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_hdfs, conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user, component_group=params.user_group, cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']['ranger-hdfs-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']['ranger-hdfs-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
if stack_version and params.upgrade_direction == Direction.UPGRADE:
# when upgrading to 2.3+, this env file must be removed
if compare_versions(stack_version, '2.3', format=True) > 0:
source_file = os.path.join(params.hadoop_conf_dir, 'set-hdfs-plugin-env.sh')
target_file = source_file + ".bak"
Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}"))
else:
Logger.info('Ranger admin not installed')
def setup_ranger_hdfs(upgrade_type=None):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if upgrade_type is not None:
hdp_version = params.version
if params.retryAble:
Logger.info("HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
setup_ranger_plugin('hadoop-client', 'hdfs',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.hdfs_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_hdfs, conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user, component_group=params.user_group, cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']['ranger-hdfs-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']['ranger-hdfs-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hdfs-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
if hdp_version and params.upgrade_direction == Direction.UPGRADE:
# when upgrading to 2.3+, this env file must be removed
if compare_versions(hdp_version, '2.3', format=True) > 0:
source_file = os.path.join(params.hadoop_conf_dir, 'set-hdfs-plugin-env.sh')
target_file = source_file + ".bak"
Execute(("mv", source_file, target_file), sudo=True, only_if=format("test -f {source_file}"))
else:
Logger.info('Ranger admin not installed')
def setup_ranger_kafka():
import params
if params.has_ranger_admin:
setup_ranger_plugin('kafka-broker', 'kafka',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.kafka_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_kafka, conf_dict=params.conf_dir,
component_user=params.kafka_user, component_group=params.user_group, cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit, plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security, plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl, plugin_policymgr_ssl_attributes=params.ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
api_version = 'v2')
else:
Logger.info('Ranger admin not installed')
def setup_ranger_knox(rolling_upgrade = False):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if rolling_upgrade:
hdp_version = params.version
if params.retryAble:
Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
setup_ranger_plugin('knox-server', 'knox',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.knox_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_knox(upgrade_type=None):
import params
if params.enable_ranger_knox:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/knox",
type="directory",
action="create_on_execute",
owner=params.knox_user,
group=params.knox_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.namenode_hosts is not None and len(
params.namenode_hosts) > 1:
Logger.info(
'Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml'
)
XmlConfig("hdfs-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
owner=params.knox_user,
group=params.knox_group,
mode=0644)
else:
File(format('{knox_conf_dir}/hdfs-site.xml'),
action="delete")
if params.xml_configurations_supported:
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.knox_principal_name
if params.security_enabled else None,
component_user_keytab=params.knox_keytab_path
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.security_enabled:
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.knox_user,
component_group=params.knox_group,
create_core_site_path=params.knox_conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
file_name='core-site.xml')
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.knox_user,
component_group=params.knox_group,
create_core_site_path=params.knox_conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos' if params.security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Knox plugin is not enabled')
def setup_ranger_hdfs(upgrade_type=None):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
setup_ranger_plugin(
'hadoop-client',
'hdfs',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.hdfs_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hdfs,
conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user,
component_group=params.user_group,
cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']
['ranger-hdfs-audit'],
plugin_audit_attributes=params.config['configuration_attributes']
['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hdfs-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hdfs-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configuration_attributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_nifi(upgrade_type=None):
import params, os
if params.has_ranger_admin and params.enable_ranger_nifi:
stack_version = params.stack_version_buildnum
File(format(
'{stack_root}/{stack_version}/{service_name}/ext/ranger/scripts/ranger_credential_helper.py'
),
owner=params.nifi_user,
group=params.nifi_group,
mode=0750)
cred_lib_prefix_path = format(
'{stack_root}/{stack_version}/{service_name}/ext/ranger/install/lib/*'
)
cred_setup_prefix_path = (format(
'{stack_root}/{stack_version}/{service_name}/ext/ranger/scripts/ranger_credential_helper.py'
), '-l', cred_lib_prefix_path)
if params.retryAble:
Logger.info(
"nifi: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"nifi: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
# create ranger nifi audit directory
if params.xa_audit_hdfs_is_enabled and params.has_namenode and params.has_hdfs_client_on_node and upgrade_type is None:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/nifi",
type="directory",
action="create_on_execute",
owner=params.nifi_user,
group=params.nifi_group,
mode=0750,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'nifi',
params.service_name,
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.nifi_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_nifi,
conf_dict=params.nifi_config_dir,
component_user=params.nifi_user,
component_group=params.nifi_group,
cache_service_list=['nifi'],
plugin_audit_properties=params.config['configurations']
['ranger-nifi-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-nifi-audit'],
plugin_security_properties=params.config['configurations']
['ranger-nifi-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-nifi-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-nifi-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-nifi-policymgr-ssl'],
component_list=[],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.ranger_nifi_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_nifi_keytab
if params.security_enabled else None,
cred_lib_path_override=cred_lib_prefix_path,
cred_setup_prefix_override=cred_setup_prefix_path)
#change permissions of ranger xml that were written to 0400
File(os.path.join(params.nifi_config_dir, 'ranger-nifi-audit.xml'),
owner=params.nifi_user,
group=params.nifi_group,
mode=0400)
File(os.path.join(params.nifi_config_dir, 'ranger-nifi-security.xml'),
owner=params.nifi_user,
group=params.nifi_group,
mode=0400)
File(os.path.join(params.nifi_config_dir, 'ranger-policymgr-ssl.xml'),
owner=params.nifi_user,
group=params.nifi_group,
mode=0400)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.enable_ranger_storm and params.security_enabled:
site_files_create_path = format(
'{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf'
)
Directory(site_files_create_path,
owner=params.storm_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access='a')
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.has_namenode and params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.is_ranger_kms_ssl_enabled:
Logger.info(
'Ranger KMS is ssl enabled, configuring ssl-client for hdfs audits.'
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['ssl-client'],
configuration_attributes=params.
config['configurationAttributes']['ssl-client'],
file_name='ssl-client.xml')
else:
Logger.info(
'Ranger KMS is not ssl enabled, skipping ssl-client for hdfs audits.'
)
except Exception, err:
Logger.exception(
"Audit directory creation in DDPS for STORM Ranger plugin failed with error:\n{0}"
.format(err))
api_version = 'v2'
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.ranger_storm_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_storm_keytab
if params.security_enabled else None)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.security_enabled:
if params.has_namenode:
mount_table_xml_inclusion_file_full_path = None
mount_table_content = None
if 'viewfs-mount-table' in params.config['configurations']:
xml_inclusion_file_name = 'viewfs-mount-table.xml'
mount_table = params.config['configurations'][
'viewfs-mount-table']
if 'content' in mount_table and mount_table[
'content'].strip():
mount_table_xml_inclusion_file_full_path = os.path.join(
site_files_create_path, xml_inclusion_file_name)
mount_table_content = mount_table['content']
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'],
file_name='core-site.xml',
xml_include_file=mount_table_xml_inclusion_file_full_path,
xml_include_file_content=mount_table_content)
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations={
'hadoop.security.authentication':
'kerberos' if params.security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
if len(params.namenode_hosts) > 1:
Logger.info(
'Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml'
)
setup_configuration_file_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configurationAttributes']['hdfs-site'],
file_name='hdfs-site.xml')
else:
Logger.info(
'Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml'
)
File(format('{site_files_create_path}/hdfs-site.xml'),
action="delete")
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
def setup_ranger_knox(upgrade_type=None):
import params
if params.has_ranger_admin:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info("Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/knox",
type="directory",
action="create_on_execute",
owner=params.knox_user,
group=params.knox_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
if params.xml_configurations_supported:
api_version=None
if params.stack_supports_ranger_kerberos and params.security_enabled:
api_version='v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin('knox-server', 'knox',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.knox_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble,api_version=api_version,
is_security_enabled = params.security_enabled,
is_stack_supports_ranger_kerberos = params.stack_supports_ranger_kerberos,
component_user_principal=params.knox_principal_name if params.security_enabled else None,
component_user_keytab=params.knox_keytab_path if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin('knox-server', 'knox',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java_home,
params.repo_name, params.knox_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_knox, conf_dict=params.knox_conf_dir,
component_user=params.knox_user, component_group=params.knox_group, cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']['ranger-knox-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']['ranger-knox-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-knox-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
stack_version_override = stack_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.enable_ranger_storm and params.security_enabled:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_storm and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.xml_configurations_supported:
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.ranger_storm_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_storm_keytab
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
site_files_create_path = format(
'{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf'
)
Directory(site_files_create_path,
owner=params.storm_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access='a')
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.has_namenode and params.security_enabled:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions"
)
setup_core_site_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
config=params.config)
if len(params.namenode_hosts) > 1:
Logger.info(
'Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml'
)
XmlConfig("hdfs-site.xml",
conf_dir=site_files_create_path,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.storm_user,
group=params.user_group,
mode=0644)
else:
Logger.info(
'Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml'
)
File(format('{site_files_create_path}/hdfs-site.xml'),
action="delete")
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Storm plugin is not enabled')
def setup_ranger_kafka():
import params
if params.enable_ranger_kafka:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.has_namenode and params.xa_audit_hdfs_is_enabled:
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.is_ranger_kms_ssl_enabled:
Logger.info(
'Ranger KMS is ssl enabled, configuring ssl-client for hdfs audits.'
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['ssl-client'],
configuration_attributes=params.
config['configurationAttributes']['ssl-client'],
file_name='ssl-client.xml')
else:
Logger.info(
'Ranger KMS is not ssl enabled, skipping ssl-client for hdfs audits.'
)
except Exception, err:
Logger.exception(
"Audit directory creation in DDPS for KAFKA Ranger plugin failed with error:\n{0}"
.format(err))
setup_ranger_plugin(
'kafka-broker',
'kafka',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.kerberos_security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.kafka_jaas_principal
if params.kerberos_security_enabled else None,
component_user_keytab=params.kafka_keytab_path
if params.kerberos_security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination',
params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.kerberos_security_enabled:
# sometimes this is a link for missing /etc/hdp directory, just remove link/file and create regular file.
Execute(
('rm', '-f', os.path.join(params.conf_dir, "core-site.xml")),
sudo=True)
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configurationAttributes']['core-site'],
file_name='core-site.xml',
xml_include_file=params.
mount_table_xml_inclusion_file_full_path,
xml_include_file_content=params.mount_table_content)
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_configuration_file_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos'
if params.kerberos_security_enabled else 'simple'
},
configuration_attributes={},
file_name='core-site.xml')
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
def setup_ranger_kafka():
import params
if params.enable_ranger_kafka:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_kafka and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
setup_ranger_plugin(
'kafka-broker',
'kafka',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.kerberos_security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.kafka_jaas_principal
if params.kerberos_security_enabled else None,
component_user_keytab=params.kafka_keytab_path
if params.kerberos_security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination',
params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.kerberos_security_enabled:
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_core_site_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'])
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_core_site_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos'
if params.kerberos_security_enabled else 'simple'
},
configuration_attributes={})
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Kafka plugin is not enabled')
def setup_ranger_hdfs(upgrade_type=None):
import params
if params.enable_ranger_hdfs:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"HDFS: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"HDFS: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
setup_ranger_plugin(
'hadoop-client',
'hdfs',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.hdfs_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hdfs,
conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user,
component_group=params.user_group,
cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']
['ranger-hdfs-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hdfs-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hdfs-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version=api_version,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.nn_principal_name
if params.security_enabled else None,
component_user_keytab=params.nn_keytab
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'hadoop-client',
'hdfs',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.hdfs_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hdfs,
conf_dict=params.hadoop_conf_dir,
component_user=params.hdfs_user,
component_group=params.user_group,
cache_service_list=['hdfs'],
plugin_audit_properties=params.config['configurations']
['ranger-hdfs-audit'],
plugin_audit_attributes=params.
config['configurationAttributes']['ranger-hdfs-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hdfs-security'],
plugin_security_attributes=params.
config['configurationAttributes']['ranger-hdfs-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hdfs-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hdfs-policymgr-ssl'],
component_list=['hadoop-client'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
if stack_version and params.upgrade_direction == Direction.UPGRADE:
# when upgrading to stack remove_ranger_hdfs_plugin_env, this env file must be removed
if check_stack_feature(StackFeature.REMOVE_RANGER_HDFS_PLUGIN_ENV,
stack_version):
source_file = os.path.join(params.hadoop_conf_dir,
'set-hdfs-plugin-env.sh')
target_file = source_file + ".bak"
Execute(("mv", source_file, target_file),
sudo=True,
only_if=format("test -f {source_file}"))
else:
Logger.info('Ranger Hdfs plugin is not enabled')
def setup_ranger_hive_interactive(upgrade_type=None):
import params
if params.enable_ranger_hive:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Hive2: Setup ranger: command retry enabled thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Hive2: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.xa_audit_hdfs_is_enabled:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/hive2",
type="directory",
action="create_on_execute",
owner=params.hive_user,
group=params.hive_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'hive-server2-hive2',
'hive',
params.ranger_previous_jdbc_jar,
params.ranger_downloaded_custom_connector,
params.ranger_driver_curl_source,
params.ranger_driver_curl_target,
params.java64_home,
params.repo_name,
params.hive_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hive,
conf_dict=params.hive_server_interactive_conf_dir,
component_user=params.hive_user,
component_group=params.user_group,
cache_service_list=['hive-server2-hive2'],
plugin_audit_properties=params.config['configurations']
['ranger-hive-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-hive-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hive-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-hive-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hive-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hive-policymgr-ssl'],
component_list=[
'hive-client', 'hive-metastore', 'hive-server2',
'hive-server2-hive2'
],
audit_db_is_enabled=False,
credential_file=params.credential_file,
xa_audit_db_password=None,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version='v2',
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.hive_principal
if params.security_enabled else None,
component_user_keytab=params.hive_server2_keytab
if params.security_enabled else None)
else:
Logger.info('Ranger Hive plugin is not enabled')
def setup_ranger_hbase(upgrade_type=None):
import params
if params.has_ranger_admin:
if params.xml_configurations_supported:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
hdp_version = None
if upgrade_type is not None:
hdp_version = params.version
if params.retryAble:
Logger.info("HBase: Setup ranger: command retry enables thus retrying if ranger admin is down !")
else:
Logger.info("HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !")
if params.xml_configurations_supported and params.enable_ranger_hbase and params.xa_audit_hdfs_is_enabled:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/hbaseMaster",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource("/ranger/audit/hbaseRegional",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True
)
params.HdfsResource(None, action="execute")
setup_ranger_plugin('hbase-client', 'hbase',
params.downloaded_custom_connector, params.driver_curl_source,
params.driver_curl_target, params.java64_home,
params.repo_name, params.hbase_ranger_plugin_repo,
params.ranger_env, params.ranger_plugin_properties,
params.policy_user, params.policymgr_mgr_url,
params.enable_ranger_hbase, conf_dict=params.hbase_conf_dir,
component_user=params.hbase_user, component_group=params.user_group, cache_service_list=['hbaseMaster', 'hbaseRegional'],
plugin_audit_properties=params.config['configurations']['ranger-hbase-audit'], plugin_audit_attributes=params.config['configuration_attributes']['ranger-hbase-audit'],
plugin_security_properties=params.config['configurations']['ranger-hbase-security'], plugin_security_attributes=params.config['configuration_attributes']['ranger-hbase-security'],
plugin_policymgr_ssl_properties=params.config['configurations']['ranger-hbase-policymgr-ssl'], plugin_policymgr_ssl_attributes=params.config['configuration_attributes']['ranger-hbase-policymgr-ssl'],
component_list=['hbase-client', 'hbase-master', 'hbase-regionserver'], audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file, xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password, ssl_keystore_password=params.ssl_keystore_password,
hdp_version_override = hdp_version, skip_if_rangeradmin_down= not params.retryAble)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_solr():
import params
Execute(
'ln -sf /usr/adh/current/ranger-admin/contrib/solr_for_audit_setup/ /usr/adh/current/solr-server/server/solr/configsets/ranger_audit_configs'
)
if params.has_ranger_admin and params.security_enabled:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Solr: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Solr: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_solr and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/solr",
type="directory",
action="create_on_execute",
owner=params.solr_user,
group=params.solr_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
File(os.path.join(params.stack_root, params.stack_version,
"ranger-solr-plugin", "ranger_credential_helper.py"),
mode=0755)
jar_files = os.listdir(
os.path.join(params.stack_root, params.stack_version,
"ranger-solr-plugin/lib"))
for jar_file in jar_files:
plugin_dir = os.path.join(params.stack_root, params.stack_version,
"ranger-solr-plugin/lib", jar_file)
Execute(
('ln', '-sf', plugin_dir,
os.path.join(params.stack_root, params.stack_version,
"solr/server/solr-webapp/webapp/WEB-INF/lib",
jar_file)),
only_if=format('ls {plugin_dir}'),
sudo=True)
setup_ranger_plugin(
'solr-server',
'solr',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.solr_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_solr,
conf_dict=params.solr_conf,
component_user=params.solr_user,
component_group=params.user_group,
cache_service_list=['solr'],
plugin_audit_properties=params.ranger_solr_audit,
plugin_audit_attributes=params.ranger_solr_audit_attrs,
plugin_security_properties=params.ranger_solr_security,
plugin_security_attributes=params.ranger_solr_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_solr_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_solr_policymgr_ssl_attrs,
component_list=['solr'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.solr_kerberos_principal
if params.security_enabled else None,
component_user_keytab=params.solr_kerberos_keytab
if params.security_enabled else None)
properties_files = os.listdir(format('/etc/solr/conf'))
if params.security_enabled and params.enable_ranger_solr:
solr_classes_dir = os.path.join(
params.stack_root, params.stack_version,
"solr/server/solr-webapp/webapp/WEB-INF/classes")
Directory(solr_classes_dir,
owner=params.solr_user,
group=params.user_group,
ignore_failures=True)
Execute(format('mkdir {solr_classes_dir}'),
not_if=format('ls {solr_classes_dir}'))
for properties_file in properties_files:
Execute(
('ln', '-sf', format('/etc/solr/conf/{properties_file}'),
os.path.join(
params.stack_root, params.stack_version,
"solr/server/solr-webapp/webapp/WEB-INF/classes",
properties_file)),
only_if=format('ls /etc/solr/conf/{properties_file}'),
sudo=True)
zk_port = ":" + params.zookeeper_port + ","
if params.enable_ranger_solr:
zookeeper_hosts_ip = zk_port.join(
params.zookeeper_hosts_list) + ":" + params.zookeeper_port
zookeeper_script = os.path.join(
params.stack_root, params.stack_version,
"solr/server/scripts/cloud-scripts/zkcli.sh")
set_solr_ranger_authorizer = format(
'{zookeeper_script} -zkhost {zookeeper_hosts_ip} ' +
'-cmd put /solr/security.json \'{{\"authentication":{{\"class\":\"org.apache.solr.security.KerberosPlugin\"}},\"authorization\":{{\"class\": '
+
'\"org.apache.ranger.authorization.solr.authorizer.RangerSolrAuthorizer\"}}}}\''
)
Execute(set_solr_ranger_authorizer)
else:
Logger.info('Ranger admin not installed')
def setup_ranger_hbase(upgrade_type=None, service_name="hbase-master"):
import params
if params.enable_ranger_hbase:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"HBase: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"HBase: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xa_audit_hdfs_is_enabled and service_name == 'hbase-master':
try:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/hbaseMaster",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/hbaseRegional",
type="directory",
action="create_on_execute",
owner=params.hbase_user,
group=params.hbase_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
except Exception, err:
Logger.exception(
"Audit directory creation in HDFS for HBASE Ranger plugin failed with error:\n{0}"
.format(err))
api_version = 'v2'
setup_ranger_plugin(
'hbase-client',
'hbase',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.hbase_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_hbase,
conf_dict=params.hbase_conf_dir,
component_user=params.hbase_user,
component_group=params.user_group,
cache_service_list=['hbaseMaster', 'hbaseRegional'],
plugin_audit_properties=params.config['configurations']
['ranger-hbase-audit'],
plugin_audit_attributes=params.config['configurationAttributes']
['ranger-hbase-audit'],
plugin_security_properties=params.config['configurations']
['ranger-hbase-security'],
plugin_security_attributes=params.config['configurationAttributes']
['ranger-hbase-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-hbase-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.
config['configurationAttributes']['ranger-hbase-policymgr-ssl'],
component_list=[
'hbase-client', 'hbase-master', 'hbase-regionserver'
],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos
if params.security_enabled else None,
component_user_principal=params.ranger_hbase_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_hbase_keytab
if params.security_enabled else None)
