def setup_ranger_knox(upgrade_type=None):
import params
if params.enable_ranger_knox:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Knox: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Knox: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_knox and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/knox",
type="directory",
action="create_on_execute",
owner=params.knox_user,
group=params.knox_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.namenode_hosts is not None and len(
params.namenode_hosts) > 1:
Logger.info(
'Ranger Knox plugin is enabled in NameNode HA environment along with audit to Hdfs enabled, creating hdfs-site.xml'
)
XmlConfig("hdfs-site.xml",
conf_dir=params.knox_conf_dir,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.knox_user,
group=params.knox_group,
mode=0644)
else:
File(format('{knox_conf_dir}/hdfs-site.xml'),
action="delete")
if params.xml_configurations_supported:
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.knox_principal_name
if params.security_enabled else None,
component_user_keytab=params.knox_keytab_path
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'knox-server',
'knox',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java_home,
params.repo_name,
params.knox_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_knox,
conf_dict=params.knox_conf_dir,
component_user=params.knox_user,
component_group=params.knox_group,
cache_service_list=['knox'],
plugin_audit_properties=params.config['configurations']
['ranger-knox-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-knox-audit'],
plugin_security_properties=params.config['configurations']
['ranger-knox-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-knox-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-knox-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-knox-policymgr-ssl'],
component_list=['knox-server'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_knox and params.has_namenode and params.security_enabled:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin, creating core-site.xml from namenode configuraitions"
)
setup_core_site_for_required_plugins(
component_user=params.knox_user,
component_group=params.knox_group,
create_core_site_path=params.knox_conf_dir,
config=params.config)
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Knox plugin is not enabled')
def setup_ranger_kafka():
import params
if params.enable_ranger_kafka:
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_kafka and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
setup_ranger_plugin(
'kafka-broker',
'kafka',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.kerberos_security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.kafka_jaas_principal
if params.kerberos_security_enabled else None,
component_user_keytab=params.kafka_keytab_path
if params.kerberos_security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '--remove-destination',
params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_kafka and params.kerberos_security_enabled:
if params.has_namenode:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is installed, creating create core-site.xml from namenode configurations"
)
setup_core_site_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations=params.config['configurations']
['core-site'],
configuration_attributes=params.
config['configuration_attributes']['core-site'])
else:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin and Namenode is not installed, creating create core-site.xml from default configurations"
)
setup_core_site_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
configurations={
'hadoop.security.authentication':
'kerberos'
if params.kerberos_security_enabled else 'simple'
},
configuration_attributes={})
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Kafka plugin is not enabled')
def setup_ranger_kafka():
import params
if params.enable_ranger_kafka:
import sys, os
script_path = os.path.realpath(__file__).split(
'/services')[0] + '/hooks/before-INSTALL/scripts/ranger'
sys.path.append(script_path)
from setup_ranger_plugin_xml import setup_ranger_plugin
if params.retryAble:
Logger.info(
"Kafka: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Kafka: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_kafka and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/kafka",
type="directory",
action="create_on_execute",
owner=params.kafka_user,
group=params.kafka_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
setup_ranger_plugin(
'kafka',
'kafka',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.kafka_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_kafka,
conf_dict=params.conf_dir,
component_user=params.kafka_user,
component_group=params.user_group,
cache_service_list=['kafka'],
plugin_audit_properties=params.ranger_kafka_audit,
plugin_audit_attributes=params.ranger_kafka_audit_attrs,
plugin_security_properties=params.ranger_kafka_security,
plugin_security_attributes=params.ranger_kafka_security_attrs,
plugin_policymgr_ssl_properties=params.ranger_kafka_policymgr_ssl,
plugin_policymgr_ssl_attributes=params.
ranger_kafka_policymgr_ssl_attrs,
component_list=['kafka-broker'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
api_version='v2',
skip_if_rangeradmin_down=not params.retryAble,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.kafka_jaas_principal
if params.security_enabled else None,
component_user_keytab=params.kafka_keytab_path
if params.security_enabled else None)
if params.enable_ranger_kafka:
Execute(('cp', '-rf', params.setup_ranger_env_sh_source,
params.setup_ranger_env_sh_target),
not_if=format("test -f {setup_ranger_env_sh_target}"),
sudo=True)
File(params.setup_ranger_env_sh_target,
owner=params.kafka_user,
group=params.user_group,
mode=0755)
if params.enable_ranger_kafka and params.has_namenode and params.security_enabled:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions"
)
setup_core_site_for_required_plugins(
component_user=params.kafka_user,
component_group=params.user_group,
create_core_site_path=params.conf_dir,
config=params.config)
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Kafka plugin is not enabled')
def setup_ranger_storm(upgrade_type=None):
"""
:param upgrade_type: Upgrade Type such as "rolling" or "nonrolling"
"""
import params
if params.enable_ranger_storm and params.security_enabled:
stack_version = None
if upgrade_type is not None:
stack_version = params.version
if params.retryAble:
Logger.info(
"Storm: Setup ranger: command retry enables thus retrying if ranger admin is down !"
)
else:
Logger.info(
"Storm: Setup ranger: command retry not enabled thus skipping if ranger admin is down !"
)
if params.xml_configurations_supported and params.enable_ranger_storm and params.xa_audit_hdfs_is_enabled:
if params.has_namenode:
params.HdfsResource("/ranger/audit",
type="directory",
action="create_on_execute",
owner=params.hdfs_user,
group=params.hdfs_user,
mode=0755,
recursive_chmod=True)
params.HdfsResource("/ranger/audit/storm",
type="directory",
action="create_on_execute",
owner=params.storm_user,
group=params.storm_user,
mode=0700,
recursive_chmod=True)
params.HdfsResource(None, action="execute")
if params.xml_configurations_supported:
api_version = None
if params.stack_supports_ranger_kerberos:
api_version = 'v2'
from resource_management.libraries.functions.setup_ranger_plugin_xml import setup_ranger_plugin
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble,
api_version=api_version,
is_security_enabled=params.security_enabled,
is_stack_supports_ranger_kerberos=params.
stack_supports_ranger_kerberos,
component_user_principal=params.ranger_storm_principal
if params.security_enabled else None,
component_user_keytab=params.ranger_storm_keytab
if params.security_enabled else None)
else:
from resource_management.libraries.functions.setup_ranger_plugin import setup_ranger_plugin
setup_ranger_plugin(
'storm-nimbus',
'storm',
params.previous_jdbc_jar,
params.downloaded_custom_connector,
params.driver_curl_source,
params.driver_curl_target,
params.java64_home,
params.repo_name,
params.storm_ranger_plugin_repo,
params.ranger_env,
params.ranger_plugin_properties,
params.policy_user,
params.policymgr_mgr_url,
params.enable_ranger_storm,
conf_dict=params.conf_dir,
component_user=params.storm_user,
component_group=params.user_group,
cache_service_list=['storm'],
plugin_audit_properties=params.config['configurations']
['ranger-storm-audit'],
plugin_audit_attributes=params.
config['configuration_attributes']['ranger-storm-audit'],
plugin_security_properties=params.config['configurations']
['ranger-storm-security'],
plugin_security_attributes=params.
config['configuration_attributes']['ranger-storm-security'],
plugin_policymgr_ssl_properties=params.config['configurations']
['ranger-storm-policymgr-ssl'],
plugin_policymgr_ssl_attributes=params.config[
'configuration_attributes']['ranger-storm-policymgr-ssl'],
component_list=['storm-client', 'storm-nimbus'],
audit_db_is_enabled=params.xa_audit_db_is_enabled,
credential_file=params.credential_file,
xa_audit_db_password=params.xa_audit_db_password,
ssl_truststore_password=params.ssl_truststore_password,
ssl_keystore_password=params.ssl_keystore_password,
stack_version_override=stack_version,
skip_if_rangeradmin_down=not params.retryAble)
site_files_create_path = format(
'{storm_component_home_dir}/extlib-daemon/ranger-storm-plugin-impl/conf'
)
Directory(site_files_create_path,
owner=params.storm_user,
group=params.user_group,
mode=0775,
create_parents=True,
cd_access='a')
if params.stack_supports_core_site_for_ranger_plugin and params.enable_ranger_storm and params.has_namenode and params.security_enabled:
Logger.info(
"Stack supports core-site.xml creation for Ranger plugin, creating create core-site.xml from namenode configuraitions"
)
setup_core_site_for_required_plugins(
component_user=params.storm_user,
component_group=params.user_group,
create_core_site_path=site_files_create_path,
config=params.config)
if len(params.namenode_hosts) > 1:
Logger.info(
'Ranger Storm plugin is enabled along with security and NameNode is HA , creating hdfs-site.xml'
)
XmlConfig("hdfs-site.xml",
conf_dir=site_files_create_path,
configurations=params.config['configurations']
['hdfs-site'],
configuration_attributes=params.
config['configuration_attributes']['hdfs-site'],
owner=params.storm_user,
group=params.user_group,
mode=0644)
else:
Logger.info(
'Ranger Storm plugin is not enabled or security is disabled, removing hdfs-site.xml'
)
File(format('{site_files_create_path}/hdfs-site.xml'),
action="delete")
else:
Logger.info(
"Stack does not support core-site.xml creation for Ranger plugin, skipping core-site.xml configurations"
)
else:
Logger.info('Ranger Storm plugin is not enabled')
