def get_role_component_current_hdp_version():
"""
Gets the current HDP version of the component that this role command is for.
:return: the current HDP version of the specified component or None
"""
hdp_select_component = None
role = default("/role", "")
role_command = default("/roleCommand", "")
if role in SERVER_ROLE_DIRECTORY_MAP:
hdp_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
hdp_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
if hdp_select_component is None:
return None
current_hdp_version = get_hdp_version(hdp_select_component)
if current_hdp_version is None:
Logger.warning("Unable to determine hdp-select version for {0}".format(
hdp_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
hdp_select_component, current_hdp_version))
return current_hdp_version
def get_role_component_current_hdp_version():
"""
Gets the current HDP version of the component that this role command is for.
:return: the current HDP version of the specified component or None
"""
hdp_select_component = None
role = default("/role", "")
role_command = default("/roleCommand", "")
if role in SERVER_ROLE_DIRECTORY_MAP:
hdp_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
hdp_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
if hdp_select_component is None:
return None
current_hdp_version = get_hdp_version(hdp_select_component)
if current_hdp_version is None:
Logger.warning("Unable to determine hdp-select version for {0}".format(
hdp_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
hdp_select_component, current_hdp_version))
return current_hdp_version
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
custom_plugin_properties = dict()
custom_plugin_properties['CUSTOM_USER'] = component_user
custom_plugin_properties['CUSTOM_GROUP'] = component_group
ModifyPropertiesFile(file_path,properties = custom_plugin_properties)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def setup_ranger_plugin(component_select_name, service_name,
downloaded_custom_connector, driver_curl_source,
driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled):
File(downloaded_custom_connector,
content = DownloadSource(driver_curl_source)
)
Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target),
not_if=format("test -f {driver_curl_target}"),
sudo=True
)
hdp_version = get_hdp_version(component_select_name)
file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties')
if not os.path.isfile(file_path):
raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}'))
ModifyPropertiesFile(file_path,
properties = plugin_properties
)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'),)
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'),)
cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')}
Execute(cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
from status_params import *
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import conf_select
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
upgrade_direction = default("/commandParams/upgrade_direction", None)
version = default("/commandParams/version", None)
if version is None:
version = get_hdp_version('knox-server')
# E.g., 2.3.2.0
version_formatted = format_hdp_stack_version(version)
# E.g., 2.3
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# This is the version whose state is CURRENT. During an RU, this is the source version.
# DO NOT format it since we need the build number too.
upgrade_from_version = default("/hostLevelParams/current_version", None)
# server configurations
# Default value used in HDP 2.3.0.0 and earlier.
knox_data_dir = '/var/lib/knox/data'
audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}')
jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver"
downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}")
driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}")
driver_curl_target = format("{kafka_home}libs/{jdbc_jar_name}")
ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls']
xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None
ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None
hdp_version = get_hdp_version('kafka-broker')
setup_ranger_env_sh_source = format('/usr/hdp/{hdp_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh')
setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh")
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
xa_audit_db_is_enabled = False
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None
hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None
hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None
default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
def setup_ranger_plugin(component_select_name,
service_name,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
hdp_version_override=None,
skip_if_rangeradmin_down=True):
if audit_db_is_enabled:
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
if hdp_version_override is not None:
hdp_version = hdp_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled,
hdp_version, credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password, component_user,
component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete")
def refresh_tez_state_dependent_params():
global tez_home_dir, tez_conf_dir, hdp_stack_version
tez_home_dir = os.environ["TEZ_HOME"]
tez_conf_dir = os.path.join(tez_home_dir, "conf")
# this is not available on INSTALL action because hdp-select is not available
hdp_stack_version = get_hdp_version("tez")
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.script.script import Script
from status_params import *
from resource_management.libraries.resources.hdfs_resource import HdfsResource
from resource_management.libraries.functions import hdp_select
from resource_management.libraries.functions import conf_select
# server configurations
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_name = default("/hostLevelParams/stack_name", None)
upgrade_direction = default("/commandParams/upgrade_direction", None)
version = default("/commandParams/version", None)
if version is None:
version = get_hdp_version('knox-server')
# E.g., 2.3.2.0
version_formatted = format_hdp_stack_version(version)
# E.g., 2.3
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
hdp_stack_version = format_hdp_stack_version(stack_version_unformatted)
# This is the version whose state is CURRENT. During an RU, this is the source version.
# DO NOT format it since we need the build number too.
upgrade_from_version = default("/hostLevelParams/current_version", None)
# server configurations
# Default value used in HDP 2.3.0.0 and earlier.
knox_data_dir = '/var/lib/knox/data'
xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit'][
'xasecure.audit.destination.db'] if xml_configurations_supported else None
xa_audit_hdfs_is_enabled = config['configurations']['ranger-kafka-audit'][
'xasecure.audit.destination.hdfs'] if xml_configurations_supported else None
ssl_keystore_password = unicode(
config['configurations']['ranger-kafka-policymgr-ssl']
['xasecure.policymgr.clientssl.keystore.password']
) if xml_configurations_supported else None
ssl_truststore_password = unicode(
config['configurations']['ranger-kafka-policymgr-ssl']
['xasecure.policymgr.clientssl.truststore.password']
) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks'
) if xml_configurations_supported else None
hdp_version = get_hdp_version('kafka-broker')
setup_ranger_env_sh_source = format(
'/usr/hdp/{hdp_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh'
)
setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh")
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
xa_audit_db_is_enabled = False
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
hdfs_user = config['configurations']['hadoop-env'][
'hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env'][
def refresh_tez_state_dependent_params():
global tez_home_dir, tez_conf_dir, hdp_stack_version
tez_home_dir = os.environ["TEZ_HOME"]
tez_conf_dir = os.path.join(tez_home_dir, "conf")
# this is not available on INSTALL action because hdp-select is not available
hdp_stack_version = get_hdp_version("tez")
def setup_ranger_plugin(component_select_name, service_name,
component_downloaded_custom_connector, component_driver_curl_source,
component_driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, conf_dict, component_user, component_group,
cache_service_list, plugin_audit_properties, plugin_audit_attributes,
plugin_security_properties, plugin_security_attributes,
plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes,
component_list, audit_db_is_enabled, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, api_version=None, hdp_version_override = None, skip_if_rangeradmin_down = True):
if audit_db_is_enabled:
File(component_downloaded_custom_connector,
content = DownloadSource(component_driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(component_driver_curl_target, mode=0644)
hdp_version = get_hdp_version(component_select_name)
if hdp_version_override is not None:
hdp_version = hdp_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
current_datetime = datetime.now()
File(format('{component_conf_dir}/ranger-security.xml'),
owner = component_user,
group = component_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')],
owner = component_user,
group = component_group,
mode=0775,
recursive = True,
cd_access = 'a'
)
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')),
owner = component_user,
group = component_group,
mode = 0644
)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner = component_user,
group = component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner = component_user,
group = component_group,
mode=0744)
if str(service_name).lower() == 'yarn' :
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
else :
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file,
xa_audit_db_password, ssl_truststore_password, ssl_keystore_password,
component_user, component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete"
)
