def get_role_component_current_stack_version():
"""
Gets the current HDP version of the component that this role command is for.
:return: the current HDP version of the specified component or None
"""
role = default("/role", "")
role_command = default("/roleCommand", "")
stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
Logger.info("Checking version for {0} via {1}".format(role, stack_selector_name))
if role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
else:
stack_select_component = get_package_name()
if stack_select_component is None:
if not role:
Logger.error("No role information available.")
elif not role.lower().endswith("client"):
Logger.error("Mapping unavailable for role {0}. Skip checking its version.".format(role))
return None
current_stack_version = get_stack_version(stack_select_component)
if current_stack_version is None:
Logger.warning("Unable to determine {0} version for {1}".format(
stack_selector_name, stack_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
stack_select_component, current_stack_version))
return current_stack_version
def get_role_component_current_stack_version():
"""
Gets the current HDP version of the component that this role command is for.
:return: the current HDP version of the specified component or None
"""
stack_select_component = None
role = default("/role", "")
role_command = default("/roleCommand", "")
stack_selector_name = stack_tools.get_stack_tool_name(
stack_tools.STACK_SELECTOR_NAME)
if role in SERVER_ROLE_DIRECTORY_MAP:
stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
if stack_select_component is None:
return None
current_stack_version = get_stack_version(stack_select_component)
if current_stack_version is None:
Logger.warning("Unable to determine {0} version for {1}".format(
stack_selector_name, stack_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
stack_select_component, current_stack_version))
return current_stack_version
def get_role_component_current_stack_version():
"""
Gets the current HDP version of the component that this role command is for.
:return: the current HDP version of the specified component or None
"""
stack_select_component = None
role = default("/role", "")
role_command = default("/roleCommand", "")
stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME)
if role in SERVER_ROLE_DIRECTORY_MAP:
stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role]
elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP:
stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role]
if stack_select_component is None:
return None
current_stack_version = get_stack_version(stack_select_component)
if current_stack_version is None:
Logger.warning("Unable to determine {0} version for {1}".format(
stack_selector_name, stack_select_component))
else:
Logger.info("{0} is currently at version {1}".format(
stack_select_component, current_stack_version))
return current_stack_version
'ranger-kafka-audit']['xasecure.audit.destination.db']
xa_audit_hdfs_is_enabled = default(
'/configurations/ranger-kafka-audit/xasecure.audit.destination.hdfs',
False)
ssl_keystore_password = unicode(
config['configurations']['ranger-kafka-policymgr-ssl']
['xasecure.policymgr.clientssl.keystore.password']
) if xml_configurations_supported else None
ssl_truststore_password = unicode(
config['configurations']['ranger-kafka-policymgr-ssl']
['xasecure.policymgr.clientssl.truststore.password']
) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks'
) if xml_configurations_supported else None
stack_version = get_stack_version('kafka-broker')
setup_ranger_env_sh_source = format(
'{stack_root}/{stack_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh'
)
setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh")
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
xa_audit_db_is_enabled = False
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
hdfs_user = config['configurations']['hadoop-env'][
'hdfs_user'] if has_namenode else None
hdfs_user_keytab = config['configurations']['hadoop-env'][
spark_yarn_historyServer_address = default(spark_history_server_host,
"localhost")
spark_history_ui_port = config['configurations']['spark-defaults'][
'spark.history.ui.port']
spark_env_sh = config['configurations']['spark-env']['content']
spark_log4j_properties = config['configurations']['spark-log4j-properties'][
'content']
spark_metrics_properties = config['configurations'][
'spark-metrics-properties']['content']
hive_server_host = default("/clusterHostInfo/hive_server_hosts", [])
is_hive_installed = not len(hive_server_host) == 0
full_stack_version = get_stack_version('spark-client')
spark_javaopts_properties = default(
"/configurations/spark-javaopts-properties/content", " ")
if spark_javaopts_properties.find('-Dhdp.version') == -1:
spark_javaopts_properties = spark_javaopts_properties + ' -Dhdp.version=' + str(
full_stack_version)
else:
lists = spark_javaopts_properties.split(" ")
for idx, val in enumerate(lists):
if (val.startswith("-Dhdp.version=")):
lists[idx] = "-Dhdp.version=" + str(full_stack_version)
spark_javaopts_properties = " ".join(lists)
security_enabled = config['configurations']['cluster-env']['security_enabled']
kinit_path_local = get_kinit_path(
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
if stack_version_override is None:
stack_version = get_stack_version(component_select_name)
else:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = False
policycache_path = os.path.join('/etc', 'ranger', repo_name,
'policycache')
try:
for cache_service in cache_service_list:
policycache_json_file = format(
'{policycache_path}/{cache_service}_{repo_name}.json')
if os.path.isfile(policycache_json_file) and os.path.getsize(
policycache_json_file) > 0:
with open(policycache_json_file) as json_file:
json_data = json.load(json_file)
if 'serviceName' in json_data and json_data[
'serviceName'] == repo_name:
service_name_exist = True
Logger.info(
"Skipping Ranger API calls, as policy cache file exists for {0}"
.format(service_name))
Logger.warning(
"If service name for {0} is not created on Ranger Admin UI, then to re-create it delete policy cache file: {1}"
.format(service_name, policycache_json_file))
break
except Exception, err:
Logger.error(
"Error occurred while fetching service name from policy cache file.\nError: {0}"
.format(err))
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
# creating symblink should be done by rpm package
# setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(
service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user, component_group, java_home,
cred_lib_path_override, cred_setup_prefix_override)
zeppelin_kerberos_keytab = config['configurations']['zeppelin-env'][
'zeppelin.server.kerberos.keytab']
zeppelin_kerberos_principal = config['configurations']['zeppelin-env'][
'zeppelin.server.kerberos.principal']
# e.g. 2.3
stack_version_unformatted = config['hostLevelParams']['stack_version']
# e.g. 2.3.0.0
stack_version_formatted = format_stack_version(stack_version_unformatted)
# e.g. 2.3.0.0-2130
full_stack_version = default("/commandParams/version", None)
spark_client_version = get_stack_version('spark-client')
hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", [])
livy_hosts = default("/clusterHostInfo/livy_server_hosts", [])
livy_livyserver_host = None
livy_livyserver_port = None
if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and \
len(livy_hosts) > 0:
livy_livyserver_host = str(livy_hosts[0])
livy_livyserver_port = config['configurations']['livy-conf'][
'livy.server.port']
hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
security_enabled = config['configurations']['cluster-env']['security_enabled']
hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
xa_audit_hdfs_is_enabled = default(
'/configurations/ranger-solr-audit/xasecure.audit.destination.hdfs',
False)
ssl_keystore_password = unicode(
config['configurations']['ranger-solr-policymgr-ssl']
['xasecure.policymgr.clientssl.keystore.password']
) if xml_configurations_supported else None
ssl_truststore_password = unicode(
config['configurations']['ranger-solr-policymgr-ssl']
['xasecure.policymgr.clientssl.truststore.password']
) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks'
) if xml_configurations_supported else None
stack_version = get_stack_version(
'hadoop-hdfs-datanode'
) # use hadoop-hdfs-datanode as its required for solr
setup_ranger_env_sh_source = format(
'{stack_root}/{stack_version}/ranger-solr-plugin/install/conf.templates/enable/solr-ranger-env.sh'
)
setup_ranger_env_sh_target = format("{solr_conf}/solr-ranger-env.sh")
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
xa_audit_db_is_enabled = False
namenode_hosts = default("/clusterHostInfo/namenode_hosts", [])
has_namenode = not len(namenode_hosts) == 0
# *********************** end RANGER PLUGIN CHANGES ****************
smokeuser = config['configurations']['cluster-env']['smokeuser']
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
component_downloaded_custom_connector,
component_driver_curl_source,
component_driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
conf_dict,
component_user,
component_group,
cache_service_list,
plugin_audit_properties,
plugin_audit_attributes,
plugin_security_properties,
plugin_security_attributes,
plugin_policymgr_ssl_properties,
plugin_policymgr_ssl_attributes,
component_list,
audit_db_is_enabled,
credential_file,
xa_audit_db_password,
ssl_truststore_password,
ssl_keystore_password,
api_version=None,
stack_version_override=None,
skip_if_rangeradmin_down=True,
is_security_enabled=False,
is_stack_supports_ranger_kerberos=False,
component_user_principal=None,
component_user_keytab=None,
cred_lib_path_override=None,
cred_setup_prefix_override=None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith(
"/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content=DownloadSource(component_driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination',
component_downloaded_custom_connector,
component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
if stack_version_override is None:
stack_version = get_stack_version(component_select_name)
else:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
service_name_exist = get_policycache_service_name(
service_name, repo_name, cache_service_list)
if not service_name_exist:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user,
is_security_enabled, is_stack_supports_ranger_kerberos,
component_user, component_user_principal,
component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
current_datetime = datetime.now()
File(
format('{component_conf_dir}/ranger-security.xml'),
owner=component_user,
group=component_group,
mode=0644,
content=InlineTemplate(
format(
'<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'
)))
Directory([
os.path.join('/etc', 'ranger', repo_name),
os.path.join('/etc', 'ranger', repo_name, 'policycache')
],
owner=component_user,
group=component_group,
mode=0775,
create_parents=True,
cd_access='a')
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',
format('{cache_service}_{repo_name}.json')),
owner=component_user,
group=component_group,
mode=0644)
# remove plain-text password from xml configs
plugin_audit_password_property = 'xasecure.audit.destination.db.password'
plugin_audit_properties_copy = {}
plugin_audit_properties_copy.update(plugin_audit_properties)
if plugin_audit_password_property in plugin_audit_properties_copy:
plugin_audit_properties_copy[
plugin_audit_password_property] = "crypted"
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties_copy,
configuration_attributes=plugin_audit_attributes,
owner=component_user,
group=component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner=component_user,
group=component_group,
mode=0744)
# remove plain-text password from xml configs
plugin_password_properties = [
'xasecure.policymgr.clientssl.keystore.password',
'xasecure.policymgr.clientssl.truststore.password'
]
plugin_policymgr_ssl_properties_copy = {}
plugin_policymgr_ssl_properties_copy.update(
plugin_policymgr_ssl_properties)
for prop in plugin_password_properties:
if prop in plugin_policymgr_ssl_properties_copy:
plugin_policymgr_ssl_properties_copy[prop] = "crypted"
if str(service_name).lower() == 'yarn':
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
else:
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties_copy,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner=component_user,
group=component_group,
mode=0744)
setup_ranger_plugin_keystore(
service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, component_user, component_group, java_home,
cred_lib_path_override, cred_setup_prefix_override)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete")
'xasecure.audit.destination.db']
xa_audit_hdfs_is_enabled = default(
'/configurations/ranger-solr-audit/xasecure.audit.destination.hdfs',
False)
ssl_keystore_password = unicode(
config['configurations']['ranger-solr-policymgr-ssl']
['xasecure.policymgr.clientssl.keystore.password']
) if xml_configurations_supported else None
ssl_truststore_password = unicode(
config['configurations']['ranger-solr-policymgr-ssl']
['xasecure.policymgr.clientssl.truststore.password']
) if xml_configurations_supported else None
credential_file = format('/etc/ranger/{repo_name}/cred.jceks'
) if xml_configurations_supported else None
stack_version = get_stack_version('solr-server')
setup_ranger_env_sh_source = format(
'{stack_root}/{stack_version}/ranger-solr-plugin/install/conf.templates/enable/solr-ranger-env.sh'
)
setup_ranger_env_sh_target = format("{solr_conf}/solr-ranger-env.sh")
#For SQLA explicitly disable audit to DB for Ranger
if xa_audit_db_flavor == 'sqla':
xa_audit_db_is_enabled = False
namenode_hosts = default("/clusterHostInfo/namenode_host", [])
has_namenode = not len(namenode_hosts) == 0
# *********************** end RANGER PLUGIN CHANGES ****************
smokeuser = config['configurations']['cluster-env']['smokeuser']
smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
def setup_ranger_plugin(component_select_name, service_name,
component_downloaded_custom_connector, component_driver_curl_source,
component_driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, conf_dict, component_user, component_group,
cache_service_list, plugin_audit_properties, plugin_audit_attributes,
plugin_security_properties, plugin_security_attributes,
plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes,
component_list, audit_db_is_enabled, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, api_version=None, stack_version_override = None, skip_if_rangeradmin_down = True):
if audit_db_is_enabled:
File(component_downloaded_custom_connector,
content = DownloadSource(component_driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(component_driver_curl_target, mode=0644)
stack_version = get_stack_version(component_select_name)
if stack_version_override is not None:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
current_datetime = datetime.now()
File(format('{component_conf_dir}/ranger-security.xml'),
owner = component_user,
group = component_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')],
owner = component_user,
group = component_group,
mode=0775,
create_parents = True,
cd_access = 'a'
)
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')),
owner = component_user,
group = component_group,
mode = 0644
)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner = component_user,
group = component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner = component_user,
group = component_group,
mode=0744)
if str(service_name).lower() == 'yarn' :
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
else :
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password, ssl_keystore_password,
component_user, component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete"
)
def setup_ranger_plugin(component_select_name,
service_name,
previous_jdbc_jar,
downloaded_custom_connector,
driver_curl_source,
driver_curl_target,
java_home,
repo_name,
plugin_repo_dict,
ranger_env_properties,
plugin_properties,
policy_user,
policymgr_mgr_url,
plugin_enabled,
component_user,
component_group,
api_version=None,
skip_if_rangeradmin_down=True,
**kwargs):
if driver_curl_source and not driver_curl_source.endswith("/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(downloaded_custom_connector,
content=DownloadSource(driver_curl_source),
mode=0644)
Execute(('cp', '--remove-destination', downloaded_custom_connector,
driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True)
File(driver_curl_target, mode=0644)
stack_root = Script.get_stack_root()
stack_version = get_stack_version(component_select_name)
file_path = format(
'{stack_root}/{stack_version}/ranger-{service_name}-plugin/install.properties'
)
if not os.path.isfile(file_path):
raise Fail(
format(
'Ranger {service_name} plugin install.properties file does not exist at {file_path}'
))
ModifyPropertiesFile(file_path, properties=plugin_properties)
custom_plugin_properties = dict()
custom_plugin_properties['CUSTOM_USER'] = component_user
custom_plugin_properties['CUSTOM_GROUP'] = component_group
ModifyPropertiesFile(file_path, properties=custom_plugin_properties)
if plugin_enabled:
cmd = (format('enable-{service_name}-plugin.sh'), )
if api_version == 'v2' and api_version is not None:
ranger_adm_obj = RangeradminV2(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
else:
ranger_adm_obj = Rangeradmin(
url=policymgr_mgr_url,
skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(
service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'],
ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'],
ranger_env_properties['admin_password'], policy_user)
else:
cmd = (format('disable-{service_name}-plugin.sh'), )
cmd_env = {
'JAVA_HOME': java_home,
'PWD':
format('{stack_root}/{stack_version}/ranger-{service_name}-plugin'),
'PATH':
format('{stack_root}/{stack_version}/ranger-{service_name}-plugin')
}
Execute(
cmd,
environment=cmd_env,
logoutput=True,
sudo=True,
)
def refresh_tez_state_dependent_params():
global tez_home_dir, tez_conf_dir, stack_version_formatted
tez_home_dir = os.environ["TEZ_HOME"]
tez_conf_dir = os.path.join(tez_home_dir, "conf")
# this is not available on INSTALL action because hdp-select is not available
stack_version_formatted = get_stack_version("tez")
from resource_management.libraries.functions.format import format
from resource_management.libraries.functions.version import format_stack_version, get_major_version
from resource_management.libraries.functions.default import default
from resource_management.libraries.functions import get_kinit_path
from resource_management.libraries.functions.get_stack_version import get_stack_version
config = Script.get_config()
tmp_dir = Script.get_tmp_dir()
stack_root= Script.get_stack_root()
stack_name = default("/hostLevelParams/stack_name", None)
stack_version_unformatted = str(config['hostLevelParams']['stack_version'])
stack_version_formatted = format_stack_version(stack_version_unformatted)
major_stack_version = get_major_version(stack_version_formatted)
full_stack_version = get_stack_version('titan-client')
# New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade
version = default("/commandParams/version", None)
titan_user = config['configurations']['titan-env']['titan_user']
user_group = config['configurations']['cluster-env']['user_group']
titan_log_dir = config['configurations']['titan-env']['titan_log_dir']
titan_server_port = config['configurations']['titan-env']['titan_server_port']
titan_hdfs_home_dir = config['configurations']['titan-env']['titan_hdfs_home_dir']
titan_log_file = format("{titan_log_dir}/titan-{titan_server_port}.log")
titan_err_file = format("{titan_log_dir}/titan-{titan_server_port}.err")
smokeuser = config['configurations']['cluster-env']['smokeuser']
smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar,
component_downloaded_custom_connector, component_driver_curl_source,
component_driver_curl_target, java_home,
repo_name, plugin_repo_dict,
ranger_env_properties, plugin_properties,
policy_user, policymgr_mgr_url,
plugin_enabled, conf_dict, component_user, component_group,
cache_service_list, plugin_audit_properties, plugin_audit_attributes,
plugin_security_properties, plugin_security_attributes,
plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes,
component_list, audit_db_is_enabled, credential_file,
xa_audit_db_password, ssl_truststore_password,
ssl_keystore_password, api_version=None, stack_version_override = None, skip_if_rangeradmin_down = True,
is_security_enabled = False, is_stack_supports_ranger_kerberos = False,
component_user_principal = None, component_user_keytab = None):
if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith("/None"):
if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar):
File(previous_jdbc_jar, action='delete')
File(component_downloaded_custom_connector,
content = DownloadSource(component_driver_curl_source),
mode = 0644
)
Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target),
path=["/bin", "/usr/bin/"],
sudo=True
)
File(component_driver_curl_target, mode=0644)
if policymgr_mgr_url.endswith('/'):
policymgr_mgr_url = policymgr_mgr_url.rstrip('/')
stack_version = get_stack_version(component_select_name)
if stack_version_override is not None:
stack_version = stack_version_override
component_conf_dir = conf_dict
if plugin_enabled:
if api_version is not None and api_version == 'v2':
ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user,is_security_enabled,is_stack_supports_ranger_kerberos,component_user,component_user_principal,component_user_keytab)
else:
ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down)
ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict,
ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'],
ranger_env_properties['admin_username'], ranger_env_properties['admin_password'],
policy_user)
current_datetime = datetime.now()
File(format('{component_conf_dir}/ranger-security.xml'),
owner = component_user,
group = component_group,
mode = 0644,
content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>'))
)
Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')],
owner = component_user,
group = component_group,
mode=0775,
create_parents = True,
cd_access = 'a'
)
for cache_service in cache_service_list:
File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')),
owner = component_user,
group = component_group,
mode = 0644
)
XmlConfig(format('ranger-{service_name}-audit.xml'),
conf_dir=component_conf_dir,
configurations=plugin_audit_properties,
configuration_attributes=plugin_audit_attributes,
owner = component_user,
group = component_group,
mode=0744)
XmlConfig(format('ranger-{service_name}-security.xml'),
conf_dir=component_conf_dir,
configurations=plugin_security_properties,
configuration_attributes=plugin_security_attributes,
owner = component_user,
group = component_group,
mode=0744)
if str(service_name).lower() == 'yarn' :
XmlConfig("ranger-policymgr-ssl-yarn.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
else :
XmlConfig("ranger-policymgr-ssl.xml",
conf_dir=component_conf_dir,
configurations=plugin_policymgr_ssl_properties,
configuration_attributes=plugin_policymgr_ssl_attributes,
owner = component_user,
group = component_group,
mode=0744)
#This should be done by rpm
#setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list)
setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file,
xa_audit_db_password, ssl_truststore_password, ssl_keystore_password,
component_user, component_group, java_home)
else:
File(format('{component_conf_dir}/ranger-security.xml'),
action="delete"
)
