def get_role_component_current_hdp_version(): """ Gets the current HDP version of the component that this role command is for. :return: the current HDP version of the specified component or None """ hdp_select_component = None role = default("/role", "") role_command = default("/roleCommand", "") if role in SERVER_ROLE_DIRECTORY_MAP: hdp_select_component = SERVER_ROLE_DIRECTORY_MAP[role] elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP: hdp_select_component = SERVICE_CHECK_DIRECTORY_MAP[role] if hdp_select_component is None: return None current_hdp_version = get_hdp_version(hdp_select_component) if current_hdp_version is None: Logger.warning("Unable to determine hdp-select version for {0}".format( hdp_select_component)) else: Logger.info("{0} is currently at version {1}".format( hdp_select_component, current_hdp_version)) return current_hdp_version
def setup_ranger_plugin(component_select_name, service_name, downloaded_custom_connector, driver_curl_source, driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down = True, **kwargs): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True ) File(driver_curl_target, mode=0644) hdp_version = get_hdp_version(component_select_name) file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties') if not os.path.isfile(file_path): raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}')) ModifyPropertiesFile(file_path, properties = plugin_properties ) custom_plugin_properties = dict() custom_plugin_properties['CUSTOM_USER'] = component_user custom_plugin_properties['CUSTOM_GROUP'] = component_group ModifyPropertiesFile(file_path,properties = custom_plugin_properties) if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'),) if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down = skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) else: cmd = (format('disable-{service_name}-plugin.sh'),) cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')} Execute(cmd, environment=cmd_env, logoutput=True, sudo=True, )
def setup_ranger_plugin(component_select_name, service_name, downloaded_custom_connector, driver_curl_source, driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled): File(downloaded_custom_connector, content = DownloadSource(driver_curl_source) ) Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target), not_if=format("test -f {driver_curl_target}"), sudo=True ) hdp_version = get_hdp_version(component_select_name) file_path = format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin/install.properties') if not os.path.isfile(file_path): raise Fail(format('Ranger {service_name} plugin install.properties file does not exist at {file_path}')) ModifyPropertiesFile(file_path, properties = plugin_properties ) if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'),) ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) else: cmd = (format('disable-{service_name}-plugin.sh'),) cmd_env = {'JAVA_HOME': java_home, 'PWD': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin'), 'PATH': format('/usr/hdp/{hdp_version}/ranger-{service_name}-plugin')} Execute(cmd, environment=cmd_env, logoutput=True, sudo=True, )
from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.script.script import Script from status_params import * from resource_management.libraries.resources.hdfs_resource import HdfsResource from resource_management.libraries.functions import hdp_select from resource_management.libraries.functions import conf_select # server configurations config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_name = default("/hostLevelParams/stack_name", None) upgrade_direction = default("/commandParams/upgrade_direction", None) version = default("/commandParams/version", None) if version is None: version = get_hdp_version('knox-server') # E.g., 2.3.2.0 version_formatted = format_hdp_stack_version(version) # E.g., 2.3 stack_version_unformatted = str(config['hostLevelParams']['stack_version']) hdp_stack_version = format_hdp_stack_version(stack_version_unformatted) # This is the version whose state is CURRENT. During an RU, this is the source version. # DO NOT format it since we need the build number too. upgrade_from_version = default("/hostLevelParams/current_version", None) # server configurations # Default value used in HDP 2.3.0.0 and earlier. knox_data_dir = '/var/lib/knox/data'
audit_jdbc_url = format('jdbc:sqlanywhere:database={xa_audit_db_name};host={xa_db_host}') jdbc_driver = "sap.jdbc4.sqlanywhere.IDriver" downloaded_custom_connector = format("{tmp_dir}/{jdbc_jar_name}") driver_curl_source = format("{jdk_location}/{jdbc_symlink_name}") driver_curl_target = format("{kafka_home}libs/{jdbc_jar_name}") ranger_audit_solr_urls = config['configurations']['ranger-admin-site']['ranger.audit.solr.urls'] xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-kafka-audit']['xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.keystore.password']) if xml_configurations_supported else None ssl_truststore_password = unicode(config['configurations']['ranger-kafka-policymgr-ssl']['xasecure.policymgr.clientssl.truststore.password']) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks') if xml_configurations_supported else None hdp_version = get_hdp_version('kafka-broker') setup_ranger_env_sh_source = format('/usr/hdp/{hdp_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh') setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh") #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site']['fs.defaultFS'] if has_namenode else None
def setup_ranger_plugin(component_select_name, service_name, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, hdp_version_override=None, skip_if_rangeradmin_down=True): if audit_db_is_enabled: File(component_downloaded_custom_connector, content=DownloadSource(component_driver_curl_source), mode=0644) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(component_driver_curl_target, mode=0644) hdp_version = get_hdp_version(component_select_name) if hdp_version_override is not None: hdp_version = hdp_version_override component_conf_dir = conf_dict if plugin_enabled: if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File( format('{component_conf_dir}/ranger-security.xml'), owner=component_user, group=component_group, mode=0644, content=InlineTemplate( format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>' ))) Directory([ os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache') ], owner=component_user, group=component_group, mode=0775, create_parents=True, cd_access='a') for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache', format('{cache_service}_{repo_name}.json')), owner=component_user, group=component_group, mode=0644) XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties, configuration_attributes=plugin_audit_attributes, owner=component_user, group=component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner=component_user, group=component_group, mode=0744) if str(service_name).lower() == 'yarn': XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) else: XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) #This should be done by rpm #setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home) else: File(format('{component_conf_dir}/ranger-security.xml'), action="delete")
def refresh_tez_state_dependent_params(): global tez_home_dir, tez_conf_dir, hdp_stack_version tez_home_dir = os.environ["TEZ_HOME"] tez_conf_dir = os.path.join(tez_home_dir, "conf") # this is not available on INSTALL action because hdp-select is not available hdp_stack_version = get_hdp_version("tez")
xa_audit_db_is_enabled = config['configurations']['ranger-kafka-audit'][ 'xasecure.audit.destination.db'] if xml_configurations_supported else None xa_audit_hdfs_is_enabled = config['configurations']['ranger-kafka-audit'][ 'xasecure.audit.destination.hdfs'] if xml_configurations_supported else None ssl_keystore_password = unicode( config['configurations']['ranger-kafka-policymgr-ssl'] ['xasecure.policymgr.clientssl.keystore.password'] ) if xml_configurations_supported else None ssl_truststore_password = unicode( config['configurations']['ranger-kafka-policymgr-ssl'] ['xasecure.policymgr.clientssl.truststore.password'] ) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks' ) if xml_configurations_supported else None hdp_version = get_hdp_version('kafka-broker') setup_ranger_env_sh_source = format( '/usr/hdp/{hdp_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh' ) setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh") #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env'][ 'hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env'][
def setup_ranger_plugin(component_select_name, service_name, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, hdp_version_override = None, skip_if_rangeradmin_down = True): if audit_db_is_enabled: File(component_downloaded_custom_connector, content = DownloadSource(component_driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True ) File(component_driver_curl_target, mode=0644) hdp_version = get_hdp_version(component_select_name) if hdp_version_override is not None: hdp_version = hdp_version_override component_conf_dir = conf_dict if plugin_enabled: if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File(format('{component_conf_dir}/ranger-security.xml'), owner = component_user, group = component_group, mode = 0644, content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) ) Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')], owner = component_user, group = component_group, mode=0775, recursive = True, cd_access = 'a' ) for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')), owner = component_user, group = component_group, mode = 0644 ) XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties, configuration_attributes=plugin_audit_attributes, owner = component_user, group = component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner = component_user, group = component_group, mode=0744) if str(service_name).lower() == 'yarn' : XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) else : XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) #This should be done by rpm #setup_ranger_plugin_jar_symblink(hdp_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, hdp_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home) else: File(format('{component_conf_dir}/ranger-security.xml'), action="delete" )