def get_role_component_current_stack_version(): """ Gets the current HDP version of the component that this role command is for. :return: the current HDP version of the specified component or None """ role = default("/role", "") role_command = default("/roleCommand", "") stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME) Logger.info("Checking version for {0} via {1}".format(role, stack_selector_name)) if role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP: stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role] else: stack_select_component = get_package_name() if stack_select_component is None: if not role: Logger.error("No role information available.") elif not role.lower().endswith("client"): Logger.error("Mapping unavailable for role {0}. Skip checking its version.".format(role)) return None current_stack_version = get_stack_version(stack_select_component) if current_stack_version is None: Logger.warning("Unable to determine {0} version for {1}".format( stack_selector_name, stack_select_component)) else: Logger.info("{0} is currently at version {1}".format( stack_select_component, current_stack_version)) return current_stack_version
def get_role_component_current_stack_version(): """ Gets the current HDP version of the component that this role command is for. :return: the current HDP version of the specified component or None """ stack_select_component = None role = default("/role", "") role_command = default("/roleCommand", "") stack_selector_name = stack_tools.get_stack_tool_name( stack_tools.STACK_SELECTOR_NAME) if role in SERVER_ROLE_DIRECTORY_MAP: stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role] elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP: stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role] if stack_select_component is None: return None current_stack_version = get_stack_version(stack_select_component) if current_stack_version is None: Logger.warning("Unable to determine {0} version for {1}".format( stack_selector_name, stack_select_component)) else: Logger.info("{0} is currently at version {1}".format( stack_select_component, current_stack_version)) return current_stack_version
def get_role_component_current_stack_version(): """ Gets the current HDP version of the component that this role command is for. :return: the current HDP version of the specified component or None """ stack_select_component = None role = default("/role", "") role_command = default("/roleCommand", "") stack_selector_name = stack_tools.get_stack_tool_name(stack_tools.STACK_SELECTOR_NAME) if role in SERVER_ROLE_DIRECTORY_MAP: stack_select_component = SERVER_ROLE_DIRECTORY_MAP[role] elif role_command == "SERVICE_CHECK" and role in SERVICE_CHECK_DIRECTORY_MAP: stack_select_component = SERVICE_CHECK_DIRECTORY_MAP[role] if stack_select_component is None: return None current_stack_version = get_stack_version(stack_select_component) if current_stack_version is None: Logger.warning("Unable to determine {0} version for {1}".format( stack_selector_name, stack_select_component)) else: Logger.info("{0} is currently at version {1}".format( stack_select_component, current_stack_version)) return current_stack_version
'ranger-kafka-audit']['xasecure.audit.destination.db'] xa_audit_hdfs_is_enabled = default( '/configurations/ranger-kafka-audit/xasecure.audit.destination.hdfs', False) ssl_keystore_password = unicode( config['configurations']['ranger-kafka-policymgr-ssl'] ['xasecure.policymgr.clientssl.keystore.password'] ) if xml_configurations_supported else None ssl_truststore_password = unicode( config['configurations']['ranger-kafka-policymgr-ssl'] ['xasecure.policymgr.clientssl.truststore.password'] ) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks' ) if xml_configurations_supported else None stack_version = get_stack_version('kafka-broker') setup_ranger_env_sh_source = format( '{stack_root}/{stack_version}/ranger-kafka-plugin/install/conf.templates/enable/kafka-ranger-env.sh' ) setup_ranger_env_sh_target = format("{conf_dir}/kafka-ranger-env.sh") #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env'][ 'hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env'][
spark_yarn_historyServer_address = default(spark_history_server_host, "localhost") spark_history_ui_port = config['configurations']['spark-defaults'][ 'spark.history.ui.port'] spark_env_sh = config['configurations']['spark-env']['content'] spark_log4j_properties = config['configurations']['spark-log4j-properties'][ 'content'] spark_metrics_properties = config['configurations'][ 'spark-metrics-properties']['content'] hive_server_host = default("/clusterHostInfo/hive_server_hosts", []) is_hive_installed = not len(hive_server_host) == 0 full_stack_version = get_stack_version('spark-client') spark_javaopts_properties = default( "/configurations/spark-javaopts-properties/content", " ") if spark_javaopts_properties.find('-Dhdp.version') == -1: spark_javaopts_properties = spark_javaopts_properties + ' -Dhdp.version=' + str( full_stack_version) else: lists = spark_javaopts_properties.split(" ") for idx, val in enumerate(lists): if (val.startswith("-Dhdp.version=")): lists[idx] = "-Dhdp.version=" + str(full_stack_version) spark_javaopts_properties = " ".join(lists) security_enabled = config['configurations']['cluster-env']['security_enabled'] kinit_path_local = get_kinit_path(
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, stack_version_override=None, skip_if_rangeradmin_down=True, is_security_enabled=False, is_stack_supports_ranger_kerberos=False, component_user_principal=None, component_user_keytab=None, cred_lib_path_override=None, cred_setup_prefix_override=None): if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith( "/None"): if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar): File(previous_jdbc_jar, action='delete') File(component_downloaded_custom_connector, content=DownloadSource(component_driver_curl_source), mode=0644) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(component_driver_curl_target, mode=0644) if policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') if stack_version_override is None: stack_version = get_stack_version(component_select_name) else: stack_version = stack_version_override component_conf_dir = conf_dict if plugin_enabled: service_name_exist = False policycache_path = os.path.join('/etc', 'ranger', repo_name, 'policycache') try: for cache_service in cache_service_list: policycache_json_file = format( '{policycache_path}/{cache_service}_{repo_name}.json') if os.path.isfile(policycache_json_file) and os.path.getsize( policycache_json_file) > 0: with open(policycache_json_file) as json_file: json_data = json.load(json_file) if 'serviceName' in json_data and json_data[ 'serviceName'] == repo_name: service_name_exist = True Logger.info( "Skipping Ranger API calls, as policy cache file exists for {0}" .format(service_name)) Logger.warning( "If service name for {0} is not created on Ranger Admin UI, then to re-create it delete policy cache file: {1}" .format(service_name, policycache_json_file)) break except Exception, err: Logger.error( "Error occurred while fetching service name from policy cache file.\nError: {0}" .format(err)) if not service_name_exist: if api_version is not None and api_version == 'v2': ranger_adm_obj = RangeradminV2( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user, is_security_enabled, is_stack_supports_ranger_kerberos, component_user, component_user_principal, component_user_keytab) else: ranger_adm_obj = Rangeradmin( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File( format('{component_conf_dir}/ranger-security.xml'), owner=component_user, group=component_group, mode=0644, content=InlineTemplate( format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>' ))) Directory([ os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache') ], owner=component_user, group=component_group, mode=0775, create_parents=True, cd_access='a') for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache', format('{cache_service}_{repo_name}.json')), owner=component_user, group=component_group, mode=0644) # remove plain-text password from xml configs plugin_audit_password_property = 'xasecure.audit.destination.db.password' plugin_audit_properties_copy = {} plugin_audit_properties_copy.update(plugin_audit_properties) if plugin_audit_password_property in plugin_audit_properties_copy: plugin_audit_properties_copy[ plugin_audit_password_property] = "crypted" XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties_copy, configuration_attributes=plugin_audit_attributes, owner=component_user, group=component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner=component_user, group=component_group, mode=0744) # remove plain-text password from xml configs plugin_password_properties = [ 'xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password' ] plugin_policymgr_ssl_properties_copy = {} plugin_policymgr_ssl_properties_copy.update( plugin_policymgr_ssl_properties) for prop in plugin_password_properties: if prop in plugin_policymgr_ssl_properties_copy: plugin_policymgr_ssl_properties_copy[prop] = "crypted" if str(service_name).lower() == 'yarn': XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties_copy, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) else: XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties_copy, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) # creating symblink should be done by rpm package # setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list) setup_ranger_plugin_keystore( service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home, cred_lib_path_override, cred_setup_prefix_override)
zeppelin_kerberos_keytab = config['configurations']['zeppelin-env'][ 'zeppelin.server.kerberos.keytab'] zeppelin_kerberos_principal = config['configurations']['zeppelin-env'][ 'zeppelin.server.kerberos.principal'] # e.g. 2.3 stack_version_unformatted = config['hostLevelParams']['stack_version'] # e.g. 2.3.0.0 stack_version_formatted = format_stack_version(stack_version_unformatted) # e.g. 2.3.0.0-2130 full_stack_version = default("/commandParams/version", None) spark_client_version = get_stack_version('spark-client') hbase_master_hosts = default("/clusterHostInfo/hbase_master_hosts", []) livy_hosts = default("/clusterHostInfo/livy_server_hosts", []) livy_livyserver_host = None livy_livyserver_port = None if stack_version_formatted and check_stack_feature(StackFeature.SPARK_LIVY, stack_version_formatted) and \ len(livy_hosts) > 0: livy_livyserver_host = str(livy_hosts[0]) livy_livyserver_port = config['configurations']['livy-conf'][ 'livy.server.port'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] security_enabled = config['configurations']['cluster-env']['security_enabled'] hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab']
xa_audit_hdfs_is_enabled = default( '/configurations/ranger-solr-audit/xasecure.audit.destination.hdfs', False) ssl_keystore_password = unicode( config['configurations']['ranger-solr-policymgr-ssl'] ['xasecure.policymgr.clientssl.keystore.password'] ) if xml_configurations_supported else None ssl_truststore_password = unicode( config['configurations']['ranger-solr-policymgr-ssl'] ['xasecure.policymgr.clientssl.truststore.password'] ) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks' ) if xml_configurations_supported else None stack_version = get_stack_version( 'hadoop-hdfs-datanode' ) # use hadoop-hdfs-datanode as its required for solr setup_ranger_env_sh_source = format( '{stack_root}/{stack_version}/ranger-solr-plugin/install/conf.templates/enable/solr-ranger-env.sh' ) setup_ranger_env_sh_target = format("{solr_conf}/solr-ranger-env.sh") #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_hosts", []) has_namenode = not len(namenode_hosts) == 0 # *********************** end RANGER PLUGIN CHANGES **************** smokeuser = config['configurations']['cluster-env']['smokeuser']
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, stack_version_override=None, skip_if_rangeradmin_down=True, is_security_enabled=False, is_stack_supports_ranger_kerberos=False, component_user_principal=None, component_user_keytab=None, cred_lib_path_override=None, cred_setup_prefix_override=None): if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith( "/None"): if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar): File(previous_jdbc_jar, action='delete') File(component_downloaded_custom_connector, content=DownloadSource(component_driver_curl_source), mode=0644) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(component_driver_curl_target, mode=0644) if policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') if stack_version_override is None: stack_version = get_stack_version(component_select_name) else: stack_version = stack_version_override component_conf_dir = conf_dict if plugin_enabled: service_name_exist = get_policycache_service_name( service_name, repo_name, cache_service_list) if not service_name_exist: if api_version is not None and api_version == 'v2': ranger_adm_obj = RangeradminV2( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user, is_security_enabled, is_stack_supports_ranger_kerberos, component_user, component_user_principal, component_user_keytab) else: ranger_adm_obj = Rangeradmin( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File( format('{component_conf_dir}/ranger-security.xml'), owner=component_user, group=component_group, mode=0644, content=InlineTemplate( format( '<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>' ))) Directory([ os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache') ], owner=component_user, group=component_group, mode=0775, create_parents=True, cd_access='a') for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache', format('{cache_service}_{repo_name}.json')), owner=component_user, group=component_group, mode=0644) # remove plain-text password from xml configs plugin_audit_password_property = 'xasecure.audit.destination.db.password' plugin_audit_properties_copy = {} plugin_audit_properties_copy.update(plugin_audit_properties) if plugin_audit_password_property in plugin_audit_properties_copy: plugin_audit_properties_copy[ plugin_audit_password_property] = "crypted" XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties_copy, configuration_attributes=plugin_audit_attributes, owner=component_user, group=component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner=component_user, group=component_group, mode=0744) # remove plain-text password from xml configs plugin_password_properties = [ 'xasecure.policymgr.clientssl.keystore.password', 'xasecure.policymgr.clientssl.truststore.password' ] plugin_policymgr_ssl_properties_copy = {} plugin_policymgr_ssl_properties_copy.update( plugin_policymgr_ssl_properties) for prop in plugin_password_properties: if prop in plugin_policymgr_ssl_properties_copy: plugin_policymgr_ssl_properties_copy[prop] = "crypted" if str(service_name).lower() == 'yarn': XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties_copy, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) else: XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties_copy, configuration_attributes=plugin_policymgr_ssl_attributes, owner=component_user, group=component_group, mode=0744) setup_ranger_plugin_keystore( service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home, cred_lib_path_override, cred_setup_prefix_override) else: File(format('{component_conf_dir}/ranger-security.xml'), action="delete")
'xasecure.audit.destination.db'] xa_audit_hdfs_is_enabled = default( '/configurations/ranger-solr-audit/xasecure.audit.destination.hdfs', False) ssl_keystore_password = unicode( config['configurations']['ranger-solr-policymgr-ssl'] ['xasecure.policymgr.clientssl.keystore.password'] ) if xml_configurations_supported else None ssl_truststore_password = unicode( config['configurations']['ranger-solr-policymgr-ssl'] ['xasecure.policymgr.clientssl.truststore.password'] ) if xml_configurations_supported else None credential_file = format('/etc/ranger/{repo_name}/cred.jceks' ) if xml_configurations_supported else None stack_version = get_stack_version('solr-server') setup_ranger_env_sh_source = format( '{stack_root}/{stack_version}/ranger-solr-plugin/install/conf.templates/enable/solr-ranger-env.sh' ) setup_ranger_env_sh_target = format("{solr_conf}/solr-ranger-env.sh") #For SQLA explicitly disable audit to DB for Ranger if xa_audit_db_flavor == 'sqla': xa_audit_db_is_enabled = False namenode_hosts = default("/clusterHostInfo/namenode_host", []) has_namenode = not len(namenode_hosts) == 0 # *********************** end RANGER PLUGIN CHANGES **************** smokeuser = config['configurations']['cluster-env']['smokeuser'] smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
def setup_ranger_plugin(component_select_name, service_name, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, stack_version_override = None, skip_if_rangeradmin_down = True): if audit_db_is_enabled: File(component_downloaded_custom_connector, content = DownloadSource(component_driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True ) File(component_driver_curl_target, mode=0644) stack_version = get_stack_version(component_select_name) if stack_version_override is not None: stack_version = stack_version_override component_conf_dir = conf_dict if plugin_enabled: if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File(format('{component_conf_dir}/ranger-security.xml'), owner = component_user, group = component_group, mode = 0644, content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) ) Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')], owner = component_user, group = component_group, mode=0775, create_parents = True, cd_access = 'a' ) for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')), owner = component_user, group = component_group, mode = 0644 ) XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties, configuration_attributes=plugin_audit_attributes, owner = component_user, group = component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner = component_user, group = component_group, mode=0744) if str(service_name).lower() == 'yarn' : XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) else : XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) #This should be done by rpm #setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home) else: File(format('{component_conf_dir}/ranger-security.xml'), action="delete" )
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, downloaded_custom_connector, driver_curl_source, driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, component_user, component_group, api_version=None, skip_if_rangeradmin_down=True, **kwargs): if driver_curl_source and not driver_curl_source.endswith("/None"): if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar): File(previous_jdbc_jar, action='delete') File(downloaded_custom_connector, content=DownloadSource(driver_curl_source), mode=0644) Execute(('cp', '--remove-destination', downloaded_custom_connector, driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True) File(driver_curl_target, mode=0644) stack_root = Script.get_stack_root() stack_version = get_stack_version(component_select_name) file_path = format( '{stack_root}/{stack_version}/ranger-{service_name}-plugin/install.properties' ) if not os.path.isfile(file_path): raise Fail( format( 'Ranger {service_name} plugin install.properties file does not exist at {file_path}' )) ModifyPropertiesFile(file_path, properties=plugin_properties) custom_plugin_properties = dict() custom_plugin_properties['CUSTOM_USER'] = component_user custom_plugin_properties['CUSTOM_GROUP'] = component_group ModifyPropertiesFile(file_path, properties=custom_plugin_properties) if plugin_enabled: cmd = (format('enable-{service_name}-plugin.sh'), ) if api_version == 'v2' and api_version is not None: ranger_adm_obj = RangeradminV2( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) else: ranger_adm_obj = Rangeradmin( url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository( service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) else: cmd = (format('disable-{service_name}-plugin.sh'), ) cmd_env = { 'JAVA_HOME': java_home, 'PWD': format('{stack_root}/{stack_version}/ranger-{service_name}-plugin'), 'PATH': format('{stack_root}/{stack_version}/ranger-{service_name}-plugin') } Execute( cmd, environment=cmd_env, logoutput=True, sudo=True, )
def refresh_tez_state_dependent_params(): global tez_home_dir, tez_conf_dir, stack_version_formatted tez_home_dir = os.environ["TEZ_HOME"] tez_conf_dir = os.path.join(tez_home_dir, "conf") # this is not available on INSTALL action because hdp-select is not available stack_version_formatted = get_stack_version("tez")
from resource_management.libraries.functions.format import format from resource_management.libraries.functions.version import format_stack_version, get_major_version from resource_management.libraries.functions.default import default from resource_management.libraries.functions import get_kinit_path from resource_management.libraries.functions.get_stack_version import get_stack_version config = Script.get_config() tmp_dir = Script.get_tmp_dir() stack_root= Script.get_stack_root() stack_name = default("/hostLevelParams/stack_name", None) stack_version_unformatted = str(config['hostLevelParams']['stack_version']) stack_version_formatted = format_stack_version(stack_version_unformatted) major_stack_version = get_major_version(stack_version_formatted) full_stack_version = get_stack_version('titan-client') # New Cluster Stack Version that is defined during the RESTART of a Rolling Upgrade version = default("/commandParams/version", None) titan_user = config['configurations']['titan-env']['titan_user'] user_group = config['configurations']['cluster-env']['user_group'] titan_log_dir = config['configurations']['titan-env']['titan_log_dir'] titan_server_port = config['configurations']['titan-env']['titan_server_port'] titan_hdfs_home_dir = config['configurations']['titan-env']['titan_hdfs_home_dir'] titan_log_file = format("{titan_log_dir}/titan-{titan_server_port}.log") titan_err_file = format("{titan_log_dir}/titan-{titan_server_port}.err") smokeuser = config['configurations']['cluster-env']['smokeuser'] smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
def setup_ranger_plugin(component_select_name, service_name, previous_jdbc_jar, component_downloaded_custom_connector, component_driver_curl_source, component_driver_curl_target, java_home, repo_name, plugin_repo_dict, ranger_env_properties, plugin_properties, policy_user, policymgr_mgr_url, plugin_enabled, conf_dict, component_user, component_group, cache_service_list, plugin_audit_properties, plugin_audit_attributes, plugin_security_properties, plugin_security_attributes, plugin_policymgr_ssl_properties, plugin_policymgr_ssl_attributes, component_list, audit_db_is_enabled, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, api_version=None, stack_version_override = None, skip_if_rangeradmin_down = True, is_security_enabled = False, is_stack_supports_ranger_kerberos = False, component_user_principal = None, component_user_keytab = None): if audit_db_is_enabled and component_driver_curl_source is not None and not component_driver_curl_source.endswith("/None"): if previous_jdbc_jar and os.path.isfile(previous_jdbc_jar): File(previous_jdbc_jar, action='delete') File(component_downloaded_custom_connector, content = DownloadSource(component_driver_curl_source), mode = 0644 ) Execute(('cp', '--remove-destination', component_downloaded_custom_connector, component_driver_curl_target), path=["/bin", "/usr/bin/"], sudo=True ) File(component_driver_curl_target, mode=0644) if policymgr_mgr_url.endswith('/'): policymgr_mgr_url = policymgr_mgr_url.rstrip('/') stack_version = get_stack_version(component_select_name) if stack_version_override is not None: stack_version = stack_version_override component_conf_dir = conf_dict if plugin_enabled: if api_version is not None and api_version == 'v2': ranger_adm_obj = RangeradminV2(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user,is_security_enabled,is_stack_supports_ranger_kerberos,component_user,component_user_principal,component_user_keytab) else: ranger_adm_obj = Rangeradmin(url=policymgr_mgr_url, skip_if_rangeradmin_down=skip_if_rangeradmin_down) ranger_adm_obj.create_ranger_repository(service_name, repo_name, plugin_repo_dict, ranger_env_properties['ranger_admin_username'], ranger_env_properties['ranger_admin_password'], ranger_env_properties['admin_username'], ranger_env_properties['admin_password'], policy_user) current_datetime = datetime.now() File(format('{component_conf_dir}/ranger-security.xml'), owner = component_user, group = component_group, mode = 0644, content = InlineTemplate(format('<ranger>\n<enabled>{current_datetime}</enabled>\n</ranger>')) ) Directory([os.path.join('/etc', 'ranger', repo_name), os.path.join('/etc', 'ranger', repo_name, 'policycache')], owner = component_user, group = component_group, mode=0775, create_parents = True, cd_access = 'a' ) for cache_service in cache_service_list: File(os.path.join('/etc', 'ranger', repo_name, 'policycache',format('{cache_service}_{repo_name}.json')), owner = component_user, group = component_group, mode = 0644 ) XmlConfig(format('ranger-{service_name}-audit.xml'), conf_dir=component_conf_dir, configurations=plugin_audit_properties, configuration_attributes=plugin_audit_attributes, owner = component_user, group = component_group, mode=0744) XmlConfig(format('ranger-{service_name}-security.xml'), conf_dir=component_conf_dir, configurations=plugin_security_properties, configuration_attributes=plugin_security_attributes, owner = component_user, group = component_group, mode=0744) if str(service_name).lower() == 'yarn' : XmlConfig("ranger-policymgr-ssl-yarn.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) else : XmlConfig("ranger-policymgr-ssl.xml", conf_dir=component_conf_dir, configurations=plugin_policymgr_ssl_properties, configuration_attributes=plugin_policymgr_ssl_attributes, owner = component_user, group = component_group, mode=0744) #This should be done by rpm #setup_ranger_plugin_jar_symblink(stack_version, service_name, component_list) setup_ranger_plugin_keystore(service_name, audit_db_is_enabled, stack_version, credential_file, xa_audit_db_password, ssl_truststore_password, ssl_keystore_password, component_user, component_group, java_home) else: File(format('{component_conf_dir}/ranger-security.xml'), action="delete" )