security_enabled = config['configurations']['cluster-env']['security_enabled'] storm_ui_host = default("/clusterHostInfo/storm_ui_server_hosts", []) if security_enabled: _hostname_lowercase = config['hostname'].lower() _storm_principal_name = config['configurations']['storm-env']['storm_principal_name'] storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase) storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] if stack_is_hdp22_or_further: storm_ui_keytab_path = config['configurations']['storm-env']['storm_ui_keytab'] _storm_ui_jaas_principal_name = config['configurations']['storm-env']['storm_ui_principal_name'] storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',_hostname_lowercase) storm_bare_jaas_principal = get_bare_principal(_storm_principal_name) _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name'] nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST', _hostname_lowercase) nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name) nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab'] kafka_bare_jaas_principal = None if stack_is_hdp22_or_further: if security_enabled: storm_thrift_transport = config['configurations']['storm-site']['_storm.thrift.secure.transport'] # generate KafkaClient jaas config if kafka is kerberoized _kafka_principal_name = default("/configurations/kafka-env/kafka_principal_name", None) kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name) else:
if has_ranger_tagsync: ranger_tagsync_principal = config['configurations']['ranger-tagsync-site']['ranger.tagsync.kerberos.principal'] tagsync_jaas_principal = ranger_tagsync_principal.replace('_HOST', current_host.lower()) tagsync_keytab_path = config['configurations']['ranger-tagsync-site']['ranger.tagsync.kerberos.keytab'] # logic to create core-site.xml if hdfs not installed if stack_supports_ranger_kerberos and not has_namenode: core_site_property = { 'hadoop.security.authentication': 'kerberos' if security_enabled else 'simple' } realm = 'EXAMPLE.COM' if security_enabled: ranger_admin_principal = config['configurations']['ranger-admin-site']['ranger.admin.kerberos.principal'] ranger_usersync_principal = config['configurations']['ranger-ugsync-site']['ranger.usersync.kerberos.principal'] ranger_admin_bare_principal = get_bare_principal(ranger_admin_principal) ranger_usersync_bare_principal = get_bare_principal(ranger_usersync_principal) realm = config['configurations']['kerberos-env']['realm'] rule_dict = [ {'principal': ranger_admin_bare_principal, 'user': unix_user}, {'principal': ranger_usersync_bare_principal, 'user': '******'}, ] if has_ranger_tagsync: ranger_tagsync_bare_principal = get_bare_principal(ranger_tagsync_principal) rule_dict.append({'principal': ranger_tagsync_bare_principal, 'user': '******'}) core_site_auth_to_local_property = '' for item in range(len(rule_dict)): rule_line = 'RULE:[2:$1@$0]({0}@{1})s/.*/{2}/\n'.format(rule_dict[item]['principal'], realm, rule_dict[item]['user'])
_hostname_lowercase = config['hostname'].lower() _storm_principal_name = config['configurations']['storm-env'][ 'storm_principal_name'] storm_jaas_principal = _storm_principal_name.replace( '_HOST', _hostname_lowercase) storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] if stack_is_hdp22_or_further: storm_ui_keytab_path = config['configurations']['storm-env'][ 'storm_ui_keytab'] _storm_ui_jaas_principal_name = config['configurations']['storm-env'][ 'storm_ui_principal_name'] storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace( '_HOST', _hostname_lowercase) storm_bare_jaas_principal = get_bare_principal(_storm_principal_name) _nimbus_principal_name = config['configurations']['storm-env'][ 'nimbus_principal_name'] nimbus_jaas_principal = _nimbus_principal_name.replace( '_HOST', _hostname_lowercase) nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name) nimbus_keytab_path = config['configurations']['storm-env'][ 'nimbus_keytab'] kafka_bare_jaas_principal = None if stack_is_hdp22_or_further: if security_enabled: storm_thrift_transport = config['configurations']['storm-site'][ '_storm.thrift.secure.transport'] # generate KafkaClient jaas config if kafka is kerberoized
master_hosts = default('/clusterHostInfo/accumulo_master_hosts', []) monitor_hosts = default('/clusterHostInfo/accumulo_monitor_hosts', []) gc_hosts = default('/clusterHostInfo/accumulo_gc_hosts', []) tracer_hosts = default('/clusterHostInfo/accumulo_tracer_hosts', []) # security properties accumulo_user_keytab = config['configurations']['accumulo-env'][ 'accumulo_user_keytab'] accumulo_principal_name = config['configurations']['accumulo-env'][ 'accumulo_principal_name'] # kinit properties kinit_path_local = status_params.kinit_path_local if security_enabled: bare_accumulo_principal = get_bare_principal( config['configurations']['accumulo-site'] ['general.kerberos.principal']) kinit_cmd = format( "{kinit_path_local} -kt {accumulo_user_keytab} {accumulo_principal_name};" ) else: kinit_cmd = "" #for create_hdfs_directory hostname = status_params.hostname hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] hdfs_site = config['configurations']['hdfs-site']
_hostname_lowercase = config['agentLevelParams']['hostname'].lower() _storm_principal_name = config['configurations']['storm-env'][ 'storm_principal_name'] storm_jaas_principal = _storm_principal_name.replace( '_HOST', _hostname_lowercase) _ambari_principal_name = default( '/configurations/cluster-env/ambari_principal_name', None) storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] storm_ui_keytab_path = config['configurations']['storm-env'][ 'storm_ui_keytab'] _storm_ui_jaas_principal_name = config['configurations']['storm-env'][ 'storm_ui_principal_name'] storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace( '_HOST', _hostname_lowercase) storm_bare_jaas_principal = get_bare_principal(_storm_principal_name) if _ambari_principal_name: ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name) _nimbus_principal_name = config['configurations']['storm-env'][ 'nimbus_principal_name'] nimbus_jaas_principal = _nimbus_principal_name.replace( '_HOST', _hostname_lowercase) nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name) nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab'] if streamline_installed and 'streamline_principal_name' in config[ 'configurations']['streamline-env']: _streamline_principal_name = config['configurations'][ 'streamline-env']['streamline_principal_name'] streamline_bare_jaas_principal = get_bare_principal( _streamline_principal_name)
hadoop_lib_home = stack_root + '/hadoop/lib/share/hadoop/common' kafka_hosts = default('/clusterHostInfo/kafka_hosts', []) from random import shuffle shuffle(kafka_hosts) has_kafka = len(kafka_hosts) > 0 # Kafka Jaas configs kafka_bare_jaas_principal = None druid_jaas_file = format('{druid_conf_dir}/druid_jaas.conf') if security_enabled and has_kafka and 'kafka_principal_name' in config[ 'configurations']['kafka-env']: # generate KafkaClient jaas config if kafka is kerberoized _kafka_principal_name = default( "/configurations/kafka-env/kafka_principal_name", None) kafka_bare_jaas_principal = get_bare_principal(_kafka_principal_name) _hostname_lowercase = config['agentLevelParams']['hostname'].lower() _druid_principal_name = config['configurations']['druid-common'][ 'druid.escalator.internalClientPrincipal'] druid_jaas_principal = _druid_principal_name.replace( '_HOST', _hostname_lowercase) druid_keytab_path = config['configurations']['druid-common'][ 'druid.escalator.internalClientKeytab'] import os import multiprocessing cpu_count = multiprocessing.cpu_count() mem_bytes = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES') mem_gib = int(mem_bytes / (1024**3) * 0.8)
storm_ui_host = default("/clusterHostInfo/storm_ui_server_hosts", []) storm_user_nofile_limit = default('/configurations/storm-env/storm_user_nofile_limit', 128000) storm_user_nproc_limit = default('/configurations/storm-env/storm_user_noproc_limit', 65536) if security_enabled: _hostname_lowercase = config['agentLevelParams']['hostname'].lower() _storm_principal_name = config['configurations']['storm-env']['storm_principal_name'] storm_jaas_principal = _storm_principal_name.replace('_HOST',_hostname_lowercase) _ambari_principal_name = default('/configurations/cluster-env/ambari_principal_name', None) storm_keytab_path = config['configurations']['storm-env']['storm_keytab'] storm_ui_keytab_path = config['configurations']['storm-env']['storm_ui_keytab'] _storm_ui_jaas_principal_name = config['configurations']['storm-env']['storm_ui_principal_name'] storm_ui_jaas_principal = _storm_ui_jaas_principal_name.replace('_HOST',_hostname_lowercase) storm_bare_jaas_principal = get_bare_principal(_storm_principal_name) if _ambari_principal_name: ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name) _nimbus_principal_name = config['configurations']['storm-env']['nimbus_principal_name'] nimbus_jaas_principal = _nimbus_principal_name.replace('_HOST', _hostname_lowercase) nimbus_bare_jaas_principal = get_bare_principal(_nimbus_principal_name) nimbus_keytab_path = config['configurations']['storm-env']['nimbus_keytab'] if streamline_installed and 'streamline_principal_name' in config['configurations']['streamline-env']: _streamline_principal_name = config['configurations']['streamline-env']['streamline_principal_name'] streamline_bare_jaas_principal = get_bare_principal(_streamline_principal_name) kafka_bare_jaas_principal = None if security_enabled: # generate KafkaClient jaas config if kafka is kerberized _kafka_principal_name = default("/configurations/kafka-env/kafka_principal_name", None)
"/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password") hms_partition_passwd = default( "/configurations/kms-env/hsm_partition_password", None) # kms kerberos from stack 2.5 onward rangerkms_bare_principal = 'rangerkms' if stack_supports_ranger_kerberos: if security_enabled: rangerkms_principal = config['configurations']['dbks-site'][ 'ranger.ks.kerberos.principal'] rangerkms_keytab = config['configurations']['dbks-site'][ 'ranger.ks.kerberos.keytab'] if not is_empty(rangerkms_principal) and rangerkms_principal != '': rangerkms_bare_principal = get_bare_principal(rangerkms_principal) rangerkms_principal = rangerkms_principal.replace( '_HOST', kms_host.lower()) kms_plugin_config['policy.download.auth.users'] = format( 'keyadmin,{rangerkms_bare_principal}') custom_ranger_service_config = generate_ranger_service_config( config['configurations']['kms-properties']) if len(custom_ranger_service_config) > 0: kms_plugin_config.update(custom_ranger_service_config) kms_ranger_plugin_repo = { 'isEnabled': 'true', 'configs': kms_plugin_config, 'description': 'kms repo', 'name': repo_name,
if security_enabled: _hostname_lowercase = config['agentLevelParams']['hostname'].lower() _flink_principal_name = config['configurations']['flink-env'][ 'flink_principal_name'] flink_jaas_principal = _flink_principal_name.replace( '_HOST', _hostname_lowercase) _ambari_principal_name = default( '/configurations/cluster-env/ambari_principal_name', None) flink_keytab_path = config['configurations']['flink-env']['flink_keytab'] flink_kerberos_keytab = config['configurations']['flink-site'][ 'security.kerberos.login.keytab'] flink_kerberos_principal = config['configurations']['flink-site'][ 'security.kerberos.login.principal'] if _ambari_principal_name: ambari_bare_jaas_principal = get_bare_principal(_ambari_principal_name) jdk_location = config['ambariLevelParams']['jdk_location'] namenode_hosts = default("/clusterHostInfo/namenode_hosts", []) has_namenode = not len(namenode_hosts) == 0 hdfs_user = config['configurations']['hadoop-env'][ 'hdfs_user'] if has_namenode else None hdfs_user_keytab = config['configurations']['hadoop-env'][ 'hdfs_user_keytab'] if has_namenode else None hdfs_principal_name = config['configurations']['hadoop-env'][ 'hdfs_principal_name'] if has_namenode else None hdfs_site = config['configurations']['hdfs-site'] if has_namenode else None default_fs = config['configurations']['core-site'][ 'fs.defaultFS'] if has_namenode else None hadoop_bin_dir = stack_select.get_hadoop_dir("bin") if has_namenode else None
core_site_property = { 'hadoop.security.authentication': 'kerberos' if security_enabled else 'simple' } if security_enabled: realm = 'EXAMPLE.COM' ranger_admin_bare_principal = 'rangeradmin' ranger_usersync_bare_principal = 'rangerusersync' ranger_tagsync_bare_principal = 'rangertagsync' ranger_usersync_principal = config['configurations'][ 'ranger-ugsync-site']['ranger.usersync.kerberos.principal'] if not is_empty( ranger_admin_principal) and ranger_admin_principal != '': ranger_admin_bare_principal = get_bare_principal( ranger_admin_principal) if not is_empty( ranger_usersync_principal) and ranger_usersync_principal != '': ranger_usersync_bare_principal = get_bare_principal( ranger_usersync_principal) realm = config['configurations']['kerberos-env']['realm'] rule_dict = [ { 'principal': ranger_admin_bare_principal, 'user': unix_user }, { 'principal': ranger_usersync_bare_principal, 'user': '******' },
"/configurations/dbks-site/ranger.ks.hsm.partition.password.alias", "ranger.kms.hsm.partition.password") hms_partition_passwd = default( "/configurations/kms-env/hsm_partition_password", None) # kms kerberos from stack 2.5 onward rangerkms_bare_principal = 'rangerkms' if stack_supports_ranger_kerberos: if security_enabled: rangerkms_principal = config['configurations']['dbks-site'][ 'ranger.ks.kerberos.principal'] rangerkms_keytab = config['configurations']['dbks-site'][ 'ranger.ks.kerberos.keytab'] if not is_empty(rangerkms_principal) and rangerkms_principal != '': rangerkms_bare_principal = get_bare_principal(rangerkms_principal) rangerkms_principal = rangerkms_principal.replace( '_HOST', kms_host.lower()) kms_plugin_config['policy.download.auth.users'] = format( 'keyadmin,{rangerkms_bare_principal}') custom_ranger_service_config = generate_ranger_service_config( config['configurations']['kms-properties']) if len(custom_ranger_service_config) > 0: kms_plugin_config.update(custom_ranger_service_config) kms_ranger_plugin_repo = { 'isEnabled': 'true', 'configs': kms_plugin_config, 'description': 'kms repo', 'name': repo_name,
def recommendStormConfigurationsFromHDP25(self, configurations, clusterData, services, hosts): storm_site = self.getServicesSiteProperties(services, "storm-site") storm_env = self.getServicesSiteProperties(services, "storm-env") putStormSiteProperty = self.putProperty(configurations, "storm-site", services) putStormSiteAttributes = self.putPropertyAttribute( configurations, "storm-site") security_enabled = self.isSecurityEnabled(services) if storm_env and storm_site: if security_enabled: _storm_principal_name = storm_env[ 'storm_principal_name'] if 'storm_principal_name' in storm_env else None storm_bare_jaas_principal = get_bare_principal( _storm_principal_name) if 'nimbus.impersonation.acl' in storm_site: storm_nimbus_impersonation_acl = storm_site[ "nimbus.impersonation.acl"] storm_nimbus_impersonation_acl.replace( '{{storm_bare_jaas_principal}}', storm_bare_jaas_principal) putStormSiteProperty('nimbus.impersonation.acl', storm_nimbus_impersonation_acl) else: if 'nimbus.impersonation.acl' in storm_site: putStormSiteAttributes('nimbus.impersonation.acl', 'delete', 'true') if 'nimbus.impersonation.authorizer' in storm_site: putStormSiteAttributes('nimbus.impersonation.authorizer', 'delete', 'true') rangerPluginEnabled = '' if 'ranger-storm-plugin-properties' in configurations and 'ranger-storm-plugin-enabled' in configurations[ 'ranger-storm-plugin-properties']['properties']: rangerPluginEnabled = configurations[ 'ranger-storm-plugin-properties']['properties'][ 'ranger-storm-plugin-enabled'] elif 'ranger-storm-plugin-properties' in services[ 'configurations'] and 'ranger-storm-plugin-enabled' in services[ 'configurations']['ranger-storm-plugin-properties'][ 'properties']: rangerPluginEnabled = services['configurations'][ 'ranger-storm-plugin-properties']['properties'][ 'ranger-storm-plugin-enabled'] storm_authorizer_class = 'org.apache.storm.security.auth.authorizer.SimpleACLAuthorizer' ranger_authorizer_class = 'org.apache.ranger.authorization.storm.authorizer.RangerStormAuthorizer' # Cluster is kerberized if security_enabled: if rangerPluginEnabled and (rangerPluginEnabled.lower() == 'Yes'.lower()): putStormSiteProperty('nimbus.authorizer', ranger_authorizer_class) else: putStormSiteProperty('nimbus.authorizer', storm_authorizer_class) else: putStormSiteAttributes('nimbus.authorizer', 'delete', 'true') servicesList = [ service["StackServices"]["service_name"] for service in services["services"] ] # Storm AMS integration if 'AMBARI_METRICS' in servicesList: putStormSiteProperty( 'storm.cluster.metrics.consumer.register', '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsReporter"}]' ) putStormSiteProperty( 'topology.metrics.consumer.register', '[{"class": "org.apache.hadoop.metrics2.sink.storm.StormTimelineMetricsSink", ' '"parallelism.hint": 1, ' '"whitelist": ["kafkaOffset\\\..+/", "__complete-latency", "__process-latency", ' '"__execute-latency", ' '"__receive\\\.population$", "__sendqueue\\\.population$", "__execute-count", "__emit-count", ' '"__ack-count", "__fail-count", "memory/heap\\\.usedBytes$", "memory/nonHeap\\\.usedBytes$", ' '"GC/.+\\\.count$", "GC/.+\\\.timeMs$"]}]') else: putStormSiteProperty('storm.cluster.metrics.consumer.register', 'null') putStormSiteProperty('topology.metrics.consumer.register', 'null')
def recommendStreamlineConfigurations(self, configurations, clusterData, services, hosts): super(HDF30StackAdvisor, self).recommendRangerConfigurations(configurations, clusterData, services, hosts) servicesList = [ service["StackServices"]["service_name"] for service in services["services"] ] security_enabled = self.isSecurityEnabled(services) if 'STORM' in servicesList and security_enabled: storm_site = self.getServicesSiteProperties(services, "storm-site") if storm_site is not None: putStormSiteProperty = self.putProperty( configurations, "storm-site", services) putStormSiteAttributes = self.putPropertyAttribute( configurations, "storm-site") storm_env = self.getServicesSiteProperties( services, "storm-env") storm_nimbus_impersonation_acl = storm_site[ "nimbus.impersonation.acl"] if "nimbus.impersonation.acl" in storm_site else None streamline_env = self.getServicesSiteProperties( services, "streamline-env") _streamline_principal_name = streamline_env[ 'streamline_principal_name'] if 'streamline_principal_name' in streamline_env else None if _streamline_principal_name is not None and storm_nimbus_impersonation_acl is not None: streamline_bare_principal = get_bare_principal( _streamline_principal_name) storm_nimbus_impersonation_acl = storm_nimbus_impersonation_acl.replace( '{{streamline_bare_principal}}', streamline_bare_principal) putStormSiteProperty('nimbus.impersonation.acl', storm_nimbus_impersonation_acl) storm_nimbus_autocred_plugin_classes = storm_site[ "nimbus.autocredential.plugins.classes"] if "nimbus.autocredential.plugins.classes" in storm_site else None if storm_nimbus_autocred_plugin_classes is not None: new_storm_nimbus_autocred_plugin_classes = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = DefaultStackAdvisor.appendToYamlString( storm_nimbus_autocred_plugin_classes, new_storm_nimbus_autocred_plugin_classes) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.autocredential.plugins.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) storm_nimbus_credential_renewer_classes = storm_site[ "nimbus.credential.renewers.classes"] if "nimbus.credential.renewers.classes" in storm_site else None if storm_nimbus_credential_renewer_classes is not None: new_storm_nimbus_credential_renewer_classes_array = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = DefaultStackAdvisor.appendToYamlString( storm_nimbus_credential_renewer_classes, new_storm_nimbus_credential_renewer_classes_array) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.credential.renewers.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) putStormSiteProperty("nimbus.credential.renewers.freq.secs", "82800")
def getServiceConfigurationRecommendations(self, configurations, clusterData, services, hosts): Logger.info( "Class: %s, Method: %s. Get Service Configuration Recommendations." % (self.__class__.__name__, inspect.stack()[0][3])) servicesList = [ service["StackServices"]["service_name"] for service in services["services"] ] security_enabled = self.isSecurityEnabled(services) if 'AMBARI_METRICS' in servicesList: putAmsSiteProperty = self.putProperty(configurations, "ams-site") putAmsSiteProperty( 'timeline.metrics.downsampler.event.metric.patterns', 'topology\.%') if 'STORM' in servicesList and security_enabled: storm_site = self.getServicesSiteProperties(services, "storm-site") streamline_env = self.getServicesSiteProperties( services, "streamline-env") if storm_site is not None and streamline_env is not None: putStormSiteProperty = self.putProperty( configurations, "storm-site", services) putStormSiteAttributes = self.putPropertyAttribute( configurations, "storm-site") storm_env = self.getServicesSiteProperties( services, "storm-env") storm_nimbus_impersonation_acl = storm_site[ "nimbus.impersonation.acl"] if "nimbus.impersonation.acl" in storm_site else None streamline_env = self.getServicesSiteProperties( services, "streamline-env") _streamline_principal_name = streamline_env[ 'streamline_principal_name'] if 'streamline_principal_name' in streamline_env else None if _streamline_principal_name is not None and storm_nimbus_impersonation_acl is not None: streamline_bare_principal = get_bare_principal( _streamline_principal_name) storm_nimbus_impersonation_acl = "{ " + streamline_bare_principal + " : {hosts: ['*'], groups: ['*']}, {{storm_bare_jaas_principal}} : {hosts: ['*'], groups: ['*']}}" putStormSiteProperty('nimbus.impersonation.acl', storm_nimbus_impersonation_acl) storm_nimbus_autocred_plugin_classes = storm_site[ "nimbus.autocredential.plugins.classes"] if "nimbus.autocredential.plugins.classes" in storm_site else None # Here storm_nimbus_autocred_plugin_classes is resulting in none. There is no nimbus.autocredential.plugins.classes in storm-site.xml if storm_nimbus_autocred_plugin_classes is not None: new_storm_nimbus_autocred_plugin_classes = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = self.appendToYamlString( storm_nimbus_autocred_plugin_classes, new_storm_nimbus_autocred_plugin_classes) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.autocredential.plugins.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) storm_nimbus_credential_renewer_classes = storm_site[ "nimbus.credential.renewers.classes"] if "nimbus.credential.renewers.classes" in storm_site else None if storm_nimbus_credential_renewer_classes is not None: new_storm_nimbus_credential_renewer_classes_array = [ 'org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive' ] new_conf = self.appendToYamlString( storm_nimbus_credential_renewer_classes, new_storm_nimbus_credential_renewer_classes_array) putStormSiteProperty( "nimbus.autocredential.plugins.classes", new_conf) else: putStormSiteProperty( "nimbus.credential.renewers.classes", "['org.apache.storm.hdfs.security.AutoHDFS', 'org.apache.storm.hbase.security.AutoHBase', 'org.apache.storm.hive.security.AutoHive']" ) putStormSiteProperty("nimbus.credential.renewers.freq.secs", "82800") properties = get_ambari_properties() ambari_version = get_ambari_version(properties) if not (ambari_version) or not (ambari_version.startswith('2.5')): putStreamlineLogSearchConfAttribute = self.putPropertyAttribute( configurations, "streamline-logsearch-conf") putStreamlineLogSearchConfAttribute('service_name', 'visible', 'false') putStreamlineLogSearchConfAttribute('component_mappings', 'visible', 'false') putStreamlineLogSearchConfAttribute('content', 'visible', 'false') pass
tserver_hosts = default('/clusterHostInfo/accumulo_tserver_hosts', '/clusterHostInfo/slave_hosts') else: tserver_hosts = default('/clusterHostInfo/accumulo_tserver_hosts', '/clusterHostInfo/all_hosts') master_hosts = default('/clusterHostInfo/accumulo_master_hosts', []) monitor_hosts = default('/clusterHostInfo/accumulo_monitor_hosts', []) gc_hosts = default('/clusterHostInfo/accumulo_gc_hosts', []) tracer_hosts = default('/clusterHostInfo/accumulo_tracer_hosts', []) # security properties accumulo_user_keytab = config['configurations']['accumulo-env']['accumulo_user_keytab'] accumulo_principal_name = config['configurations']['accumulo-env']['accumulo_principal_name'] # kinit properties kinit_path_local = status_params.kinit_path_local if security_enabled: bare_accumulo_principal = get_bare_principal(config['configurations']['accumulo-site']['general.kerberos.principal']) kinit_cmd = format("{kinit_path_local} -kt {accumulo_user_keytab} {accumulo_principal_name};") else: kinit_cmd = "" host_sys_prepped = default("/hostLevelParams/host_sys_prepped", False) #for create_hdfs_directory hostname = status_params.hostname hdfs_user_keytab = config['configurations']['hadoop-env']['hdfs_user_keytab'] hdfs_user = config['configurations']['hadoop-env']['hdfs_user'] hdfs_principal_name = config['configurations']['hadoop-env']['hdfs_principal_name'] hdfs_site = config['configurations']['hdfs-site']