def __init__(self, cluster, conf=None, on=None): """ @param cluster Current cluster. @param conf Configuration dict. port Port at which OauthbearerOIDCApp should be bound (optional). A (random) free port will be chosen otherwise. @param on Node name to run on. """ super(OauthbearerOIDCApp, self).__init__(cluster, conf=conf, on=on) self.conf['port'] = trivup.TcpPortAllocator(self.cluster).next( self, port_base=self.conf.get('port', None)) self.conf['valid_url'] = 'http://localhost:%d/retrieve' % \ self.conf['port'] self.conf['badformat_url'] = 'http://localhost:%d/retrieve/badformat' \ % self.conf['port'] self.conf['expired_url'] = 'http://localhost:%d/retrieve/expire' % \ self.conf['port'] self.conf['jwks_url'] = 'http://localhost:%d/keys' % self.conf['port'] self.conf['sasl_oauthbearer_method'] = 'OIDC' self.conf['sasl_oauthbearer_client_id'] = '123' self.conf['sasl_oauthbearer_client_secret'] = 'abc' self.conf['sasl_oauthbearer_scope'] = 'test' self.conf['sasl_oauthbearer_extensions'] = \ 'ExtensionworkloadIdentity=develC348S,Extensioncluster=lkc123'
def __init__(self, cluster, realm, conf=None, on=None): """ @param cluster Current cluster @param realm Realm name @param conf Configuration dict, ignored. @param on Node name to run on """ super(KerberosKdcApp, self).__init__(cluster, conf=conf, on=on) self.conf['port'] = trivup.TcpPortAllocator(self.cluster).next(self) self.conf['realm'] = realm self.conf['address'] = '%(nodename)s:%(port)d' % self.conf self.conf['dbpath'] = self.mkpath('database') self.conf['admin_keytab'] = self.mkpath('admin_keytab') self.conf['stash_file'] = self.mkpath('stash_file') # Generate config files self.conf['krb5_conf'] = self.create_file_from_template( 'krb5.conf', self.conf) self.env_add('KRB5_CONFIG', self.conf['krb5_conf']) self.conf['kdc_conf'] = self.create_file_from_template( 'kdc.conf', self.conf) self.env_add('KRB5_KDC_PROFILE', self.conf['kdc_conf']) # Create database and stash file r = self.execute( 'kdb5_util -P "" -r %(realm)s -d "%(dbpath)s" -sf "%(stash_file)s" create -s' % self.conf).wait() if r != 0: raise Exception('Failed to create kdb5 database') self.conf['start_cmd'] = '/usr/sbin/krb5kdc -n' self.conf['stop_cmd'] = None # Ctrl-C
def __init__(self, cluster, conf=None, on=None, bin_path=None): """ @param cluster Current cluster @param on Node name to run on Config: bindir Path to zookeeper-server-start.sh directory (optional) Falls back to Kafka bindir Exposes 'address' (host:port) for other apps. """ super(ZookeeperApp, self).__init__(cluster, conf=conf, on=on) self.conf['port'] = trivup.TcpPortAllocator(self.cluster).next(self) self.conf['datadir'] = self.create_dir('datadir') self.conf['address'] = '%(nodename)s:%(port)d' % self.conf # Generate config file self.conf['conf_file'] = self.create_file_from_template('zookeeper.properties', self.conf) # noqa: E501
def __init__(self, cluster, realm, conf=None, on=None): """ @param cluster Current cluster @param realm Realm name @param conf Configuration dict, optional. "port": port to bind to. "cross_realms": "realm1=kdc1:port,realm2=kdc2:port" - cross-realm. The first cross realm is the $default_realm. "renew_lifetime": see krb5.conf docs (default 10 min) "ticket_lifetime": see krb5.conf docs (default 60 min) @param on Node name to run on """ super(KerberosKdcApp, self).__init__(cluster, conf=conf, on=on) self.conf['realm'] = realm if self.conf.get('port', None) is None: self.conf['port'] = trivup.TcpPortAllocator(self.cluster).next(self) # noqa: E501 self.conf['address'] = '%(nodename)s:%(port)d' % self.conf self.conf['dbpath'] = self.mkpath('database') self.conf['admin_keytab'] = self.mkpath('admin_keytab') self.conf['stash_file'] = self.mkpath('stash_file') if self.conf.get('renew_lifetime', None) is None: self.conf['renew_lifetime'] = '12h' if self.conf.get('ticket_lifetime', None) is None: self.conf['ticket_lifetime'] = '30m' # Set up cross-realm trusts, if desired. cross_realms = self.conf.get('cross_realms', '').split(',') if len(cross_realms) > 0 and cross_realms[0] != '': cross_realms_conf = "" capaths_conf = "" for crinfo in cross_realms: crealm, ckdc = crinfo.split('=') if crealm == realm: continue cross_realms_conf += " %s = {\n kdc = %s\n admin_server = %s\n }\n" % (crealm, ckdc, ckdc) # noqa: E501 capaths_conf += " %s = {\n %s = .\n }\n" % (crealm, realm) capaths_conf += " %s = {\n %s = .\n }\n" % (realm, crealm) self.conf['default_realm'] = cross_realms[0].split('=')[0] self.conf['cross_realms'] = cross_realms_conf self.conf['capaths'] = capaths_conf else: self.conf['default_realm'] = realm self.conf['cross_realms'] = '' self.conf['capaths'] = '' # Generate config files self.conf['krb5_conf'] = self.create_file_from_template('krb5.conf', self.conf) self.env_add('KRB5_CONFIG', self.conf['krb5_conf']) self.conf['kdc_conf'] = self.create_file_from_template('kdc.conf', self.conf) self.env_add('KRB5_KDC_PROFILE', self.conf['kdc_conf']) # Create database and stash file r = self.execute('kdb5_util -P "" -r %(realm)s -d "%(dbpath)s" -sf "%(stash_file)s" create -s' % self.conf).wait() # noqa: E501 if r != 0: raise Exception('Failed to create kdb5 database') self.conf['start_cmd'] = '/usr/sbin/krb5kdc -n' self.conf['stop_cmd'] = None # Ctrl-C
def __init__(self, cluster, conf=None, on=None): """ @param cluster Current cluster @param conf Configuration dict, see below. @param on Node name to run on Supported conf keys: * version - Kafka version to use, will build 'trunk' from kafka_path, otherwise the version is taken to be a formal release which will be downloaded and deployed. * listeners - CSV list of listener types: PLAINTEXT,SSL,SASL,SASL_SSL * listener_host - alternative listener host instead of node name (e.g., '*') * advertised_hostname - hostname to use for advertised.listeners (defaults to 'on' node) * sasl_mechanisms - CSV list of SASL mechanisms to enable: GSSAPI,PLAIN,SCRAM-SHA-n,OAUTHBEARER SASL listeners will be added automatically. KerberosKdcApp is required for GSSAPI. * sasl_users - CSV list of SASL PLAIN/SCRAM of user=pass for authenticating clients * ssl_client_auth - ssl.client.auth broker property (def: required) * num_partitions - Topic auto-create partition count (3) * replication_Factor - Topic auto-create replication factor (1) * port_base - Low TCP port base to start allocating from (random) * kafka_path - Path to Kafka build tree (for trunk usage) * fdlimit - RLIMIT_NOFILE (or "max") (default: max) * conf - arbitary server.properties config as a list of strings. * realm - Kerberos realm to use when sasl_mechanisms contains GSSAPI """ super(KafkaBrokerApp, self).__init__(cluster, conf=conf, on=on) self.zk = cluster.find_app('ZookeeperApp') if self.zk is None: raise Exception('ZookeeperApp required') # Kafka repo uses SVN nomenclature if self.conf['version'] == 'master': self.conf['version'] = 'trunk' if 'fdlimit' not in self.conf: self.conf['fdlimit'] = 'max' listener_host = self.conf.get('listener_host', self.conf.get('nodename')) # Kafka Configuration properties self.conf['log_dirs'] = self.create_dir('logs') if 'num_partitions' not in self.conf: self.conf['num_partitions'] = 3 self.conf['zk_connect'] = self.zk.get('address', None) if 'replication_factor' not in self.conf: self.conf['replication_factor'] = 1 # Kafka paths if self.conf.get('kafka_path', None) is not None: self.conf['destdir'] = self.conf['kafka_path'] self.conf['bindir'] = os.path.join(self.conf['destdir'], 'bin') start_sh = os.path.join(self.conf['bindir'], 'kafka-server-start.sh') kafka_configs_sh = os.path.join(self.conf['bindir'], 'kafka-configs.sh') else: start_sh = 'kafka-server-start.sh' kafka_configs_sh = 'kafka-configs.sh' # Arbitrary (non-template) configuration statements conf_blob = self.conf.get('conf', list()) jaas_blob = list() # # Configure listeners, SSL and SASL # listeners = self.conf.get('listeners', 'PLAINTEXT').split(',') # SASL support sasl_mechs = [ x for x in self.conf.get('sasl_mechanisms', '').replace( ' ', '').split(',') if len(x) > 0 ] if len(sasl_mechs) > 0: listeners.append('SASL_PLAINTEXT') # SSL support ssl = cluster.find_app(SslApp) if ssl is not None: # Add SSL listeners listeners.append('SSL') if len(sasl_mechs) > 0: listeners.append('SASL_SSL') listener_map = 'listener.security.protocol.map=' + \ 'PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:' + \ 'SASL_PLAINTEXT,SASL_SSL:SASL_SSL' can_docker = self.conf['version'] == 'trunk' or \ int(self.conf['version'].split('.')[0]) > 0 if can_docker: # Map DOCKER listener to PLAINTEXT security protocol listener_map += ',DOCKER:PLAINTEXT' conf_blob.append(listener_map) # Create listeners ports = [(x, trivup.TcpPortAllocator(self.cluster).next( self, self.conf.get('port_base', None))) for x in sorted(set(listeners))] self.conf['port'] = ports[0][1] # "Default" port if can_docker: # Add docker listener to allow services (e.g, SchemaRegistry) in # docker-containers to reach the on-host Kafka. docker_port = trivup.TcpPortAllocator(self.cluster).next(self) docker_host = '%s:%d' % (cluster.get_docker_host(), docker_port) self.conf['address'] = '%s:%d' % (listener_host, self.conf['port']) listeners = ['%s://%s:%d' % (x[0], "0.0.0.0", x[1]) for x in ports] if can_docker: listeners.append('%s://%s:%d' % ('DOCKER', "0.0.0.0", docker_port)) self.conf['listeners'] = ','.join(listeners) if 'advertised_hostname' not in self.conf: self.conf['advertised_hostname'] = self.conf['nodename'] advertised_listeners = [ '%s://%s:%d' % (x[0], self.conf['advertised_hostname'], x[1]) for x in ports ] if can_docker: # Expose service to docker containers as well. advertised_listeners.append('DOCKER://%s' % docker_host) self.conf['docker_advertised_listeners'] = 'PLAINTEXT://%s' % \ docker_host self.conf['advertised.listeners'] = ','.join(advertised_listeners) self.conf['advertised_listeners'] = self.conf['advertised.listeners'] self.conf['auto_create_topics'] = self.conf.get( 'auto_create_topics', 'true') self.dbg('Listeners: %s' % self.conf['listeners']) self.dbg('Advertised Listeners: %s' % self.conf['advertised.listeners']) if len(sasl_mechs) > 0: self.dbg('SASL mechanisms: %s' % sasl_mechs) jaas_blob.append('KafkaServer {') conf_blob.append('sasl.enabled.mechanisms=%s' % ','.join(sasl_mechs)) # Handle PLAIN and SCRAM-.. the same way for mech in sasl_mechs: if mech.find('SCRAM') != -1: plugin = 'scram.Scram' elif mech == 'PLAIN': plugin = 'plain.Plain' else: continue sasl_users = self.conf.get('sasl_users', '') if len(sasl_users) == 0: self.log(('WARNING: No sasl_users configured for %s, ' 'expected CSV of user=pass,..') % plugin) else: jaas_blob.append( ('org.apache.kafka.common.security.' '%sLoginModule required debug=true') % plugin) for up in sasl_users.split(','): u, p = up.split('=') if plugin == 'plain.Plain': jaas_blob.append(' user_%s="%s"' % (u, p)) elif plugin == 'scram.Scram': jaas_blob.append(' username="******" password="******"' % (u, p)) # SCRAM users are set up in ZK # with kafka-configs.sh self.post_start_cmds.append( ('%s --zookeeper %s --alter --add-config ' '\'%s=[iterations=4096,password=%s]\' ' '--entity-type users --entity-name \'%s\'') % ( kafka_configs_sh, self.conf['zk_connect'], # noqa: E501 mech, p, u)) jaas_blob[-1] += ';' if 'GSSAPI' in sasl_mechs: conf_blob.append('sasl.kerberos.service.name=%s' % 'kafka') realm = self.conf.get('realm', None) if realm is None: kdc = self.cluster.find_app(KerberosKdcApp) self.conf['realm'] = kdc.conf.get('realm', None) else: kdc = self.cluster.find_app(KerberosKdcApp, ('realm', realm)) # If a realm was specified it is most likely because # we're operating in a cross-realm scenario. # Add a principal mapping client principals without # hostname ("admin" rather than "admin/localhost") # to a local user. # This is not compatible with "admin/localhost" principals. conf_blob.append( 'authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer' ) # noqa: E501 conf_blob.append('allow.everyone.if.no.acl.found=true') conf_blob.append( 'sasl.kerberos.principal.to.local.rules=RULE:[1:admin](.*)s/^.*/admin/' ) # noqa: E501 assert kdc is not None, \ "No KerberosKdcApp found (realm={})".format(realm) self.env_add('KRB5_CONFIG', kdc.conf['krb5_conf']) self.env_add( 'KAFKA_OPTS', '-Djava.security.krb5.conf=%s' % kdc.conf['krb5_conf']) self.env_add('KAFKA_OPTS', '-Dsun.security.krb5.debug=true') self.kerberos_principal, self.kerberos_keytab = kdc.add_principal( 'kafka', self.conf['advertised_hostname']) # noqa: E501 jaas_blob.append( 'com.sun.security.auth.module.Krb5LoginModule required' ) # noqa: E501 jaas_blob.append( 'useKeyTab=true storeKey=true doNotPrompt=true' ) # noqa: E501 jaas_blob.append('keyTab="%s"' % self.kerberos_keytab) jaas_blob.append('debug=true') jaas_blob.append('principal="%s";' % self.kerberos_principal) if 'OAUTHBEARER' in sasl_mechs: # Use the unsecure JSON web token. # Client should be configured with # 'sasl.oauthbearer.config=scope=requiredScope principal=admin' # Change requiredScope to something else to trigger auth error. conf_blob.append('super.users=User:admin') conf_blob.append('allow.everyone.if.no.acl.found=true') conf_blob.append( 'authorizer.class.name=kafka.security.auth.SimpleAclAuthorizer' ) # noqa: E501 jaas_blob.append( 'org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required' ) # noqa: E501 jaas_blob.append(' unsecuredLoginLifetimeSeconds="3600"') jaas_blob.append(' unsecuredLoginStringClaim_sub="admin"') jaas_blob.append( ' unsecuredValidatorRequiredScope="requiredScope"' ) # noqa: E501 jaas_blob.append(';') jaas_blob.append('};\n') self.conf['jaas_file'] = self.create_file( 'jaas_broker.conf', data='\n'.join(jaas_blob)) self.env_add( 'KAFKA_OPTS', '-Djava.security.auth.login.config=%s' % self.conf['jaas_file']) self.env_add('KAFKA_OPTS', '-Djava.security.debug=all') # SSL config and keys (et.al.) if ssl is not None: keystore, truststore, _, _ = ssl.create_keystore('broker%s' % self.appid) conf_blob.append('ssl.protocol=TLS') conf_blob.append('ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1') conf_blob.append('ssl.keystore.type = JKS') conf_blob.append('ssl.keystore.location = %s' % keystore) conf_blob.append('ssl.keystore.password = %s ' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.key.password = %s' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.truststore.type = JKS') conf_blob.append('ssl.truststore.location = %s' % truststore) conf_blob.append('ssl.truststore.password = %s' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.client.auth = %s' % self.conf.get('ssl_client_auth', 'required')) # Generate config file self.conf['conf_file'] = self.create_file_from_template( 'server.properties', # noqa: E501 self.conf, append_data=Template('\n'.join(conf_blob)).substitute( self.conf)) # noqa: E501 # Generate LOG4J file (if app debug is enabled) if self.debug: self.conf['log4j_file'] = self.create_file_from_template( 'log4j.properties', self.conf, subst=False) # noqa: E501 self.env_add( 'KAFKA_LOG4J_OPTS', '-Dlog4j.configuration=file:%s' % self.conf['log4j_file']) self.env_add('LOG_DIR', self.mkpath('debug')) # Runs in foreground, stopped by Ctrl-C # This is the default for no-deploy use: # will be overwritten by deploy() if enabled. self.conf['start_cmd'] = '%s %s' % (start_sh, self.conf['conf_file']) self.conf['stop_cmd'] = None # Ctrl-C
def __init__(self, cluster, conf=None, on=None): """ @param cluster Current cluster @param conf Configuration dict, see below. @param on Node name to run on Supported conf keys: * version - Confluent Platform version to use. * port_base - Low TCP port base to start allocating from (random) * image - docker image to use * conf - schema-registry docker image config strings (NOT USED) """ super(SchemaRegistryApp, self).__init__(cluster, conf=conf, on=on) if self.conf.get('image', '') == '': self.conf['image'] = self.default_image self.conf['container_id'] = 'trivup_sr_%s' % str(uuid.uuid4())[0:7] kafka = cluster.find_app(KafkaBrokerApp) if kafka is None: raise Exception('KafkaBrokerApp required') bootstrap_servers = kafka.conf.get('docker_advertised_listeners') if bootstrap_servers is None: raise Exception('KafkaBrokerApp required') # Create listener port = trivup.TcpPortAllocator(self.cluster).next( self, self.conf.get('port_base', None)) docker_args = '' if cluster.platform == 'linux': # Let container bind to host localhost self.conf['extport'] = port self.conf['intport'] = port docker_args = '--network=host' elif cluster.platform == 'darwin': # On OSX localhost binds are not possible, so set up a # port forwarding. self.conf['extport'] = port self.conf['intport'] = 8081 docker_args = '-p %d:%d' % (self.conf['extport'], self.conf['intport']) # This is the listener address inside the docker container self.conf['listeners'] = 'http://0.0.0.0:%d' % self.conf.get('intport') # This is the listener address outside the docker container, # using port-forwarding self.conf['url'] = 'http://localhost:%d' % self.conf['extport'] # Run in foreground. self.conf[ 'start_cmd'] = 'docker run %s --name %s -e SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS=%s -e SCHEMA_REGISTRY_HOST_NAME=localhost -e SCHEMA_REGISTRY_LISTENERS=%s -e SCHEMA_REGISTRY_DEBUG=true %s' % ( # noqa: E501 docker_args, self.conf.get('container_id'), bootstrap_servers, self.conf.get('listeners'), self.conf.get('image')) # Stop through docker self.conf['stop_cmd'] = 'docker stop %s' % \ self.conf.get('container_id')
def __init__(self, cluster, conf=None, on=None): """ @param cluster Current cluster @param conf Configuration dict, see below. @param on Node name to run on Supported conf keys: * version - Kafka version to use, will build 'trunk' from kafka_path, otherwise the version is taken to be a formal release which will be downloaded and deployed. * listeners - CSV list of listener types: PLAINTEXT,SSL,SASL,SASL_SSL * listener_host - alternative listener host instead of node name (e.g., '*') * advertised_hostname - hostname to use for advertised.listeners (defaults to 'on' node) * sasl_mechanisms - CSV list of SASL mechanisms to enable: GSSAPI,PLAIN,SCRAM-SHA-n SASL listeners will be added automatically. KerberosKdcApp is required for GSSAPI. * sasl_users - CSV list of SASL PLAIN/SCRAM of user=pass for authenticating clients * ssl_client_auth - ssl.client.auth broker property (def: required) * num_partitions - Topic auto-create partition count (3) * replication_Factor - Topic auto-create replication factor (1) * port_base - Low TCP port base to start allocating from (random) * kafka_path - Path to Kafka build tree (for trunk usage) * fdlimit - RLIMIT_NOFILE (or "max") (default: max) * conf - arbitary server.properties config as a list of strings. """ super(KafkaBrokerApp, self).__init__(cluster, conf=conf, on=on) self.zk = cluster.find_app('ZookeeperApp') if self.zk is None: raise Exception('ZookeeperApp required') # Kafka repo uses SVN nomenclature if self.conf['version'] == 'master': self.conf['version'] = 'trunk' if 'fdlimit' not in self.conf: self.conf['fdlimit'] = 'max' listener_host = self.conf.get('listener_host', self.conf.get('nodename')) # Kafka Configuration properties self.conf['log_dirs'] = self.create_dir('logs') if 'num_partitions' not in self.conf: self.conf['num_partitions'] = 3 self.conf['zk_connect'] = self.zk.get('address', None) if 'replication_factor' not in self.conf: self.conf['replication_factor'] = 1 # Kafka paths if self.conf.get('kafka_path', None) is not None: self.conf['destdir'] = self.conf['kafka_path'] self.conf['bindir'] = os.path.join(self.conf['destdir'], 'bin') start_sh = os.path.join(self.conf['bindir'], 'kafka-server-start.sh') kafka_configs_sh = os.path.join(self.conf['bindir'], 'kafka-configs.sh') else: start_sh = 'kafka-server-start.sh' kafka_configs_sh = 'kafka-configs.sh' # Arbitrary (non-template) configuration statements conf_blob = self.conf.get('conf', list()) jaas_blob = list() # # Configure listeners, SSL and SASL # listeners = self.conf.get('listeners', 'PLAINTEXT').split(',') # SASL support sasl_mechs = [ x for x in self.conf.get('sasl_mechanisms', '').replace( ' ', '').split(',') if len(x) > 0 ] if len(sasl_mechs) > 0: listeners.append('SASL_PLAINTEXT') # SSL support if getattr(cluster, 'ssl', None) is not None: # Add SSL listeners listeners.append('SSL') if len(sasl_mechs) > 0: listeners.append('SASL_SSL') # Create listeners ports = [(x, trivup.TcpPortAllocator(self.cluster).next( self, self.conf.get('port_base', None))) for x in sorted(set(listeners))] self.conf['port'] = ports[0][1] # "Default" port self.conf['address'] = '%s:%d' % (listener_host, self.conf['port']) self.conf['listeners'] = ','.join( ['%s://%s:%d' % (x[0], listener_host, x[1]) for x in ports]) if 'advertised_hostname' not in self.conf: self.conf['advertised_hostname'] = self.conf['nodename'] self.conf['advertised.listeners'] = ','.join([ '%s://%s:%d' % (x[0], self.conf['advertised_hostname'], x[1]) for x in ports ]) self.conf['auto_create_topics'] = self.conf.get( 'auto_create_topics', 'true') self.dbg('Listeners: %s' % self.conf['listeners']) self.dbg('Advertised Listeners: %s' % self.conf['advertised.listeners']) if len(sasl_mechs) > 0: self.dbg('SASL mechanisms: %s' % sasl_mechs) jaas_blob.append('KafkaServer {') conf_blob.append('sasl.enabled.mechanisms=%s' % ','.join(sasl_mechs)) # Handle PLAIN and SCRAM-.. the same way for mech in sasl_mechs: if mech.find('SCRAM') != -1: plugin = 'scram.Scram' elif mech == 'PLAIN': plugin = 'plain.Plain' else: continue sasl_users = self.conf.get('sasl_users', '') if len(sasl_users) == 0: self.log( 'WARNING: No sasl_users configured for %s, expected CSV of user=pass,..' % plugin) else: jaas_blob.append( 'org.apache.kafka.common.security.%sLoginModule required debug=true' % plugin) for up in sasl_users.split(','): u, p = up.split('=') if plugin is 'plain.Plain': jaas_blob.append(' user_%s="%s"' % (u, p)) elif plugin is 'scram.Scram': jaas_blob.append(' username="******" password="******"' % (u, p)) # SCRAM users are set up in ZK with kafka-configs.sh self.post_start_cmds.append('%s --zookeeper %s --alter --add-config \'%s=[iterations=4096,password=%s]\' --entity-type users --entity-name \'%s\'' % \ (kafka_configs_sh, self.conf['zk_connect'], mech, p, u)) jaas_blob[-1] += ';' if 'GSSAPI' in sasl_mechs: conf_blob.append('sasl.kerberos.service.name=%s' % 'kafka') kdc = self.cluster.find_app(KerberosKdcApp) self.env_add('KRB5_CONFIG', kdc.conf['krb5_conf']) self.env_add( 'KAFKA_OPTS', '-Djava.security.krb5.conf=%s' % kdc.conf['krb5_conf']) self.env_add('KAFKA_OPTS', '-Dsun.security.krb5.debug=true') self.kerberos_principal, self.kerberos_keytab = kdc.add_principal( 'kafka', self.conf['advertised_hostname']) jaas_blob.append( 'com.sun.security.auth.module.Krb5LoginModule required') jaas_blob.append( 'useKeyTab=true storeKey=true doNotPrompt=true') jaas_blob.append('keyTab="%s"' % self.kerberos_keytab) jaas_blob.append('debug=true') jaas_blob.append('principal="%s";' % self.kerberos_principal) jaas_blob.append('};\n') self.conf['jaas_file'] = self.create_file( 'jaas_broker.conf', data='\n'.join(jaas_blob)) self.env_add( 'KAFKA_OPTS', '-Djava.security.auth.login.config=%s' % self.conf['jaas_file']) self.env_add('KAFKA_OPTS', '-Djava.security.debug=all') # SSL config and keys (et.al.) if getattr(cluster, 'ssl', None) is not None: ssl = cluster.ssl keystore, truststore, _, _ = ssl.create_keystore('broker%s' % self.appid) conf_blob.append('ssl.protocol=TLS') conf_blob.append('ssl.enabled.protocols=TLSv1.2,TLSv1.1,TLSv1') conf_blob.append('ssl.keystore.type = JKS') conf_blob.append('ssl.keystore.location = %s' % keystore) conf_blob.append('ssl.keystore.password = %s ' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.key.password = %s' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.truststore.type = JKS') conf_blob.append('ssl.truststore.location = %s' % truststore) conf_blob.append('ssl.truststore.password = %s' % ssl.conf.get('ssl_key_pass')) conf_blob.append('ssl.client.auth = %s' % self.conf.get('ssl_client_auth', 'required')) # Generate config file self.conf['conf_file'] = self.create_file_from_template( 'server.properties', self.conf, append_data='\n'.join(conf_blob)) # Generate LOG4J file (if app debug is enabled) if self.debug: self.conf['log4j_file'] = self.create_file_from_template( 'log4j.properties', self.conf, subst=False) self.env_add( 'KAFKA_LOG4J_OPTS', '-Dlog4j.configuration=file:%s' % self.conf['log4j_file']) self.env_add('LOG_DIR', self.mkpath('debug')) # Runs in foreground, stopped by Ctrl-C # This is the default for no-deploy use: will be overwritten by deploy() if enabled. self.conf['start_cmd'] = '%s %s' % (start_sh, self.conf['conf_file']) self.conf['stop_cmd'] = None # Ctrl-C