def __init__(self, version, conf={}, num_brokers=3, kafka_path=None, debug=False): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Supported \p conf keys: * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SSL_SASL \p conf dict is passed to KafkaBrokerApp classes, etc. """ super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, 'tmp', debug=debug) self.brokers = list() # One ZK (from Kafka repo) ZookeeperApp(self, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') # Start Kerberos KDC if GSSAPI (Kerberos) is configured if 'GSSAPI' in conf.get('sasl_mechanisms', []): kdc = KerberosKdcApp(self, 'MYREALM') # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() # Brokers defconf = {'replication_factor': min(num_brokers, 3), 'num_partitions': 4, 'version': version, 'security.protocol': 'PLAINTEXT'} defconf.update(conf) self.conf = defconf for n in range(0, num_brokers): self.brokers.append(KafkaBrokerApp(self, defconf, kafka_path=kafka_path))
def __init__(self, version, conf={}, num_brokers=3, debug=False, scenario="default"): """ @brief Create, deploy and start a Kafka cluster using Kafka \\p version Supported \\p conf keys: * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL \\p conf dict is passed to KafkaBrokerApp classes, etc. """ super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, os.environ.get('TRIVUP_ROOT', 'tmp'), debug=debug) # Read trivup config from scenario definition. defconf = read_scenario_conf(scenario) defconf.update(conf) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): self.ssl = SslApp(self, defconf) self.brokers = list() # One ZK (from Kafka repo) ZookeeperApp(self) # Start Kerberos KDC if GSSAPI (Kerberos) is configured if 'GSSAPI' in defconf.get('sasl_mechanisms', []): kdc = KerberosKdcApp(self, 'MYREALM') # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() # Brokers defconf.update({ 'replication_factor': min(num_brokers, 3), 'version': version, 'security.protocol': 'PLAINTEXT' }) self.conf = defconf for n in range(0, num_brokers): # Configure rack & replica selector if broker supports # fetch-from-follower if version_as_list(version) >= [2, 4, 0]: defconf.update({ 'conf': [ 'broker.rack=RACK${appid}', 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector' ] }) # noqa: E501 self.brokers.append(KafkaBrokerApp(self, defconf))
def _setup_kerberos(self): """ Set up Kerberos KDCs """ # Create KDCs for each realm. # First realm will be the default / broker realm. # realm_cnt = int(self.conf.get('realm_cnt', 1)) # No point in having more than two realms assert realm_cnt > 0 and realm_cnt < 3 realms = ['REALM{}.TRIVUP'.format(x + 1) for x in range(0, realm_cnt)] # Pre-Allocate ports for the KDCs so they can reference eachother # in the krb5.conf configuration. kdc_ports = {x: TcpPortAllocator( self.cluster).next("dummy") for x in realms} # Set up realm=kdc:port cross-realm mappings cross_realms = ",".join(["{}={}:{}".format( x, self.cluster.get_node().name, kdc_ports[x]) for x in realms]) kdcs = dict() for realm in realms: kdc = KerberosKdcApp( self.cluster, realm, conf={'port': kdc_ports[realm], 'cross_realms': cross_realms, 'renew_lifetime': int(self.conf.get('krb_renew_lifetime')), 'ticket_lifetime': int(self.conf.get('krb_ticket_lifetime'))}) # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() kdcs[realm] = kdc self.broker_realm = realms[0] self.client_realm = realms[-1] self.broker_kdc = kdcs[self.broker_realm] self.client_kdc = kdcs[self.client_realm] # Add cross-realm TGTs if realm_cnt > 1: KerberosKdcApp.add_cross_realm_tgts(kdcs) # Add client envs and configuration self.env['KRB5CCNAME'] = self.client_kdc.mkpath('krb5cc') self.env['KRB5_CONFIG'] = self.client_kdc.conf['krb5_conf'] self.env['KRB5_KDC_PROFILE'] = self.client_kdc.conf['kdc_conf'] principal, keytab = self.client_kdc.add_principal('admin') self._client_conf['sasl.kerberos.keytab'] = keytab self._client_conf['sasl.kerberos.principal'] = principal.split('@')[0] # Refresh ticket 60s before renew timeout. self._client_conf['sasl.kerberos.min.time.before.relogin'] = \ max(1, int(self.conf.get('krb_renew_lifetime')) - 60) * 1000
def __init__(self, version, conf={}, num_brokers=3, kafka_path=None, debug=False): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Supported \p conf keys: * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL \p conf dict is passed to KafkaBrokerApp classes, etc. """ super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, 'tmp', debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): self.ssl = SslApp(self, conf) self.brokers = list() # One ZK (from Kafka repo) ZookeeperApp(self, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') # Start Kerberos KDC if GSSAPI (Kerberos) is configured if 'GSSAPI' in conf.get('sasl_mechanisms', []): kdc = KerberosKdcApp(self, 'MYREALM') # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() # Brokers defconf = { 'replication_factor': min(num_brokers, 3), 'num_partitions': 4, 'version': version, 'security.protocol': 'PLAINTEXT' } defconf.update(conf) self.conf = defconf for n in range(0, num_brokers): self.brokers.append( KafkaBrokerApp(self, defconf, kafka_path=kafka_path))
def test_kerberos_cross_realm(): """ Test Kerberos cross-realm trusts """ topic = "test" cluster = Cluster('KafkaCluster', root_path=os.environ.get('TRIVUP_ROOT', 'tmp'), debug=True) ZookeeperApp(cluster) # # Create KDCs for each realm. # First realm will be the default / broker realm. # realm_cnt = 2 realms = ["REALM{}.COM".format(x + 1) for x in range(0, realm_cnt)] # Pre-Allocate ports for the KDCs so they can reference eachother # in the krb5.conf configuration. kdc_ports = {x: TcpPortAllocator(cluster).next("dummy") for x in realms} # Set up realm=kdc:port cross-realm mappings cross_realms = ",".join(["{}={}:{}".format(x, cluster.get_node().name, kdc_ports[x]) for x in realms]) kdcs = dict() for realm in realms: kdc = KerberosKdcApp(cluster, realm, conf={'port': kdc_ports[realm], 'cross_realms': cross_realms, 'renew_lifetime': '30', 'ticket_lifetime': '120'}) kdc.start() kdcs[realm] = kdc broker_realm = realms[0] client_realm = realms[1] broker_kdc = kdcs[broker_realm] client_kdc = kdcs[client_realm] # Create broker_cnt brokers broker_cnt = 4 brokerconf = {'replication_factor': min(3, int(broker_cnt)), 'num_partitions': broker_cnt * 2, 'version': '2.2.0', 'sasl_mechanisms': 'GSSAPI', 'realm': broker_realm, 'conf': ['connections.max.idle.ms=60000']} brokers = dict() for n in range(0, broker_cnt): broker = KafkaBrokerApp(cluster, brokerconf) brokers[broker.appid] = broker # Get bootstrap server list security_protocol = 'SASL_PLAINTEXT' all_listeners = (','.join(cluster.get_all( 'listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) assert len(bootstrap_servers) > 0, "no bootstrap servers" print("## Deploying cluster") cluster.deploy() print("## Starting cluster") cluster.start(timeout=30) # Add cross-realm TGTs for realm in realms: for crealm in [x for x in realms if x != realm]: kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), crealm, realm)).wait() kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), realm, crealm)).wait() # Create client base configuration client_config = { 'bootstrap.servers': bootstrap_servers, 'enable.sparse.connections': False, 'broker.address.family': 'v4', 'sasl.mechanisms': 'GSSAPI', 'security.protocol': security_protocol, 'debug': 'broker,security' } os.environ['KRB5CCNAME'] = client_kdc.mkpath('krb5cc') os.environ['KRB5_CONFIG'] = client_kdc.conf['krb5_conf'] os.environ['KRB5_KDC_PROFILE'] = client_kdc.conf['kdc_conf'] principal,keytab = client_kdc.add_principal("admin") client_config['sasl.kerberos.keytab'] = keytab client_config['sasl.kerberos.principal'] = principal.split('@')[0] client_config['sasl.kerberos.min.time.before.relogin'] = 120*1000*3 print(client_config) print("bootstraps: {}".format(client_config['bootstrap.servers'])) p = Producer(client_config) time.sleep(10) for n in range(1, 100): p.produce(topic, "msg #{}".format(n)) p.poll(1.0) p.flush(1.0) print("####### {} messages remaining\n\n\n".format(len(p))) start = time.time() end = start + (90*60) until = start + (12*60) while time.time() < end: now = time.time() if until < now: print("### Producing 2 messages") for n in range(1, 2): p.produce(topic, "msg #{}".format(n)) until = now + (12*60) p.poll(1.0) del p cluster.stop()