def test_version(version, cmd=None, deploy=True): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ cluster = Cluster('librdkafkaInteractiveBrokerVersionTests', 'tmp') # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') zk_address = zk1.get('address') # Two brokers conf = {'replication_factor': 3, 'num_partitions': 4, 'version': version} broker1 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) broker2 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) broker3 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) bootstrap_servers = ','.join(cluster.get_all('address', '', KafkaBrokerApp)) # Generate test config file fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) if version != 'trunk': os.write(fd, ('broker.version=%s\n' % version).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster') cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): raise TimeoutError('Cluster did not go operational') print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env = 'RDKAFKA_TEST_CONF=%s ZK_ADDRESS=%s KAFKA_VERSION=%s' % ( test_conf_file, zk_address, version) if not cmd: cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % ( cluster.name, version) subprocess.call('%s %s' % (cmd_env, cmd), shell=True, executable='/bin/bash') os.remove(test_conf_file) cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return True
def test_version(version): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ cluster = Cluster('librdkafkaBrokerVersionTests', 'tmp') # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') zk_address = zk1.get('address') # Two brokers conf = {'replication_factor': 3, 'num_partitions': 4, 'version': version} broker1 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) broker2 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) broker3 = KafkaBrokerApp(cluster, conf, kafka_path=kafka_path) bootstrap_servers = ','.join(cluster.get_all('address', '', KafkaBrokerApp)) # Generate test config file fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, 'bootstrap.servers=%s\n' % bootstrap_servers) os.close(fd) print('# Deploying cluster') cluster.deploy() print('# Starting cluster') cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): raise TimeoutError('Cluster did not go operational') print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) print('\033[32mCluster started.. Executing librdkafka tests\033[0m') r = subprocess.call( 'TEST_LEVEL=%d RDKAFKA_TEST_CONF=%s ZK_ADDRESS=%s make' % (test_level, test_conf_file, zk_address), shell=True) if r == 0: print('\033[37;42mTests PASSED on broker version %s\033[0m' % version) ret = True else: print('\033[33;41mTests FAILED on broker version %s (ret %d)\033[0m' % (version, r)) ret = False os.remove(test_conf_file) cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return ret
def __init__(self, version, conf={}, num_brokers=3, debug=False, scenario="default"): """ @brief Create, deploy and start a Kafka cluster using Kafka \\p version Supported \\p conf keys: * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL \\p conf dict is passed to KafkaBrokerApp classes, etc. """ super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, os.environ.get('TRIVUP_ROOT', 'tmp'), debug=debug) # Read trivup config from scenario definition. defconf = read_scenario_conf(scenario) defconf.update(conf) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): self.ssl = SslApp(self, defconf) self.brokers = list() # One ZK (from Kafka repo) ZookeeperApp(self) # Start Kerberos KDC if GSSAPI (Kerberos) is configured if 'GSSAPI' in defconf.get('sasl_mechanisms', []): kdc = KerberosKdcApp(self, 'MYREALM') # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() # Brokers defconf.update({ 'replication_factor': min(num_brokers, 3), 'version': version, 'security.protocol': 'PLAINTEXT' }) self.conf = defconf for n in range(0, num_brokers): # Configure rack & replica selector if broker supports # fetch-from-follower if version_as_list(version) >= [2, 4, 0]: defconf.update({ 'conf': [ 'broker.rack=RACK${appid}', 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector' ] }) # noqa: E501 self.brokers.append(KafkaBrokerApp(self, defconf))
def __init__(self, version, conf={}, num_brokers=3, kafka_path=None, debug=False): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Supported \p conf keys: * security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL \p conf dict is passed to KafkaBrokerApp classes, etc. """ super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, 'tmp', debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): self.ssl = SslApp(self, conf) self.brokers = list() # One ZK (from Kafka repo) ZookeeperApp(self, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') # Start Kerberos KDC if GSSAPI (Kerberos) is configured if 'GSSAPI' in conf.get('sasl_mechanisms', []): kdc = KerberosKdcApp(self, 'MYREALM') # Kerberos needs to be started prior to Kafka so that principals # and keytabs are available at the time of Kafka config generation. kdc.start() # Brokers defconf = { 'replication_factor': min(num_brokers, 3), 'num_partitions': 4, 'version': version, 'security.protocol': 'PLAINTEXT' } defconf.update(conf) self.conf = defconf for n in range(0, num_brokers): self.brokers.append( KafkaBrokerApp(self, defconf, kafka_path=kafka_path))
def __init__(self, broker_cnt=3, kafka_version='2.3.0'): super(KafkaCluster, self).__init__() self.cluster = Cluster('KafkaCluster', root_path=os.environ.get('TRIVUP_ROOT', 'tmp'), debug=True) self.apps = dict() # Create a single ZK for the cluster ZookeeperApp(self.cluster) # Create broker_cnt brokers brokerconf = { 'replication_factor': min(3, int(broker_cnt)), 'num_partitions': 4, 'version': kafka_version } self.brokers = dict() for n in range(0, broker_cnt): broker = KafkaBrokerApp(self.cluster, brokerconf) self.brokers[broker.appid] = broker # Get bootstrap server list security_protocol = 'PLAINTEXT' all_listeners = (','.join( self.cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join( [x for x in all_listeners if x.startswith(security_protocol)]) # Create client base configuration self._client_config = { 'bootstrap.servers': bootstrap_servers, 'broker.address.family': 'v4' } self.cluster.deploy()
def test_version(version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1, root_path='tmp', broker_cnt=3): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('LibrdkafkaTestCluster', root_path, debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster) zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() defconf = { 'replication_factor': min(broker_cnt, 3), 'num_partitions': 4, 'version': version } defconf.update(conf) print('conf: ', defconf) brokers = [] for n in range(0, broker_cnt): brokers.append(KafkaBrokerApp(cluster, defconf)) # Generate test config file security_protocol = 'PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('test.sql.command=sqlite3 rdktests\n').encode('ascii')) os.write(fd, 'broker.address.family=v4\n'.encode('ascii')) if version != 'trunk': os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) else: os.write(fd, 'api.version.request=true\n'.encode('ascii')) # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, ('sasl.mechanisms=%s\n' % mech).encode('ascii')) if mech == 'PLAIN' or mech.find('SCRAM') != -1: print('# Writing SASL %s client config to %s' % (mech, test_conf_file)) security_protocol = 'SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u, p = up.split('=') os.write(fd, ('sasl.username=%s\n' % u).encode('ascii')) os.write(fd, ('sasl.password=%s\n' % p).encode('ascii')) break else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key, req, pem = ssl.create_key('librdkafka') os.write(fd, ('ssl.ca.location=%s\n' % ssl.ca_cert).encode('ascii')) os.write(fd, ('ssl.certificate.location=%s\n' % pem).encode('ascii')) os.write(fd, ('ssl.key.location=%s\n' % key).encode('ascii')) os.write(fd, ('ssl.key.password=%s\n' % ssl.conf.get('ssl_key_pass')).encode('ascii')) # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join( [x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, ('security.protocol=%s\n' % security_protocol).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster, instance path %s' % cluster.instance_path()) cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster %s did not go operational, see logs in %s/%s' % \ (cluster.name, cluster.root_path, cluster.instance)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env = 'export KAFKA_PATH="%s" RDKAFKA_TEST_CONF="%s" ZK_ADDRESS="%s" BROKERS="%s" TEST_KAFKA_VERSION="%s" TRIVUP_ROOT="%s"; ' % \ (brokers[0].conf.get('destdir'), test_conf_file, zk_address, bootstrap_servers, version, cluster.instance_path()) if not cmd: cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % ( cluster.name, version) for i in range(0, exec_cnt): subprocess.call('%s %s' % (cmd_env, cmd), shell=True, executable='/bin/bash') try: os.remove(test_conf_file) except: pass cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return True
def test_version (version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1, root_path='tmp', broker_cnt=3, scenario='default'): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('LibrdkafkaTestCluster', root_path, debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster) zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() defconf = {'version': version} defconf.update(conf) print('conf: ', defconf) brokers = [] for n in range(0, broker_cnt): # Configure rack & replica selector if broker supports fetch-from-follower if version_as_number(version) >= 2.4: defconf.update({'conf': ['broker.rack=RACK${appid}', 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector']}) brokers.append(KafkaBrokerApp(cluster, defconf)) cmd_env = os.environ.copy() # Generate test config file security_protocol='PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('test.sql.command=sqlite3 rdktests\n').encode('ascii')) os.write(fd, 'broker.address.family=v4\n'.encode('ascii')) if version.startswith('0.9') or version.startswith('0.8'): os.write(fd, 'api.version.request=false\n'.encode('ascii')) os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, ('sasl.mechanisms=%s\n' % mech).encode('ascii')) if mech == 'PLAIN' or mech.find('SCRAM') != -1: print('# Writing SASL %s client config to %s' % (mech, test_conf_file)) security_protocol='SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u,p = up.split('=') os.write(fd, ('sasl.username=%s\n' % u).encode('ascii')) os.write(fd, ('sasl.password=%s\n' % p).encode('ascii')) break elif mech == 'OAUTHBEARER': security_protocol='SASL_PLAINTEXT' os.write(fd, ('enable.sasl.oauthbearer.unsecure.jwt=true\n')) os.write(fd, ('sasl.oauthbearer.config=%s\n' % \ 'scope=requiredScope principal=admin').encode('ascii')) else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key = ssl.create_cert('librdkafka') os.write(fd, ('ssl.ca.location=%s\n' % ssl.ca['pem']).encode('ascii')) os.write(fd, ('ssl.certificate.location=%s\n' % key['pub']['pem']).encode('ascii')) os.write(fd, ('ssl.key.location=%s\n' % key['priv']['pem']).encode('ascii')) os.write(fd, ('ssl.key.password=%s\n' % key['password']).encode('ascii')) for k, v in ssl.ca.iteritems(): cmd_env['RDK_SSL_ca_{}'.format(k)] = v # Set envs for all generated keys so tests can find them. for k, v in key.iteritems(): if type(v) is dict: for k2, v2 in v.iteritems(): # E.g. "RDK_SSL_priv_der=path/to/librdkafka-priv.der" cmd_env['RDK_SSL_{}_{}'.format(k, k2)] = v2 else: cmd_env['RDK_SSL_{}'.format(k)] = v # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, ('security.protocol=%s\n' % security_protocol).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster, instance path %s' % cluster.instance_path()) cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster %s did not go operational, see logs in %s/%s' % \ (cluster.name, cluster.root_path, cluster.instance)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env['KAFKA_PATH'] = brokers[0].conf.get('destdir') cmd_env['RDKAFKA_TEST_CONF'] = test_conf_file cmd_env['ZK_ADDRESS'] = zk_address cmd_env['BROKERS'] = bootstrap_servers cmd_env['TEST_KAFKA_VERSION'] = version cmd_env['TRIVUP_ROOT'] = cluster.instance_path() cmd_env['TEST_SCENARIO'] = scenario # Per broker env vars for b in [x for x in cluster.apps if isinstance(x, KafkaBrokerApp)]: cmd_env['BROKER_ADDRESS_%d' % b.appid] = b.conf['address'] # Add each broker pid as an env so they can be killed indivdidually. cmd_env['BROKER_PID_%d' % b.appid] = str(b.proc.pid) # JMX port, if available jmx_port = b.conf.get('jmx_port', None) if jmx_port is not None: cmd_env['BROKER_JMX_PORT_%d' % b.appid] = str(jmx_port) if not cmd: cmd_env['PS1'] = '[TRIVUP:%s@%s] \\u@\\h:\w$ ' % (cluster.name, version) cmd = 'bash --rcfile <(cat ~/.bashrc)' ret = True for i in range(0, exec_cnt): retcode = subprocess.call(cmd, env=cmd_env, shell=True, executable='/bin/bash') if retcode != 0: print('# Command failed with returncode %d: %s' % (retcode, cmd)) ret = False try: os.remove(test_conf_file) except: pass cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return ret
def __init__(self, **kwargs): """ Create and start a KafkaCluster. See default_conf above for parameters. """ super(KafkaCluster, self).__init__() conf = kwargs self.conf = deepcopy(self.default_conf) if conf is not None: self.conf.update(conf) self.version = self.conf.get('version') self.version_num = [int(x) for x in self.version.split('.')][:3] self.kraft = self.conf.get('kraft') # Create trivup Cluster self.cluster = Cluster(self.__class__.__name__, os.environ.get( 'TRIVUP_ROOT', 'tmp-%s' % self.__class__.__name__), debug=bool(self.conf.get('debug', False)), cleanup=bool(self.conf.get('cleanup', True))) self._client_conf = dict() self.env = dict() self.sasl_mechanism = self.conf.get('sasl_mechanism') # Add OIDC server app if bool(self.conf.get('oidc', False)): self.oidc = OauthbearerOIDCApp(self.cluster) if not self.sasl_mechanism: self.sasl_mechanism = 'OAUTHBEARER' elif self.sasl_mechanism.upper() != 'OAUTHBEARER': raise RuntimeError( f"OIDC requires sasl.mechanism OAUTHBEARER, not '{self.sasl_mechanism}'" ) # Generate SSL certs if enabled if bool(self.conf.get('with_ssl')): self.ssl = SslApp(self.cluster, self.conf) else: self.ssl = None # Map mechanism and SSL to security protocol self.security_protocol = { (True, True): 'SASL_SSL', (True, False): 'SASL_PLAINTEXT', (False, True): 'SSL', (False, False): 'PLAINTEXT' }[(bool(self.sasl_mechanism), bool(self.ssl is not None))] if not self.kraft: # Create single ZK for the cluster (don't start yet) self.zk = ZookeeperApp(self.cluster) else: self.zk = None # Allocate (but don't use) a dummy appid so that the brokers get # the same appid/nodeid for both KRaft and ZK modes. Allocator(self.cluster).next() # Broker configuration broker_cnt = int(self.conf.get('broker_cnt')) self.broker_conf = { 'replication_factor': min(3, broker_cnt), 'num_partitions': 4, 'version': self.version, 'sasl_mechanisms': self.sasl_mechanism, 'sasl_users': self.conf.get('sasl_users'), 'conf': self.conf.get('broker_conf', []), 'kafka_path': self.conf.get('kafka_path', None) } # Start Kerberos KDCs if GSSAPI (Kerberos) is configured if self.sasl_mechanism == 'GSSAPI': self._setup_kerberos() self.broker_conf['realm'] = self.broker_realm self.broker_conf['listener_host'] = 'localhost' # Create brokers (don't start yet) self.brokers = dict() for n in range(0, broker_cnt): bconf = copy.deepcopy(self.broker_conf) if self.version_num >= [2, 4, 0]: # Configure rack & replica selector if broker supports # fetch-from-follower bconf.update({ 'conf': [ 'broker.rack=RACK${appid}', 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector' ] }) # noqa: E501 broker = KafkaBrokerApp(self.cluster, bconf) self.brokers[broker.appid] = broker # Generate bootstrap servers list all_listeners = (','.join( self.cluster.get_all('advertised_listeners', '', KafkaBrokerApp))).split(',') self.bootstrap_servers = ','.join( [x for x in all_listeners if x.startswith(self.security_protocol)]) assert len(self.bootstrap_servers) >= broker_cnt, \ "{} < {} expected bootstrap servers".format( len(self.bootstrap_servers), broker_cnt) # Create SchemaRegistry if enabled if bool(self.conf.get('with_sr', False)): self.sr = SchemaRegistryApp( self.cluster, {'version': self.conf.get('cp_version')}) self.env['SR_URL'] = self.sr.get('url') # Create librdkafka client configuration self._setup_client_conf() # Deploy cluster self.cluster.deploy() # Start cluster self.start()
def __init__(self, **kwargs): """ Create and start a KafkaCluster. See default_conf above for parameters. """ super(KafkaCluster, self).__init__() conf = kwargs self.conf = deepcopy(self.default_conf) if conf is not None: self.conf.update(conf) self.version = self.conf.get('version') # Create trivup Cluster self.cluster = Cluster(self.__class__.__name__, os.environ.get( 'TRIVUP_ROOT', 'tmp-%s' % self.__class__.__name__), debug=bool(self.conf.get('debug', False))) self._client_conf = dict() self.env = dict() self.sasl_mechanism = self.conf.get('sasl_mechanism') # Generate SSL certs if enabled if bool(self.conf.get('with_ssl')): self.ssl = SslApp(self.cluster, self.conf) else: self.ssl = None # Map mechanism and SSL to security protocol self.security_protocol = { (True, True): 'SASL_SSL', (True, False): 'SASL_PLAINTEXT', (False, True): 'SSL', (False, False): 'PLAINTEXT' }[(bool(self.sasl_mechanism), bool(self.ssl is not None))] # Create single ZK for the cluster (don't start yet') self.zk = ZookeeperApp(self.cluster) # Broker configuration broker_cnt = int(self.conf.get('broker_cnt')) self.broker_conf = { 'replication_factor': min(3, broker_cnt), 'num_partitions': 4, 'version': self.version, 'sasl_mechanisms': self.sasl_mechanism, 'sasl_users': self.conf.get('sasl_users'), 'conf': self.conf.get('broker_conf', []) } # Start Kerberos KDCs if GSSAPI (Kerberos) is configured if self.sasl_mechanism == 'GSSAPI': self._setup_kerberos() self.broker_conf['realm'] = self.broker_realm # Create brokers (don't start yet) self.brokers = dict() for n in range(0, broker_cnt): broker = KafkaBrokerApp(self.cluster, self.broker_conf) self.brokers[broker.appid] = broker # Generate bootstrap servers list all_listeners = (','.join( self.cluster.get_all('advertised_listeners', '', KafkaBrokerApp))).split(',') self.bootstrap_servers = ','.join( [x for x in all_listeners if x.startswith(self.security_protocol)]) assert len(self.bootstrap_servers) >= broker_cnt, \ "{} < {} expected bootstrap servers".format( len(self.bootstrap_servers), broker_cnt) # Create SchemaRegistry if enabled if bool(self.conf.get('with_sr', False)): self.sr = SchemaRegistryApp( self.cluster, {'version': self.conf.get('cp_version')}) self.env['SR_URL'] = self.sr.get('url') # Create librdkafka client configuration self._setup_client_conf() # Deploy cluster self.cluster.deploy() # Start cluster self.start()
from trivup.apps.KafkaBrokerApp import KafkaBrokerApp import subprocess if __name__ == '__main__': cluster = Cluster('TestCluster', 'tmp') # One ZK zk1 = ZookeeperApp( cluster, bin_path='/home/maglun/src/kafka/bin/zookeeper-server-start.sh' ) # noqa: E501 # Two brokers conf = {'replication_factor': 3, 'num_partitions': 4} broker1 = KafkaBrokerApp(cluster, conf) broker2 = KafkaBrokerApp(cluster, conf) broker3 = KafkaBrokerApp(cluster, conf) bootstrap_servers = ','.join(cluster.get_all('address', '', KafkaBrokerApp)) print('# Deploying cluster') cluster.deploy() print('# Starting cluster') cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): print('# Cluster did not go operational: letting you troubleshoot ' +
def test_kerberos_cross_realm(): """ Test Kerberos cross-realm trusts """ topic = "test" cluster = Cluster('KafkaCluster', root_path=os.environ.get('TRIVUP_ROOT', 'tmp'), debug=True) ZookeeperApp(cluster) # # Create KDCs for each realm. # First realm will be the default / broker realm. # realm_cnt = 2 realms = ["REALM{}.COM".format(x + 1) for x in range(0, realm_cnt)] # Pre-Allocate ports for the KDCs so they can reference eachother # in the krb5.conf configuration. kdc_ports = {x: TcpPortAllocator(cluster).next("dummy") for x in realms} # Set up realm=kdc:port cross-realm mappings cross_realms = ",".join(["{}={}:{}".format(x, cluster.get_node().name, kdc_ports[x]) for x in realms]) kdcs = dict() for realm in realms: kdc = KerberosKdcApp(cluster, realm, conf={'port': kdc_ports[realm], 'cross_realms': cross_realms, 'renew_lifetime': '30', 'ticket_lifetime': '120'}) kdc.start() kdcs[realm] = kdc broker_realm = realms[0] client_realm = realms[1] broker_kdc = kdcs[broker_realm] client_kdc = kdcs[client_realm] # Create broker_cnt brokers broker_cnt = 4 brokerconf = {'replication_factor': min(3, int(broker_cnt)), 'num_partitions': broker_cnt * 2, 'version': '2.2.0', 'sasl_mechanisms': 'GSSAPI', 'realm': broker_realm, 'conf': ['connections.max.idle.ms=60000']} brokers = dict() for n in range(0, broker_cnt): broker = KafkaBrokerApp(cluster, brokerconf) brokers[broker.appid] = broker # Get bootstrap server list security_protocol = 'SASL_PLAINTEXT' all_listeners = (','.join(cluster.get_all( 'listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) assert len(bootstrap_servers) > 0, "no bootstrap servers" print("## Deploying cluster") cluster.deploy() print("## Starting cluster") cluster.start(timeout=30) # Add cross-realm TGTs for realm in realms: for crealm in [x for x in realms if x != realm]: kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), crealm, realm)).wait() kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), realm, crealm)).wait() # Create client base configuration client_config = { 'bootstrap.servers': bootstrap_servers, 'enable.sparse.connections': False, 'broker.address.family': 'v4', 'sasl.mechanisms': 'GSSAPI', 'security.protocol': security_protocol, 'debug': 'broker,security' } os.environ['KRB5CCNAME'] = client_kdc.mkpath('krb5cc') os.environ['KRB5_CONFIG'] = client_kdc.conf['krb5_conf'] os.environ['KRB5_KDC_PROFILE'] = client_kdc.conf['kdc_conf'] principal,keytab = client_kdc.add_principal("admin") client_config['sasl.kerberos.keytab'] = keytab client_config['sasl.kerberos.principal'] = principal.split('@')[0] client_config['sasl.kerberos.min.time.before.relogin'] = 120*1000*3 print(client_config) print("bootstraps: {}".format(client_config['bootstrap.servers'])) p = Producer(client_config) time.sleep(10) for n in range(1, 100): p.produce(topic, "msg #{}".format(n)) p.poll(1.0) p.flush(1.0) print("####### {} messages remaining\n\n\n".format(len(p))) start = time.time() end = start + (90*60) until = start + (12*60) while time.time() < end: now = time.time() if until < now: print("### Producing 2 messages") for n in range(1, 2): p.produce(topic, "msg #{}".format(n)) until = now + (12*60) p.poll(1.0) del p cluster.stop()
from trivup.apps.ZookeeperApp import ZookeeperApp from trivup.apps.KafkaBrokerApp import KafkaBrokerApp import subprocess import time if __name__ == '__main__': cluster = Cluster('TestCluster', 'tmp') # One ZK zk1 = ZookeeperApp(cluster, bin_path='/home/maglun/src/kafka/bin/zookeeper-server-start.sh') # Two brokers conf = {'replication_factor': 3, 'num_partitions': 4} broker1 = KafkaBrokerApp(cluster, conf, kafka_path='/home/maglun/src/kafka') broker2 = KafkaBrokerApp(cluster, conf, kafka_path='/home/maglun/src/kafka') broker3 = KafkaBrokerApp(cluster, conf, kafka_path='/home/maglun/src/kafka') bootstrap_servers = ','.join(cluster.get_all('address','',KafkaBrokerApp)) print('# Deploying cluster') cluster.deploy() print('# Starting cluster') cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): print('# Cluster did not go operational: letting you troubleshoot in shell') print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers)