def test_version(version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1, root_path='tmp', broker_cnt=3): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('LibrdkafkaTestCluster', root_path, debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster) zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() defconf = { 'replication_factor': min(broker_cnt, 3), 'num_partitions': 4, 'version': version } defconf.update(conf) print('conf: ', defconf) brokers = [] for n in range(0, broker_cnt): brokers.append(KafkaBrokerApp(cluster, defconf)) # Generate test config file security_protocol = 'PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('test.sql.command=sqlite3 rdktests\n').encode('ascii')) os.write(fd, 'broker.address.family=v4\n'.encode('ascii')) if version != 'trunk': os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) else: os.write(fd, 'api.version.request=true\n'.encode('ascii')) # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, ('sasl.mechanisms=%s\n' % mech).encode('ascii')) if mech == 'PLAIN' or mech.find('SCRAM') != -1: print('# Writing SASL %s client config to %s' % (mech, test_conf_file)) security_protocol = 'SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u, p = up.split('=') os.write(fd, ('sasl.username=%s\n' % u).encode('ascii')) os.write(fd, ('sasl.password=%s\n' % p).encode('ascii')) break else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key, req, pem = ssl.create_key('librdkafka') os.write(fd, ('ssl.ca.location=%s\n' % ssl.ca_cert).encode('ascii')) os.write(fd, ('ssl.certificate.location=%s\n' % pem).encode('ascii')) os.write(fd, ('ssl.key.location=%s\n' % key).encode('ascii')) os.write(fd, ('ssl.key.password=%s\n' % ssl.conf.get('ssl_key_pass')).encode('ascii')) # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join( [x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, ('security.protocol=%s\n' % security_protocol).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster, instance path %s' % cluster.instance_path()) cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster %s did not go operational, see logs in %s/%s' % \ (cluster.name, cluster.root_path, cluster.instance)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env = 'export KAFKA_PATH="%s" RDKAFKA_TEST_CONF="%s" ZK_ADDRESS="%s" BROKERS="%s" TEST_KAFKA_VERSION="%s" TRIVUP_ROOT="%s"; ' % \ (brokers[0].conf.get('destdir'), test_conf_file, zk_address, bootstrap_servers, version, cluster.instance_path()) if not cmd: cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % ( cluster.name, version) for i in range(0, exec_cnt): subprocess.call('%s %s' % (cmd_env, cmd), shell=True, executable='/bin/bash') try: os.remove(test_conf_file) except: pass cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return True
def test_version (version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('librdkafkaInteractiveBrokerVersionTests', 'tmp', debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster, bin_path=kafka_path + '/bin/zookeeper-server-start.sh') zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() # Three brokers defconf = {'replication_factor': 3, 'num_partitions': 4, 'version': version} defconf.update(conf) print('conf: ', defconf) broker1 = KafkaBrokerApp(cluster, defconf, kafka_path=kafka_path) broker2 = KafkaBrokerApp(cluster, defconf, kafka_path=kafka_path) broker3 = KafkaBrokerApp(cluster, defconf, kafka_path=kafka_path) # Generate test config file security_protocol='PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) if version != 'trunk': os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) else: os.write(fd, 'api.version.request=true\n') # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, 'sasl.mechanisms=%s\n' % mech) if mech == 'PLAIN': print('# Writing SASL PLAIN client config to %s' % test_conf_file) security_protocol='SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u,p = up.split('=') os.write(fd, 'sasl.username=%s\n' % u) os.write(fd, 'sasl.password=%s\n' % p) break else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key, req, pem = ssl.create_key('librdkafka') os.write(fd, 'ssl.ca.location=%s\n' % ssl.ca_cert) os.write(fd, 'ssl.certificate.location=%s\n' % pem) os.write(fd, 'ssl.key.location=%s\n' % key) os.write(fd, 'ssl.key.password=%s\n' % ssl.conf.get('ssl_key_pass')) # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, 'security.protocol=%s\n' % security_protocol) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster') cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster did not go operational, see logs in %s' % \ (cluster.root_path)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env = 'export KAFKA_PATH=%s RDKAFKA_TEST_CONF=%s ZK_ADDRESS=%s BROKERS=%s TEST_KAFKA_VERSION=%s;' % \ (broker1.conf.get('destdir'), test_conf_file, zk_address, bootstrap_servers, version) if not cmd: cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % (cluster.name, version) for i in range(0, exec_cnt): subprocess.call('%s %s' % (cmd_env, cmd), shell=True, executable='/bin/bash') os.remove(test_conf_file) cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return True
def test_version (version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1, root_path='tmp', broker_cnt=3, scenario='default'): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('LibrdkafkaTestCluster', root_path, debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster) zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() defconf = {'version': version} defconf.update(conf) print('conf: ', defconf) brokers = [] for n in range(0, broker_cnt): # Configure rack & replica selector if broker supports fetch-from-follower if version_as_number(version) >= 2.4: defconf.update({'conf': ['broker.rack=RACK${appid}', 'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector']}) brokers.append(KafkaBrokerApp(cluster, defconf)) cmd_env = os.environ.copy() # Generate test config file security_protocol='PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('test.sql.command=sqlite3 rdktests\n').encode('ascii')) os.write(fd, 'broker.address.family=v4\n'.encode('ascii')) if version.startswith('0.9') or version.startswith('0.8'): os.write(fd, 'api.version.request=false\n'.encode('ascii')) os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, ('sasl.mechanisms=%s\n' % mech).encode('ascii')) if mech == 'PLAIN' or mech.find('SCRAM') != -1: print('# Writing SASL %s client config to %s' % (mech, test_conf_file)) security_protocol='SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u,p = up.split('=') os.write(fd, ('sasl.username=%s\n' % u).encode('ascii')) os.write(fd, ('sasl.password=%s\n' % p).encode('ascii')) break elif mech == 'OAUTHBEARER': security_protocol='SASL_PLAINTEXT' os.write(fd, ('enable.sasl.oauthbearer.unsecure.jwt=true\n')) os.write(fd, ('sasl.oauthbearer.config=%s\n' % \ 'scope=requiredScope principal=admin').encode('ascii')) else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key = ssl.create_cert('librdkafka') os.write(fd, ('ssl.ca.location=%s\n' % ssl.ca['pem']).encode('ascii')) os.write(fd, ('ssl.certificate.location=%s\n' % key['pub']['pem']).encode('ascii')) os.write(fd, ('ssl.key.location=%s\n' % key['priv']['pem']).encode('ascii')) os.write(fd, ('ssl.key.password=%s\n' % key['password']).encode('ascii')) for k, v in ssl.ca.iteritems(): cmd_env['RDK_SSL_ca_{}'.format(k)] = v # Set envs for all generated keys so tests can find them. for k, v in key.iteritems(): if type(v) is dict: for k2, v2 in v.iteritems(): # E.g. "RDK_SSL_priv_der=path/to/librdkafka-priv.der" cmd_env['RDK_SSL_{}_{}'.format(k, k2)] = v2 else: cmd_env['RDK_SSL_{}'.format(k)] = v # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, ('security.protocol=%s\n' % security_protocol).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster, instance path %s' % cluster.instance_path()) cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster %s did not go operational, see logs in %s/%s' % \ (cluster.name, cluster.root_path, cluster.instance)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env['KAFKA_PATH'] = brokers[0].conf.get('destdir') cmd_env['RDKAFKA_TEST_CONF'] = test_conf_file cmd_env['ZK_ADDRESS'] = zk_address cmd_env['BROKERS'] = bootstrap_servers cmd_env['TEST_KAFKA_VERSION'] = version cmd_env['TRIVUP_ROOT'] = cluster.instance_path() cmd_env['TEST_SCENARIO'] = scenario # Per broker env vars for b in [x for x in cluster.apps if isinstance(x, KafkaBrokerApp)]: cmd_env['BROKER_ADDRESS_%d' % b.appid] = b.conf['address'] # Add each broker pid as an env so they can be killed indivdidually. cmd_env['BROKER_PID_%d' % b.appid] = str(b.proc.pid) # JMX port, if available jmx_port = b.conf.get('jmx_port', None) if jmx_port is not None: cmd_env['BROKER_JMX_PORT_%d' % b.appid] = str(jmx_port) if not cmd: cmd_env['PS1'] = '[TRIVUP:%s@%s] \\u@\\h:\w$ ' % (cluster.name, version) cmd = 'bash --rcfile <(cat ~/.bashrc)' ret = True for i in range(0, exec_cnt): retcode = subprocess.call(cmd, env=cmd_env, shell=True, executable='/bin/bash') if retcode != 0: print('# Command failed with returncode %d: %s' % (retcode, cmd)) ret = False try: os.remove(test_conf_file) except: pass cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return ret
def test_version (version, cmd=None, deploy=True, conf={}, debug=False, exec_cnt=1, root_path='tmp', broker_cnt=3): """ @brief Create, deploy and start a Kafka cluster using Kafka \p version Then run librdkafka's regression tests. """ print('## Test version %s' % version) cluster = Cluster('LibrdkafkaTestCluster', root_path, debug=debug) # Enable SSL if desired if 'SSL' in conf.get('security.protocol', ''): cluster.ssl = SslApp(cluster, conf) # One ZK (from Kafka repo) zk1 = ZookeeperApp(cluster) zk_address = zk1.get('address') # Start Kerberos KDC if GSSAPI is configured if 'GSSAPI' in args.conf.get('sasl_mechanisms', []): KerberosKdcApp(cluster, 'MYREALM').start() defconf = {'replication_factor': min(int(conf.get('replication_factor', broker_cnt)), 3), 'num_partitions': 4, 'version': version} defconf.update(conf) print('conf: ', defconf) brokers = [] for n in range(0, broker_cnt): brokers.append(KafkaBrokerApp(cluster, defconf)) # Generate test config file security_protocol='PLAINTEXT' fd, test_conf_file = tempfile.mkstemp(prefix='test_conf', text=True) os.write(fd, ('test.sql.command=sqlite3 rdktests\n').encode('ascii')) os.write(fd, 'broker.address.family=v4\n'.encode('ascii')) if version != 'trunk': os.write(fd, ('broker.version.fallback=%s\n' % version).encode('ascii')) else: os.write(fd, 'api.version.request=true\n'.encode('ascii')) # SASL (only one mechanism supported) mech = defconf.get('sasl_mechanisms', '').split(',')[0] if mech != '': os.write(fd, ('sasl.mechanisms=%s\n' % mech).encode('ascii')) if mech == 'PLAIN' or mech.find('SCRAM') != -1: print('# Writing SASL %s client config to %s' % (mech, test_conf_file)) security_protocol='SASL_PLAINTEXT' # Use first user as SASL user/pass for up in defconf.get('sasl_users', '').split(','): u,p = up.split('=') os.write(fd, ('sasl.username=%s\n' % u).encode('ascii')) os.write(fd, ('sasl.password=%s\n' % p).encode('ascii')) break elif mech == 'OAUTHBEARER': security_protocol='SASL_PLAINTEXT' os.write(fd, ('sasl.oauthbearer.config=%s\n' % \ 'scope=requiredScope principal=admin').encode('ascii')) else: print('# FIXME: SASL %s client config not written to %s' % (mech, test_conf_file)) # SSL support ssl = getattr(cluster, 'ssl', None) if ssl is not None: if 'SASL' in security_protocol: security_protocol = 'SASL_SSL' else: security_protocol = 'SSL' key, req, pem = ssl.create_key('librdkafka') os.write(fd, ('ssl.ca.location=%s\n' % ssl.ca_cert).encode('ascii')) os.write(fd, ('ssl.certificate.location=%s\n' % pem).encode('ascii')) os.write(fd, ('ssl.key.location=%s\n' % key).encode('ascii')) os.write(fd, ('ssl.key.password=%s\n' % ssl.conf.get('ssl_key_pass')).encode('ascii')) # Define bootstrap brokers based on selected security protocol print('# Using client security.protocol=%s' % security_protocol) all_listeners = (','.join(cluster.get_all('listeners', '', KafkaBrokerApp))).split(',') bootstrap_servers = ','.join([x for x in all_listeners if x.startswith(security_protocol)]) os.write(fd, ('bootstrap.servers=%s\n' % bootstrap_servers).encode('ascii')) os.write(fd, ('security.protocol=%s\n' % security_protocol).encode('ascii')) os.close(fd) if deploy: print('# Deploying cluster') cluster.deploy() else: print('# Not deploying') print('# Starting cluster, instance path %s' % cluster.instance_path()) cluster.start() print('# Waiting for brokers to come up') if not cluster.wait_operational(30): cluster.stop(force=True) raise Exception('Cluster %s did not go operational, see logs in %s/%s' % \ (cluster.name, cluster.root_path, cluster.instance)) print('# Connect to cluster with bootstrap.servers %s' % bootstrap_servers) cmd_env = os.environ.copy() cmd_env['KAFKA_PATH'] = brokers[0].conf.get('destdir') cmd_env['RDKAFKA_TEST_CONF'] = test_conf_file cmd_env['ZK_ADDRESS'] = zk_address cmd_env['BROKERS'] = bootstrap_servers cmd_env['TEST_KAFKA_VERSION'] = version cmd_env['TRIVUP_ROOT'] = cluster.instance_path() # Add each broker pid as an env so they can be killed indivdidually. for b in [x for x in cluster.apps if isinstance(x, KafkaBrokerApp)]: cmd_env['BROKER_PID_%d' % b.appid] = str(b.proc.pid) if not cmd: cmd = 'bash --rcfile <(cat ~/.bashrc; echo \'PS1="[TRIVUP:%s@%s] \\u@\\h:\w$ "\')' % (cluster.name, version) ret = True for i in range(0, exec_cnt): retcode = subprocess.call(cmd, env=cmd_env, shell=True, executable='/bin/bash') if retcode != 0: print('# Command failed with returncode %d: %s' % (retcode, cmd)) ret = False try: os.remove(test_conf_file) except: pass cluster.stop(force=True) cluster.cleanup(keeptypes=['log']) return ret