def __init__(self, version, conf={}, num_brokers=3, kafka_path=None,
debug=False):
"""
@brief Create, deploy and start a Kafka cluster using Kafka \p version
Supported \p conf keys:
* security.protocol - PLAINTEXT, SASL_PLAINTEXT, SSL_SASL
\p conf dict is passed to KafkaBrokerApp classes, etc.
"""
super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__, 'tmp', debug=debug)
self.brokers = list()
# One ZK (from Kafka repo)
ZookeeperApp(self, bin_path=kafka_path + '/bin/zookeeper-server-start.sh')
# Start Kerberos KDC if GSSAPI (Kerberos) is configured
if 'GSSAPI' in conf.get('sasl_mechanisms', []):
kdc = KerberosKdcApp(self, 'MYREALM')
# Kerberos needs to be started prior to Kafka so that principals
# and keytabs are available at the time of Kafka config generation.
kdc.start()
# Brokers
defconf = {'replication_factor': min(num_brokers, 3), 'num_partitions': 4, 'version': version,
'security.protocol': 'PLAINTEXT'}
defconf.update(conf)
self.conf = defconf
for n in range(0, num_brokers):
self.brokers.append(KafkaBrokerApp(self, defconf, kafka_path=kafka_path))
def __init__(self,
version,
conf={},
num_brokers=3,
debug=False,
scenario="default"):
"""
@brief Create, deploy and start a Kafka cluster using Kafka \\p version
Supported \\p conf keys:
* security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL
\\p conf dict is passed to KafkaBrokerApp classes, etc.
"""
super(LibrdkafkaTestCluster,
self).__init__(self.__class__.__name__,
os.environ.get('TRIVUP_ROOT', 'tmp'),
debug=debug)
# Read trivup config from scenario definition.
defconf = read_scenario_conf(scenario)
defconf.update(conf)
# Enable SSL if desired
if 'SSL' in conf.get('security.protocol', ''):
self.ssl = SslApp(self, defconf)
self.brokers = list()
# One ZK (from Kafka repo)
ZookeeperApp(self)
# Start Kerberos KDC if GSSAPI (Kerberos) is configured
if 'GSSAPI' in defconf.get('sasl_mechanisms', []):
kdc = KerberosKdcApp(self, 'MYREALM')
# Kerberos needs to be started prior to Kafka so that principals
# and keytabs are available at the time of Kafka config generation.
kdc.start()
# Brokers
defconf.update({
'replication_factor': min(num_brokers, 3),
'version': version,
'security.protocol': 'PLAINTEXT'
})
self.conf = defconf
for n in range(0, num_brokers):
# Configure rack & replica selector if broker supports
# fetch-from-follower
if version_as_list(version) >= [2, 4, 0]:
defconf.update({
'conf': [
'broker.rack=RACK${appid}',
'replica.selector.class=org.apache.kafka.common.replica.RackAwareReplicaSelector'
]
}) # noqa: E501
self.brokers.append(KafkaBrokerApp(self, defconf))
def _setup_kerberos(self):
""" Set up Kerberos KDCs """
# Create KDCs for each realm.
# First realm will be the default / broker realm.
#
realm_cnt = int(self.conf.get('realm_cnt', 1))
# No point in having more than two realms
assert realm_cnt > 0 and realm_cnt < 3
realms = ['REALM{}.TRIVUP'.format(x + 1) for x in range(0, realm_cnt)]
# Pre-Allocate ports for the KDCs so they can reference eachother
# in the krb5.conf configuration.
kdc_ports = {x: TcpPortAllocator(
self.cluster).next("dummy") for x in realms}
# Set up realm=kdc:port cross-realm mappings
cross_realms = ",".join(["{}={}:{}".format(
x, self.cluster.get_node().name, kdc_ports[x]) for x in realms])
kdcs = dict()
for realm in realms:
kdc = KerberosKdcApp(
self.cluster, realm,
conf={'port': kdc_ports[realm],
'cross_realms': cross_realms,
'renew_lifetime':
int(self.conf.get('krb_renew_lifetime')),
'ticket_lifetime':
int(self.conf.get('krb_ticket_lifetime'))})
# Kerberos needs to be started prior to Kafka so that principals
# and keytabs are available at the time of Kafka config generation.
kdc.start()
kdcs[realm] = kdc
self.broker_realm = realms[0]
self.client_realm = realms[-1]
self.broker_kdc = kdcs[self.broker_realm]
self.client_kdc = kdcs[self.client_realm]
# Add cross-realm TGTs
if realm_cnt > 1:
KerberosKdcApp.add_cross_realm_tgts(kdcs)
# Add client envs and configuration
self.env['KRB5CCNAME'] = self.client_kdc.mkpath('krb5cc')
self.env['KRB5_CONFIG'] = self.client_kdc.conf['krb5_conf']
self.env['KRB5_KDC_PROFILE'] = self.client_kdc.conf['kdc_conf']
principal, keytab = self.client_kdc.add_principal('admin')
self._client_conf['sasl.kerberos.keytab'] = keytab
self._client_conf['sasl.kerberos.principal'] = principal.split('@')[0]
# Refresh ticket 60s before renew timeout.
self._client_conf['sasl.kerberos.min.time.before.relogin'] = \
max(1, int(self.conf.get('krb_renew_lifetime')) - 60) * 1000
def __init__(self,
version,
conf={},
num_brokers=3,
kafka_path=None,
debug=False):
"""
@brief Create, deploy and start a Kafka cluster using Kafka \p version
Supported \p conf keys:
* security.protocol - PLAINTEXT, SASL_PLAINTEXT, SASL_SSL
\p conf dict is passed to KafkaBrokerApp classes, etc.
"""
super(LibrdkafkaTestCluster, self).__init__(self.__class__.__name__,
'tmp',
debug=debug)
# Enable SSL if desired
if 'SSL' in conf.get('security.protocol', ''):
self.ssl = SslApp(self, conf)
self.brokers = list()
# One ZK (from Kafka repo)
ZookeeperApp(self,
bin_path=kafka_path + '/bin/zookeeper-server-start.sh')
# Start Kerberos KDC if GSSAPI (Kerberos) is configured
if 'GSSAPI' in conf.get('sasl_mechanisms', []):
kdc = KerberosKdcApp(self, 'MYREALM')
# Kerberos needs to be started prior to Kafka so that principals
# and keytabs are available at the time of Kafka config generation.
kdc.start()
# Brokers
defconf = {
'replication_factor': min(num_brokers, 3),
'num_partitions': 4,
'version': version,
'security.protocol': 'PLAINTEXT'
}
defconf.update(conf)
self.conf = defconf
for n in range(0, num_brokers):
self.brokers.append(
KafkaBrokerApp(self, defconf, kafka_path=kafka_path))
def test_kerberos_cross_realm():
""" Test Kerberos cross-realm trusts """
topic = "test"
cluster = Cluster('KafkaCluster',
root_path=os.environ.get('TRIVUP_ROOT', 'tmp'),
debug=True)
ZookeeperApp(cluster)
#
# Create KDCs for each realm.
# First realm will be the default / broker realm.
#
realm_cnt = 2
realms = ["REALM{}.COM".format(x + 1) for x in range(0, realm_cnt)]
# Pre-Allocate ports for the KDCs so they can reference eachother
# in the krb5.conf configuration.
kdc_ports = {x: TcpPortAllocator(cluster).next("dummy") for x in realms}
# Set up realm=kdc:port cross-realm mappings
cross_realms = ",".join(["{}={}:{}".format(x, cluster.get_node().name, kdc_ports[x]) for x in realms])
kdcs = dict()
for realm in realms:
kdc = KerberosKdcApp(cluster, realm,
conf={'port': kdc_ports[realm],
'cross_realms': cross_realms,
'renew_lifetime': '30',
'ticket_lifetime': '120'})
kdc.start()
kdcs[realm] = kdc
broker_realm = realms[0]
client_realm = realms[1]
broker_kdc = kdcs[broker_realm]
client_kdc = kdcs[client_realm]
# Create broker_cnt brokers
broker_cnt = 4
brokerconf = {'replication_factor': min(3, int(broker_cnt)),
'num_partitions': broker_cnt * 2,
'version': '2.2.0',
'sasl_mechanisms': 'GSSAPI',
'realm': broker_realm,
'conf': ['connections.max.idle.ms=60000']}
brokers = dict()
for n in range(0, broker_cnt):
broker = KafkaBrokerApp(cluster, brokerconf)
brokers[broker.appid] = broker
# Get bootstrap server list
security_protocol = 'SASL_PLAINTEXT'
all_listeners = (','.join(cluster.get_all(
'listeners', '', KafkaBrokerApp))).split(',')
bootstrap_servers = ','.join([x for x in all_listeners
if x.startswith(security_protocol)])
assert len(bootstrap_servers) > 0, "no bootstrap servers"
print("## Deploying cluster")
cluster.deploy()
print("## Starting cluster")
cluster.start(timeout=30)
# Add cross-realm TGTs
for realm in realms:
for crealm in [x for x in realms if x != realm]:
kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), crealm, realm)).wait()
kdcs[realm].execute('kadmin.local -d "{}" -q "addprinc -requires_preauth -pw password krbtgt/{}@{}"'.format(kdcs[realm].conf.get('dbpath'), realm, crealm)).wait()
# Create client base configuration
client_config = {
'bootstrap.servers': bootstrap_servers,
'enable.sparse.connections': False,
'broker.address.family': 'v4',
'sasl.mechanisms': 'GSSAPI',
'security.protocol': security_protocol,
'debug': 'broker,security'
}
os.environ['KRB5CCNAME'] = client_kdc.mkpath('krb5cc')
os.environ['KRB5_CONFIG'] = client_kdc.conf['krb5_conf']
os.environ['KRB5_KDC_PROFILE'] = client_kdc.conf['kdc_conf']
principal,keytab = client_kdc.add_principal("admin")
client_config['sasl.kerberos.keytab'] = keytab
client_config['sasl.kerberos.principal'] = principal.split('@')[0]
client_config['sasl.kerberos.min.time.before.relogin'] = 120*1000*3
print(client_config)
print("bootstraps: {}".format(client_config['bootstrap.servers']))
p = Producer(client_config)
time.sleep(10)
for n in range(1, 100):
p.produce(topic, "msg #{}".format(n))
p.poll(1.0)
p.flush(1.0)
print("####### {} messages remaining\n\n\n".format(len(p)))
start = time.time()
end = start + (90*60)
until = start + (12*60)
while time.time() < end:
now = time.time()
if until < now:
print("### Producing 2 messages")
for n in range(1, 2):
p.produce(topic, "msg #{}".format(n))
until = now + (12*60)
p.poll(1.0)
del p
cluster.stop()