def create_partition(topic, num_partitions):
"""
Creates partitions for a given kafka topic
:param topic: string
:param num_partitions: int
:return: None
"""
admin.NewPartitions(topic=topic, new_total_count=num_partitions)
def verify_admin():
""" Verify Admin API """
a = admin.AdminClient({'bootstrap.servers': bootstrap_servers})
our_topic = topic + '_admin_' + str(uuid.uuid4())
num_partitions = 2
topic_config = {"compression.type": "gzip"}
#
# First iteration: validate our_topic creation.
# Second iteration: create topic.
#
for validate in (True, False):
fs = a.create_topics([
admin.NewTopic(our_topic,
num_partitions=num_partitions,
config=topic_config,
replication_factor=1)
],
validate_only=validate,
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Find the topic in list_topics
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Increase the partition count
#
num_partitions += 3
fs = a.create_partitions(
[admin.NewPartitions(our_topic, new_total_count=num_partitions)],
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Verify with list_topics.
#
verify_topic_metadata(a, {our_topic: num_partitions})
def verify_config(expconfig, configs):
"""
Verify that the config key,values in expconfig are found
and matches the ConfigEntry in configs.
"""
for key, expvalue in expconfig.items():
entry = configs.get(key, None)
assert entry is not None, "Config {} not found in returned configs".format(
key)
assert entry.value == str(expvalue), \
"Config {} with value {} does not match expected value {}".format(key, entry, expvalue)
#
# Get current topic config
#
resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic)
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Now change the config.
#
topic_config["file.delete.delay.ms"] = 12345
topic_config["compression.type"] = "snappy"
for key, value in topic_config.items():
resource.set_config(key, value)
fs = a.alter_configs([resource])
fs[resource].result() # will raise exception on failure
#
# Read the config back again and verify.
#
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Delete the topic
#
fs = a.delete_topics([our_topic])
fs[our_topic].result() # will raise exception on failure
print("Topic {} marked for deletion".format(our_topic))
def verify_admin():
""" Verify Admin API """
a = admin.AdminClient({'bootstrap.servers': bootstrap_servers})
our_topic = topic + '_admin_' + str(uuid.uuid4())
num_partitions = 2
topic_config = {"compression.type": "gzip"}
#
# First iteration: validate our_topic creation.
# Second iteration: create topic.
#
for validate in (True, False):
fs = a.create_topics([
admin.NewTopic(our_topic,
num_partitions=num_partitions,
config=topic_config,
replication_factor=1)
],
validate_only=validate,
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Find the topic in list_topics
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Increase the partition count
#
num_partitions += 3
fs = a.create_partitions(
[admin.NewPartitions(our_topic, new_total_count=num_partitions)],
operation_timeout=10.0)
for topic2, f in fs.items():
f.result() # trigger exception if there was an error
#
# Verify with list_topics.
#
verify_topic_metadata(a, {our_topic: num_partitions})
#
# Verify with list_groups.
#
# Produce some messages
p = confluent_kafka.Producer({"bootstrap.servers": bootstrap_servers})
p.produce(our_topic, 'Hello Python!', headers=produce_headers)
p.produce(our_topic, key='Just a key and headers', headers=produce_headers)
def consume_messages(group_id):
# Consume messages
conf = {
'bootstrap.servers': bootstrap_servers,
'group.id': group_id,
'session.timeout.ms': 6000,
'enable.auto.commit': False,
'on_commit': print_commit_result,
'error_cb': error_cb,
'auto.offset.reset': 'earliest',
'enable.partition.eof': True
}
c = confluent_kafka.Consumer(conf)
c.subscribe([our_topic])
eof_reached = dict()
while True:
msg = c.poll()
if msg is None:
raise Exception(
'Got timeout from poll() without a timeout set: %s' % msg)
if msg.error():
if msg.error().code(
) == confluent_kafka.KafkaError._PARTITION_EOF:
print('Reached end of %s [%d] at offset %d' %
(msg.topic(), msg.partition(), msg.offset()))
eof_reached[(msg.topic(), msg.partition())] = True
if len(eof_reached) == len(c.assignment()):
print(
'EOF reached for all assigned partitions: exiting')
break
else:
print('Consumer error: %s: ignoring' % msg.error())
break
# Commit offset
c.commit(msg, asynchronous=False)
group1 = 'test-group-1'
group2 = 'test-group-2'
consume_messages(group1)
consume_messages(group2)
# list_groups without group argument
groups = set(group.id for group in a.list_groups(timeout=10))
assert group1 in groups, "Consumer group {} not found".format(group1)
assert group2 in groups, "Consumer group {} not found".format(group2)
# list_groups with group argument
groups = set(group.id for group in a.list_groups(group1))
assert group1 in groups, "Consumer group {} not found".format(group1)
groups = set(group.id for group in a.list_groups(group2))
assert group2 in groups, "Consumer group {} not found".format(group2)
def verify_config(expconfig, configs):
"""
Verify that the config key,values in expconfig are found
and matches the ConfigEntry in configs.
"""
for key, expvalue in expconfig.items():
entry = configs.get(key, None)
assert entry is not None, "Config {} not found in returned configs".format(
key)
assert entry.value == str(expvalue), \
"Config {} with value {} does not match expected value {}".format(key, entry, expvalue)
#
# Get current topic config
#
resource = admin.ConfigResource(admin.RESOURCE_TOPIC, our_topic)
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Now change the config.
#
topic_config["file.delete.delay.ms"] = 12345
topic_config["compression.type"] = "snappy"
for key, value in topic_config.items():
resource.set_config(key, value)
fs = a.alter_configs([resource])
fs[resource].result() # will raise exception on failure
#
# Read the config back again and verify.
#
fs = a.describe_configs([resource])
configs = fs[resource].result() # will raise exception on failure
# Verify config matches our expectations
verify_config(topic_config, configs)
#
# Delete the topic
#
fs = a.delete_topics([our_topic])
fs[our_topic].result() # will raise exception on failure
print("Topic {} marked for deletion".format(our_topic))