def create_partiton(self):
from kafka.admin import KafkaAdminClient, NewPartitions
try:
adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker)
partitions = dict()
pn = self.newpn_input.get()
if pn and pn != "":
pn = int(pn)
newp = NewPartitions(total_count=pn)
print(self.cur_topic)
partitions[self.cur_topic] = newp
adminClient.create_partitions(partitions)
except Exception as e:
logger.error("{}".format(e))
class WarriorKafkaClient():
"""
This class contains all kafka admin client related
methods
"""
def __init__(self, **configs):
"""
create a kafka client
"""
print_info("Creating kafka client")
try:
self.kafka_client = KafkaAdminClient(**configs)
except KafkaError as exc:
print_error("kafka client - Exception during connecting to broker- {}".format(exc))
def create_topics(self, topic_sets, **kwargs):
"""
create topics for the producer or consumer to use
Arguments:
topic_sets(list) : list of
['topic_name', 'num_partitions', 'replication_factor'] lists
example : ['topic1',1,1]
timeout(int): time in milliseconds
Returns:
result(bool) : False if exception occures, True otherwise
None.
"""
timeout = kwargs.get("timeout", None)
validate = kwargs.get("validate", False)
new_topics = [NewTopic(name=tup[0], num_partitions=tup[1],\
replication_factor=tup[2]) for tup in topic_sets]
print_info("creating topics")
try:
self.kafka_client.create_topics(new_topics=new_topics,
timeout_ms=timeout,
validate_only=validate)
result = True
except KafkaError as exc:
print_error("Exception during creating topics - {}".format(exc))
result = False
return result
def delete_topics(self, topics, timeout=None):
"""
Delete topics
Arguments:
topics(list): list of topic names
timeout(int): timeout in milliseconds
Returns:
result(bool) : False if exception occures, True otherwise
"""
print_info("deleting topics {}".format(topics))
try:
self.kafka_client.delete_topics(topics=topics,
timeout_ms=timeout)
result = True
except KafkaError as exc:
print_error("Exception during deleting topics - {}".format(exc))
result = False
return result
def create_partitions_in_topic(self, partitions, **kwargs):
"""
create partitions in topic
Arguments:
partitions(list) : list of ['topic_name','num_partitions'] lists
example : [['topic1',4], ['topic2',5]]
timeout(int): timeout in milliseconds
Returns:
result(bool) : False if exception occures, True otherwise
"""
timeout = kwargs.get("timeout", None)
validate = kwargs.get("validate", False)
topic_partitions = {tup[0]:NewPartitions(total_count=tup[1]) for tup in partitions}
print_info("creating partitions in topic")
try:
self.kafka_client.create_partitions(topic_partitions=topic_partitions,
timeout_ms=timeout,
validate_only=validate)
result = True
except KafkaError as exc:
print_error("Exception during creating partitions - {}".format(exc))
result = False
return result
import time
kafka_broker = "localhost:9092"
topic = "test"
num_partitions = 4
admin_client = KafkaAdminClient(bootstrap_servers=kafka_broker,
client_id='admin-client')
try:
topic_list = []
topic_list.append(
NewTopic(name=topic,
num_partitions=num_partitions,
replication_factor=1))
admin_client.create_topics(new_topics=topic_list, validate_only=False)
except Exception as e:
# if topic already exists, set partitions
try:
admin_client.create_partitions({topic: NewPartitions(num_partitions)})
except:
pass
producer = KafkaProducer(bootstrap_servers=[kafka_broker])
i = 0
while True:
i += 1
partition = i % num_partitions
producer.send(topic, partition=partition, value=str(i).encode('utf-8'))
print(f"Sent {i} on partition {partition}")
time.sleep(1)
client.poll(future=update)
metadata = client.cluster
with open('configuration/kafka_topics.yaml', 'r') as topicsdata:
topics_params = yaml.safe_load(topicsdata)
topicsdata.close()
for topic in topics_params['topics']:
if topic['name'] not in metadata.topics():
print('Topic ' + topic['name'] + ' will be created')
topic_list.append(
NewTopic(name=topic['name'],
num_partitions=topic['partitions'],
replication_factor=topic['replication-factor']))
if topic['name'] in metadata.topics():
if len(metadata.partitions_for_topic(
topic['name'])) < topic['partitions']:
print('New partitions for topic: ' + topic['name'] +
' will be created')
num_partitions = NewPartitions(topic['partitions'])
new_partitions = {topic['name']: num_partitions}
admin_client.create_partitions(new_partitions)
print('Additional partitions created for topic: ' + topic['name'])
if topic_list:
admin_client.create_topics(new_topics=topic_list, validate_only=False)
print('Topics successfully created')
else:
print('No topics to create')