def create_partiton(self): from kafka.admin import KafkaAdminClient, NewPartitions try: adminClient = KafkaAdminClient(bootstrap_servers=self.cur_broker) partitions = dict() pn = self.newpn_input.get() if pn and pn != "": pn = int(pn) newp = NewPartitions(total_count=pn) print(self.cur_topic) partitions[self.cur_topic] = newp adminClient.create_partitions(partitions) except Exception as e: logger.error("{}".format(e))
class WarriorKafkaClient(): """ This class contains all kafka admin client related methods """ def __init__(self, **configs): """ create a kafka client """ print_info("Creating kafka client") try: self.kafka_client = KafkaAdminClient(**configs) except KafkaError as exc: print_error("kafka client - Exception during connecting to broker- {}".format(exc)) def create_topics(self, topic_sets, **kwargs): """ create topics for the producer or consumer to use Arguments: topic_sets(list) : list of ['topic_name', 'num_partitions', 'replication_factor'] lists example : ['topic1',1,1] timeout(int): time in milliseconds Returns: result(bool) : False if exception occures, True otherwise None. """ timeout = kwargs.get("timeout", None) validate = kwargs.get("validate", False) new_topics = [NewTopic(name=tup[0], num_partitions=tup[1],\ replication_factor=tup[2]) for tup in topic_sets] print_info("creating topics") try: self.kafka_client.create_topics(new_topics=new_topics, timeout_ms=timeout, validate_only=validate) result = True except KafkaError as exc: print_error("Exception during creating topics - {}".format(exc)) result = False return result def delete_topics(self, topics, timeout=None): """ Delete topics Arguments: topics(list): list of topic names timeout(int): timeout in milliseconds Returns: result(bool) : False if exception occures, True otherwise """ print_info("deleting topics {}".format(topics)) try: self.kafka_client.delete_topics(topics=topics, timeout_ms=timeout) result = True except KafkaError as exc: print_error("Exception during deleting topics - {}".format(exc)) result = False return result def create_partitions_in_topic(self, partitions, **kwargs): """ create partitions in topic Arguments: partitions(list) : list of ['topic_name','num_partitions'] lists example : [['topic1',4], ['topic2',5]] timeout(int): timeout in milliseconds Returns: result(bool) : False if exception occures, True otherwise """ timeout = kwargs.get("timeout", None) validate = kwargs.get("validate", False) topic_partitions = {tup[0]:NewPartitions(total_count=tup[1]) for tup in partitions} print_info("creating partitions in topic") try: self.kafka_client.create_partitions(topic_partitions=topic_partitions, timeout_ms=timeout, validate_only=validate) result = True except KafkaError as exc: print_error("Exception during creating partitions - {}".format(exc)) result = False return result
import time kafka_broker = "localhost:9092" topic = "test" num_partitions = 4 admin_client = KafkaAdminClient(bootstrap_servers=kafka_broker, client_id='admin-client') try: topic_list = [] topic_list.append( NewTopic(name=topic, num_partitions=num_partitions, replication_factor=1)) admin_client.create_topics(new_topics=topic_list, validate_only=False) except Exception as e: # if topic already exists, set partitions try: admin_client.create_partitions({topic: NewPartitions(num_partitions)}) except: pass producer = KafkaProducer(bootstrap_servers=[kafka_broker]) i = 0 while True: i += 1 partition = i % num_partitions producer.send(topic, partition=partition, value=str(i).encode('utf-8')) print(f"Sent {i} on partition {partition}") time.sleep(1)
client.poll(future=update) metadata = client.cluster with open('configuration/kafka_topics.yaml', 'r') as topicsdata: topics_params = yaml.safe_load(topicsdata) topicsdata.close() for topic in topics_params['topics']: if topic['name'] not in metadata.topics(): print('Topic ' + topic['name'] + ' will be created') topic_list.append( NewTopic(name=topic['name'], num_partitions=topic['partitions'], replication_factor=topic['replication-factor'])) if topic['name'] in metadata.topics(): if len(metadata.partitions_for_topic( topic['name'])) < topic['partitions']: print('New partitions for topic: ' + topic['name'] + ' will be created') num_partitions = NewPartitions(topic['partitions']) new_partitions = {topic['name']: num_partitions} admin_client.create_partitions(new_partitions) print('Additional partitions created for topic: ' + topic['name']) if topic_list: admin_client.create_topics(new_topics=topic_list, validate_only=False) print('Topics successfully created') else: print('No topics to create')