def init_partitions(settings: Settings): client = KafkaAdminClient(bootstrap_servers=settings.kafka_server, ) try: client.create_partitions( topic_partitions={ settings.kafka_topic: NewPartitions(total_count=len(settings.schema_table.keys())) }) except Exception as e: logger.warning(f"init_partitions error:{e}")
def init_partitions(): client = KafkaAdminClient(bootstrap_servers=settings.KAFKA_SERVER, ) try: client.create_partitions( topic_partitions={ settings.KAFKA_TOPIC: NewPartitions(total_count=len(settings.PARTITIONS.keys())) }) except Exception as e: logger.warning(f'init_partitions error:{e}')
def _init_partitions(cls, settings: Settings): client = KafkaAdminClient(bootstrap_servers=settings.kafka_servers, ) try: client.create_partitions( topic_partitions={ settings.kafka_topic: NewPartitions( total_count=len(settings.kafka_partitions.keys())) }) except Exception as e: logger.debug(f"init_partitions error:{e}")
def test_admin_client(self): """ This test verifies that Kafka Admin Client can still be used to manage Kafka. """ admin_client = KafkaAdminClient( bootstrap_servers=KafkaBrokerIntegrationTest.kafka_address()) # Create a topic with 3 partitions. new_topic_spec = NewTopic(name='test_admin_client', num_partitions=3, replication_factor=1) create_response = admin_client.create_topics([new_topic_spec]) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) # Alter topic (change some Kafka-level property). config_resource = ConfigResource(ConfigResourceType.TOPIC, new_topic_spec.name, {'flush.messages': 42}) alter_response = admin_client.alter_configs([config_resource]) error_data = alter_response.resources self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0][0], 0) # Add 2 more partitions to topic. new_partitions_spec = {new_topic_spec.name: NewPartitions(5)} new_partitions_response = admin_client.create_partitions( new_partitions_spec) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) # Delete a topic. delete_response = admin_client.delete_topics([new_topic_spec.name]) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) self.metrics.collect_final_metrics() self.metrics.assert_metric_increase('create_topics', 1) self.metrics.assert_metric_increase('alter_configs', 1) self.metrics.assert_metric_increase('create_partitions', 1) self.metrics.assert_metric_increase('delete_topics', 1)
from kafka import KafkaAdminClient from kafka.admin import NewPartitions admin_client = KafkaAdminClient(bootstrap_servers=['localhost:9092']) topic_partitions = {'userInfo': NewPartitions(total_count=4)} admin_client.create_partitions(topic_partitions)