def init_partitions(settings: Settings):
client = KafkaAdminClient(bootstrap_servers=settings.kafka_server, )
try:
client.create_partitions(
topic_partitions={
settings.kafka_topic:
NewPartitions(total_count=len(settings.schema_table.keys()))
})
except Exception as e:
logger.warning(f"init_partitions error:{e}")
def init_partitions():
client = KafkaAdminClient(bootstrap_servers=settings.KAFKA_SERVER, )
try:
client.create_partitions(
topic_partitions={
settings.KAFKA_TOPIC:
NewPartitions(total_count=len(settings.PARTITIONS.keys()))
})
except Exception as e:
logger.warning(f'init_partitions error:{e}')
def _init_partitions(cls, settings: Settings):
client = KafkaAdminClient(bootstrap_servers=settings.kafka_servers, )
try:
client.create_partitions(
topic_partitions={
settings.kafka_topic:
NewPartitions(
total_count=len(settings.kafka_partitions.keys()))
})
except Exception as e:
logger.debug(f"init_partitions error:{e}")
def test_admin_client(self):
"""
This test verifies that Kafka Admin Client can still be used to manage Kafka.
"""
admin_client = KafkaAdminClient(
bootstrap_servers=KafkaBrokerIntegrationTest.kafka_address())
# Create a topic with 3 partitions.
new_topic_spec = NewTopic(name='test_admin_client',
num_partitions=3,
replication_factor=1)
create_response = admin_client.create_topics([new_topic_spec])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Alter topic (change some Kafka-level property).
config_resource = ConfigResource(ConfigResourceType.TOPIC,
new_topic_spec.name,
{'flush.messages': 42})
alter_response = admin_client.alter_configs([config_resource])
error_data = alter_response.resources
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0][0], 0)
# Add 2 more partitions to topic.
new_partitions_spec = {new_topic_spec.name: NewPartitions(5)}
new_partitions_response = admin_client.create_partitions(
new_partitions_spec)
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Delete a topic.
delete_response = admin_client.delete_topics([new_topic_spec.name])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
self.metrics.collect_final_metrics()
self.metrics.assert_metric_increase('create_topics', 1)
self.metrics.assert_metric_increase('alter_configs', 1)
self.metrics.assert_metric_increase('create_partitions', 1)
self.metrics.assert_metric_increase('delete_topics', 1)
from kafka import KafkaAdminClient
from kafka.admin import NewPartitions
admin_client = KafkaAdminClient(bootstrap_servers=['localhost:9092'])
topic_partitions = {'userInfo': NewPartitions(total_count=4)}
admin_client.create_partitions(topic_partitions)
