def test_describe_configs_mixed_resources_returns_configs(topic, kafka_admin_client):
"""Tests that describe config returns configs for mixed resource types (topic + broker)
"""
broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
configs = kafka_admin_client.describe_configs([
ConfigResource(ConfigResourceType.TOPIC, topic),
ConfigResource(ConfigResourceType.BROKER, broker_id)])
assert len(configs) == 2
for config in configs:
assert (config.resources[0][2] == ConfigResourceType.TOPIC
and config.resources[0][3] == topic) or \
(config.resources[0][2] == ConfigResourceType.BROKER
and config.resources[0][3] == str(broker_id))
assert len(config.resources[0][4]) > 1
def test_describe_configs_invalid_broker_id_raises(kafka_admin_client):
"""Tests that describe config raises exception on non-integer broker id
"""
broker_id = "str"
with pytest.raises(ValueError):
configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
def get(self, topicName):
"""
Get Topic Configuration.
"""
app.logger.info(
"Request to get Configuration for topic {0}.".format(topicName))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
config_list = []
config = ConfigResource(ConfigResourceType.TOPIC, topicName)
topic_configs = admin.describe_configs([config])
topic_config = topic_configs[0].resources[0]
for c in topic_config[4]:
config_list.append({'key': c[0], 'value': c[1]})
return config_list
except Exception as e:
ns_topic.abort(500, str(e.args))
finally:
admin.close()
def put(self, topicName):
"""
Update Topic Configuration.
"""
ckey = request.json['key']
cvalue = request.json['value']
app.logger.info(
"Request to update configuration for topic {0} for key {1} and value {2}."
.format(topicName, ckey, cvalue))
try:
admin = KafkaAdminClient(
bootstrap_servers=config['cluster.broker.listeners'],
security_protocol=config['cluster.security.protocol'],
ssl_cafile=config['cluster.ssl.cafile'],
ssl_certfile=config['cluster.ssl.certfile'],
ssl_keyfile=config['cluster.ssl.keyfile'])
new_config = ConfigResource(ConfigResourceType.TOPIC, topicName,
{ckey: cvalue})
result = admin.alter_configs([new_config])
except UnknownTopicOrPartitionError as e:
api.abort(400, e.description)
except Exception as e:
ns_topic.abort(500, str(e.args))
finally:
admin.close()
if result.resources[0][0] == 0:
return {"configured": topicName}
else:
api.abort(400, "Bad Request(" + result.resources[0][1] + ")")
def alter_topic_config(admin, topic, config_var, value):
"""Alter TOPIC config."""
resource = ConfigResource(ConfigResourceType.TOPIC,
topic,
configs={config_var: value})
click.echo(f"setting {config_var}:{value}")
admin.admin.alter_configs([resource])
def update_topic_config(admin, topic_name, body):
alter_config_response = admin.alter_configs(
[ConfigResource(resource_type='topic', name=topic_name,
configs=body)]).to_object()
if alter_config_response['resources'][0]['error_code'] != 0:
raise Exception(
f"Failed to update topic {topic_name}: {alter_config_response['resources'][0]['error_message']}"
)
def test_describe_configs_topic_resource_returns_configs(topic, kafka_admin_client):
"""Tests that describe config returns configs for topic
"""
configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.TOPIC, topic)])
assert len(configs) == 1
assert configs[0].resources[0][2] == ConfigResourceType.TOPIC
assert configs[0].resources[0][3] == topic
assert len(configs[0].resources[0][4]) > 1
def alter_topics(admin, prefix, config_var, value):
"""Alter multiple topic configs, selecting topics by prefix."""
topics = admin.get_topics_by_prefix(prefix)
for topic in topics:
resource = ConfigResource(ConfigResourceType.TOPIC,
topic,
configs={config_var: value})
click.echo(f"altering topic config for: {topic}, {config_var}:{value}")
admin.admin.alter_configs([resource])
def describe_topic_config(admin, topic, config_var):
"""Show TOPIC config."""
configs = {}
if config_var is not None:
configs = {config_var: True}
resource = ConfigResource(ConfigResourceType.TOPIC, topic, configs=configs)
topic_config = admin.admin.describe_configs([resource])
configuration = topic_config[0].resources[0][4]
pprint(configuration)
def test_describe_configs_broker_resource_returns_configs(kafka_admin_client):
"""Tests that describe config returns configs for broker
"""
broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
assert len(configs) == 1
assert configs[0].resources[0][2] == ConfigResourceType.BROKER
assert configs[0].resources[0][3] == str(broker_id)
assert len(configs[0].resources[0][4]) > 1
def describeConfig():
try:
string_input = str(input("Please enter a topic:")).strip()
listOfConfigResourceObject = [
ConfigResource(i, name)
for i, name in zip([ConfigResourceType.TOPIC], [string_input])
]
pprint.pprint(adminClient.describe_configs(listOfConfigResourceObject))
except Exception as e:
print(e)
def get_topic_config_value(admin, topic_name, config_key):
described_config = admin.describe_configs([ConfigResource(resource_type='topic', name=topic_name)])[0].to_object()
if described_config['resources'][0]['error_code'] != 0:
raise Exception(
f"Failed to get topic {topic_name} config: {described_config['resources'][0]['error_message']}")
config_value = next((config['config_value'] for config in described_config['resources'][0]['config_entries'] if
config['config_names'] == config_key))
return config_value
def get_topic_config(self, topic: str) -> dict:
config_version = self._matching_api_version(DescribeConfigsRequest)
req_cfgs = [ConfigResource(ConfigResourceType.TOPIC, topic)]
cfgs = self.describe_configs(req_cfgs)
assert len(cfgs) == 1
assert len(cfgs[0].resources) == 1
err, _, _, _, config_values = cfgs[0].resources[0]
if err != 0:
raise for_code(err)
topic_config = {}
for cv in config_values:
if config_version == 0:
name, val, _, _, _ = cv
else:
name, val, _, _, _, _ = cv
topic_config[name] = val
return topic_config
def test_admin_client(self):
"""
This test verifies that Kafka Admin Client can still be used to manage Kafka.
"""
admin_client = KafkaAdminClient(
bootstrap_servers=KafkaBrokerIntegrationTest.kafka_address())
# Create a topic with 3 partitions.
new_topic_spec = NewTopic(name='test_admin_client',
num_partitions=3,
replication_factor=1)
create_response = admin_client.create_topics([new_topic_spec])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Alter topic (change some Kafka-level property).
config_resource = ConfigResource(ConfigResourceType.TOPIC,
new_topic_spec.name,
{'flush.messages': 42})
alter_response = admin_client.alter_configs([config_resource])
error_data = alter_response.resources
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0][0], 0)
# Add 2 more partitions to topic.
new_partitions_spec = {new_topic_spec.name: NewPartitions(5)}
new_partitions_response = admin_client.create_partitions(
new_partitions_spec)
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
# Delete a topic.
delete_response = admin_client.delete_topics([new_topic_spec.name])
error_data = create_response.topic_errors
self.assertEqual(len(error_data), 1)
self.assertEqual(error_data[0], (new_topic_spec.name, 0, None))
self.metrics.collect_final_metrics()
self.metrics.assert_metric_increase('create_topics', 1)
self.metrics.assert_metric_increase('alter_configs', 1)
self.metrics.assert_metric_increase('create_partitions', 1)
self.metrics.assert_metric_increase('delete_topics', 1)