def test_describe_configs_mixed_resources_returns_configs(topic, kafka_admin_client): """Tests that describe config returns configs for mixed resource types (topic + broker) """ broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId configs = kafka_admin_client.describe_configs([ ConfigResource(ConfigResourceType.TOPIC, topic), ConfigResource(ConfigResourceType.BROKER, broker_id)]) assert len(configs) == 2 for config in configs: assert (config.resources[0][2] == ConfigResourceType.TOPIC and config.resources[0][3] == topic) or \ (config.resources[0][2] == ConfigResourceType.BROKER and config.resources[0][3] == str(broker_id)) assert len(config.resources[0][4]) > 1
def test_describe_configs_invalid_broker_id_raises(kafka_admin_client): """Tests that describe config raises exception on non-integer broker id """ broker_id = "str" with pytest.raises(ValueError): configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
def get(self, topicName): """ Get Topic Configuration. """ app.logger.info( "Request to get Configuration for topic {0}.".format(topicName)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) config_list = [] config = ConfigResource(ConfigResourceType.TOPIC, topicName) topic_configs = admin.describe_configs([config]) topic_config = topic_configs[0].resources[0] for c in topic_config[4]: config_list.append({'key': c[0], 'value': c[1]}) return config_list except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close()
def put(self, topicName): """ Update Topic Configuration. """ ckey = request.json['key'] cvalue = request.json['value'] app.logger.info( "Request to update configuration for topic {0} for key {1} and value {2}." .format(topicName, ckey, cvalue)) try: admin = KafkaAdminClient( bootstrap_servers=config['cluster.broker.listeners'], security_protocol=config['cluster.security.protocol'], ssl_cafile=config['cluster.ssl.cafile'], ssl_certfile=config['cluster.ssl.certfile'], ssl_keyfile=config['cluster.ssl.keyfile']) new_config = ConfigResource(ConfigResourceType.TOPIC, topicName, {ckey: cvalue}) result = admin.alter_configs([new_config]) except UnknownTopicOrPartitionError as e: api.abort(400, e.description) except Exception as e: ns_topic.abort(500, str(e.args)) finally: admin.close() if result.resources[0][0] == 0: return {"configured": topicName} else: api.abort(400, "Bad Request(" + result.resources[0][1] + ")")
def alter_topic_config(admin, topic, config_var, value): """Alter TOPIC config.""" resource = ConfigResource(ConfigResourceType.TOPIC, topic, configs={config_var: value}) click.echo(f"setting {config_var}:{value}") admin.admin.alter_configs([resource])
def update_topic_config(admin, topic_name, body): alter_config_response = admin.alter_configs( [ConfigResource(resource_type='topic', name=topic_name, configs=body)]).to_object() if alter_config_response['resources'][0]['error_code'] != 0: raise Exception( f"Failed to update topic {topic_name}: {alter_config_response['resources'][0]['error_message']}" )
def test_describe_configs_topic_resource_returns_configs(topic, kafka_admin_client): """Tests that describe config returns configs for topic """ configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.TOPIC, topic)]) assert len(configs) == 1 assert configs[0].resources[0][2] == ConfigResourceType.TOPIC assert configs[0].resources[0][3] == topic assert len(configs[0].resources[0][4]) > 1
def alter_topics(admin, prefix, config_var, value): """Alter multiple topic configs, selecting topics by prefix.""" topics = admin.get_topics_by_prefix(prefix) for topic in topics: resource = ConfigResource(ConfigResourceType.TOPIC, topic, configs={config_var: value}) click.echo(f"altering topic config for: {topic}, {config_var}:{value}") admin.admin.alter_configs([resource])
def describe_topic_config(admin, topic, config_var): """Show TOPIC config.""" configs = {} if config_var is not None: configs = {config_var: True} resource = ConfigResource(ConfigResourceType.TOPIC, topic, configs=configs) topic_config = admin.admin.describe_configs([resource]) configuration = topic_config[0].resources[0][4] pprint(configuration)
def test_describe_configs_broker_resource_returns_configs(kafka_admin_client): """Tests that describe config returns configs for broker """ broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)]) assert len(configs) == 1 assert configs[0].resources[0][2] == ConfigResourceType.BROKER assert configs[0].resources[0][3] == str(broker_id) assert len(configs[0].resources[0][4]) > 1
def describeConfig(): try: string_input = str(input("Please enter a topic:")).strip() listOfConfigResourceObject = [ ConfigResource(i, name) for i, name in zip([ConfigResourceType.TOPIC], [string_input]) ] pprint.pprint(adminClient.describe_configs(listOfConfigResourceObject)) except Exception as e: print(e)
def get_topic_config_value(admin, topic_name, config_key): described_config = admin.describe_configs([ConfigResource(resource_type='topic', name=topic_name)])[0].to_object() if described_config['resources'][0]['error_code'] != 0: raise Exception( f"Failed to get topic {topic_name} config: {described_config['resources'][0]['error_message']}") config_value = next((config['config_value'] for config in described_config['resources'][0]['config_entries'] if config['config_names'] == config_key)) return config_value
def get_topic_config(self, topic: str) -> dict: config_version = self._matching_api_version(DescribeConfigsRequest) req_cfgs = [ConfigResource(ConfigResourceType.TOPIC, topic)] cfgs = self.describe_configs(req_cfgs) assert len(cfgs) == 1 assert len(cfgs[0].resources) == 1 err, _, _, _, config_values = cfgs[0].resources[0] if err != 0: raise for_code(err) topic_config = {} for cv in config_values: if config_version == 0: name, val, _, _, _ = cv else: name, val, _, _, _, _ = cv topic_config[name] = val return topic_config
def test_admin_client(self): """ This test verifies that Kafka Admin Client can still be used to manage Kafka. """ admin_client = KafkaAdminClient( bootstrap_servers=KafkaBrokerIntegrationTest.kafka_address()) # Create a topic with 3 partitions. new_topic_spec = NewTopic(name='test_admin_client', num_partitions=3, replication_factor=1) create_response = admin_client.create_topics([new_topic_spec]) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) # Alter topic (change some Kafka-level property). config_resource = ConfigResource(ConfigResourceType.TOPIC, new_topic_spec.name, {'flush.messages': 42}) alter_response = admin_client.alter_configs([config_resource]) error_data = alter_response.resources self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0][0], 0) # Add 2 more partitions to topic. new_partitions_spec = {new_topic_spec.name: NewPartitions(5)} new_partitions_response = admin_client.create_partitions( new_partitions_spec) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) # Delete a topic. delete_response = admin_client.delete_topics([new_topic_spec.name]) error_data = create_response.topic_errors self.assertEqual(len(error_data), 1) self.assertEqual(error_data[0], (new_topic_spec.name, 0, None)) self.metrics.collect_final_metrics() self.metrics.assert_metric_increase('create_topics', 1) self.metrics.assert_metric_increase('alter_configs', 1) self.metrics.assert_metric_increase('create_partitions', 1) self.metrics.assert_metric_increase('delete_topics', 1)