def test_describe_configs_api():
""" describe_configs() tests, these wont really do anything since there
is no broker configured. """
a = AdminClient({"socket.timeout.ms": 10})
fs = a.describe_configs(
[ConfigResource(confluent_kafka.admin.RESOURCE_BROKER, "3")])
# ignore the result
with pytest.raises(Exception):
a.describe_configs(None)
with pytest.raises(Exception):
a.describe_configs("something")
with pytest.raises(Exception):
a.describe_configs([])
with pytest.raises(ValueError):
a.describe_configs([
None,
ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, "mytopic")
])
fs = a.describe_configs([
ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, "mytopic"),
ConfigResource(confluent_kafka.admin.RESOURCE_GROUP, "mygroup")
],
request_timeout=0.123)
with pytest.raises(KafkaException):
for f in concurrent.futures.as_completed(iter(fs.values())):
f.result(timeout=1)
def alter_config_for_topic(
self, topic: Topic, dry_run=False, respect_existing_config=False
):
"""
Alter the configuration of a single topic.
:topic a Topic instance
:respect_existing_config merge existing conig into new
:dry_run perform dry-run only
"""
new_config = {}
# First get existing configs.. so really "old config" at this stage
existing_config = {
val.name: val.value
for (key, val) in self.describe_topic(topic.name).items()
}
if respect_existing_config:
new_config.update(existing_config)
# And update with changed values to get real new config
new_config.update(topic.configs)
new_config = self._sanitize_topic_config(new_config)
# get a config delta
config_delta = self._get_config_diff(existing_config, new_config)
resource = ConfigResource(restype=Type.TOPIC, name=topic.name)
for key, value in new_config.items():
resource.set_config(key, value)
fs = self.adminclient.alter_configs(
[resource], request_timeout=30, validate_only=dry_run
)
# check result
configs_altered = []
configs_failed = {}
for res, future in fs.items():
try:
future.result()
configs_altered.append(res)
except Exception as e:
configs_failed[res] = str(e)
if topic.name not in self.dry_run_plan:
self._update_plan(
topic.name,
{
"topic": topic,
"reason": None,
"config_delta": config_delta,
},
)
self._update_plan(
topic.name,
{"configs_altered": configs_altered, "configs_failed": configs_failed},
)
return (configs_altered, configs_failed, config_delta)
def modify_config(topic, new_config):
# type: (str, dict)
"""Modify topic-config.
Keyword arguments:
topic -- topicname
new_config -- dictionary with new config
"""
resource = [ConfigResource("TOPIC", topic)]
try:
des = admin.describe_configs(resource)
except KafkaException as e:
msg = ("Can not retrieve topic-config from topic %s: %s" \
%(topic, e)
)
y = list(des.values())
old_conf = y[0].result()
for config, newvalue in new_config.items(): #iterate trough new-config-dict and set them on topic-resource
resource[0].set_config(config, newvalue)
try:
des = admin.alter_configs(resource) #alter topic with new config
y = list(des.values())
y[0].result() #use .result-func for finalizing
except KafkaException as e:
msg = ("Failed to finalize config-change for topic %s: %s" \
%(topic, e)
)
fail_module(msg)
def compare_config(topic, new_config):
# type: (str, dict) -> bool
"""Compare the defined config in the playbook with the one set at the moment for this topic.
Keyword arguments:
topic -- topicname
new_config -- dictionary with new config and values
Return:
bool -- True if change is needed, else False
"""
resource = [ConfigResource("TOPIC", topic)]
try:
des = admin.describe_configs(resource)
except KafkaException as e:
msg = ("Can not retrieve topic-config from topic %s: %s" \
%(topic, e)
)
y = list(des.values())
old_conf = y[0].result()
#iterate trough new-config-dict and compare with old-config-dict, using config as key
for config, newvalue in new_config.items():
if str(newvalue) != old_conf[config].value:
return True
return False
def config_resource(self, restype, name, **kwargs):
"""Represents a resource that has configuration, and (optionally) a collection of configuration properties
for that resource. Used by describe_configs() and alter_configs().
- ``restype`` (ConfigResource.Type): The resource type.
- ``name`` (str): The resource name, which depends on the resource type. For RESOURCE_BROKER,
the resource name is the broker id.
"""
return ConfigResource(restype=restype, name=name, **kwargs)
def test_alter_configs_api():
""" alter_configs() tests, these wont really do anything since there
is no broker configured. """
a = AdminClient({"socket.timeout.ms": 10})
fs = a.alter_configs([
ConfigResource(confluent_kafka.admin.RESOURCE_BROKER,
"3",
set_config={"some": "config"})
])
# ignore the result
with pytest.raises(Exception):
a.alter_configs(None)
with pytest.raises(Exception):
a.alter_configs("something")
with pytest.raises(ValueError):
a.alter_configs([])
with pytest.raises(ValueError):
a.alter_configs([
None,
ConfigResource("topic",
"mytopic",
add_config={"something": "else"})
])
fs = a.alter_configs([
ConfigResource("topic",
"mytopic",
set_config={
"set": "this",
"and": "this"
},
add_config={"add": "this"},
del_config=["this"]),
ConfigResource(confluent_kafka.admin.RESOURCE_GROUP, "mygroup")
],
request_timeout=0.123)
with pytest.raises(KafkaException):
for f in concurrent.futures.as_completed(iter(fs.values())):
f.result(timeout=1)
def example_alter_configs(a, args):
""" Alter configs atomically, replacing non-specified
configuration properties with their default values.
"""
resources = []
for restype, resname, configs in zip(args[0::3], args[1::3], args[2::3]):
resource = ConfigResource(restype, resname)
resources.append(resource)
for k, v in [conf.split('=') for conf in configs.split(',')]:
resource.set_config(k, v)
fs = a.alter_configs(resources)
# Wait for operation to finish.
for res, f in fs.items():
try:
f.result() # empty, but raises exception on failure
print("{} configuration successfully altered".format(res))
except Exception:
raise
def cluster_config(bootstrap_server, topic):
conf = {'bootstrap.servers': bootstrap_server}
adminClient = AdminClient(conf)
topic_configResource = adminClient.describe_configs(
[ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, topic)])
for j in concurrent.futures.as_completed(
iter(topic_configResource.values())):
config_response = j.result(timeout=1)
dict_config = {}
for key in config_response:
dict_config[key] = str(config_response[key])
return dict_config
def cluster_config(bootstrap_server, topic):
"""
Shows the cluster configuration. It requires a bootstrap_server ip and the name of a existing topic
"""
conf = {'bootstrap.servers': bootstrap_server}
adminClient = AdminClient(conf)
topic_configResource = adminClient.describe_configs(
[ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, topic)])
for j in concurrent.futures.as_completed(
iter(topic_configResource.values())):
config_response = j.result(timeout=1)
print config_response
def test_types():
ConfigResource(confluent_kafka.admin.RESOURCE_BROKER, "2")
ConfigResource("broker", "2")
ConfigResource(confluent_kafka.admin.RESOURCE_GROUP, "mygroup")
ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, "")
with pytest.raises(ValueError):
ConfigResource("doesnt exist", "hi")
with pytest.raises(ValueError):
ConfigResource(confluent_kafka.admin.RESOURCE_TOPIC, None)
def get_config(topic, admin):
"""Get raw topic-config for other functions to use
Keyword Arguments:
topic -- topicname
admin -- adminclient for fetching config
Return:
konf -- dict containing raw topic-config
"""
resources = [ConfigResource("TOPIC", topic)]
des = admin.describe_configs(resources)
y = list(des.values())
konf = y[0].result()
return konf
def kafka_GetInfo(self, p_szTopicName):
if self.__kafka_servers__ is None:
raise SQLCliException(
"Missed kafka server information. Please use \"kafka set server\" first .."
)
a = AdminClient({'bootstrap.servers': self.__kafka_servers__})
configs = a.describe_configs(resources=[
ConfigResource(ConfigResource.Type["TOPIC"], p_szTopicName)
])
nReturnMessages = None
for f in concurrent.futures.as_completed(iter(configs.values())):
if nReturnMessages is None:
nReturnMessages = "ConfigProperties: \n" + \
" " + str(f.result(timeout=1)["retention.ms"])
else:
nReturnMessages = nReturnMessages + "\n" + \
"ConfigProperties: \n" + \
" " + str(f.result(timeout=1)["retention.ms"])
return nReturnMessages
def describe_topic(self, topic: str):
"""
Return a list of configs for the provided topic name
"""
resource = ConfigResource(restype=Type.TOPIC, name=topic)
fs = self.adminclient.describe_configs([resource])
if len(fs) != 1:
return (
f"describe_configs for topic {topic} did not return a single response."
)
for res, future in fs.items():
try:
configs = future.result()
# for config in configs.values():
# print(f"Topic {topic} -> {config.name} = {config.value}")
return configs
except KafkaException as e:
print(f"Failed to to describe config for {res} with error {e}")
return {}
def get_topic_config(topic):
# type: (str) -> dict
"""Get Topic configuration.
Keyword arguments:
topic -- topicname
Return:
old_conf -- dict containing topic configuration
"""
resource = [ConfigResource("TOPIC", topic)]
try:
des = admin.describe_configs(resource)
except KafkaException as e:
msg = ("Can not retrieve topic-config from topic %s: %s" % (topic, e))
y = list(des.values())
old_conf = y[0].result()
return old_conf
def example_delta_alter_configs(a, args):
"""
The AlterConfigs Kafka API requires all configuration to be passed,
any left out configuration properties will revert to their default settings.
This example shows how to just modify the supplied configuration entries
by first reading the configuration from the broker, updating the supplied
configuration with the broker configuration (without overwriting), and
then writing it all back.
The async nature of futures is also show-cased, which makes this example
a bit more complex than it needs to be in the synchronous case.
"""
# Convert supplied config to resources.
# We can reuse the same resources both for describe_configs and
# alter_configs.
resources = []
for restype, resname, configs in zip(args[0::3], args[1::3], args[2::3]):
resource = ConfigResource(restype, resname)
resources.append(resource)
for k, v in [conf.split('=') for conf in configs.split(',')]:
resource.set_config(k, v)
# Set up a locked counter and an Event (for signaling) to track when the
# second level of futures are done. This is a bit of contrived example
# due to no other asynchronous mechanism being used, so we'll need
# to wait on something to signal completion.
class WaitZero(object):
def __init__(self, waitcnt):
self.cnt = waitcnt
self.lock = threading.Lock()
self.event = threading.Event()
def decr(self):
""" Decrement cnt by 1"""
with self.lock:
assert self.cnt > 0
self.cnt -= 1
self.event.set()
def wait(self):
""" Wait until cnt reaches 0 """
self.lock.acquire()
while self.cnt > 0:
self.lock.release()
self.event.wait()
self.event.clear()
self.lock.acquire()
self.lock.release()
def __len__(self):
with self.lock:
return self.cnt
wait_zero = WaitZero(len(resources))
# Read existing configuration from cluster
fs = a.describe_configs(resources)
def delta_alter_configs_done(fut, resource):
e = fut.exception()
if e is not None:
print("Config update for {} failed: {}".format(resource, e))
else:
print("Config for {} updated".format(resource))
wait_zero.decr()
def delta_alter_configs(resource, remote_config):
print("Updating {} supplied config entries {} with {} config entries read from cluster".format(
len(resource), resource, len(remote_config)))
# Only set configuration that is not default
for k, entry in [(k, v) for k, v in remote_config.items() if not v.is_default]:
resource.set_config(k, entry.value, overwrite=False)
fs = a.alter_configs([resource])
fs[resource].add_done_callback(lambda fut: delta_alter_configs_done(fut, resource))
# For each resource's future set up a completion callback
# that in turn calls alter_configs() on that single resource.
# This is ineffective since the resources can usually go in
# one single alter_configs() call, but we're also show-casing
# the futures here.
for res, f in fs.items():
f.add_done_callback(lambda fut, resource=res: delta_alter_configs(resource, fut.result()))
# Wait for done callbacks to be triggered and operations to complete.
print("Waiting for {} resource updates to finish".format(len(wait_zero)))
wait_zero.wait()
