def test_alter_topic_config_works(topic_controller: TopicController,
topic_id: str):
initial_topic = Topic(topic_id, config={"cleanup.policy": "delete"})
topic_controller.create_topics([initial_topic])
topic_controller.update_from_cluster(initial_topic)
config = initial_topic.config
assert config.get("cleanup.policy") == "delete"
change_topic = Topic(topic_id, config={"cleanup.policy": "compact"})
topic_controller.alter_configs([change_topic])
topic_controller.update_from_cluster(change_topic)
after_changes_applied_topic = topic_controller.get_cluster_topic(topic_id)
final_config = after_changes_applied_topic.config
assert final_config.get("cleanup.policy") == "compact"
def ping(state, times, wait):
topic_controller = state.cluster.topic_controller
deltas = []
try:
try:
topic_controller.create_topics(
[topic_controller.get_cluster_topic(PING_TOPIC)])
except TopicAlreadyExistsException:
click.echo("Topic already exists.")
producer = PingProducer()
consumer = PingConsumer(PING_GROUP_ID, PING_TOPIC, True)
click.echo(f"Ping with {state.cluster.bootstrap_servers}")
for i in range(times):
producer.produce(PING_TOPIC)
_, delta = consumer.consume()
deltas.append(delta)
click.echo(f"m_seq={i} time={delta:.2f}ms")
sleep(wait)
except KeyboardInterrupt:
pass
finally:
topic_controller.delete_topic(Topic(PING_TOPIC))
click.echo("--- statistics ---")
click.echo(f"{len(deltas)} messages sent/received")
click.echo(
f"min/avg/max = {min(deltas):.2f}/{(sum(deltas) / len(deltas)):.2f}/{max(deltas):.2f} ms"
)
def delete_topic(state: State, topic_name: str):
topic_controller = state.cluster.topic_controller
if ensure_approval("Are you sure?", no_verify=state.no_verify):
topic_controller.delete_topic(Topic(topic_name))
assert topic_name not in (t.name
for t in topic_controller.list_topics())
def create_topic(state: State, topic_name: str):
if not ensure_approval("Are you sure?", no_verify=state.no_verify):
click.echo("Aborted")
return
topic_controller = state.cluster.topic_controller
topic_controller.create_topics([Topic(topic_name)])
def test_topic_creation_raises_for_wrong_config(
topic_controller: TopicController,
confluent_admin_client: confluent_kafka.admin.AdminClient,
topic_id: str):
topics = confluent_admin_client.list_topics(timeout=5).topics.keys()
assert topic_id not in topics
# We only have 1 broker for tests, so a higher replication should fail
with pytest.raises(KafkaException):
topic_controller.create_topics([Topic(topic_id, replication_factor=2)])
def test_topic_deletion_works(
topic_controller: TopicController,
confluent_admin_client: confluent_kafka.admin.AdminClient, topic: str):
topics = confluent_admin_client.list_topics(timeout=5).topics.keys()
assert topic in topics
topic_controller.delete_topic(Topic(topic))
# Invalidate cache
confluent_admin_client.poll(timeout=1)
topics = confluent_admin_client.list_topics(timeout=5).topics.keys()
assert topic not in topics
def update_from_cluster(self, topic: Topic):
"""Takes a topic and, based on its name, updates all attributes from the cluster"""
confluent_topic: ConfluentTopic = self._get_client_topic(
topic.name, ClientTypes.Confluent)
pykafka_topic: PyKafkaTopic = self._get_client_topic(
topic.name, ClientTypes.PyKafka)
low_watermarks = pykafka_topic.earliest_available_offsets()
high_watermarks = pykafka_topic.latest_available_offsets()
topic.partition_data = self._get_partition_data(
confluent_topic, low_watermarks, high_watermarks)
topic.config = self.cluster.retrieve_config(ConfigResource.Type.TOPIC,
topic.name)
topic.is_only_local = False
return topic
def test_topic_creation_works(
topic_controller: TopicController,
confluent_admin_client: confluent_kafka.admin.AdminClient,
topic_id: str):
topics = confluent_admin_client.list_topics(timeout=5).topics.keys()
assert topic_id not in topics
topic_controller.create_topics([Topic(topic_id, replication_factor=1)])
# invalidate cache
confluent_admin_client.poll(timeout=1)
topics = confluent_admin_client.list_topics(timeout=5).topics.keys()
assert topic_id in topics
def test_topic_diff(topic_controller: TopicController, topic_id: str):
# the value we get from cluster configs is as string
# testing against this is important to ensure consistency
default_delete_retention = "86400000"
topic_conf = {
"name": topic_id,
"replication_factor": 1,
"num_partitions": 50,
"config": {
"cleanup.policy": "compact"
},
}
get_diff = topic_controller.diff_with_cluster
conf = json.loads(json.dumps(topic_conf))
topic = Topic.from_dict(conf)
topic_controller.create_topics([topic])
assert not get_diff(
topic).has_changes, "Shouldn't have diff on just created topic"
conf = json.loads(json.dumps(topic_conf))
conf["config"]["cleanup.policy"] = "delete"
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("cleanup.policy", "compact", "delete")
assert get_diff(topic) == diff, "Should have a diff on cleanup.policy"
conf = json.loads(json.dumps(topic_conf))
conf["config"]["delete.retention.ms"] = 1500
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("delete.retention.ms",
default_delete_retention, 1500)
assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms"
# the same as before, but this time with string values
conf = json.loads(json.dumps(topic_conf))
conf["config"]["delete.retention.ms"] = "1500"
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("delete.retention.ms",
default_delete_retention, "1500")
assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms"
conf = json.loads(json.dumps(topic_conf))
conf["num_partitions"] = 3
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("num_partitions", 50, 3)
assert get_diff(topic) == diff, "Should have a diff on num_partitions"
conf = json.loads(json.dumps(topic_conf))
conf["replication_factor"] = 3
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("replication_factor", 1, 3)
assert get_diff(topic) == diff, "Should have a diff on replication_factor"
def changed_topic_object(cluster, topic):
yield Topic(topic, 1, 3, {"cleanup.policy": "compact"})
def test_apply(topic_controller: TopicController, topic_id: str):
runner = CliRunner()
topic_name = f"apply_{topic_id}"
topic_1 = {
"name": topic_name + "_1",
"replication_factor": 1,
"num_partitions": 50,
"config": {
"cleanup.policy": "compact"
},
}
topic_2 = {
"name": topic_name + "_2",
"replication_factor": 1,
"num_partitions": 5,
"config": {
"cleanup.policy": "delete",
"delete.retention.ms": 50000
},
}
apply_conf = {"topics": [topic_1]}
# 1: topic creation
path = save_yaml(topic_id, apply_conf)
result = runner.invoke(apply, ["-f", path], input="Y\n")
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 2: change cleanup policy to delete
topic_1["config"]["cleanup.policy"] = "delete"
path = save_yaml(topic_id, apply_conf)
result = runner.invoke(apply, ["-f", path], input="Y\n")
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 3: add another topic and change the first one again
apply_conf["topics"].append(topic_2)
topic_1["config"]["cleanup.policy"] = "compact"
path = save_yaml(topic_id, apply_conf)
result = runner.invoke(apply, ["-f", path], input="Y\n")
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 4: no changes
result = runner.invoke(apply, ["-f", path])
assert (result.exit_code == 0 and "No changes detected, aborting"
in result.output), f"Calling apply failed, error: {result.output}"
# 5: change partitions - this attempt should be cancelled
topic_1["num_partitions"] = 3
topic_1["config"]["cleanup.policy"] = "delete"
path = save_yaml(topic_id, apply_conf)
result = runner.invoke(apply, ["-f", path], input="Y\n")
assert (result.exit_code == 0
and "to `replication_factor` and `num_partitions`"
in result.output), f"Calling apply failed, error: {result.output}"
# reset config to the old settings again
topic_1["num_partitions"] = 50
topic_1["config"]["cleanup.policy"] = "compact"
# final: check results in the cluster to make sure they match
for topic_conf in apply_conf["topics"]:
topic_from_conf = Topic.from_dict(topic_conf)
assert not topic_controller.diff_with_cluster(
topic_from_conf
).has_changes, f"Topic configs don't match, diff is {topic_controller.diff_with_cluster(topic_from_conf)}"
def apply(state: State, file: str):
# Get topic data based on the YAML
yaml_topic_configs = yaml.safe_load(open(file)).get("topics")
yaml_topics = [Topic.from_dict(conf) for conf in yaml_topic_configs]
yaml_topic_names = [t.name for t in yaml_topics]
if not len(yaml_topic_names) == len(set(yaml_topic_names)):
raise ValueError("Duplicate topic names in the YAML!")
# Get topic data based on the cluster state
topic_controller = state.cluster.topic_controller
cluster_topics = topic_controller.list_topics(
search_string="|".join(yaml_topic_names))
cluster_topic_names = [t.name for t in cluster_topics]
# Calculate changes
to_create = [
yaml_topic for yaml_topic in yaml_topics
if yaml_topic.name not in cluster_topic_names
]
to_edit = [
yaml_topic for yaml_topic in yaml_topics if yaml_topic not in to_create
and topic_controller.diff_with_cluster(yaml_topic).has_changes
]
to_edit_diffs = {
t.name: topic_controller.diff_with_cluster(t)
for t in to_edit
}
to_ignore = [
yaml_topic for yaml_topic in yaml_topics
if yaml_topic not in to_create and yaml_topic not in to_edit
]
# Sanity check - the 3 groups of topics should be complete and have no overlap
assert (set(to_create).isdisjoint(set(to_edit))
and set(to_create).isdisjoint(set(to_ignore))
and set(to_edit).isdisjoint(set(to_ignore)) and
len(to_create) + len(to_edit) + len(to_ignore) == len(yaml_topics))
# Print diffs so the user can check
click.echo(pretty_unchanged_topic_configs(to_ignore))
click.echo(pretty_new_topic_configs(to_create))
click.echo(pretty_topic_diffs(to_edit_diffs))
# Check for actionable changes
if len(to_edit) + len(to_create) == 0:
click.echo("No changes detected, aborting")
return
# Warn users & abort when replication & num_partition changes are attempted
if any(not diff.is_valid for _, diff in to_edit_diffs.items()):
click.echo(
"Changes to `replication_factor` and `num_partitions` can not be applied on already existing topics"
)
click.echo("Cancelling due to invalid changes")
return
# Get approval
if not ensure_approval("Apply changes?", no_verify=state.no_verify):
click.echo("Cancelling changes")
return
# apply changes
topic_controller.create_topics(to_create)
topic_controller.alter_configs(to_edit)
# output confirmation
changes = {
"unchanged": len(to_ignore),
"created": len(to_create),
"changed": len(to_edit)
}
click.echo(
click.style(pretty({"Successfully applied changes": changes}),
fg="green"))
def get_cluster_topic(self, topic_name: str) -> Topic:
"""Convenience function getting an existing topic based on topic_name"""
return self.update_from_cluster(Topic(topic_name))