def test_topic_diff(topic_controller: TopicController, topic_id: str):
# the value we get from cluster configs is as string
# testing against this is important to ensure consistency
default_delete_retention = "86400000"
topic_conf = {
"name": topic_id,
"replication_factor": 1,
"num_partitions": 50,
"config": {
"cleanup.policy": "compact"
},
}
get_diff = topic_controller.diff_with_cluster
conf = json.loads(json.dumps(topic_conf))
topic = Topic.from_dict(conf)
topic_controller.create_topics([topic])
assert not get_diff(
topic).has_changes, "Shouldn't have diff on just created topic"
conf = json.loads(json.dumps(topic_conf))
conf["config"]["cleanup.policy"] = "delete"
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("cleanup.policy", "compact", "delete")
assert get_diff(topic) == diff, "Should have a diff on cleanup.policy"
conf = json.loads(json.dumps(topic_conf))
conf["config"]["delete.retention.ms"] = 1500
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("delete.retention.ms",
default_delete_retention, 1500)
assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms"
# the same as before, but this time with string values
conf = json.loads(json.dumps(topic_conf))
conf["config"]["delete.retention.ms"] = "1500"
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("delete.retention.ms",
default_delete_retention, "1500")
assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms"
conf = json.loads(json.dumps(topic_conf))
conf["num_partitions"] = 3
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("num_partitions", 50, 3)
assert get_diff(topic) == diff, "Should have a diff on num_partitions"
conf = json.loads(json.dumps(topic_conf))
conf["replication_factor"] = 3
topic = Topic.from_dict(conf)
diff = TopicDiff().set_diff("replication_factor", 1, 3)
assert get_diff(topic) == diff, "Should have a diff on replication_factor"
def apply(state: State, file: str):
"""Apply a set of topic configurations.
Create new topics and apply changes to existing topics, as specified in the config yaml file <file>.
"""
# Get topic data based on the YAML
yaml_topic_configs = yaml.safe_load(open(file)).get("topics")
yaml_topics = [Topic.from_dict(conf) for conf in yaml_topic_configs]
yaml_topic_names = [t.name for t in yaml_topics]
if not len(yaml_topic_names) == len(set(yaml_topic_names)):
raise ValidationException("Duplicate topic names in the YAML!")
# Get topic data based on the cluster state
topic_controller = state.cluster.topic_controller
cluster_topics = topic_controller.list_topics(search_string="|".join(yaml_topic_names))
cluster_topic_names = [t.name for t in cluster_topics]
# Calculate changes
to_create = [yaml_topic for yaml_topic in yaml_topics if yaml_topic.name not in cluster_topic_names]
to_edit = [
yaml_topic
for yaml_topic in yaml_topics
if yaml_topic not in to_create and topic_controller.diff_with_cluster(yaml_topic).has_changes
]
to_edit_diffs = {t.name: topic_controller.diff_with_cluster(t) for t in to_edit}
to_ignore = [yaml_topic for yaml_topic in yaml_topics if yaml_topic not in to_create and yaml_topic not in to_edit]
# Sanity check - the 3 groups of topics should be complete and have no overlap
assert (
set(to_create).isdisjoint(set(to_edit))
and set(to_create).isdisjoint(set(to_ignore))
and set(to_edit).isdisjoint(set(to_ignore))
and len(to_create) + len(to_edit) + len(to_ignore) == len(yaml_topics)
)
# Print diffs so the user can check
click.echo(pretty_unchanged_topic_configs(to_ignore))
click.echo(pretty_new_topic_configs(to_create))
click.echo(pretty_topic_diffs(to_edit_diffs))
# Check for actionable changes
if len(to_edit) + len(to_create) == 0:
click.echo("No changes detected, aborting!")
return
# Warn users & abort when replication & num_partition changes are attempted
if any(not diff.is_valid for _, diff in to_edit_diffs.items()):
click.echo(
"Changes to `replication_factor` and `num_partitions` can not be applied on already existing topics."
)
click.echo("Cancelling due to invalid changes")
return
# Get approval
if not ensure_approval("Apply changes?", no_verify=state.no_verify):
click.echo("Cancelling changes")
return
# apply changes
topic_controller.create_topics(to_create)
topic_controller.alter_configs(to_edit)
# output confirmation
changes = {"unchanged": len(to_ignore), "created": len(to_create), "changed": len(to_edit)}
click.echo(click.style(pretty({"Successfully applied changes": changes}), fg="green"))
def test_apply(interactive_cli_runner: CliRunner,
topic_controller: TopicController, topic_id: str):
topic_name = f"apply_{topic_id}"
topic_1 = {
"name": topic_name + "_1",
"replication_factor": 1,
"num_partitions": 50,
"config": {
"cleanup.policy": "compact"
},
}
topic_2 = {
"name": topic_name + "_2",
"replication_factor": 1,
"num_partitions": 5,
"config": {
"cleanup.policy": "delete",
"delete.retention.ms": 50000
},
}
apply_conf = {"topics": [topic_1]}
# 1: topic creation
path = save_yaml(topic_id, apply_conf)
result = interactive_cli_runner.invoke(esque,
args=["apply", "-f", path],
input="Y\n",
catch_exceptions=False)
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 2: change cleanup policy to delete
topic_1["config"]["cleanup.policy"] = "delete"
path = save_yaml(topic_id, apply_conf)
result = interactive_cli_runner.invoke(esque,
args=["apply", "-f", path],
input="Y\n",
catch_exceptions=False)
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 3: add another topic and change the first one again
apply_conf["topics"].append(topic_2)
topic_1["config"]["cleanup.policy"] = "compact"
path = save_yaml(topic_id, apply_conf)
result = interactive_cli_runner.invoke(esque,
args=["apply", "-f", path],
input="Y\n",
catch_exceptions=False)
assert (result.exit_code == 0 and "Successfully applied changes"
in result.output), f"Calling apply failed, error: {result.output}"
# 4: no changes
result = interactive_cli_runner.invoke(esque,
args=["apply", "-f", path],
catch_exceptions=False)
assert (result.exit_code == 0 and "No changes detected, aborting"
in result.output), f"Calling apply failed, error: {result.output}"
# 5: change partitions - this attempt should be cancelled
topic_1["num_partitions"] = 3
topic_1["config"]["cleanup.policy"] = "delete"
path = save_yaml(topic_id, apply_conf)
result = interactive_cli_runner.invoke(esque,
args=["apply", "-f", path],
input="Y\n",
catch_exceptions=False)
assert (result.exit_code == 1
and "to `replication_factor` and `num_partitions`"
in result.output), f"Calling apply failed, error: {result.output}"
# reset config to the old settings again
topic_1["num_partitions"] = 50
topic_1["config"]["cleanup.policy"] = "compact"
# final: check results in the cluster to make sure they match
for topic_conf in apply_conf["topics"]:
topic_from_conf = Topic.from_dict(topic_conf)
assert not topic_controller.diff_with_cluster(
topic_from_conf
).has_changes, f"Topic configs don't match, diff is {topic_controller.diff_with_cluster(topic_from_conf)}"
