def test_get_topics_with_prefix( non_interactive_cli_runner: CliRunner, topic_controller: TopicController, confluent_admin_client: confluent_kafka.admin.AdminClient, ): topic_base = "".join(random.choices(ascii_letters, k=5)) prefix_1 = "ab" prefix_2 = "fx" new_topics = [ prefix_1 + topic_base, prefix_2 + topic_base, prefix_1 + prefix_2 + topic_base ] topic_controller.create_topics( [Topic(new_topic, replication_factor=1) for new_topic in new_topics]) confluent_admin_client.poll(timeout=1) result = non_interactive_cli_runner.invoke(get_topics, ["-p", prefix_1, "-o", "json"], catch_exceptions=False) assert result.exit_code == 0 retrieved_topics = json.loads(result.output) assert len(retrieved_topics) > 1 for retrieved_topic in retrieved_topics: assert retrieved_topic.startswith(prefix_1)
def create_consumer_group_offset_change_plan( self, consumer_id: str, topic_name: str, offset_to_value: Optional[int], offset_by_delta: Optional[int], offset_to_timestamp: Optional[str], offset_from_group: Optional[str], ) -> List[ConsumerGroupOffsetPlan]: consumer_group_state, offset_plans = self._read_current_consumergroup_offsets( consumer_id=consumer_id, topic_name_expression=topic_name) if consumer_group_state == "Dead": self._logger.error( "The consumer group {} does not exist.".format(consumer_id)) return None elif consumer_group_state == "Empty": if offset_to_value: for plan_element in offset_plans.values(): (allowed_offset, error, message) = self._select_new_offset_for_consumer( offset_to_value, plan_element) plan_element.proposed_offset = allowed_offset if error: self._logger.error(message) elif offset_by_delta: for plan_element in offset_plans.values(): requested_offset = plan_element.current_offset + offset_by_delta (allowed_offset, error, message) = self._select_new_offset_for_consumer( requested_offset, plan_element) plan_element.proposed_offset = allowed_offset if error: self._logger.error(message) elif offset_to_timestamp: timestamp_limit = pendulum.parse(offset_to_timestamp) current_offset_dict = TopicController( self.cluster, None).get_offsets_closest_to_timestamp( topic_name=topic_name, timestamp_limit=timestamp_limit) for plan_element in offset_plans.values(): plan_element.current_offset = current_offset_dict.get( plan_element.partition_id, 0) elif offset_from_group: _, mirror_consumer_group = self._read_current_consumergroup_offsets( consumer_id=offset_from_group, topic_name_expression=topic_name) for key, value in mirror_consumer_group.items(): if key in offset_plans.keys(): offset_plans[ key].proposed_offset = value.current_offset else: value.current_offset = 0 offset_plans[key] = value return list(offset_plans.values()) else: self._logger.error( "Consumergroup {} is not empty. Use the {} option if you want to override this safety mechanism." .format(consumer_id, red_bold("--force"))) return list(offset_plans.values())
def test_topic_creation_raises_for_wrong_config( topic_controller: TopicController, confluent_admin_client: confluent_kafka.admin.AdminClient, topic_id: str): topics = confluent_admin_client.list_topics(timeout=5).topics.keys() assert topic_id not in topics # We only have 1 broker for tests, so a higher replication should fail with pytest.raises(KafkaException): topic_controller.create_topics([Topic(topic_id, replication_factor=2)])
def test_topic_creation_works( topic_controller: TopicController, confluent_admin_client: confluent_kafka.admin.AdminClient, topic_id: str): topics = confluent_admin_client.list_topics(timeout=5).topics.keys() assert topic_id not in topics topic_controller.create_topics([Topic(topic_id, replication_factor=1)]) topics = confluent_admin_client.list_topics(timeout=5).topics.keys() assert topic_id in topics
def test_topic_diff(topic_controller: TopicController, topic_id: str): # the value we get from cluster configs is as string # testing against this is important to ensure consistency default_delete_retention = "86400000" topic_conf = { "name": topic_id, "replication_factor": 1, "num_partitions": 50, "config": { "cleanup.policy": "compact" }, } get_diff = topic_controller.diff_with_cluster conf = json.loads(json.dumps(topic_conf)) topic = Topic.from_dict(conf) topic_controller.create_topics([topic]) assert not get_diff( topic).has_changes, "Shouldn't have diff on just created topic" conf = json.loads(json.dumps(topic_conf)) conf["config"]["cleanup.policy"] = "delete" topic = Topic.from_dict(conf) diff = TopicDiff().set_diff("cleanup.policy", "compact", "delete") assert get_diff(topic) == diff, "Should have a diff on cleanup.policy" conf = json.loads(json.dumps(topic_conf)) conf["config"]["delete.retention.ms"] = 1500 topic = Topic.from_dict(conf) diff = TopicDiff().set_diff("delete.retention.ms", default_delete_retention, 1500) assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms" # the same as before, but this time with string values conf = json.loads(json.dumps(topic_conf)) conf["config"]["delete.retention.ms"] = "1500" topic = Topic.from_dict(conf) diff = TopicDiff().set_diff("delete.retention.ms", default_delete_retention, "1500") assert get_diff(topic) == diff, "Should have a diff on delete.retention.ms" conf = json.loads(json.dumps(topic_conf)) conf["num_partitions"] = 3 topic = Topic.from_dict(conf) diff = TopicDiff().set_diff("num_partitions", 50, 3) assert get_diff(topic) == diff, "Should have a diff on num_partitions" conf = json.loads(json.dumps(topic_conf)) conf["replication_factor"] = 3 topic = Topic.from_dict(conf) diff = TopicDiff().set_diff("replication_factor", 1, 3) assert get_diff(topic) == diff, "Should have a diff on replication_factor"
def test_edit_topic_works( interactive_cli_runner: CliRunner, monkeypatch: MonkeyPatch, topic_controller: TopicController, confluent_admin_client: confluent_kafka.admin.AdminClient, topic: str, ): topics = confluent_admin_client.list_topics(timeout=5).topics.keys() assert topic in topics config_dict = { "config": { "cleanup.policy": "delete", "compression.type": "producer", "delete.retention.ms": "123456789", "file.delete.delay.ms": "60000", "flush.messages": "123456789", "flush.ms": "9223372036854775807", "follower.replication.throttled.replicas": "", "index.interval.bytes": "4096", "leader.replication.throttled.replicas": "", "max.message.bytes": "1000012", "message.downconversion.enable": "true", "message.format.version": "2.2-IV1", "message.timestamp.difference.max.ms": "123456789", "message.timestamp.type": "CreateTime", "min.cleanable.dirty.ratio": "0.5", "min.compaction.lag.ms": "0", "min.insync.replicas": "1", "preallocate": "false", "retention.bytes": "-1", "retention.ms": "123456789", "segment.bytes": "123456789", "segment.index.bytes": "123456789", "segment.jitter.ms": "0", "segment.ms": "123456789", "unclean.leader.election.enable": "true", } } def mock_edit_function(text=None, editor=None, env=None, require_save=None, extension=None, filename=None): return yaml.dump(config_dict, default_flow_style=False) monkeypatch.setattr(click, "edit", mock_edit_function) result = interactive_cli_runner.invoke(edit_topic, topic, input="y\n", catch_exceptions=False) assert result.exit_code == 0 topic_config_dict = topic_controller.get_cluster_topic(topic).as_dict( only_editable=True) for key, value in config_dict["config"].items(): assert (key, topic_config_dict["config"][key]) == (key, value)
def _set_offset_to_timestamp(self, offset_plans, offset_to_timestamp, topic_name): timestamp_limit = pendulum.parse(offset_to_timestamp) proposed_offset_dict = TopicController( self.cluster).get_offsets_closest_to_timestamp( topic_name=topic_name, timestamp=timestamp_limit) for plan_element in offset_plans.values(): plan_element.proposed_offset = proposed_offset_dict[ plan_element.partition_id].offset
def test_alter_topic_config_works(topic_controller: TopicController, topic_id: str): initial_topic = Topic(topic_id, config={"cleanup.policy": "delete"}) topic_controller.create_topics([initial_topic]) topic_controller.update_from_cluster(initial_topic) config = initial_topic.config assert config.get("cleanup.policy") == "delete" change_topic = Topic(topic_id, config={"cleanup.policy": "compact"}) topic_controller.alter_configs([change_topic]) topic_controller.update_from_cluster(change_topic) after_changes_applied_topic = topic_controller.get_cluster_topic(topic_id) final_config = after_changes_applied_topic.config assert final_config.get("cleanup.policy") == "compact"
def test_correct_amount_of_messages(mocker, non_interactive_cli_runner: CliRunner, topic_controller: TopicController): delete_topic_mock = mocker.patch.object(TopicController, "delete_topic", mocker.Mock()) result = non_interactive_cli_runner.invoke(ping, catch_exceptions=False) assert result.exit_code == 0 assert delete_topic_mock.call_count == 1 ping_topic = topic_controller.get_cluster_topic(config.PING_TOPIC) assert ping_topic.watermarks[0].high == 10
def test_alter_topic_config_only_changes_mentioned_attributes( topic_controller: TopicController, topic_id: str): initial_topic = Topic(topic_id, config={ "cleanup.policy": "delete", "min.compaction.lag.ms": "1000000" }) topic_controller.create_topics([initial_topic]) topic_controller.update_from_cluster(initial_topic) config = initial_topic.config assert config.get("cleanup.policy") == "delete" assert config.get("min.compaction.lag.ms") == "1000000" change_topic = Topic(topic_id, config={"cleanup.policy": "compact"}) topic_controller.alter_configs([change_topic]) topic_controller.update_from_cluster(change_topic) after_changes_applied_topic = topic_controller.get_cluster_topic(topic_id) final_config = after_changes_applied_topic.config assert final_config.get("cleanup.policy") == "compact" assert final_config.get("min.compaction.lag.ms") == "1000000"
def topic_from_template( template_topic: str, partitions: Optional[int], replication_factor: Optional[int], topic_controller: TopicController, topic_name: str, ) -> Topic: template_config = topic_controller.get_cluster_topic(template_topic) if partitions is None: partitions = template_config.num_partitions if replication_factor is None: replication_factor = template_config.replication_factor config = template_config.config topic = Topic(topic_name, num_partitions=partitions, replication_factor=replication_factor, config=config) return topic
def test_topic_object_works(topic_controller: TopicController, topic: str): topic = topic_controller.get_cluster_topic(topic) assert isinstance(topic, Topic) assert len(topic.watermarks) != 0
def test_topic_listing_works(topic_controller: TopicController, topic: str): topics = topic_controller.list_topics() assert topic in [t.name for t in topics]
def test_watermarks(filled_topic: Topic, topic_controller: TopicController): topic_controller.update_from_cluster(filled_topic) assert filled_topic.watermarks == {0: Watermark(10, 0)}
def topic_controller(self) -> TopicController: if self.__topic_controller is None: self.__topic_controller = TopicController(self, self._config) return self.__topic_controller
def test_apply(interactive_cli_runner: CliRunner, topic_controller: TopicController, topic_id: str): topic_name = f"apply_{topic_id}" topic_1 = { "name": topic_name + "_1", "replication_factor": 1, "num_partitions": 50, "config": { "cleanup.policy": "compact" }, } topic_2 = { "name": topic_name + "_2", "replication_factor": 1, "num_partitions": 5, "config": { "cleanup.policy": "delete", "delete.retention.ms": 50000 }, } apply_conf = {"topics": [topic_1]} # 1: topic creation path = save_yaml(topic_id, apply_conf) result = interactive_cli_runner.invoke(esque, args=["apply", "-f", path], input="Y\n", catch_exceptions=False) assert (result.exit_code == 0 and "Successfully applied changes" in result.output), f"Calling apply failed, error: {result.output}" # 2: change cleanup policy to delete topic_1["config"]["cleanup.policy"] = "delete" path = save_yaml(topic_id, apply_conf) result = interactive_cli_runner.invoke(esque, args=["apply", "-f", path], input="Y\n", catch_exceptions=False) assert (result.exit_code == 0 and "Successfully applied changes" in result.output), f"Calling apply failed, error: {result.output}" # 3: add another topic and change the first one again apply_conf["topics"].append(topic_2) topic_1["config"]["cleanup.policy"] = "compact" path = save_yaml(topic_id, apply_conf) result = interactive_cli_runner.invoke(esque, args=["apply", "-f", path], input="Y\n", catch_exceptions=False) assert (result.exit_code == 0 and "Successfully applied changes" in result.output), f"Calling apply failed, error: {result.output}" # 4: no changes result = interactive_cli_runner.invoke(esque, args=["apply", "-f", path], catch_exceptions=False) assert (result.exit_code == 0 and "No changes detected, aborting" in result.output), f"Calling apply failed, error: {result.output}" # 5: change partitions - this attempt should be cancelled topic_1["num_partitions"] = 3 topic_1["config"]["cleanup.policy"] = "delete" path = save_yaml(topic_id, apply_conf) result = interactive_cli_runner.invoke(esque, args=["apply", "-f", path], input="Y\n", catch_exceptions=False) assert (result.exit_code == 1 and "to `replication_factor` and `num_partitions`" in result.output), f"Calling apply failed, error: {result.output}" # reset config to the old settings again topic_1["num_partitions"] = 50 topic_1["config"]["cleanup.policy"] = "compact" # final: check results in the cluster to make sure they match for topic_conf in apply_conf["topics"]: topic_from_conf = Topic.from_dict(topic_conf) assert not topic_controller.diff_with_cluster( topic_from_conf ).has_changes, f"Topic configs don't match, diff is {topic_controller.diff_with_cluster(topic_from_conf)}"