def describe_consumergroup(state: State, consumer_id: str, all_partitions: bool, output_format: str):
"""Return information on group coordinator, offsets, watermarks, lag, and various metadata
for consumer group CONSUMER_GROUP."""
consumer_group = ConsumerGroupController(state.cluster).get_consumergroup(consumer_id)
consumer_group_desc = consumer_group.describe(verbose=all_partitions)
click.echo(format_output(consumer_group_desc, output_format))
def test_set_offsets_offset_to_delta_all_topics(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
interactive_cli_runner.invoke(
esque,
args=["set", "offsets", consumer_group, "--offset-by-delta", "-2"],
input="y\n",
catch_exceptions=False)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 8
def describe_topic(state: State, topic_name: str, consumers: bool, output_format: str):
"""Describe a topic.
Returns information on a given topic and its partitions, with the option of including
all consumer groups that read from the topic.
"""
topic = state.cluster.topic_controller.get_cluster_topic(topic_name)
output_dict = {
"topic": topic_name,
"partitions": [partition.as_dict() for partition in topic.partitions],
"config": topic.config,
}
if consumers:
consumergroup_controller = ConsumerGroupController(state.cluster)
groups = consumergroup_controller.list_consumer_groups()
consumergroups = [
group_name
for group_name in groups
if topic_name in consumergroup_controller.get_consumergroup(group_name).topics
]
output_dict["consumergroups"] = consumergroups
click.echo(format_output(output_dict, output_format))
def test_consumer_group_deletions_piped(
non_interactive_cli_runner: CliRunner,
consumergroup_controller: ConsumerGroupController,
filled_topic,
unittest_config: Config,
):
consumer_groups_to_delete = [
randomly_generated_consumer_groups(filled_topic, unittest_config)
for _ in range(2)
]
remaining_consumer_group = randomly_generated_consumer_groups(
filled_topic, unittest_config)
consumer_groups_pre_deletion = consumergroup_controller.list_consumer_groups(
)
assert all(group in consumer_groups_pre_deletion
for group in consumer_groups_to_delete)
assert remaining_consumer_group in consumer_groups_pre_deletion
assert "not_in_the_list_of_consumers" not in consumer_groups_pre_deletion
result = non_interactive_cli_runner.invoke(
esque,
args=["delete", "consumergroup", "--no-verify"],
input="\n".join(consumer_groups_to_delete +
["not_in_the_list_of_consumers"]),
catch_exceptions=False,
)
assert result.exit_code == 0
consumer_groups_post_deletion = consumergroup_controller.list_consumer_groups(
)
assert all(group not in consumer_groups_post_deletion
for group in consumer_groups_to_delete)
assert remaining_consumer_group in consumer_groups_post_deletion
assert all(existing_group in consumer_groups_pre_deletion
for existing_group in consumer_groups_post_deletion)
def test_edit_offsets(
monkeypatch: MonkeyPatch,
interactive_cli_runner,
topic: str,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(consumer_id=consumer_group).describe(
partitions=True
)
offset_config = {"offsets": [{"topic": topic, "partition": 0, "offset": 1}]}
def mock_edit_function(text=None, editor=None, env=None, require_save=None, extension=None, filename=None):
return yaml.dump(offset_config, default_flow_style=False)
monkeypatch.setattr(click, "edit", mock_edit_function)
result = interactive_cli_runner.invoke(
esque, args=["edit", "offsets", consumer_group, "-t", topic], input="y\n", catch_exceptions=False
)
assert result.exit_code == 0
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(consumer_id=consumer_group).describe(
partitions=True
)
assert consumergroup_desc_before["offsets"][topic][0]["consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0]["consumer_offset"] == 1
def test_delete_nonexistent_consumer_groups(
partly_read_consumer_group: str,
consumergroup_controller: ConsumerGroupController):
groups_before = consumergroup_controller.list_consumer_groups()
consumergroup_controller.delete_consumer_groups(
consumer_ids=["definitely_nonexistent"])
groups_after = consumergroup_controller.list_consumer_groups()
assert groups_before == groups_after
def test_delete_consumer_groups(
partly_read_consumer_group: str,
consumergroup_controller: ConsumerGroupController):
groups_before_deletion = consumergroup_controller.list_consumer_groups()
assert partly_read_consumer_group in groups_before_deletion
consumergroup_controller.delete_consumer_groups(
consumer_ids=[partly_read_consumer_group])
groups_after_deletion = consumergroup_controller.list_consumer_groups()
assert partly_read_consumer_group not in groups_after_deletion
def test_consumer_group_offset_set(
consumergroup_controller: ConsumerGroupController,
filled_topic: Topic):
topic = TopicPartition(topic=filled_topic.name, offset=5, partition=0)
consumer_group_name = "non_existing"
consumergroup_controller.commit_offsets(consumer_group_name, [topic])
consumer_group: ConsumerGroup = consumergroup_controller.get_consumer_group(
consumer_group_name)
offsets = consumer_group.get_offsets()
assert offsets[filled_topic.name][0] == 5
def set_offsets(
state: State,
consumer_id: str,
topic_name: str,
offset_to_value: int,
offset_by_delta: int,
offset_to_timestamp: str,
offset_from_group: str,
):
"""Set consumer group offsets.
Change or set the offset of a consumer group for a topic, i.e. the message number the consumer group will read next.
This can be done by specifying an explicit offset (--offset-to-value), a delta to shift the current offset forwards
or backwards (--offset-by-delta), a timestamp in which the offset of the first message on or after the timestamp is
taken (--offset-by-timestamp), or a group from which to copy the offsets from. In the case that the consumer group
reads from more than one topic, a regular expression can be given to specify the offset of which topic to change.
NOTE: the default is to change the offset for all topics."""
logger = logging.getLogger(__name__)
consumergroup_controller = ConsumerGroupController(state.cluster)
offset_plan = consumergroup_controller.create_consumer_group_offset_change_plan(
consumer_id=consumer_id,
topic_name=topic_name if topic_name else ".*",
offset_to_value=offset_to_value,
offset_by_delta=offset_by_delta,
offset_to_timestamp=offset_to_timestamp,
offset_from_group=offset_from_group,
)
if offset_plan and len(offset_plan) > 0:
click.echo(green_bold("Proposed offset changes: "))
offset_plan.sort(key=attrgetter("topic_name", "partition_id"))
for topic_name, group in groupby(offset_plan, attrgetter("topic_name")):
group = list(group)
max_proposed = max(len(str(elem.proposed_offset)) for elem in group)
max_current = max(len(str(elem.current_offset)) for elem in group)
for plan_element in group:
new_offset = str(plan_element.proposed_offset).rjust(max_proposed)
format_args = dict(
topic_name=plan_element.topic_name,
partition_id=plan_element.partition_id,
current_offset=plan_element.current_offset,
new_offset=new_offset if plan_element.offset_equal else red_bold(new_offset),
max_current=max_current,
)
click.echo(
"Topic: {topic_name}, partition {partition_id:2}, current offset: {current_offset:{max_current}}, new offset: {new_offset}".format(
**format_args
)
)
if ensure_approval("Are you sure?", no_verify=state.no_verify):
consumergroup_controller.edit_consumer_group_offsets(consumer_id=consumer_id, offset_plan=offset_plan)
else:
logger.info("No changes proposed.")
return
def delete_consumergroup(state: State, consumergroup_id: Tuple[str]):
"""Delete consumer groups"""
consumer_groups = list(consumergroup_id) + get_piped_stdin_arguments()
consumergroup_controller: ConsumerGroupController = ConsumerGroupController(
state.cluster)
current_consumergroups = consumergroup_controller.list_consumer_groups()
existing_consumer_groups: List[str] = []
for group in consumer_groups:
if group in current_consumergroups:
click.echo(f"Deleting {click.style(group, fg='green')}")
existing_consumer_groups.append(group)
else:
click.echo(
f"Skipping {click.style(group, fg='yellow')} — does not exist")
if not existing_consumer_groups:
click.echo(
click.style(
"The provided list contains no existing consumer groups.",
fg="red"))
else:
if ensure_approval("Are you sure?", no_verify=state.no_verify):
consumergroup_controller.delete_consumer_groups(
existing_consumer_groups)
current_consumergroups = consumergroup_controller.list_consumer_groups(
)
assert all(consumer_group not in current_consumergroups
for consumer_group in existing_consumer_groups)
click.echo(
click.style(
f"Consumer groups '{existing_consumer_groups}' successfully deleted.",
fg="green"))
def list_consumergroups(ctx, args, incomplete):
state = ctx.ensure_object(State)
return [
group
for group in ConsumerGroupController(state.cluster).list_consumer_groups()
if group.startswith(incomplete)
]
def test_set_offsets_offset_to_timestamp_value(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
messages = produce_text_test_messages(producer=producer,
topic_name=topic,
amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
fifth_message = messages[4]
timestamp = fifth_message.timestamp
dt = pendulum.from_timestamp(round(timestamp / 1000) - 1)
interactive_cli_runner.invoke(
esque,
args=[
"set",
"offsets",
consumer_group,
"--topic-name",
topic,
"--offset-to-timestamp",
dt.format("YYYY-MM-DDTHH:mm:ss"),
],
input="y\n",
catch_exceptions=False,
)
# Check assertions:
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 4
def create_consumergroup(state: State, consumergroup_id: str, topics: str):
"""
Create consumer group for several topics using format <topic_name>[partition]=offset.
[partition] and offset are optional.
Default value for offset is 0.
If there is no partition, consumer group will be assigned to all topic partitions.
"""
pattern = re.compile(
r"(?P<topic_name>[\w.-]+)(?:\[(?P<partition>\d+)\])?(?:=(?P<offset>\d+))?"
)
topic_controller = state.cluster.topic_controller
clean_topics: List[TopicPartition] = []
msg = ""
for topic in topics:
match = pattern.match(topic)
if not match:
raise ValidationException("Topic name should be present")
topic = match.group("topic_name")
partition_match = match.group("partition")
offset_match = match.group("offset")
offset = int(offset_match) if offset_match else 0
if not partition_match:
topic_config = topic_controller.get_cluster_topic(topic)
watermarks = topic_config.watermarks
for part, wm in watermarks.items():
offset = offset if wm.high >= offset else 0
clean_topics.append(
TopicPartition(topic=topic, partition=part, offset=offset))
msg += f"{topic}[{part}]={offset}\n"
else:
partition = int(partition_match)
clean_topics.append(
TopicPartition(topic=topic, partition=partition,
offset=offset))
msg += f"{topic}[{partition}]={offset}\n"
if not ensure_approval(
f"This will create the consumer group '{consumergroup_id}' with initial offsets:\n"
+ msg + "\nAre you sure?",
no_verify=state.no_verify,
):
click.echo(click.style("Aborted!", bg="red"))
return
consumergroup_controller: ConsumerGroupController = ConsumerGroupController(
state.cluster)
created_consumergroup: ConsumerGroup = consumergroup_controller.create_consumer_group(
consumergroup_id, offsets=clean_topics)
click.echo(
click.style(
f"Consumer group '{created_consumergroup.id}' was successfully created",
fg="green"))
def set_offsets(
state: State,
consumer_id: str,
topic_name: str,
offset_to_value: int,
offset_by_delta: int,
offset_to_timestamp: str,
offset_from_group: str,
):
"""Set consumer group offsets.
Change or set the offset of a consumer group for a topic, i.e. the message number the consumer group will read next.
This can be done by specifying an explicit offset (--offset-to-value), a delta to shift the current offset forwards
or backwards (--offset-by-delta), a timestamp in which the offset of the first message on or after the timestamp is
taken (--offset-by-timestamp), or a group from which to copy the offsets from. In the case that the consumer group
reads from more than one topic, a regular expression can be given to specify the offset of which topic to change.
NOTE: the default is to change the offset for all topics."""
logger = logging.getLogger(__name__)
consumergroup_controller = ConsumerGroupController(state.cluster)
offset_plan = consumergroup_controller.create_consumer_group_offset_change_plan(
consumer_id=consumer_id,
topic_name=topic_name if topic_name else ".*",
offset_to_value=offset_to_value,
offset_by_delta=offset_by_delta,
offset_to_timestamp=offset_to_timestamp,
offset_from_group=offset_from_group,
)
if offset_plan and len(offset_plan) > 0:
click.echo(green_bold("Proposed offset changes: "))
pretty_offset_plan(offset_plan)
if ensure_approval("Are you sure?", no_verify=state.no_verify):
consumergroup_controller.edit_consumer_group_offsets(
consumer_id=consumer_id, offset_plan=offset_plan)
else:
logger.info("No changes proposed.")
return
def test_offset_not_committed(
non_interactive_cli_runner: CliRunner,
consumergroup_controller: ConsumerGroupController):
result = non_interactive_cli_runner.invoke(esque,
args=["ping", "--no-verify"],
catch_exceptions=False)
assert result.exit_code == 0
# cannot use pytest.raises(ConsumerGroupDoesNotExistException) because other tests may have committed offsets
# for this group
try:
data = consumergroup_controller.get_consumer_group(
config.ESQUE_GROUP_ID).describe(partitions=True)
assert config.PING_TOPIC.encode() not in data["offsets"]
except ConsumerGroupDoesNotExistException:
pass
def test_set_offsets_offset_from_group(
topic: str,
interactive_cli_runner,
producer: ConfluenceProducer,
consumer_group: str,
target_consumer_group: str,
consumergroup_controller: ConsumerGroupController,
):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
consumergroup_controller.commit_offsets(
consumer_group, [TopicPartition(topic=topic, partition=0, offset=10)])
consumergroup_desc_before = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
interactive_cli_runner.invoke(
esque,
args=["set", "offsets", consumer_group, "--offset-by-delta", "-2"],
input="y\n",
catch_exceptions=False)
consumergroup_desc_after = consumergroup_controller.get_consumer_group(
consumer_id=consumer_group).describe(partitions=True)
# create a new consumer in a separate group and consume just one message
consumergroup_controller.commit_offsets(
target_consumer_group,
[TopicPartition(topic=topic, partition=0, offset=1)])
interactive_cli_runner.invoke(
esque,
args=[
"set", "offsets", target_consumer_group, "--offset-from-group",
consumer_group
],
input="y\n",
catch_exceptions=False,
)
consumergroup_desc_target = consumergroup_controller.get_consumer_group(
consumer_id=target_consumer_group).describe(partitions=True)
assert consumergroup_desc_before["offsets"][topic][0][
"consumer_offset"] == 10
assert consumergroup_desc_after["offsets"][topic][0][
"consumer_offset"] == 8
assert consumergroup_desc_target["offsets"][topic][0][
"consumer_offset"] == 8
def test_describe_topic_last_timestamp_does_not_commit(
non_interactive_cli_runner: CliRunner, topic: str,
consumergroup_controller: ConsumerGroupController, producer):
produce_text_test_messages(producer=producer, topic_name=topic, amount=10)
result = non_interactive_cli_runner.invoke(
esque,
args=["describe", "topic", topic, "--last-timestamp"],
catch_exceptions=False)
assert result.exit_code == 0
output = result.output
check_described_topic(output)
# cannot use pytest.raises(ConsumerGroupDoesNotExistException) because other tests may have committed offsets
# for this group
try:
data = consumergroup_controller.get_consumer_group(
config.ESQUE_GROUP_ID).describe(partitions=True)
assert topic.encode() not in data["offsets"]
except ConsumerGroupDoesNotExistException:
pass
def edit_offsets(state: State, consumer_id: str, topic_name: str):
"""Edit a topic.
Open the offsets of the consumer group in the default editor. If the user saves upon exiting the editor,
all the offsets will be set to the given values.
"""
logger = logging.getLogger(__name__)
consumergroup_controller = ConsumerGroupController(state.cluster)
consumer_group_state, offset_plans = consumergroup_controller.read_current_consumer_group_offsets(
consumer_id=consumer_id,
topic_name_expression=topic_name if topic_name else ".*")
if consumer_group_state != "Empty":
logger.error(
"Consumergroup {} is not empty. Setting offsets is only allowed for empty consumer groups."
.format(consumer_id))
sorted_offset_plan = list(offset_plans.values())
sorted_offset_plan.sort(key=attrgetter("topic_name", "partition_id"))
offset_plan_as_yaml = {
"offsets": [{
"topic": element.topic_name,
"partition": element.partition_id,
"offset": element.current_offset
} for element in sorted_offset_plan]
}
_, new_conf = edit_yaml(str(offset_plan_as_yaml),
validator=validation.validate_offset_config)
for new_offset in new_conf["offsets"]:
plan_key: str = f"{new_offset['topic']}::{new_offset['partition']}"
if plan_key in offset_plans:
final_value, error, message = ConsumerGroupController.select_new_offset_for_consumer(
requested_offset=new_offset["offset"],
offset_plan=offset_plans[plan_key])
if error:
logger.error(message)
offset_plans[plan_key].proposed_offset = final_value
if offset_plans and len(offset_plans) > 0:
click.echo(green_bold("Proposed offset changes: "))
pretty_offset_plan(list(offset_plans.values()))
if ensure_approval("Are you sure?", no_verify=state.no_verify):
consumergroup_controller.edit_consumer_group_offsets(
consumer_id=consumer_id,
offset_plan=list(offset_plans.values()))
else:
logger.info("No changes proposed.")
return
def test_offset_not_committed(
avro_producer: AvroProducer,
source_topic: Tuple[str, int],
non_interactive_cli_runner: CliRunner,
consumergroup_controller: ConsumerGroupController,
):
source_topic_id, _ = source_topic
produce_avro_test_messages(avro_producer, topic_name=source_topic_id)
non_interactive_cli_runner.invoke(esque,
args=[
"consume", "--stdout", "--numbers",
"10", "--avro", source_topic_id
],
catch_exceptions=False)
# cannot use pytest.raises(ConsumerGroupDoesNotExistException) because other tests may have committed offsets
# for this group
try:
data = consumergroup_controller.get_consumer_group(
config.ESQUE_GROUP_ID).describe(partitions=True)
assert source_topic_id.encode() not in data["offsets"]
except ConsumerGroupDoesNotExistException:
pass
def test_get_consumer_group(partly_read_consumer_group: str,
consumergroup_controller: ConsumerGroupController):
instance = consumergroup_controller.get_consumer_group(
partly_read_consumer_group)
assert isinstance(instance, ConsumerGroup)
def consumergroup_controller(cluster: Cluster):
yield ConsumerGroupController(cluster)
def consumergroup_instance(partly_read_consumer_group: str,
consumergroup_controller: ConsumerGroupController):
yield consumergroup_controller.get_consumer_group(
partly_read_consumer_group)
def test_list_consumer_groups(
partly_read_consumer_group: str,
consumergroup_controller: ConsumerGroupController):
groups = consumergroup_controller.list_consumer_groups()
assert partly_read_consumer_group in groups
def get_consumergroups(state: State, output_format: str):
"""List all consumer groups."""
groups = ConsumerGroupController(state.cluster).list_consumer_groups()
click.echo(format_output(groups, output_format))