def get_timestamp(state: State, topic_name: str, offset: str,
output_format: str):
"""Get Timestamps for given offset.
Gets the timestamp for the message(s) at or right after OFFSET in topic TOPIC_NAME.
If the topic has multiple partitions, the OFFSET wil be used for every partition.
If there is no message at OFFSET, the next available offset will be used.
If there is no message at all after OFFSET, offset will be `-1` and timestamp will be `None` in the output for the
corresponding partition.
In the Kafka world, -1 corresponds to the position after the last known message i.e. the end of the topic partition
_not including_ the last message in the partition.
OFFSET must be a positive integer or `first`/`last` in order to read the first or last message from the partition(s).
"""
if isinstance(offset, str) and (offset.lower() in ["first", "last"]):
offset = offset.lower()
elif offset.isdigit():
offset = int(offset)
else:
raise ValidationException(
'Offset must be a positive integer, "first" or "last"')
offsets = state.cluster.topic_controller.get_timestamp_of_closest_offset(
topic_name=topic_name, offset=offset)
output_offset_data(offsets, output_format)
def create_topic(
state: State, topic_name: str, template_topic: str, partitions: Optional[int], replication_factor: Optional[int]
):
"""Create a topic.
Create a topic called TOPIC_NAME with the option of providing a template topic, <template_topic>,
from which all the configuration options will be copied.
If both <template_topic> and any of the <partitions> or <replication-factor> options are given, then <partitions>
or <replication-factor> takes precedence over corresponding attributes of <template_topic>.
"""
topic_controller = state.cluster.topic_controller
if topic_controller.topic_exists(topic_name):
raise ValidationException(f"Topic {topic_name!r} already exists.")
if template_topic:
topic = topic_from_template(template_topic, partitions, replication_factor, topic_controller, topic_name)
else:
topic = topic_with_defaults(partitions, replication_factor, state, topic_name)
if not ensure_approval(
f"Create topic {blue_bold(topic.name)} "
+ f"with replication factor {blue_bold(str(topic.replication_factor))} "
+ f"and {blue_bold(str(topic.num_partitions))} partition"
+ ("s" if topic.num_partitions != 1 else "")
+ f" in context {blue_bold(state.config.current_context)}?",
no_verify=state.no_verify,
):
click.echo(click.style("Aborted!", bg="red"))
return
topic_controller.create_topics([topic])
click.echo(click.style(f"Topic with '{topic.name}' successfully created.", fg="green"))
def validate_esque_config(esque_config: Dict) -> None:
validate(esque_config, SCHEMA_DIR / "esque_config.yaml")
ctx = esque_config["current_context"]
all_ctx = sorted(esque_config["contexts"])
if ctx not in all_ctx:
raise ValidationException(
f"Context {ctx} does not exist. Available contexts {all_ctx}")
def create_consumergroup(state: State, consumergroup_id: str, topics: str):
"""
Create consumer group for several topics using format <topic_name>[partition]=offset.
[partition] and offset are optional.
Default value for offset is 0.
If there is no partition, consumer group will be assigned to all topic partitions.
"""
pattern = re.compile(
r"(?P<topic_name>[\w.-]+)(?:\[(?P<partition>\d+)\])?(?:=(?P<offset>\d+))?"
)
topic_controller = state.cluster.topic_controller
clean_topics: List[TopicPartition] = []
msg = ""
for topic in topics:
match = pattern.match(topic)
if not match:
raise ValidationException("Topic name should be present")
topic = match.group("topic_name")
partition_match = match.group("partition")
offset_match = match.group("offset")
offset = int(offset_match) if offset_match else 0
if not partition_match:
topic_config = topic_controller.get_cluster_topic(topic)
watermarks = topic_config.watermarks
for part, wm in watermarks.items():
offset = offset if wm.high >= offset else 0
clean_topics.append(
TopicPartition(topic=topic, partition=part, offset=offset))
msg += f"{topic}[{part}]={offset}\n"
else:
partition = int(partition_match)
clean_topics.append(
TopicPartition(topic=topic, partition=partition,
offset=offset))
msg += f"{topic}[{partition}]={offset}\n"
if not ensure_approval(
f"This will create the consumer group '{consumergroup_id}' with initial offsets:\n"
+ msg + "\nAre you sure?",
no_verify=state.no_verify,
):
click.echo(click.style("Aborted!", bg="red"))
return
consumergroup_controller: ConsumerGroupController = ConsumerGroupController(
state.cluster)
created_consumergroup: ConsumerGroup = consumergroup_controller.create_consumer_group(
consumergroup_id, offsets=clean_topics)
click.echo(
click.style(
f"Consumer group '{created_consumergroup.id}' was successfully created",
fg="green"))
def describe_broker(state: State, broker: str, output_format: str):
"""Return configuration options for broker BROKER. BROKER can be given with broker id (integer),
the host name (if hostname is unique), or socket address ('hostname:port')"""
if broker.isdigit():
broker = Broker.from_id(state.cluster, broker).describe()
elif ":" not in broker:
broker = Broker.from_host(state.cluster, broker).describe()
else:
try:
host, port = broker.split(":")
broker = Broker.from_host_and_port(state.cluster, host, int(port)).describe()
except ValueError:
raise ValidationException("BROKER must either be the broker id, the hostname, or in the form 'host:port'")
click.echo(format_output(broker, output_format))
def apply(state: State, file: str):
"""Apply a set of topic configurations.
Create new topics and apply changes to existing topics, as specified in the config yaml file <file>.
"""
# Get topic data based on the YAML
yaml_topic_configs = yaml.safe_load(open(file)).get("topics")
yaml_topics = [Topic.from_dict(conf) for conf in yaml_topic_configs]
yaml_topic_names = [t.name for t in yaml_topics]
if not len(yaml_topic_names) == len(set(yaml_topic_names)):
raise ValidationException("Duplicate topic names in the YAML!")
# Get topic data based on the cluster state
topic_controller = state.cluster.topic_controller
cluster_topics = topic_controller.list_topics(search_string="|".join(yaml_topic_names))
cluster_topic_names = [t.name for t in cluster_topics]
# Calculate changes
to_create = [yaml_topic for yaml_topic in yaml_topics if yaml_topic.name not in cluster_topic_names]
to_edit = [
yaml_topic
for yaml_topic in yaml_topics
if yaml_topic not in to_create and topic_controller.diff_with_cluster(yaml_topic).has_changes
]
to_edit_diffs = {t.name: topic_controller.diff_with_cluster(t) for t in to_edit}
to_ignore = [yaml_topic for yaml_topic in yaml_topics if yaml_topic not in to_create and yaml_topic not in to_edit]
# Sanity check - the 3 groups of topics should be complete and have no overlap
assert (
set(to_create).isdisjoint(set(to_edit))
and set(to_create).isdisjoint(set(to_ignore))
and set(to_edit).isdisjoint(set(to_ignore))
and len(to_create) + len(to_edit) + len(to_ignore) == len(yaml_topics)
)
# Print diffs so the user can check
click.echo(pretty_unchanged_topic_configs(to_ignore))
click.echo(pretty_new_topic_configs(to_create))
click.echo(pretty_topic_diffs(to_edit_diffs))
# Check for actionable changes
if len(to_edit) + len(to_create) == 0:
click.echo("No changes detected, aborting!")
return
# Warn users & abort when replication & num_partition changes are attempted
if any(not diff.is_valid for _, diff in to_edit_diffs.items()):
click.echo(
"Changes to `replication_factor` and `num_partitions` can not be applied on already existing topics."
)
click.echo("Cancelling due to invalid changes")
return
# Get approval
if not ensure_approval("Apply changes?", no_verify=state.no_verify):
click.echo("Cancelling changes")
return
# apply changes
topic_controller.create_topics(to_create)
topic_controller.alter_configs(to_edit)
# output confirmation
changes = {"unchanged": len(to_ignore), "created": len(to_create), "changed": len(to_edit)}
click.echo(click.style(pretty({"Successfully applied changes": changes}), fg="green"))