def transfer(state: State, topic: str, from_context: str, to_context: str,
numbers: int, last: bool, avro: bool, keep_file: bool):
current_timestamp_milliseconds = int(round(time.time() * 1000))
unique_name = topic + "_" + str(current_timestamp_milliseconds)
group_id = "group_for_" + unique_name
directory_name = "message_" + unique_name
base_dir = Path(directory_name)
state.config.context_switch(from_context)
with HandleFileOnFinished(base_dir, keep_file) as working_dir:
number_consumed_messages = _consume_to_file(working_dir, topic,
group_id, from_context,
numbers, avro, last)
if number_consumed_messages == 0:
click.echo(
click.style("Execution stopped, because no messages consumed.",
fg="red"))
click.echo(
bold(
"Possible reasons: The topic is empty or the starting offset was set too high."
))
return
click.echo("\nReady to produce to context " + blue_bold(to_context) +
" and target topic " + blue_bold(topic))
if not ensure_approval("Do you want to proceed?\n",
no_verify=state.no_verify):
return
state.config.context_switch(to_context)
_produce_from_file(topic, to_context, working_dir, avro)
def describe_consumergroup(state, consumer_id, verbose):
try:
consumer_group = ConsumerGroupController(
state.cluster).get_consumergroup(consumer_id)
consumer_group_desc = consumer_group.describe(verbose=verbose)
click.echo(pretty(consumer_group_desc, break_lists=True))
except ConsumerGroupDoesNotExistException:
click.echo(bold(f"Consumer Group {consumer_id} not found."))
def ctx(state, context):
if not context:
for c in state.config.available_contexts:
if c == state.config.current_context:
click.echo(bold(c))
else:
click.echo(c)
if context:
try:
state.config.context_switch(context)
except ContextNotDefinedException:
click.echo(f"Context {context} does not exist")
def describe_topic(state, topic_name):
topic = state.cluster.topic_controller.get_cluster_topic(topic_name)
config = {"Config": topic.config}
click.echo(bold(f"Topic: {topic_name}"))
for partition in topic.partitions:
click.echo(
pretty(
{f"Partition {partition.partition_id}": partition.as_dict()},
break_lists=True))
click.echo(pretty(config))
def ctx(state: State, context: str):
"""List contexts and switch between them.
\b
USAGE:
esque ctx : list available contexts
esque ctx CONTEXT : switch to context CONTEXT
"""
if not context:
for c in state.config.available_contexts:
if c == state.config.current_context:
click.echo(bold(c))
else:
click.echo(c)
if context:
state.config.context_switch(context)
state.config.save()
click.echo(f"Switched to context: {context}.")
def consume(
state: State,
topic: str,
from_context: str,
numbers: int,
match: str,
last: bool,
avro: bool,
directory: str,
consumergroup: str,
preserve_order: bool,
write_to_stdout: bool,
):
"""Consume messages from a topic.
Read messages from a given topic in a given context. These messages can either be written
to files in an automatically generated directory (default behavior), or to STDOUT.
\b
EXAMPLES:
# Consume the first 10 messages from TOPIC in the current context and print them to STDOUT in order.
esque consume --first -n 10 --preserve-order --stdout TOPIC
\b
# Consume <n> messages, starting from the 10th, from TOPIC in the <source_ctx> context and write them to files.
esque consume --match "message.offset > 9" -n <n> TOPIC -f <source_ctx>
\b
# Copy source_topic in first context to destination_topic in second-context.
esque consume -f first-context --stdout source_topic | esque produce -t second-context --stdin destination_topic
"""
current_timestamp_milliseconds = int(round(time.time() * 1000))
consumergroup_prefix = "group_for_"
if directory and write_to_stdout:
raise ValueError("Cannot write to a directory and STDOUT, please pick one!")
if not from_context:
from_context = state.config.current_context
state.config.context_switch(from_context)
if topic not in map(attrgetter("name"), state.cluster.topic_controller.list_topics(get_topic_objects=False)):
raise TopicDoesNotExistException(f"Topic {topic} does not exist!", -1)
if not consumergroup:
consumergroup = consumergroup_prefix + topic + "_" + str(current_timestamp_milliseconds)
if not directory:
directory = Path() / "messages" / topic / str(current_timestamp_milliseconds)
output_directory = Path(directory)
if not write_to_stdout:
click.echo(f"Creating directory {blue_bold(str(output_directory))} if it does not exist.")
output_directory.mkdir(parents=True, exist_ok=True)
click.echo(f"Start consuming from topic {blue_bold(topic)} in source context {blue_bold(from_context)}.")
if preserve_order:
partitions = []
for partition in state.cluster.topic_controller.get_cluster_topic(topic).partitions:
partitions.append(partition.partition_id)
total_number_of_consumed_messages = consume_to_file_ordered(
output_directory=output_directory,
topic=topic,
group_id=consumergroup,
partitions=partitions,
numbers=numbers,
avro=avro,
match=match,
last=last,
write_to_stdout=write_to_stdout,
)
else:
total_number_of_consumed_messages = consume_to_files(
output_directory=output_directory,
topic=topic,
group_id=consumergroup,
numbers=numbers,
avro=avro,
match=match,
last=last,
write_to_stdout=write_to_stdout,
)
if not write_to_stdout:
click.echo(f"Output generated to {blue_bold(str(output_directory))}")
if total_number_of_consumed_messages == numbers or numbers == sys.maxsize:
click.echo(blue_bold(str(total_number_of_consumed_messages)) + " messages consumed.")
else:
click.echo(
"Only found "
+ bold(str(total_number_of_consumed_messages))
+ " messages in topic, out of "
+ blue_bold(str(numbers))
+ " required."
)
def consume(
state: State,
topic: str,
from_context: str,
number: Optional[int],
match: str,
last: bool,
avro: bool,
binary: bool,
directory: str,
consumergroup: str,
preserve_order: bool,
write_to_stdout: bool,
pretty_print: bool,
):
"""Consume messages from a topic.
Read messages from a given topic in a given context. These messages can either be written
to files in an automatically generated directory (default behavior), or to STDOUT.
If writing to STDOUT, then data will be represented as a JSON object with the message key and the message value
always being a string.
With the --avro option, those strings are JSON serialized objects.
With the --binary option those strings contain the base64 encoded binary data.
Without any of the two options, the data in the messages is treated utf-8 encoded strings and will be used as-is.
\b
EXAMPLES:
# Consume the first 10 messages from TOPIC in the current context and print them to STDOUT in order.
esque consume --first -n 10 --preserve-order --pretty-print --stdout TOPIC
\b
# Consume <n> messages, starting from the 10th, from TOPIC in the <source_ctx> context and write them to files.
esque consume --match "message.offset > 9" -n <n> TOPIC -f <source_ctx>
\b
# Extract json objects from keys
esque consume --stdout --avro TOPIC | jq '.key | fromjson'
\b
# Extract binary data from keys (depending on the data this could mess up your console)
esque consume --stdout --binary TOPIC | jq '.key | @base64d'
"""
if not from_context:
from_context = state.config.current_context
state.config.context_switch(from_context)
if not write_to_stdout and not directory:
directory = Path() / "messages" / topic / datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
if binary and avro:
raise ValueError("Cannot set data to be interpreted as binary AND avro.")
builder = PipelineBuilder()
input_message_serializer = create_input_serializer(avro, binary, state)
builder.with_input_message_serializer(input_message_serializer)
input_handler = create_input_handler(consumergroup, from_context, topic)
builder.with_input_handler(input_handler)
output_handler = create_output_handler(directory, write_to_stdout, binary, pretty_print)
builder.with_output_handler(output_handler)
output_message_serializer = create_output_message_serializer(write_to_stdout, directory, avro, binary)
builder.with_output_message_serializer(output_message_serializer)
if last:
start = KafkaHandler.OFFSET_AFTER_LAST_MESSAGE
else:
start = KafkaHandler.OFFSET_AT_FIRST_MESSAGE
builder.with_range(start=start, limit=number)
if preserve_order:
topic_data = Cluster().topic_controller.get_cluster_topic(topic, retrieve_partition_watermarks=False)
builder.with_stream_decorator(yield_messages_sorted_by_timestamp(len(topic_data.partitions)))
if match:
builder.with_stream_decorator(yield_only_matching_messages(match))
counter, counter_decorator = event_counter()
builder.with_stream_decorator(counter_decorator)
pipeline = builder.build()
pipeline.run_pipeline()
if not write_to_stdout:
if counter.message_count == number:
click.echo(blue_bold(str(counter.message_count)) + " messages consumed.")
else:
click.echo(
"Only found "
+ bold(str(counter.message_count))
+ " messages in topic, out of "
+ blue_bold(str(number))
+ " required."
)