def __init__(self, test_context, extra_rp_conf=dict(), num_brokers=3):
def enable_wasm_options():
return dict(
developer_mode=True,
enable_coproc=True,
)
wasm_opts = enable_wasm_options()
wasm_opts.update(extra_rp_conf)
super(WasmTest, self).__init__(test_context,
extra_rp_conf=wasm_opts,
num_brokers=num_brokers)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
def start_redpanda_nodes(self, nodes):
self.start_redpanda(num_nodes=nodes)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
class WasmPartitionMovementTest(PartitionMovementMixin, EndToEndTest):
"""
Tests to ensure that the materialized partition movement feature
is working as expected. This feature has materialized topics move to
where their respective sources are moved to.
"""
def __init__(self, ctx, *args, **kvargs):
super(WasmPartitionMovementTest, self).__init__(
ctx,
*args,
extra_rp_conf={
# Disable leader balancer, as this test is doing its own
# partition movement and the balancer would interfere
'enable_leader_balancer': False,
'enable_coproc': True,
'developer_mode': True,
'auto_create_topics_enabled': False
},
**kvargs)
self._ctx = ctx
self._rpk_tool = None
self._build_tool = None
self.result_data = []
self.mconsumer = None
def start_redpanda_nodes(self, nodes):
self.start_redpanda(num_nodes=nodes)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
def restart_random_redpanda_node(self):
node = random.sample(self.redpanda.nodes, 1)[0]
self.logger.info(f"Randomly killing redpanda node: {node.name}")
self.redpanda.restart_nodes(node)
def _deploy_identity_copro(self, inputs, outputs):
script = WasmScript(inputs=inputs,
outputs=outputs,
script=WasmTemplateRepository.IDENTITY_TRANSFORM)
self._build_tool.build_test_artifacts(script)
self._rpk_tool.wasm_deploy(
script.get_artifact(self._build_tool.work_dir), script.name,
"ducktape")
def _verify_materialized_assignments(self, topic, partition, assignments):
admin = Admin(self.redpanda)
massignments = self._get_assignments(admin, topic, partition)
self.logger.info(
f"materialized assignments for {topic}-{partition}: {massignments}"
)
self._wait_post_move(topic, partition, assignments)
def _grab_input(self, topic):
metadata = self.client().describe_topics()
selected = [x for x in metadata if x['topic'] == topic]
assert len(selected) == 1
partition = random.choice(selected[0]["partitions"])
return selected[0]["topic"], partition["partition"]
def _on_data_consumed(self, record, node):
self.result_data.append(record["value"])
def _start_mconsumer(self, materialized_topic):
self.mconsumer = VerifiableConsumer(
self.test_context,
num_nodes=1,
redpanda=self.redpanda,
topic=materialized_topic,
group_id="test_group",
on_record_consumed=self._on_data_consumed)
self.mconsumer.start()
def _await_consumer(self, limit, timeout):
wait_until(
lambda: self.mconsumer.total_consumed() >= limit,
timeout_sec=timeout,
err_msg=
"Timeout of after %ds while awaiting delivery of %d materialized records, recieved: %d"
% (timeout, limit, self.mconsumer.total_consumed()))
self.mconsumer.stop()
def _prime_env(self):
output_topic = "identity_output2"
self.start_redpanda_nodes(3)
spec = TopicSpec(name="topic2",
partition_count=3,
replication_factor=3)
self.client().create_topic(spec)
self._deploy_identity_copro([spec.name], [output_topic])
self.topic = spec.name
self.start_producer(num_nodes=1, throughput=10000)
self.start_consumer(1)
self.await_startup(min_records=500)
materialized_topic = construct_materialized_topic(
spec.name, output_topic)
def topic_created():
metadata = self.client().describe_topics()
self.logger.info(f"metadata: {metadata}")
return any([x['topic'] == materialized_topic for x in metadata])
wait_until(topic_created, timeout_sec=30, backoff_sec=2)
self._start_mconsumer(materialized_topic)
t, p = self._grab_input(spec.name)
return {
'topic': t,
'partition': p,
'materialized_topic': materialized_topic
}
@cluster(num_nodes=6)
def test_dynamic(self):
"""
Move partitions with active consumer / producer
"""
s = self._prime_env()
for _ in range(5):
_, partition, assignments = self._do_move_and_verify(
s['topic'], s['partition'])
self._verify_materialized_assignments(s['materialized_topic'],
partition, assignments)
# Vaidate input
self.run_validation(min_records=500,
enable_idempotence=False,
consumer_timeout_sec=90)
# Wait for all output
self._await_consumer(self.consumer.total_consumed(), 90)
# Validate number of records & their content
self.logger.info(
f'A: {self.mconsumer.total_consumed()} B: {self.consumer.total_consumed()}'
)
assert self.mconsumer.total_consumed() == self.consumer.total_consumed(
)
# Since the identity copro was deployed, contents of logs should be identical
assert set(self.records_consumed) == set(self.result_data)
@cluster(num_nodes=6)
def test_dynamic_with_failure(self):
s = self._prime_env()
_, partition, assignments = self._do_move_and_verify(
s['topic'], s['partition'])
# Crash a node before verifying, it has a fixed amount of seconds to restart
n = random.sample(self.redpanda.nodes, 1)[0]
self.redpanda.restart_nodes(n)
self._verify_materialized_assignments(s['materialized_topic'],
partition, assignments)
self.run_validation(min_records=500,
enable_idempotence=False,
consumer_timeout_sec=90)
# Wait for all acked, output, significant because due to failure the number of
# actual produced records may be less than expected
num_producer_acked = len(self.producer.acked)
self._await_consumer(num_producer_acked, 90)
# GTE due to the fact that upon error, copro may re-processed already processed
# data depending on when offsets were checkpointed
assert self.mconsumer.total_consumed() >= num_producer_acked
class WasmTest(RedpandaTest):
def __init__(self, test_context, extra_rp_conf=dict(), num_brokers=3):
def enable_wasm_options():
return dict(
developer_mode=True,
enable_coproc=True,
)
wasm_opts = enable_wasm_options()
wasm_opts.update(extra_rp_conf)
super(WasmTest, self).__init__(test_context,
extra_rp_conf=wasm_opts,
num_brokers=num_brokers)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
def _build_script(self, script):
# Produce all data
total_input_records = reduce(lambda acc, x: acc + x[1][0],
script.inputs, 0)
total_output_expected = reduce(lambda acc, x: acc + x[1],
script.outputs, 0)
# Build the script itself
self._build_tool.build_test_artifacts(script)
# Deploy coprocessor
self._rpk_tool.wasm_deploy(
script.get_artifact(self._build_tool.work_dir), "ducktape")
return (total_input_records, total_output_expected)
def _start(self, topic_spec, scripts):
all_materialized = all(
flat_map(
lambda x: [is_materialized_topic(x[0]) for x in x.outputs],
scripts))
if not all_materialized:
raise Exception("All output topics must be materaizlied topics")
def to_output_topic_spec(output_topics):
"""
Create a list of TopicPartitions for the set of output topics.
Must parse the materialzied topic for the input topic to determine
the number of partitions.
"""
input_lookup = dict([(x.name, x) for x in topic_spec])
result = []
for output_topic, _ in output_topics:
src = input_lookup.get(get_source_topic(output_topic))
if src is None:
raise Exception(
"Bad spec, materialized topics source must belong to "
"the input topic spec set")
result.append(
TopicSpec(name=output_topic,
partition_count=src.partition_count,
replication_factor=src.replication_factor,
cleanup_policy=src.cleanup_policy))
return result
def expand_topic_spec(etc):
"""
Convers a TopicSpec iterable to a TopicPartitions list
"""
return flat_map(
lambda spec: [
TopicPartition(spec.name, x)
for x in range(0, spec.partition_count)
], etc)
# Calcualte expected records on all inputs / outputs
total_inputs, total_outputs = reduce(
merge_two_tuple, [self._build_script(x) for x in scripts], (0, 0))
input_tps = expand_topic_spec(topic_spec)
output_tps = expand_topic_spec(
to_output_topic_spec(
flat_map(lambda script: script.outputs, scripts)))
producers = []
for script in scripts:
for topic, producer_opts in script.inputs:
num_records, records_size = producer_opts
try:
producer = NativeKafkaProducer(self.redpanda.brokers(),
topic, num_records,
records_size)
producer.start()
producers.append(producer)
except Exception as e:
self.logger.error(
f"Failed to create NativeKafkaProducer: {e}")
raise
input_consumer = None
output_consumer = None
try:
input_consumer = NativeKafkaConsumer(self.redpanda.brokers(),
input_tps, total_inputs)
output_consumer = NativeKafkaConsumer(self.redpanda.brokers(),
output_tps, total_outputs)
except Exception as e:
self.logger.error(f"Failed to create NativeKafkaConsumer: {e}")
raise
input_consumer.start()
output_consumer.start()
def all_done():
# Uncomment to periodically see the amt of data read
# self.logger.info("Input: %d" %
# input_consumer.results.num_records())
# self.logger.info("Output: %d" %
# output_consumer.results.num_records())
return input_consumer.is_finished() and \
output_consumer.is_finished()
timeout, backoff = self.wasm_test_timeout()
wait_until(all_done, timeout_sec=timeout, backoff_sec=backoff)
try:
input_consumer.join()
output_consumer.join()
[x.join() for x in producers]
except Exception as e:
self.logger.error("Exception occured in background thread: {e}")
raise e
return (input_consumer.results, output_consumer.results)
def wasm_test_timeout(self):
"""
2-tuple representing timeout(0) and backoff interval(1)
"""
return (30, 1)
class WasmTest(RedpandaTest):
def __init__(self, test_context, extra_rp_conf=dict(), num_brokers=3):
def enable_wasm_options():
return dict(
developer_mode=True,
enable_coproc=True,
)
wasm_opts = enable_wasm_options()
wasm_opts.update(extra_rp_conf)
super(WasmTest, self).__init__(test_context,
extra_rp_conf=wasm_opts,
num_brokers=num_brokers)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
self._input_consumer = None
self._output_consumer = None
self._producers = None
def _build_script(self, script):
# Build the script itself
self._build_tool.build_test_artifacts(script)
# Deploy coprocessor
self._rpk_tool.wasm_deploy(
script.get_artifact(self._build_tool.work_dir), script.name,
"ducktape")
def restart_wasm_engine(self, node):
self.logger.info(
f"Begin manually triggered restart of wasm engine on node {node}")
node.account.kill_process("bin/node", clean_shutdown=False)
self.redpanda.start_wasm_engine(node)
def restart_redpanda(self, node):
self.logger.info(
f"Begin manually triggered restart of redpanda on node {node}")
self.redpanda.restart_nodes(node)
def start(self, topic_spec, scripts):
def to_output_topic_spec(output_topics):
"""
Create a list of TopicPartitions for the set of output topics.
Must parse the materialzied topic for the input topic to determine
the number of partitions.
"""
result = []
for src, _, _ in topic_spec:
materialized_topics = [
TopicSpec(name=construct_materialized_topic(
src.name, dest),
partition_count=src.partition_count,
replication_factor=src.replication_factor,
cleanup_policy=src.cleanup_policy)
for dest in output_topics
]
result += materialized_topics
return result
def expand_topic_spec(etc):
"""
Convers a TopicSpec iterable to a TopicPartitions list
"""
return set(
flat_map(
lambda spec: [
TopicPartition(spec.name, x)
for x in range(0, spec.partition_count)
], etc))
for script in scripts:
self._build_script(script)
input_tps = expand_topic_spec([x[0] for x in topic_spec])
output_tps = expand_topic_spec(
to_output_topic_spec(
flat_map(lambda script: script.outputs, scripts)))
# Calcualte expected records on all inputs / outputs
total_inputs = reduce(lambda acc, x: acc + x[1], topic_spec, 0)
def accrue(acc, output_topic):
src = get_source_topic(output_topic)
num_records = [x[1] for x in topic_spec if x[0].name == src]
assert (len(num_records) == 1)
return acc + num_records[0]
total_outputs = reduce(accrue, set([x.topic for x in output_tps]), 0)
self.logger.info(f"Input consumer assigned: {input_tps}")
self.logger.info(f"Output consumer assigned: {output_tps}")
self._producers = []
for tp_spec, num_records, record_size in topic_spec:
try:
producer = NativeKafkaProducer(self.redpanda.brokers(),
tp_spec.name, num_records, 100,
record_size)
producer.start()
self._producers.append(producer)
except Exception as e:
self.logger.error(f"Failed to create NativeKafkaProducer: {e}")
raise
try:
self._input_consumer = NativeKafkaConsumer(self.redpanda.brokers(),
list(input_tps),
total_inputs)
self._output_consumer = NativeKafkaConsumer(
self.redpanda.brokers(), list(output_tps), total_outputs)
except Exception as e:
self.logger.error(f"Failed to create NativeKafkaConsumer: {e}")
raise
self.logger.info(
f"Waiting for {total_inputs} input records and {total_outputs}"
" result records")
self._input_consumer.start()
self._output_consumer.start()
def wait_on_results(self):
def all_done():
# Uncomment to periodically see the amt of data read
self.logger.info("Input: %d" %
self._input_consumer.results.num_records())
self.logger.info("Output: %d" %
self._output_consumer.results.num_records())
batch_total = self._input_consumer.results.num_records()
if batch_total > 0:
self.records_recieved(batch_total)
return self._input_consumer.is_finished() \
and self._output_consumer.is_finished()
timeout, backoff = self.wasm_test_timeout()
wait_until(all_done, timeout_sec=timeout, backoff_sec=backoff)
try:
[x.join() for x in self._producers]
self._input_consumer.join()
self._output_consumer.join()
except Exception as e:
self.logger.error("Exception occured in background thread: {e}")
raise e
input_consumed = self._input_consumer.results.num_records()
output_consumed = self._output_consumer.results.num_records()
self.logger.info(f"Consumed {input_consumed} input"
f" records and"
f" {output_consumed} result records")
input_expected = self._input_consumer._num_records
output_expected = self._output_consumer._num_records
if input_consumed < input_expected:
raise Exception(
f"Consumed {input_consumed} expected {input_expected} input records"
)
if output_consumed < output_expected:
raise Exception(
f"Consumed {output_consumed} expected {output_expected} output records"
)
return (self._input_consumer.results, self._output_consumer.results)
def records_recieved(self, output_recieved):
"""
Called when a traunch of records has been returned from consumers
"""
pass
def wasm_test_timeout(self):
"""
2-tuple representing timeout(0) and backoff interval(1)
"""
return (90, 1)