def __init__(self, test_context, enable_sasl=False, group=None):
# idempotence is necessary for bench example
extra_rp_conf = {
"enable_idempotence": True,
"enable_sasl": enable_sasl
}
self._ctx = test_context
super(FranzGoBase, self).__init__(test_context=test_context,
extra_rp_conf=extra_rp_conf)
self._max_records = 1000 if self.scale.local else 1000000
self._enable_sasl = enable_sasl
# In CI, it takes approx 300s to produce/consume 1mill records (or 3333.33 records/s).
# The closest factor of 1mill is 320s (or 3125 records/s).
# Also, add 30 seconds because sometimes franzgo doesn't produce/consume
# immediatly since RP is in middle of election or other
# background tasks.
self._timeout = math.ceil(self._max_records / 3125) + 30
self.logger.debug(self._timeout)
franzgo_producer = FranzGoExamples.FranzGoBenchProduce(
self.redpanda, self.topic, self._max_records, self._enable_sasl)
self._producer = ExampleRunner(self._ctx,
franzgo_producer,
timeout_sec=self._timeout)
franzgo_consumer = FranzGoExamples.FranzGoBenchConsume(
self.redpanda, self.topic, self._max_records, self._enable_sasl,
group)
self._consumer = ExampleRunner(self._ctx,
franzgo_consumer,
timeout_sec=self._timeout)
def test_sarama_http_server(self):
sarama_example = SaramaExamples.SaramaHttpServer(self.redpanda)
example = ExampleRunner(self._ctx,
sarama_example,
timeout_sec=self._timeout)
# Start the example
example.start()
# Wait for the server to load
wait_until(example.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
# Get the node the server is on and
# a ducktape node
server_name = example.node_name()
node = random.choice(self.redpanda.nodes)
# Http get request using curl
curl = f"curl -SL http://{server_name}:8080/"
def try_curl():
result = node.account.ssh_output(curl, timeout_sec=5).decode()
self.logger.debug(result)
return "Your data is stored with unique identifier" in result
# Using wait_until for auto-retry because sometimes
# redpanda is in the middle of a leadership election when
# we try to http get.
wait_until(try_curl,
timeout_sec=self._timeout,
backoff_sec=5,
err_msg="sarama http_server test failed")
def test_sarama_interceptors(self):
sarama_example = SaramaExamples.SaramaInterceptors(
self.redpanda, self.topic)
example = ExampleRunner(self._ctx,
sarama_example,
timeout_sec=self._timeout)
# Start the example
example.start()
wait_until(example.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
def create_example(self):
# This will raise TypeError if Example is undefined
example_helper = self.Example(self.redpanda, False)
example = ExampleRunner(self._ctx,
example_helper,
timeout_sec=self._timeout)
return example
class FranzGoBase(RedpandaTest):
"""
Test FranzGo bench example.
Not using any other example because bench tests the same
APIs and code that the other examples do.
"""
topics = (TopicSpec(), )
def __init__(self, test_context, enable_sasl=False, group=None):
# idempotence is necessary for bench example
extra_rp_conf = {
"enable_idempotence": True,
"enable_sasl": enable_sasl
}
self._ctx = test_context
super(FranzGoBase, self).__init__(test_context=test_context,
extra_rp_conf=extra_rp_conf)
self._max_records = 1000 if self.scale.local else 1000000
self._enable_sasl = enable_sasl
# In CI, it takes approx 300s to produce/consume 1mill records (or 3333.33 records/s).
# The closest factor of 1mill is 320s (or 3125 records/s).
# Also, add 30 seconds because sometimes franzgo doesn't produce/consume
# immediatly since RP is in middle of election or other
# background tasks.
self._timeout = math.ceil(self._max_records / 3125) + 30
self.logger.debug(self._timeout)
franzgo_producer = FranzGoExamples.FranzGoBenchProduce(
self.redpanda, self.topic, self._max_records, self._enable_sasl)
self._producer = ExampleRunner(self._ctx,
franzgo_producer,
timeout_sec=self._timeout)
franzgo_consumer = FranzGoExamples.FranzGoBenchConsume(
self.redpanda, self.topic, self._max_records, self._enable_sasl,
group)
self._consumer = ExampleRunner(self._ctx,
franzgo_consumer,
timeout_sec=self._timeout)
@cluster(num_nodes=5)
def test_franzgo_bench(self):
# Start the produce bench
self._producer.start()
self._producer.wait()
# Start the consume bench.
# Running the example sequentially because
# it's easier to debug.
self._consumer.start()
self._consumer.wait()
def test_kafka_streams(self):
example = self.create_example()
# This will raise TypeError if DriverHeler is undefined
driver_helper = self.Driver(self.redpanda, True)
driver = ExampleRunner(self._ctx,
driver_helper,
timeout_sec=self._timeout)
# Start the example
example.start()
wait_until(example.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
# Start the driver
driver.start()
wait_until(driver.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
def test_sarama_consumergroup(self):
count = 10 if self.scale.local else 5000
sarama_example = SaramaExamples.SaramaConsumerGroup(
self.redpanda, self.topic, count)
example = ExampleRunner(self._ctx,
sarama_example,
timeout_sec=self._timeout)
producer = KafProducer(self._ctx, self.redpanda, self.topic, count)
def until_partitions():
storage = self.redpanda.storage()
return len(list(storage.partitions("kafka", self.topic))) == 3
# Must wait for the paritions to materialize or else
# kaf may try to produce during leadership election.
# This results in a skipped record since kaf doesn't auto-retry.
wait_until(until_partitions,
timeout_sec=30,
backoff_sec=2,
err_msg="Expected partition did not materialize")
# Run the producer and wait for the worker
# threads to finish producing
producer.start()
producer.wait()
# Start the example
example.start()
# Wait until the example is OK to terminate
example.wait()
def test_sarama_interceptors(self):
sarama_example = SaramaExamples.SaramaInterceptors(
self.redpanda, self.topic)
example = ExampleRunner(self._ctx,
sarama_example,
timeout_sec=self._timeout)
# Start the example
example.start()
example.wait()
def test_kafka_streams(self):
example = self.create_example()
# This will raise TypeError if DriverHeler is undefined
driver_helper = self.Driver(self.redpanda, True)
driver = ExampleRunner(self._ctx,
driver_helper,
timeout_sec=self._timeout)
# Start the example
example.start()
example.wait()
# Start the driver
driver.start()
driver.wait()
driver.stop()
example.stop()
def test_kafka_streams_page_view(self):
example_jar = KafkaStreamExamples.KafkaStreamsPageView(self.redpanda,
is_driver=False)
example = ExampleRunner(self.test_context,
example_jar,
timeout_sec=self._timeout)
driver_jar = KafkaStreamExamples.KafkaStreamsPageView(self.redpanda,
is_driver=True)
driver = ExampleRunner(self.test_context,
driver_jar,
timeout_sec=self._timeout)
# Start the example
example.start()
wait_until(example.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
# Start the driver
driver.start()
wait_until(driver.condition_met,
timeout_sec=self._timeout,
backoff_sec=1)
