def verify(self, tests):
verifier_jar = "/opt/tx-verifier/tx-verifier.jar"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
rpk.create_topic("topic2")
errors = ""
for test in tests:
self.redpanda.logger.info(
"testing txn test \"{test}\"".format(test=test))
try:
cmd = "{java} -jar {verifier_jar} {test} {brokers}".format(
java="java",
verifier_jar=verifier_jar,
test=test,
brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info(
"txn test \"{test}\" passed".format(test=test))
except subprocess.CalledProcessError as e:
self.redpanda.logger.info(
"txn test \"{test}\" failed".format(test=test))
errors += test + "\n"
errors += str(e.output) + "\n"
errors += "---------------------------\n"
if len(errors) > 0:
raise DucktapeError(errors)
def test_fetch_after_committed_offset_was_removed(self,
transactions_enabled):
"""
Test fetching when consumer offset was deleted by retention
"""
self.redpanda._extra_rp_conf[
"enable_transactions"] = transactions_enabled
self.redpanda._extra_rp_conf[
"enable_idempotence"] = transactions_enabled
self.redpanda.start()
topic = TopicSpec(partition_count=1,
replication_factor=3,
cleanup_policy=TopicSpec.CLEANUP_DELETE)
self.client().create_topic(topic)
kafka_tools = KafkaCliTools(self.redpanda)
# produce until segments have been compacted
produce_until_segments(
self.redpanda,
topic=topic.name,
partition_idx=0,
count=10,
)
consumer_group = 'test'
rpk = RpkTool(self.redpanda)
def consume(n=1):
out = rpk.consume(topic.name, group=consumer_group, n=n)
split = out.split('}')
split = filter(lambda s: "{" in s, split)
return map(lambda s: json.loads(s + "}"), split)
#consume from the beggining
msgs = consume(10)
last = list(msgs).pop()
offset = last['offset']
# change retention time
kafka_tools.alter_topic_config(
topic.name, {
TopicSpec.PROPERTY_RETENTION_BYTES: 2 * self.segment_size,
})
wait_for_segments_removal(self.redpanda,
topic.name,
partition_idx=0,
count=5)
partitions = list(rpk.describe_topic(topic.name))
p = partitions[0]
assert p.start_offset > offset
# consume from the offset that doesn't exists,
# the one that was committed previously was already removed
out = list(consume(1))
assert out[0]['offset'] == p.start_offset
def test_bootstrapping_after_move(self):
"""
Move partitions with active consumer / producer
"""
self.start_redpanda(num_nodes=3)
spec = TopicSpec(name="topic", partition_count=3, replication_factor=3)
self.client().create_topic(spec)
self.topic = spec.name
self.start_producer(1)
self.start_consumer(1)
self.await_startup()
# execute single move
self._move_and_verify()
self.run_validation(enable_idempotence=False, consumer_timeout_sec=45)
# snapshot offsets
rpk = RpkTool(self.redpanda)
partitions = rpk.describe_topic(spec.name)
offset_map = {}
for p in partitions:
offset_map[p.id] = p.high_watermark
# restart all the nodes
self.redpanda.restart_nodes(self.redpanda.nodes)
def offsets_are_recovered():
return all([
offset_map[p.id] == p.high_watermark
for p in rpk.describe_topic(spec.name)
])
wait_until(offsets_are_recovered, 30, 2)
def test_reads_writes(self):
verifier_jar = "/opt/tx-verifier/tx-verifier.jar"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1", partitions=1, replicas=1)
test = "concurrent-reads-writes"
try:
cmd = "{java} -jar {verifier_jar} {test} {brokers}".format(
java="java",
verifier_jar=verifier_jar,
test=test,
brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info(
"txn test \"{test}\" passed".format(test=test))
except subprocess.CalledProcessError as e:
self.redpanda.logger.info(
"txn test \"{test}\" failed".format(test=test))
errors = ""
errors += test + "\n"
errors += str(e.output) + "\n"
errors += "---------------------------\n"
raise DucktapeError(errors)
def __init__(self, test_context):
self.s3_bucket_name = f"panda-bucket-{uuid.uuid1()}"
self._extra_rp_conf = dict(
cloud_storage_enabled=True,
cloud_storage_access_key=ArchivalTest.s3_access_key,
cloud_storage_secret_key=ArchivalTest.s3_secret_key,
cloud_storage_region=ArchivalTest.s3_region,
cloud_storage_bucket=self.s3_bucket_name,
cloud_storage_disable_tls=True,
cloud_storage_api_endpoint=ArchivalTest.s3_host_name,
cloud_storage_api_endpoint_port=9000,
cloud_storage_reconciliation_interval_ms=500,
cloud_storage_max_connections=5,
log_compaction_interval_ms=self.log_compaction_interval_ms,
log_segment_size=self.log_segment_size,
)
if test_context.function_name == "test_timeboxed_uploads":
self._extra_rp_conf.update(
log_segment_size=1024 * 1024 * 1024,
cloud_storage_segment_max_upload_interval_sec=1)
super(ArchivalTest, self).__init__(test_context=test_context,
extra_rp_conf=self._extra_rp_conf)
self.kafka_tools = KafkaCliTools(self.redpanda)
self.rpk = RpkTool(self.redpanda)
self.s3_client = S3Client(
region='panda-region',
access_key=u"panda-user",
secret_key=u"panda-secret",
endpoint=f'http://{ArchivalTest.s3_host_name}:9000',
logger=self.logger)
def describe_topic_configs(self, topic: str):
rpk = RpkTool(self._redpanda)
configs = rpk.describe_topic_configs(topic)
return {
key: TopicConfigValue(value=value[0], source=value[1])
for key, value in configs.items()
}
def alter_topic_config(self, topic: str, key: str,
value: typing.Union[str, int]):
"""
Alter a topic configuration property.
"""
rpk = RpkTool(self._redpanda)
rpk.alter_topic_config(topic, key, value)
def test_produce(self):
verifier_bin = "/opt/redpanda-tests/go/sarama/produce_test/produce_test"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
self.redpanda.logger.info("testing sarama produce")
retries = 5
for i in range(0, retries):
try:
cmd = "{verifier_bin} --brokers {brokers}".format(
verifier_bin=verifier_bin, brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info("sarama produce test passed")
break
except subprocess.CalledProcessError as e:
error = str(e.output)
self.redpanda.logger.info("sarama produce failed with " +
error)
if i + 1 != retries and NOT_LEADER_FOR_PARTITION in error:
sleep(5)
continue
raise DucktapeError("sarama produce failed with " + error)
def test_cluster_id(self):
"""
That the cluster_id exposed in Kafka metadata is automatically
populated with a uuid, that it starts with redpanda. and that
it can be overridden by setting the property to something else.
"""
rpk = RpkTool(self.redpanda)
# An example, we will compare lengths with this
uuid_example = "redpanda.87e8c0c3-7c2a-4f7b-987f-11fc1d2443a4"
def has_uuid_cluster_id():
cluster_id = rpk.cluster_metadata_id()
self.logger.info(f"cluster_id={cluster_id}")
return cluster_id is not None and len(cluster_id) == len(
uuid_example)
# This is a wait_until because the initialization of cluster_id
# is async and can happen after the cluster starts answering Kafka requests.
wait_until(has_uuid_cluster_id, timeout_sec=20, backoff_sec=1)
# Verify that the cluster_id does not change on a restart
initial_cluster_id = rpk.cluster_metadata_id()
self.redpanda.restart_nodes(self.redpanda.nodes)
assert rpk.cluster_metadata_id() == initial_cluster_id
# Verify that a manually set cluster_id is respected
manual_id = "rhubarb"
self.redpanda.set_cluster_config(values={"cluster_id": manual_id},
expect_restart=False)
assert rpk.cluster_metadata_id() == f"redpanda.{manual_id}"
def __init__(self, test_context):
super(TopicAutocreateTest, self).__init__(
test_context=test_context,
num_brokers=1,
extra_rp_conf={'auto_create_topics_enabled': False})
self.kafka_tools = KafkaCliTools(self.redpanda)
self.rpk = RpkTool(self.redpanda)
def _all_have_leaders(self, topic):
rpk = RpkTool(self.redpanda)
partitions = rpk.describe_topic(topic)
for p in partitions:
self.logger.debug(f"rpk partition: {p}")
if p.leader is None or p.leader == -1:
return False
return True
def test_consumer_rejoin(self, static_members):
"""
Test validating that re-joining static member will not casuse rebalance
"""
self.setup_producer(20)
group = 'test-gr-1'
consumers = self.create_consumers(
2,
self.topic_spec.name,
group,
static_members=static_members,
consumer_properties={"session.timeout.ms": 40000})
# wait for some messages
wait_until(lambda: ConsumerGroupTest.consumed_at_least(consumers, 50),
30, 2)
rpk = RpkTool(self.redpanda)
# at this point we have 2 consumers in stable group
self.validate_group_state(group,
expected_state="Stable",
static_members=static_members)
# stop one of the consumers
consumers[0].stop()
consumers[0].wait()
rpk_group = rpk.group_describe(group)
if static_members:
# with static members group should still be in stable state
assert rpk_group.state == "Stable"
assert rpk_group.members == 2
else:
# consumer will request group leave when shutdown gracefully and it is dynamic
self.wait_for_members(group, 1)
# start the consumer again
consumers[0].start()
consumers[0].wait_for_started()
# wait for consumer to start
if static_members:
# with static members group should be stable immediately as the
# consumer is rejoining with the same instance id
self.validate_group_state(group,
expected_state="Stable",
static_members=static_members)
else:
# group should get back to its original 2 members state
self.wait_for_members(group, 2)
self.producer.wait()
self.producer.free()
for c in consumers:
c.stop()
c.wait()
c.free()
def test_tx_init_passes(self):
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
producer = Producer({
"bootstrap.servers": self.redpanda.brokers(),
"enable.idempotence": True,
"transactional.id": "tx-id-1",
"retries": 5
})
producer.init_transactions()
def __init__(self, test_context):
si_settings = SISettings(cloud_storage_reconciliation_interval_ms=500,
cloud_storage_max_connections=5,
log_segment_size=self.log_segment_size)
super(ShadowIndexingFirewallTest,
self).__init__(test_context=test_context,
si_settings=si_settings)
self._s3_port = si_settings.cloud_storage_api_endpoint_port
self.rpk = RpkTool(self.redpanda)
def test_metadata_request_contains_all_brokers(self):
"""
Check if broker list returned from metadata request is complete
"""
wait_until(lambda: self.controller_present, 10, 1)
rpk = RpkTool(self.redpanda)
nodes = rpk.cluster_info()
assert len(nodes) == 3
all_ids = [self.redpanda.idx(n) for n in self.redpanda.nodes]
returned_node_ids = [n.id for n in nodes]
assert sorted(all_ids) == sorted(returned_node_ids)
def __init__(self, *args, **kwargs):
rp_conf = BOOTSTRAP_CONFIG.copy()
# Force verbose logging for the secret redaction test
kwargs['log_level'] = 'trace'
super(ClusterConfigTest, self).__init__(*args,
extra_rp_conf=rp_conf,
**kwargs)
self.admin = Admin(self.redpanda)
self.rpk = RpkTool(self.redpanda)
def __init__(self, *args, **kwargs):
rp_conf = BOOTSTRAP_CONFIG.copy()
# Enable our feature flag
rp_conf['enable_central_config'] = True
super(ClusterConfigTest, self).__init__(*args,
extra_rp_conf=rp_conf,
**kwargs)
self.admin = Admin(self.redpanda)
self.rpk = RpkTool(self.redpanda)
def validate_group_state(self, group, expected_state, static_members):
rpk = RpkTool(self.redpanda)
# validate group state
rpk_group = rpk.group_describe(group)
assert rpk_group.members == 2
assert rpk_group.state == expected_state
for p in rpk_group.partitions:
if static_members:
assert 'panda-consumer' in p.instance_id
else:
assert p.instance_id is None
def test_idempotent_write_passes(self):
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
producer = Producer({
"bootstrap.servers": self.redpanda.brokers(),
"enable.idempotence": True,
"retries": 5
})
producer.produce("topic1",
key="key1".encode('utf-8'),
value="value1".encode('utf-8'),
callback=on_delivery)
producer.flush()
def get_super_client(self):
if self.security.enable_mtls_identity:
return RpkTool(self.redpanda, tls_cert=self.admin_user_cert)
username, password, _ = self.redpanda.SUPERUSER_CREDENTIALS
# uses base user cert with no explicit permissions. the cert should only
# participate in tls handshake and not principal extraction.
cert = None if not self.security.tls_provider else self.base_user_cert
return RpkTool(self.redpanda,
username=username,
password=password,
sasl_mechanism=self.algorithm,
tls_cert=cert)
def test_idempotency_compacted_topic(self):
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1", config={"cleanup.policy": "compact"})
producer = Producer({
"bootstrap.servers": self.redpanda.brokers(),
"enable.idempotence": True,
"retries": 5
})
producer.produce("topic1",
key="key1".encode('utf-8'),
value="value1".encode('utf-8'),
callback=on_delivery)
producer.flush()
def _ping_pong(self):
kc = KafkaCat(self.redpanda)
rpk = RpkTool(self.redpanda)
payload = str(random.randint(0, 1000))
start = time.time()
offset = rpk.produce(self.topic, "tkey", payload, timeout=5)
consumed = kc.consume_one(self.topic, 0, offset)
latency = time.time() - start
self.logger.info(
f"_ping_pong produced '{payload}' consumed '{consumed}' in {(latency)*1000.0:.2f} ms"
)
if consumed['payload'] != payload:
raise RuntimeError(f"expected '{payload}' got '{consumed}'")
def test_delete_topic_from_ongoin_tx(self):
tx_id = "0"
producer = ck.Producer({
'bootstrap.servers': self.redpanda.brokers(),
'transactional.id': tx_id,
})
producer.init_transactions()
producer.begin_transaction()
for topic in self.topics:
for partition in range(topic.partition_count):
producer.produce(topic.name, '0', '0', partition)
producer.flush()
txs_info = self.admin.get_all_transactions()
assert len(
txs_info) == 1, "Should be only one transaction in current time"
rpk = RpkTool(self.redpanda)
topic_name = self.topics[0].name
rpk.delete_topic(topic_name)
tx = txs_info[0]
assert tx[
"transactional_id"] == tx_id, f"Expected transactional_id: {tx_id}, but got {tx['transactional_id']}"
for partition in tx["partitions"]:
assert (partition["ns"] == "kafka")
if partition["topic"] == topic_name:
self.admin.delete_partition_from_transaction(
tx["transactional_id"], partition["ns"],
partition["topic"], partition["partition_id"],
partition["etag"])
producer.commit_transaction()
producer = ck.Producer({
'bootstrap.servers': self.redpanda.brokers(),
'transactional.id': tx_id,
})
producer.init_transactions()
producer.begin_transaction()
for topic in self.topics:
if topic.name is not topic_name:
for partition in range(topic.partition_count):
producer.produce(topic.name, '0', '0', partition)
producer.commit_transaction()
def __init__(self, test_context, extra_rp_conf=dict(), num_brokers=3):
def enable_wasm_options():
return dict(
developer_mode=True,
enable_coproc=True,
)
wasm_opts = enable_wasm_options()
wasm_opts.update(extra_rp_conf)
super(WasmTest, self).__init__(test_context,
extra_rp_conf=wasm_opts,
num_brokers=num_brokers)
self._rpk_tool = RpkTool(self.redpanda)
self._build_tool = WasmBuildTool(self._rpk_tool)
def test_idempotent_write_fails(self):
username, password, algorithm = self.redpanda.SUPERUSER_CREDENTIALS
rpk = RpkTool(self.redpanda)
rpk.sasl_create_user("bob", "bob", algorithm)
try:
self.write_by_bob(algorithm)
assert False, "bob should not have access to topic1"
except AssertionError as e:
raise e
except KafkaException as e:
assert e.args[0].code(
) == TOPIC_AUTHORIZATION_FAILED, "TOPIC_AUTHORIZATION_FAILED error is expected"
class TopicAutocreateTest(RedpandaTest):
"""
Verify that autocreation works, and that the settings of an autocreated
topic match those for a topic created by hand with rpk.
"""
def __init__(self, test_context):
super(TopicAutocreateTest, self).__init__(
test_context=test_context,
num_brokers=1,
extra_rp_conf={'auto_create_topics_enabled': False})
self.kafka_tools = KafkaCliTools(self.redpanda)
self.rpk = RpkTool(self.redpanda)
@cluster(num_nodes=1)
def topic_autocreate_test(self):
auto_topic = 'autocreated'
manual_topic = "manuallycreated"
# With autocreation disabled, producing to a nonexistent topic should not work.
try:
# Use rpk rather than kafka CLI because rpk errors out promptly
self.rpk.produce(auto_topic, "foo", "bar")
except Exception:
# The write failed, and shouldn't have created a topic
assert auto_topic not in self.kafka_tools.list_topics()
else:
assert False, "Producing to a nonexistent topic should fail"
# Enable autocreation
self.redpanda.restart_nodes(self.redpanda.nodes,
{'auto_create_topics_enabled': True})
# Auto create topic
assert auto_topic not in self.kafka_tools.list_topics()
self.kafka_tools.produce(auto_topic, 1, 4096)
assert auto_topic in self.kafka_tools.list_topics()
auto_topic_spec = self.kafka_tools.describe_topic(auto_topic)
assert auto_topic_spec.retention_ms is None
assert auto_topic_spec.retention_bytes is None
# Create topic by hand, compare its properties to the autocreated one
self.rpk.create_topic(manual_topic)
manual_topic_spec = self.kafka_tools.describe_topic(auto_topic)
assert manual_topic_spec.retention_ms == auto_topic_spec.retention_ms
assert manual_topic_spec.retention_bytes == auto_topic_spec.retention_bytes
# Clear name and compare the rest of the attributes
manual_topic_spec.name = auto_topic_spec.name = None
assert manual_topic_spec == auto_topic_spec
def test_verify_compacted_topics(self):
partition_count = 1
record_count = 10000
key_cardinality = 200
v = CompactedTopicVerifier(self.redpanda)
rpk = RpkTool(self.redpanda)
res = v.produce(record_count=record_count,
p=partition_count,
key_cardinality=key_cardinality)
def records_readable():
partitions = rpk.describe_topic(v.topic)
for p in partitions:
if p.high_watermark < record_count:
return False
return True
wait_until(lambda: records_readable(),
timeout_sec=10,
backoff_sec=2,
err_msg="Records are not readable")
#TODO: added verification if segments were actually compacted
res = v.verify()
if res == None:
raise DucktapeError("Compacted topic verification failed")
class RpkClusterTest(RedpandaTest):
def __init__(self, ctx):
super(RpkClusterTest, self).__init__(test_context=ctx)
self._ctx = ctx
self._rpk = RpkTool(self.redpanda)
@cluster(num_nodes=3)
def test_cluster_info(self):
def condition():
brokers = self._rpk.cluster_info()
if len(brokers) != len(self.redpanda.nodes):
return False
advertised_addrs = self.redpanda.brokers()
ok = True
for b in brokers:
ok = ok and \
b.address in advertised_addrs
return ok
wait_until(condition,
timeout_sec=10,
backoff_sec=1,
err_msg="No brokers found or output doesn't match")
def test_topic_recreation(self):
rpk = RpkTool(self.redpanda)
topic_spec = next(filter(lambda x: x.partition_count == 1,
self.topics))
topic = topic_spec.name
group = "g0"
def check_metric():
for i in range(100):
payload = str(random.randint(0, 1000))
offset = rpk.produce(topic, "", payload, timeout=5)
metric_key = (topic, "0")
rpk.consume(topic, group=group, n=50)
metrics_offsets = self._get_offset_from_metrics(group)
assert metric_key in metrics_offsets
assert metrics_offsets[metric_key] == 50
self.redpanda.delete_topic(topic)
check_metric()
metrics_offsets = self._get_offset_from_metrics(group)
assert metrics_offsets is None
self.redpanda.create_topic(topic_spec)
check_metric()
def create_and_validate(self, name, custom_assignment):
self.redpanda.logger.info(
f"creating topic {name} with {custom_assignment}")
cli = KafkaCliTools(self.redpanda)
rpk = RpkTool(self.redpanda)
cli.create_topic_with_assignment(name, custom_assignment)
def replica_matches():
replicas_per_partition = {}
for p in rpk.describe_topic(name):
replicas_per_partition[p.id] = list(p.replicas)
self.redpanda.logger.debug(
f"requested replicas: {custom_assignment}, current replicas: {replicas_per_partition}"
)
for p_id, replicas in enumerate(custom_assignment):
if p_id not in replicas_per_partition:
return False
if set(replicas) != set(replicas_per_partition[p_id]):
return False
return True
# each assignment defines a partition
wait_until(replica_matches, 10, backoff_sec=1)
