def start_node(self, node):
node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT,
allow_fail=False)
self.security_config.setup_node(node)
node.account.create_file(self.CONFIG_FILE,
self.config_template_func(node))
node.account.create_file(
self.LOG4J_CONFIG_FILE,
self.render('connect_log4j.properties', log_file=self.LOG_FILE))
if self.connector_config_templates:
raise DucktapeError(
"Config files are not valid in distributed mode, submit connectors via the REST API"
)
self.logger.info("Starting Kafka Connect distributed process on " +
str(node.account))
if self.startup_mode == self.STARTUP_MODE_LOAD:
self.start_and_wait_to_load_plugins(node, 'distributed', '')
elif self.startup_mode == self.STARTUP_MODE_INSTANT:
self.start_and_return_immediately(node, 'distributed', '')
else:
# The default mode is to wait until the complete startup of the worker
self.start_and_wait_to_start_listening(node, 'distributed', '')
if len(self.pids(node)) == 0:
raise RuntimeError("No process ids recorded")
def start_node(self, node):
node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT,
allow_fail=False)
self.security_config.setup_node(node)
node.account.create_file(self.CONFIG_FILE,
self.config_template_func(node))
node.account.create_file(
self.LOG4J_CONFIG_FILE,
self.render('connect_log4j.properties', log_file=self.LOG_FILE))
if self.connector_config_templates:
raise DucktapeError(
"Config files are not valid in distributed mode, submit connectors via the REST API"
)
self.logger.info("Starting Kafka Connect distributed process on " +
str(node.account))
with node.account.monitor_log(self.LOG_FILE) as monitor:
node.account.ssh(self.start_cmd(node))
monitor.wait_until(
'Kafka Connect started',
timeout_sec=15,
err_msg=
"Never saw message indicating Kafka Connect finished startup on "
+ str(node.account))
if len(self.pids(node)) == 0:
raise RuntimeError("No process ids recorded")
def test_reads_writes(self):
verifier_jar = "/opt/tx-verifier/tx-verifier.jar"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1", partitions=1, replicas=1)
test = "concurrent-reads-writes"
try:
cmd = "{java} -jar {verifier_jar} {test} {brokers}".format(
java="java",
verifier_jar=verifier_jar,
test=test,
brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info(
"txn test \"{test}\" passed".format(test=test))
except subprocess.CalledProcessError as e:
self.redpanda.logger.info(
"txn test \"{test}\" failed".format(test=test))
errors = ""
errors += test + "\n"
errors += str(e.output) + "\n"
errors += "---------------------------\n"
raise DucktapeError(errors)
def verify(self, tests):
verifier_jar = "/opt/tx-verifier/tx-verifier.jar"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
rpk.create_topic("topic2")
errors = ""
for test in tests:
self.redpanda.logger.info(
"testing txn test \"{test}\"".format(test=test))
try:
cmd = "{java} -jar {verifier_jar} {test} {brokers}".format(
java="java",
verifier_jar=verifier_jar,
test=test,
brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info(
"txn test \"{test}\" passed".format(test=test))
except subprocess.CalledProcessError as e:
self.redpanda.logger.info(
"txn test \"{test}\" failed".format(test=test))
errors += test + "\n"
errors += str(e.output) + "\n"
errors += "---------------------------\n"
if len(errors) > 0:
raise DucktapeError(errors)
def test_verify_compacted_topics(self):
partition_count = 1
record_count = 10000
key_cardinality = 200
v = CompactedTopicVerifier(self.redpanda)
rpk = RpkTool(self.redpanda)
res = v.produce(record_count=record_count,
p=partition_count,
key_cardinality=key_cardinality)
def records_readable():
partitions = rpk.describe_topic(v.topic)
for p in partitions:
if p.high_watermark < record_count:
return False
return True
wait_until(lambda: records_readable(),
timeout_sec=10,
backoff_sec=2,
err_msg="Records are not readable")
#TODO: added verification if segments were actually compacted
res = v.verify()
if res == None:
raise DucktapeError("Compacted topic verification failed")
def test_produce(self):
verifier_bin = "/opt/redpanda-tests/go/sarama/produce_test/produce_test"
self.redpanda.logger.info("creating topics")
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
self.redpanda.logger.info("testing sarama produce")
retries = 5
for i in range(0, retries):
try:
cmd = "{verifier_bin} --brokers {brokers}".format(
verifier_bin=verifier_bin, brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
self.redpanda.logger.info("sarama produce test passed")
break
except subprocess.CalledProcessError as e:
error = str(e.output)
self.redpanda.logger.info("sarama produce failed with " +
error)
if i + 1 != retries and NOT_LEADER_FOR_PARTITION in error:
sleep(5)
continue
raise DucktapeError("sarama produce failed with " + error)
def __init__(self, **kwargs):
self.injected_args = kwargs
for k in self.injected_args:
try:
iter(self.injected_args[k])
except TypeError as te:
raise DucktapeError(
"Expected all values in @defaults decorator to be iterable: "
+ str(te))
def start_node(self, node):
node.account.create_file("/mnt/connect.properties", self.config_template_func(node))
if self.connector_config_templates:
raise DucktapeError("Config files are not valid in distributed mode, submit connectors via the REST API")
self.logger.info("Starting Kafka Connect distributed process")
with node.account.monitor_log("/mnt/connect.log") as monitor:
cmd = "/opt/%s/bin/connect-distributed.sh /mnt/connect.properties " % kafka_dir(node)
cmd += " 1>> /mnt/connect.log 2>> /mnt/connect.log & echo $! > /mnt/connect.pid"
node.account.ssh(cmd)
monitor.wait_until('Kafka Connect started', timeout_sec=15, err_msg="Never saw message indicating Kafka Connect finished startup")
if len(self.pids(node)) == 0:
raise RuntimeError("No process ids recorded")
def test_tx(self):
verifier_jar = "/opt/tx-verifier/tx-verifier.jar"
rpk = RpkTool(self.redpanda)
rpk.create_topic("topic1")
rpk.create_topic("topic2")
self.redpanda.logger.error("starting tx verifier")
try:
cmd = ("{java} -jar {verifier_jar} {brokers}").format(
java="java",
verifier_jar=verifier_jar,
brokers=self.redpanda.brokers())
subprocess.check_output(["/bin/sh", "-c", cmd],
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise DucktapeError("tx test failed: " + str(e.output))
