def start_node(self, node): node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT, allow_fail=False) self.security_config.setup_node(node) node.account.create_file(self.CONFIG_FILE, self.config_template_func(node)) node.account.create_file( self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE)) if self.connector_config_templates: raise DucktapeError( "Config files are not valid in distributed mode, submit connectors via the REST API" ) self.logger.info("Starting Kafka Connect distributed process on " + str(node.account)) if self.startup_mode == self.STARTUP_MODE_LOAD: self.start_and_wait_to_load_plugins(node, 'distributed', '') elif self.startup_mode == self.STARTUP_MODE_INSTANT: self.start_and_return_immediately(node, 'distributed', '') else: # The default mode is to wait until the complete startup of the worker self.start_and_wait_to_start_listening(node, 'distributed', '') if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded")
def start_node(self, node): node.account.ssh("mkdir -p %s" % self.PERSISTENT_ROOT, allow_fail=False) self.security_config.setup_node(node) node.account.create_file(self.CONFIG_FILE, self.config_template_func(node)) node.account.create_file( self.LOG4J_CONFIG_FILE, self.render('connect_log4j.properties', log_file=self.LOG_FILE)) if self.connector_config_templates: raise DucktapeError( "Config files are not valid in distributed mode, submit connectors via the REST API" ) self.logger.info("Starting Kafka Connect distributed process on " + str(node.account)) with node.account.monitor_log(self.LOG_FILE) as monitor: node.account.ssh(self.start_cmd(node)) monitor.wait_until( 'Kafka Connect started', timeout_sec=15, err_msg= "Never saw message indicating Kafka Connect finished startup on " + str(node.account)) if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded")
def test_reads_writes(self): verifier_jar = "/opt/tx-verifier/tx-verifier.jar" self.redpanda.logger.info("creating topics") rpk = RpkTool(self.redpanda) rpk.create_topic("topic1", partitions=1, replicas=1) test = "concurrent-reads-writes" try: cmd = "{java} -jar {verifier_jar} {test} {brokers}".format( java="java", verifier_jar=verifier_jar, test=test, brokers=self.redpanda.brokers()) subprocess.check_output(["/bin/sh", "-c", cmd], stderr=subprocess.STDOUT) self.redpanda.logger.info( "txn test \"{test}\" passed".format(test=test)) except subprocess.CalledProcessError as e: self.redpanda.logger.info( "txn test \"{test}\" failed".format(test=test)) errors = "" errors += test + "\n" errors += str(e.output) + "\n" errors += "---------------------------\n" raise DucktapeError(errors)
def verify(self, tests): verifier_jar = "/opt/tx-verifier/tx-verifier.jar" self.redpanda.logger.info("creating topics") rpk = RpkTool(self.redpanda) rpk.create_topic("topic1") rpk.create_topic("topic2") errors = "" for test in tests: self.redpanda.logger.info( "testing txn test \"{test}\"".format(test=test)) try: cmd = "{java} -jar {verifier_jar} {test} {brokers}".format( java="java", verifier_jar=verifier_jar, test=test, brokers=self.redpanda.brokers()) subprocess.check_output(["/bin/sh", "-c", cmd], stderr=subprocess.STDOUT) self.redpanda.logger.info( "txn test \"{test}\" passed".format(test=test)) except subprocess.CalledProcessError as e: self.redpanda.logger.info( "txn test \"{test}\" failed".format(test=test)) errors += test + "\n" errors += str(e.output) + "\n" errors += "---------------------------\n" if len(errors) > 0: raise DucktapeError(errors)
def test_verify_compacted_topics(self): partition_count = 1 record_count = 10000 key_cardinality = 200 v = CompactedTopicVerifier(self.redpanda) rpk = RpkTool(self.redpanda) res = v.produce(record_count=record_count, p=partition_count, key_cardinality=key_cardinality) def records_readable(): partitions = rpk.describe_topic(v.topic) for p in partitions: if p.high_watermark < record_count: return False return True wait_until(lambda: records_readable(), timeout_sec=10, backoff_sec=2, err_msg="Records are not readable") #TODO: added verification if segments were actually compacted res = v.verify() if res == None: raise DucktapeError("Compacted topic verification failed")
def test_produce(self): verifier_bin = "/opt/redpanda-tests/go/sarama/produce_test/produce_test" self.redpanda.logger.info("creating topics") rpk = RpkTool(self.redpanda) rpk.create_topic("topic1") self.redpanda.logger.info("testing sarama produce") retries = 5 for i in range(0, retries): try: cmd = "{verifier_bin} --brokers {brokers}".format( verifier_bin=verifier_bin, brokers=self.redpanda.brokers()) subprocess.check_output(["/bin/sh", "-c", cmd], stderr=subprocess.STDOUT) self.redpanda.logger.info("sarama produce test passed") break except subprocess.CalledProcessError as e: error = str(e.output) self.redpanda.logger.info("sarama produce failed with " + error) if i + 1 != retries and NOT_LEADER_FOR_PARTITION in error: sleep(5) continue raise DucktapeError("sarama produce failed with " + error)
def __init__(self, **kwargs): self.injected_args = kwargs for k in self.injected_args: try: iter(self.injected_args[k]) except TypeError as te: raise DucktapeError( "Expected all values in @defaults decorator to be iterable: " + str(te))
def start_node(self, node): node.account.create_file("/mnt/connect.properties", self.config_template_func(node)) if self.connector_config_templates: raise DucktapeError("Config files are not valid in distributed mode, submit connectors via the REST API") self.logger.info("Starting Kafka Connect distributed process") with node.account.monitor_log("/mnt/connect.log") as monitor: cmd = "/opt/%s/bin/connect-distributed.sh /mnt/connect.properties " % kafka_dir(node) cmd += " 1>> /mnt/connect.log 2>> /mnt/connect.log & echo $! > /mnt/connect.pid" node.account.ssh(cmd) monitor.wait_until('Kafka Connect started', timeout_sec=15, err_msg="Never saw message indicating Kafka Connect finished startup") if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded")
def test_tx(self): verifier_jar = "/opt/tx-verifier/tx-verifier.jar" rpk = RpkTool(self.redpanda) rpk.create_topic("topic1") rpk.create_topic("topic2") self.redpanda.logger.error("starting tx verifier") try: cmd = ("{java} -jar {verifier_jar} {brokers}").format( java="java", verifier_jar=verifier_jar, brokers=self.redpanda.brokers()) subprocess.check_output(["/bin/sh", "-c", cmd], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: raise DucktapeError("tx test failed: " + str(e.output))