def describe_consumer_group(self, group, node=None, new_consumer=False, command_config=None): """ Describe a consumer group. """ if node is None: node = self.nodes[0] if command_config is None: command_config = "" else: command_config = "--command-config " + command_config if new_consumer: cmd = "/opt/%s/bin/kafka-consumer-groups.sh --new-consumer --bootstrap-server %s %s --group %s --describe" % \ (kafka_dir(node), self.bootstrap_servers(self.security_protocol), command_config, group) else: cmd = "/opt/%s/bin/kafka-consumer-groups.sh --zookeeper %s %s --group %s --describe" % \ (kafka_dir(node), self.zk.connect_setting(), command_config, group) output = "" self.logger.debug(cmd) for line in node.account.ssh_capture(cmd): if not (line.startswith("SLF4J") or line.startswith("GROUP") or line.startswith("Could not fetch offset")): output += line self.logger.debug(output) return output
def start_node(self, node): node.account.ssh("mkdir -p %s" % MiniKdc.WORK_DIR, allow_fail=False) props_file = self.render('minikdc.properties', node=node) node.account.create_file(MiniKdc.PROPS_FILE, props_file) self.logger.info("minikdc.properties") self.logger.info(props_file) kafka_principals = ' '.join([ 'kafka/' + kafka_node.account.hostname for kafka_node in self.kafka_nodes ]) principals = 'client ' + kafka_principals self.logger.info("Starting MiniKdc with principals " + principals) lib_dir = "/opt/%s/core/build/dependant-testlibs" % kafka_dir(node) kdc_jars = node.account.ssh_capture("ls " + lib_dir) classpath = ":".join( [os.path.join(lib_dir, jar.strip()) for jar in kdc_jars]) cmd = "CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh org.apache.hadoop.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % ( classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) self.logger.debug( "Attempting to start MiniKdc on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(MiniKdc.LOG_FILE) as monitor: node.account.ssh(cmd) monitor.wait_until("MiniKdc Running", timeout_sec=60, backoff_sec=1, err_msg="MiniKdc didn't finish startup") node.account.scp_from(MiniKdc.KEYTAB_FILE, MiniKdc.LOCAL_KEYTAB_FILE) node.account.scp_from(MiniKdc.KRB5CONF_FILE, MiniKdc.LOCAL_KRB5CONF_FILE)
def list_consumer_groups(self, node=None, new_consumer=False, command_config=None): """ Get list of consumer groups. """ if node is None: node = self.nodes[0] if command_config is None: command_config = "" else: command_config = "--command-config " + command_config if new_consumer: cmd = "/opt/%s/bin/kafka-consumer-groups.sh --new-consumer --bootstrap-server %s %s --list" % \ (kafka_dir(node), self.bootstrap_servers(self.security_protocol), command_config) else: cmd = "/opt/%s/bin/kafka-consumer-groups.sh --zookeeper %s %s --list" % \ (kafka_dir(node), self.zk.connect_setting(), command_config) output = "" self.logger.debug(cmd) for line in node.account.ssh_capture(cmd): if not line.startswith("SLF4J"): output += line self.logger.debug(output) return output
def start_cmd(self, node): cmd = "export JMX_PORT=%d; " % self.jmx_port cmd += "export LOG_DIR=/mnt/kafka-operational-logs/; " cmd += "/opt/" + kafka_dir( node ) + "/bin/kafka-server-start.sh /mnt/kafka.properties 1>> /mnt/kafka.log 2>> /mnt/kafka.log &" return cmd
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh org.apache.kafka.tools.VerifiableLog4jAppender" cmd += " --topic %s --broker-list %s" % ( self.topic, self.kafka.bootstrap_servers(self.security_protocol)) if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) if self.security_protocol != SecurityConfig.PLAINTEXT: cmd += " --security-protocol %s" % str(self.security_protocol) if self.security_protocol == SecurityConfig.SSL or self.security_protocol == SecurityConfig.SASL_SSL: cmd += " --ssl-truststore-location %s" % str( SecurityConfig.TRUSTSTORE_PATH) cmd += " --ssl-truststore-password %s" % str( SecurityConfig.ssl_stores['ssl.truststore.password']) if self.security_protocol == SecurityConfig.SASL_PLAINTEXT or \ self.security_protocol == SecurityConfig.SASL_SSL or \ self.security_protocol == SecurityConfig.SASL_MECHANISM_GSSAPI or \ self.security_protocol == SecurityConfig.SASL_MECHANISM_PLAIN: cmd += " --sasl-kerberos-service-name %s" % str('kafka') cmd += " --client-jaas-conf-path %s" % str( SecurityConfig.JAAS_CONF_PATH) cmd += " --kerb5-conf-path %s" % str(SecurityConfig.KRB5CONF_PATH) cmd += " 2>> /mnt/kafka_log4j_appender.log | tee -a /mnt/kafka_log4j_appender.log &" return cmd
def start_cmd(self, node, idx): cmd = "" if node.version <= LATEST_0_8_2: # 0.8.2.X releases do not have VerifiableProducer.java, so cheat and add # the tools jar from trunk to the classpath cmd += "for file in /opt/%s/tools/build/libs/kafka-tools*.jar; do CLASSPATH=$CLASSPATH:$file; done; " % KAFKA_TRUNK cmd += "for file in /opt/%s/tools/build/dependant-libs-${SCALA_VERSION}*/*.jar; do CLASSPATH=$CLASSPATH:$file; done; " % KAFKA_TRUNK cmd += "export CLASSPATH; " cmd += "export LOG_DIR=%s;" % VerifiableProducer.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % VerifiableProducer.LOG4J_CONFIG cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-run-class.sh org.apache.kafka.tools.VerifiableProducer" \ " --topic %s --broker-list %s" % (self.topic, self.kafka.bootstrap_servers(self.security_config.security_protocol)) if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) if self.throughput > 0: cmd += " --throughput %s" % str(self.throughput) if self.message_validator == is_int_with_prefix: cmd += " --value-prefix %s" % str(idx) cmd += " --producer.config %s" % VerifiableProducer.CONFIG_FILE cmd += " 2>> %s | tee -a %s &" % (VerifiableProducer.STDOUT_CAPTURE, VerifiableProducer.STDOUT_CAPTURE) return cmd
def start_cmd(self, node): cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE cmd += "/opt/%s/bin/connect-distributed.sh %s " % (kafka_dir(node), self.CONFIG_FILE) cmd += " & echo $! >&3 ) 1>> %s 2>> %s 3> %s" % ( self.STDOUT_FILE, self.STDERR_FILE, self.PID_FILE) return cmd
def start_node(self, node): node.account.ssh("mkdir -p %s" % MiniKdc.WORK_DIR, allow_fail=False) props_file = self.render('minikdc.properties', node=node) node.account.create_file(MiniKdc.PROPS_FILE, props_file) self.logger.info("minikdc.properties") self.logger.info(props_file) kafka_principals = ' '.join(['kafka/' + kafka_node.account.hostname for kafka_node in self.kafka_nodes]) principals = 'client ' + kafka_principals + self.extra_principals self.logger.info("Starting MiniKdc with principals " + principals) lib_dir = "/opt/%s/core/build/dependant-testlibs" % kafka_dir(node) kdc_jars = node.account.ssh_capture("ls " + lib_dir) classpath = ":".join([os.path.join(lib_dir, jar.strip()) for jar in kdc_jars]) cmd = "CLASSPATH=%s /opt/%s/bin/kafka-run-class.sh org.apache.hadoop.minikdc.MiniKdc %s %s %s %s 1>> %s 2>> %s &" % (classpath, kafka_dir(node), MiniKdc.WORK_DIR, MiniKdc.PROPS_FILE, MiniKdc.KEYTAB_FILE, principals, MiniKdc.LOG_FILE, MiniKdc.LOG_FILE) self.logger.debug("Attempting to start MiniKdc on %s with command: %s" % (str(node.account), cmd)) with node.account.monitor_log(MiniKdc.LOG_FILE) as monitor: node.account.ssh(cmd) monitor.wait_until("MiniKdc Running", timeout_sec=60, backoff_sec=1, err_msg="MiniKdc didn't finish startup") node.account.scp_from(MiniKdc.KEYTAB_FILE, MiniKdc.LOCAL_KEYTAB_FILE) node.account.scp_from(MiniKdc.KRB5CONF_FILE, MiniKdc.LOCAL_KRB5CONF_FILE) #KDC is set to bind openly (via 0.0.0.0). Change krb5.conf to hold the specific KDC address self.replace_in_file(MiniKdc.LOCAL_KRB5CONF_FILE, '0.0.0.0', node.account.hostname)
def create_topic(self, topic_cfg, node=None): """Run the admin tool create topic command. Specifying node is optional, and may be done if for different kafka nodes have different versions, and we care where command gets run. If the node is not specified, run the command from self.nodes[0] """ if node is None: node = self.nodes[0] self.logger.info("Creating topic %s with settings %s", topic_cfg["topic"], topic_cfg) cmd = "/opt/%s/bin/kafka-topics.sh " % kafka_dir(node) cmd += "--zookeeper %(zk_connect)s --create --topic %(topic)s --partitions %(partitions)d --replication-factor %(replication)d" % { 'zk_connect': self.zk.connect_setting(), 'topic': topic_cfg.get("topic"), 'partitions': topic_cfg.get('partitions', 1), 'replication': topic_cfg.get('replication-factor', 1) } if "configs" in topic_cfg.keys() and topic_cfg["configs"] is not None: for config_name, config_value in topic_cfg["configs"].items(): cmd += " --config %s=%s" % (config_name, str(config_value)) self.logger.info("Running topic creation command...\n%s" % cmd) node.account.ssh(cmd) time.sleep(1) self.logger.info("Checking to see if topic was properly created...\n%s" % cmd) for line in self.describe_topic(topic_cfg["topic"]).split("\n"): self.logger.info(line)
def start_cmd(self, node): cmd = "export JMX_PORT=%d; " % self.jmx_port cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG cmd += "export LOG_DIR=%s; " % KafkaService.OPERATIONAL_LOG_DIR cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh %s 1>> %s 2>> %s &" % (KafkaService.CONFIG_FILE, KafkaService.STDOUT_CAPTURE, KafkaService.STDERR_CAPTURE) return cmd
def start_cmd(self, node): cmd = "export LOG_DIR=%s;" % MirrorMaker.LOG_DIR cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\";" % MirrorMaker.LOG4J_CONFIG cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += " /opt/%s/bin/kafka-run-class.sh kafka.tools.MirrorMaker" % kafka_dir( node) cmd += " --consumer.config %s" % MirrorMaker.CONSUMER_CONFIG cmd += " --producer.config %s" % MirrorMaker.PRODUCER_CONFIG cmd += " --offset.commit.interval.ms %s" % str( self.offset_commit_interval_ms) if isinstance(self.num_streams, int): cmd += " --num.streams %d" % self.num_streams else: # config num_streams separately on each node cmd += " --num.streams %d" % self.num_streams[self.idx(node) - 1] if self.whitelist is not None: cmd += " --whitelist=\"%s\"" % self.whitelist if self.blacklist is not None: cmd += " --blacklist=\"%s\"" % self.blacklist if self.new_consumer: cmd += " --new.consumer" cmd += " 1>> %s 2>> %s &" % (MirrorMaker.LOG_FILE, MirrorMaker.LOG_FILE) return cmd
def search_data_files(self, topic, messages): """Check if a set of messages made it into the Kakfa data files. Note that this method takes no account of replication. It simply looks for the payload in all the partition files of the specified topic. 'messages' should be an array of numbers. The list of missing messages is returned. """ payload_match = "payload: " + "$|payload: ".join(str(x) for x in messages) + "$" found = set([]) for node in self.nodes: # Grab all .log files in directories prefixed with this topic files = node.account.ssh_capture("find %s -regex '.*/%s-.*/[^/]*.log'" % (KafkaService.DATA_LOG_DIR, topic)) # Check each data file to see if it contains the messages we want for log in files: cmd = "/opt/%s/bin/kafka-run-class.sh kafka.tools.DumpLogSegments --print-data-log --files %s " \ "| grep -E \"%s\"" % (kafka_dir(node), log.strip(), payload_match) for line in node.account.ssh_capture(cmd, allow_fail=True): for val in messages: if line.strip().endswith("payload: "+str(val)): self.logger.debug("Found %s in data-file [%s] in line: [%s]" % (val, log.strip(), line.strip())) found.add(val) missing = list(set(messages) - found) if len(missing) > 0: self.logger.warn("The following values were not found in the data files: " + str(missing)) return missing
def execute_reassign_partitions(self, reassignment, node=None): """Run the reassign partitions admin tool in "verify" mode """ if node is None: node = self.nodes[0] json_file = "/tmp/%s_reassign.json" % str(time.time()) # reassignment to json json_str = json.dumps(reassignment) json_str = json.dumps(json_str) # create command cmd = "echo %s > %s && " % (json_str, json_file) cmd += "/opt/%s/bin/kafka-reassign-partitions.sh " % kafka_dir(node) cmd += "--zookeeper %s " % self.zk.connect_setting() cmd += "--reassignment-json-file %s " % json_file cmd += "--execute" cmd += " && sleep 1 && rm -f %s" % json_file # send command self.logger.info("Executing parition reassignment...") self.logger.debug(cmd) output = "" for line in node.account.ssh_capture(cmd): output += line self.logger.debug("Verify partition reassignment:") self.logger.debug(output)
def start_cmd(self, node): cmd = "export JMX_PORT=%d; " % self.jmx_port cmd += "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE cmd += "export LOG_DIR=/mnt/kafka-operational-logs/; " cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh /mnt/kafka.properties 1>> /mnt/kafka.log 2>> /mnt/kafka.log &" return cmd
def start_cmd(self, node): """Return the start command appropriate for the given node.""" args = self.args.copy() args['zk_connect'] = self.kafka.zk.connect_setting() args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['stderr'] = ConsoleConsumer.STDERR_CAPTURE args['log_dir'] = ConsoleConsumer.LOG_DIR args['log4j_config'] = ConsoleConsumer.LOG4J_CONFIG args['config_file'] = ConsoleConsumer.CONFIG_FILE args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['jmx_port'] = self.jmx_port args['kafka_dir'] = kafka_dir(node) args['broker_list'] = self.kafka.bootstrap_servers() cmd = "export JMX_PORT=%(jmx_port)s; " \ "export LOG_DIR=%(log_dir)s; " \ "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j_config)s\"; " \ "/opt/%(kafka_dir)s/bin/kafka-console-consumer.sh " \ "--topic %(topic)s --consumer.config %(config_file)s" % args if self.new_consumer: cmd += " --new-consumer --bootstrap-server %(broker_list)s" % args else: cmd += " --zookeeper %(zk_connect)s" % args if self.from_beginning: cmd += " --from-beginning" if self.consumer_timeout_ms is not None: # version 0.8.X and below do not support --timeout-ms option # This will be added in the properties file instead if node.version > LATEST_0_8_2: cmd += " --timeout-ms %s" % self.consumer_timeout_ms cmd += " 2>> %(stderr)s | tee -a %(stdout)s &" % args return cmd
def _worker(self, idx, node): args = self.args.copy() self.security_config.setup_node(node) if self.security_config.security_protocol != SecurityConfig.PLAINTEXT: security_config_file = SecurityConfig.CONFIG_DIR + "/security.properties" node.account.create_file(security_config_file, str(self.security_config)) else: security_config_file = "" args.update({ 'zk_connect': self.kafka.zk.connect_setting(), 'bootstrap_servers': self.kafka.bootstrap_servers(), 'security_config_file': security_config_file, 'kafka_dir': kafka_dir(node) }) cmd = "KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_dir)s/bin/kafka-run-class.sh kafka.tools.EndToEndLatency " % args cmd += "%(bootstrap_servers)s %(topic)s %(num_records)d %(acks)d 20 %(security_config_file)s" % args cmd += " | tee /mnt/end-to-end-latency.log" self.logger.debug("End-to-end latency %d command: %s", idx, cmd) results = {} for line in node.account.ssh_capture(cmd): if line.startswith("Avg latency:"): results['latency_avg_ms'] = float(line.split()[2]) if line.startswith("Percentiles"): results['latency_50th_ms'] = float(line.split()[3][:-1]) results['latency_99th_ms'] = float(line.split()[6][:-1]) results['latency_999th_ms'] = float(line.split()[9]) self.results[idx-1] = results
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh kafka.tools.ReplicaVerificationTool" cmd += " --broker-list %s --topic-white-list %s --time -2 --report-interval-ms %s" % (self.kafka.bootstrap_servers(self.security_protocol), self.topic, self.report_interval_ms) cmd += " 2>> /mnt/replica_verification_tool.log | tee -a /mnt/replica_verification_tool.log &" return cmd
def start_cmd(self, node, connector_configs): cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts cmd += "/opt/%s/bin/connect-standalone.sh %s " % (kafka_dir(node), self.CONFIG_FILE) cmd += " ".join(connector_configs) cmd += " & echo $! >&3 ) 1>> %s 2>> %s 3> %s" % (self.STDOUT_FILE, self.STDERR_FILE, self.PID_FILE) return cmd
def start_cmd(self, node): args = self.args.copy() args.update({ 'bootstrap_servers': self.kafka.bootstrap_servers(self.security_config.security_protocol), 'jmx_port': self.jmx_port, 'client_id': self.client_id, 'kafka_directory': kafka_dir(node) }) cmd = "" if node.version < TRUNK: # In order to ensure more consistent configuration between versions, always use the ProducerPerformance # tool from trunk cmd += "for file in /opt/%s/tools/build/libs/kafka-tools*.jar; do CLASSPATH=$CLASSPATH:$file; done; " % KAFKA_TRUNK cmd += "for file in /opt/%s/tools/build/dependant-libs-${SCALA_VERSION}*/*.jar; do CLASSPATH=$CLASSPATH:$file; done; " % KAFKA_TRUNK cmd += "export CLASSPATH; " cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % ProducerPerformanceService.LOG4J_CONFIG cmd += "JMX_PORT=%(jmx_port)d KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_directory)s/bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance " \ "--topic %(topic)s --num-records %(num_records)d --record-size %(record_size)d --throughput %(throughput)d --producer-props bootstrap.servers=%(bootstrap_servers)s client.id=%(client_id)s" % args self.security_config.setup_node(node) if self.security_config.security_protocol != SecurityConfig.PLAINTEXT: self.settings.update(self.security_config.properties) for key, value in self.settings.items(): cmd += " %s=%s" % (str(key), str(value)) cmd += " 2>>%s | tee %s" % (ProducerPerformanceService.STDERR_CAPTURE, ProducerPerformanceService.STDOUT_CAPTURE) return cmd
def start_cmd(self, node): args = self.args.copy() args.update({ 'zk_connect': self.kafka.zk.connect_setting(), 'bootstrap_servers': self.kafka.bootstrap_servers( self.security_config.security_protocol), 'config_file': EndToEndLatencyService.CONFIG_FILE, 'kafka_dir': kafka_dir(node) }) cmd = "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % EndToEndLatencyService.LOG4J_CONFIG if node.version >= V_0_9_0_0: cmd += "KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_dir)s/bin/kafka-run-class.sh kafka.tools.EndToEndLatency " % args cmd += "%(bootstrap_servers)s %(topic)s %(num_records)d %(acks)d %(message_bytes)d %(config_file)s" % args else: # Set fetch max wait to 0 to match behavior in later versions cmd += "KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_dir)s/bin/kafka-run-class.sh kafka.tools.TestEndToEndLatency " % args cmd += "%(bootstrap_servers)s %(zk_connect)s %(topic)s %(num_records)d 0 %(acks)d" % args cmd += " 2>> %(stderr)s | tee -a %(stdout)s" % { 'stdout': EndToEndLatencyService.STDOUT_CAPTURE, 'stderr': EndToEndLatencyService.STDERR_CAPTURE } return cmd
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh kafka.tools.SimpleConsumerShell" cmd += " --topic %s --broker-list %s --partition %s --no-wait-at-logend" % ( self.topic, self.kafka.bootstrap_servers(), self.partition) cmd += " 2>> /mnt/get_simple_consumer_shell.log | tee -a /mnt/get_simple_consumer_shell.log &" return cmd
def alter_message_format(self, topic, msg_format_version, node=None): if node is None: node = self.nodes[0] self.logger.info("Altering message format version for topic %s with format %s", topic, msg_format_version) cmd = "/opt/%s/bin/kafka-configs.sh --zookeeper %s --entity-name %s --entity-type topics --alter --add-config message.format.version=%s" % \ (kafka_dir(node), self.zk.connect_setting(), topic, msg_format_version) self.logger.info("Running alter message format command...\n%s" % cmd) node.account.ssh(cmd)
def describe_topic(self, topic, node=None): if node is None: node = self.nodes[0] cmd = "/opt/%s/bin/kafka-topics.sh --zookeeper %s --topic %s --describe" % \ (kafka_dir(node), self.zk.connect_setting(), topic) output = "" for line in node.account.ssh_capture(cmd): output += line return output
def _worker(self, idx, node): args = self.args.copy() args.update({ 'bootstrap_servers': self.kafka.bootstrap_servers(), 'jmx_port': self.jmx_port, 'client_id': self.client_id, 'kafka_directory': kafka_dir(node) }) cmd = "JMX_PORT=%(jmx_port)d KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_directory)s/bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance " \ "--topic %(topic)s --num-records %(num_records)d --record-size %(record_size)d --throughput %(throughput)d --producer-props bootstrap.servers=%(bootstrap_servers)s client.id=%(client_id)s" % args self.security_config.setup_node(node) if self.security_config.security_protocol != SecurityConfig.PLAINTEXT: self.settings.update(self.security_config.properties) for key, value in self.settings.items(): cmd += " %s=%s" % (str(key), str(value)) cmd += " | tee /mnt/producer-performance.log" self.logger.debug("Producer performance %d command: %s", idx, cmd) def parse_stats(line): parts = line.split(',') return { 'records': int(parts[0].split()[0]), 'records_per_sec': float(parts[1].split()[0]), 'mbps': float(parts[1].split('(')[1].split()[0]), 'latency_avg_ms': float(parts[2].split()[0]), 'latency_max_ms': float(parts[3].split()[0]), 'latency_50th_ms': float(parts[4].split()[0]), 'latency_95th_ms': float(parts[5].split()[0]), 'latency_99th_ms': float(parts[6].split()[0]), 'latency_999th_ms': float(parts[7].split()[0]), } last = None producer_output = node.account.ssh_capture(cmd) first_line = next(producer_output, None) if first_line is not None: self.start_jmx_tool(idx, node) for line in itertools.chain([first_line], producer_output): if self.intermediate_stats: try: self.stats[idx - 1].append(parse_stats(line)) except: # Sometimes there are extraneous log messages pass last = line try: self.results[idx - 1] = parse_stats(last) except: raise Exception( "Unable to parse aggregate performance statistics on node %d: %s" % (idx, last)) self.read_jmx_output(idx, node)
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh org.apache.kafka.tools.VerifiableLog4jAppender" cmd += " --topic %s --broker-list %s" % ( self.topic, self.kafka.bootstrap_servers()) if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) cmd += " 2>> /mnt/kafka_log4j_appender.log | tee -a /mnt/kafka_log4j_appender.log &" return cmd
def _worker(self, idx, node): args = self.args.copy() args.update({ 'bootstrap_servers': self.kafka.bootstrap_servers(self.security_config.security_protocol), 'jmx_port': self.jmx_port, 'client_id': self.client_id, 'kafka_directory': kafka_dir(node) }) cmd = "JMX_PORT=%(jmx_port)d KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_directory)s/bin/kafka-run-class.sh org.apache.kafka.tools.ProducerPerformance " \ "--topic %(topic)s --num-records %(num_records)d --record-size %(record_size)d --throughput %(throughput)d --producer-props bootstrap.servers=%(bootstrap_servers)s client.id=%(client_id)s" % args self.security_config.setup_node(node) if self.security_config.security_protocol != SecurityConfig.PLAINTEXT: self.settings.update(self.security_config.properties) for key, value in self.settings.items(): cmd += " %s=%s" % (str(key), str(value)) cmd += " | tee /mnt/producer-performance.log" self.logger.debug("Producer performance %d command: %s", idx, cmd) def parse_stats(line): parts = line.split(',') return { 'records': int(parts[0].split()[0]), 'records_per_sec': float(parts[1].split()[0]), 'mbps': float(parts[1].split('(')[1].split()[0]), 'latency_avg_ms': float(parts[2].split()[0]), 'latency_max_ms': float(parts[3].split()[0]), 'latency_50th_ms': float(parts[4].split()[0]), 'latency_95th_ms': float(parts[5].split()[0]), 'latency_99th_ms': float(parts[6].split()[0]), 'latency_999th_ms': float(parts[7].split()[0]), } last = None producer_output = node.account.ssh_capture(cmd) first_line = next(producer_output, None) if first_line is not None: self.start_jmx_tool(idx, node) for line in itertools.chain([first_line], producer_output): if self.intermediate_stats: try: self.stats[idx-1].append(parse_stats(line)) except: # Sometimes there are extraneous log messages pass last = line try: self.results[idx-1] = parse_stats(last) except: raise Exception("Unable to parse aggregate performance statistics on node %d: %s" % (idx, last)) self.read_jmx_output(idx, node)
def start_node(self, node): node.account.create_file("/mnt/connect.properties", self.config_template_func(node)) if self.connector_config_templates: raise DucktapeError("Config files are not valid in distributed mode, submit connectors via the REST API") self.logger.info("Starting Kafka Connect distributed process") with node.account.monitor_log("/mnt/connect.log") as monitor: cmd = "/opt/%s/bin/connect-distributed.sh /mnt/connect.properties " % kafka_dir(node) cmd += " 1>> /mnt/connect.log 2>> /mnt/connect.log & echo $! > /mnt/connect.pid" node.account.ssh(cmd) monitor.wait_until('Kafka Connect started', timeout_sec=15, err_msg="Never saw message indicating Kafka Connect finished startup") if len(self.pids(node)) == 0: raise RuntimeError("No process ids recorded")
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh org.apache.kafka.tools.VerifiableLog4jAppender" cmd += " --topic %s --broker-list %s" % (self.topic, self.kafka.bootstrap_servers(self.security_protocol)) if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) if self.security_protocol == SecurityConfig.SSL: cmd += " --security-protocol SSL" cmd += " --ssl-truststore-location %s" % str(SecurityConfig.TRUSTSTORE_PATH) cmd += " --ssl-truststore-password %s" % str(SecurityConfig.ssl_stores['ssl.truststore.password']) cmd += " 2>> /mnt/kafka_log4j_appender.log | tee -a /mnt/kafka_log4j_appender.log &" return cmd
def start_cmd(self, node): cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh org.apache.kafka.tools.VerifiableLog4jAppender" cmd += " --topic %s --broker-list %s" % (self.topic, self.kafka.bootstrap_servers()) if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) if self.security_protocol == SecurityConfig.SSL: cmd += " --security-protocol SSL" cmd += " --ssl-truststore-location %s" % str(SecurityConfig.TRUSTSTORE_PATH) cmd += " --ssl-truststore-password %s" % str(SecurityConfig.ssl_stores['ssl.truststore.password']) cmd += " 2>> /mnt/kafka_log4j_appender.log | tee -a /mnt/kafka_log4j_appender.log &" return cmd
def start_cmd(self, node): cmd = "" cmd += "export LOG_DIR=%s;" % VerifiableConsumer.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % VerifiableConsumer.LOG4J_CONFIG cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-run-class.sh org.apache.kafka.tools.VerifiableConsumer" \ " --group-id %s --topic %s --broker-list %s --session-timeout %s %s" % \ (self.group_id, self.topic, self.kafka.bootstrap_servers(), self.session_timeout, "--enable-autocommit" if self.enable_autocommit else "") if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) cmd += " --consumer.config %s" % VerifiableConsumer.CONFIG_FILE cmd += " 2>> %s | tee -a %s &" % (VerifiableConsumer.STDOUT_CAPTURE, VerifiableConsumer.STDOUT_CAPTURE) return cmd
def start_cmd(self, node): cmd = "" cmd += "export LOG_DIR=%s;" % VerifiableConsumer.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % VerifiableConsumer.LOG4J_CONFIG cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-run-class.sh org.apache.kafka.tools.VerifiableConsumer" \ " --group-id %s --topic %s --broker-list %s --session-timeout %s %s" % \ (self.group_id, self.topic, self.kafka.bootstrap_servers(self.security_config.security_protocol), self.session_timeout, "--enable-autocommit" if self.enable_autocommit else "") if self.max_messages > 0: cmd += " --max-messages %s" % str(self.max_messages) cmd += " --consumer.config %s" % VerifiableConsumer.CONFIG_FILE cmd += " 2>> %s | tee -a %s &" % (VerifiableConsumer.STDOUT_CAPTURE, VerifiableConsumer.STDOUT_CAPTURE) return cmd
def start_cmd(self, node): cmd = "export LOG_DIR=%s;" % ConsumerPerformanceService.LOG_DIR cmd += " export KAFKA_OPTS=%s;" % self.security_config.kafka_opts cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\";" % ConsumerPerformanceService.LOG4J_CONFIG cmd += " /opt/%s/bin/kafka-consumer-perf-test.sh" % kafka_dir(node) for key, value in self.args.items(): cmd += " --%s %s" % (key, value) cmd += " --consumer.config %s" % ConsumerPerformanceService.CONFIG_FILE for key, value in self.settings.items(): cmd += " %s=%s" % (str(key), str(value)) cmd += " 2>> %(stderr)s | tee -a %(stdout)s" % {'stdout': ConsumerPerformanceService.STDOUT_CAPTURE, 'stderr': ConsumerPerformanceService.STDERR_CAPTURE} return cmd
def start_cmd(self, node): cmd = "export LOG_DIR=%s;" % ConsumerPerformanceService.LOG_DIR cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\";" % ConsumerPerformanceService.LOG4J_CONFIG cmd += " /opt/%s/bin/kafka-consumer-perf-test.sh" % kafka_dir(node) for key, value in self.args.items(): cmd += " --%s %s" % (key, value) cmd += " --consumer.config %s" % ConsumerPerformanceService.CONFIG_FILE for key, value in self.settings.items(): cmd += " %s=%s" % (str(key), str(value)) cmd += " 2>> %(stderr)s | tee -a %(stdout)s" % { 'stdout': ConsumerPerformanceService.STDOUT_CAPTURE, 'stderr': ConsumerPerformanceService.STDERR_CAPTURE } return cmd
def start_cmd(self, node): args = self.args.copy() args['kafka'] = self.kafka.bootstrap_servers() args['zk'] = self.kafka.zk.connect_setting() args['state_dir'] = self.PERSISTENT_ROOT args['stdout'] = self.STDOUT_FILE args['stderr'] = self.STDERR_FILE args['pidfile'] = self.PID_FILE args['log4j'] = self.LOG4J_CONFIG_FILE args['kafka_dir'] = kafka_dir(node) cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j)s\"; " \ "/opt/%(kafka_dir)s/bin/streams-smoke-test.sh %(command)s %(kafka)s %(zk)s %(state_dir)s " \ " & echo $! >&3 ) 1>> %(stdout)s 2>> %(stderr)s 3> %(pidfile)s" % args return cmd
def get_offset_shell(self, topic, partitions, max_wait_ms, offsets, time): node = self.nodes[0] cmd = "/opt/%s/bin/" % kafka_dir(node) cmd += "kafka-run-class.sh kafka.tools.GetOffsetShell" cmd += " --topic %s --broker-list %s --max-wait-ms %s --offsets %s --time %s" % (topic, self.bootstrap_servers(self.security_protocol), max_wait_ms, offsets, time) if partitions: cmd += ' --partitions %s' % partitions cmd += " 2>> /mnt/get_offset_shell.log | tee -a /mnt/get_offset_shell.log &" output = "" self.logger.debug(cmd) for line in node.account.ssh_capture(cmd): output += line self.logger.debug(output) return output
def start_cmd(self, node): """Return the start command appropriate for the given node.""" args = self.args.copy() args['zk_connect'] = self.kafka.zk.connect_setting() args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['stderr'] = ConsoleConsumer.STDERR_CAPTURE args['log_dir'] = ConsoleConsumer.LOG_DIR args['log4j_config'] = ConsoleConsumer.LOG4J_CONFIG args['config_file'] = ConsoleConsumer.CONFIG_FILE args['stdout'] = ConsoleConsumer.STDOUT_CAPTURE args['jmx_port'] = self.jmx_port args['kafka_dir'] = kafka_dir(node) args['broker_list'] = self.kafka.bootstrap_servers(self.security_config.security_protocol) args['kafka_opts'] = self.security_config.kafka_opts cmd = "export JMX_PORT=%(jmx_port)s; " \ "export LOG_DIR=%(log_dir)s; " \ "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%(log4j_config)s\"; " \ "export KAFKA_OPTS=%(kafka_opts)s; " \ "/opt/%(kafka_dir)s/bin/kafka-console-consumer.sh " \ "--topic %(topic)s --consumer.config %(config_file)s" % args if self.new_consumer: cmd += " --new-consumer --bootstrap-server %(broker_list)s" % args else: cmd += " --zookeeper %(zk_connect)s" % args if self.from_beginning: cmd += " --from-beginning" if self.consumer_timeout_ms is not None: # version 0.8.X and below do not support --timeout-ms option # This will be added in the properties file instead if node.version > LATEST_0_8_2: cmd += " --timeout-ms %s" % self.consumer_timeout_ms if self.print_key: cmd += " --property print.key=true" # LoggingMessageFormatter was introduced after 0.9 if node.version > LATEST_0_9: cmd+=" --formatter kafka.tools.LoggingMessageFormatter" cmd += " --enable-systest-events" cmd += " 2>> %(stderr)s | tee -a %(stdout)s &" % args return cmd
def start_node(self, node): idx = self.idx(node) self.logger.info("Starting ZK node %d on %s", idx, node.account.hostname) node.account.ssh("mkdir -p /mnt/zookeeper") node.account.ssh("echo %d > /mnt/zookeeper/myid" % idx) config_file = self.render('zookeeper.properties') self.logger.info("zookeeper.properties:") self.logger.info(config_file) node.account.create_file("/mnt/zookeeper.properties", config_file) start_cmd = "/opt/%s/bin/zookeeper-server-start.sh " % kafka_dir(node) start_cmd += "/mnt/zookeeper.properties 1>> %(path)s 2>> %(path)s &" % self.logs[ "zk_log"] node.account.ssh(start_cmd) time.sleep(5) # give it some time to start
def start_node(self, node): idx = self.idx(node) self.logger.info("Starting ZK node %d on %s", idx, node.account.hostname) node.account.ssh("mkdir -p /mnt/zookeeper") node.account.ssh("echo %d > /mnt/zookeeper/myid" % idx) self.security_config.setup_node(node) config_file = self.render('zookeeper.properties') self.logger.info("zookeeper.properties:") self.logger.info(config_file) node.account.create_file("/mnt/zookeeper.properties", config_file) start_cmd = "export KAFKA_OPTS=\"%s\";" % self.kafka_opts start_cmd += "/opt/%s/bin/zookeeper-server-start.sh " % kafka_dir(node) start_cmd += "/mnt/zookeeper.properties 1>> %(path)s 2>> %(path)s &" % self.logs["zk_log"] node.account.ssh(start_cmd) time.sleep(5) # give it some time to start
def start_cmd(self, node): cmd = "export LOG_DIR=%s;" % MirrorMaker.LOG_DIR cmd += " export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\";" % MirrorMaker.LOG4J_CONFIG cmd += " /opt/%s/bin/kafka-run-class.sh kafka.tools.MirrorMaker" % kafka_dir(node) cmd += " --consumer.config %s" % MirrorMaker.CONSUMER_CONFIG cmd += " --producer.config %s" % MirrorMaker.PRODUCER_CONFIG cmd += " --offset.commit.interval.ms %s" % str(self.offset_commit_interval_ms) if isinstance(self.num_streams, int): cmd += " --num.streams %d" % self.num_streams else: # config num_streams separately on each node cmd += " --num.streams %d" % self.num_streams[self.idx(node) - 1] if self.whitelist is not None: cmd += " --whitelist=\"%s\"" % self.whitelist if self.blacklist is not None: cmd += " --blacklist=\"%s\"" % self.blacklist if self.new_consumer: cmd += " --new.consumer" cmd += " 1>> %s 2>> %s &" % (MirrorMaker.LOG_FILE, MirrorMaker.LOG_FILE) return cmd
def start_cmd(self, node): args = self.args.copy() args.update({ 'zk_connect': self.kafka.zk.connect_setting(), 'bootstrap_servers': self.kafka.bootstrap_servers(self.security_config.security_protocol), 'config_file': EndToEndLatencyService.CONFIG_FILE, 'kafka_dir': kafka_dir(node) }) cmd = "export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % EndToEndLatencyService.LOG4J_CONFIG if node.version >= V_0_9_0_0: cmd += "KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_dir)s/bin/kafka-run-class.sh kafka.tools.EndToEndLatency " % args cmd += "%(bootstrap_servers)s %(topic)s %(num_records)d %(acks)d %(message_bytes)d %(config_file)s" % args else: # Set fetch max wait to 0 to match behavior in later versions cmd += "KAFKA_OPTS=%(kafka_opts)s /opt/%(kafka_dir)s/bin/kafka-run-class.sh kafka.tools.TestEndToEndLatency " % args cmd += "%(bootstrap_servers)s %(zk_connect)s %(topic)s %(num_records)d 0 %(acks)d" % args cmd += " 2>> %(stderr)s | tee -a %(stdout)s" % {'stdout': EndToEndLatencyService.STDOUT_CAPTURE, 'stderr': EndToEndLatencyService.STDERR_CAPTURE} return cmd
def start_jmx_tool(self, idx, node): if self.started[idx-1] or self.jmx_object_names is None: return cmd = "/opt/%s/bin/kafka-run-class.sh kafka.tools.JmxTool " \ "--reporting-interval 1000 --jmx-url service:jmx:rmi:///jndi/rmi://127.0.0.1:%d/jmxrmi" % (kafka_dir(node), self.jmx_port) for jmx_object_name in self.jmx_object_names: cmd += " --object-name %s" % jmx_object_name for jmx_attribute in self.jmx_attributes: cmd += " --attributes %s" % jmx_attribute cmd += " | tee -a /mnt/jmx_tool.log" self.logger.debug("Start JmxTool %d command: %s", idx, cmd) jmx_output = node.account.ssh_capture(cmd, allow_fail=False) jmx_output.next() self.started[idx-1] = True
def zookeeper_migration(self, node, zk_acl): la_migra_cmd = "/opt/%s/bin/zookeeper-security-migration.sh --zookeeper.acl=%s --zookeeper.connect=%s" % (kafka_dir(node), zk_acl, self.connect_setting()) node.account.ssh(la_migra_cmd)
def start_cmd(self, node): cmd = "export JMX_PORT=%d; " % self.jmx_port cmd += "export LOG_DIR=/mnt/kafka-operational-logs/; " cmd += "export KAFKA_OPTS=%s; " % self.security_config.kafka_opts cmd += "/opt/" + kafka_dir(node) + "/bin/kafka-server-start.sh /mnt/kafka.properties 1>> /mnt/kafka.log 2>> /mnt/kafka.log &" return cmd
def acls_command(self, node, properties): cmd = "/opt/%s/bin/kafka-acls.sh %s" % (kafka_dir(node), properties) node.account.ssh(cmd)
def start_cmd(self, node): cmd = "( export KAFKA_LOG4J_OPTS=\"-Dlog4j.configuration=file:%s\"; " % self.LOG4J_CONFIG_FILE cmd += "/opt/%s/bin/connect-distributed.sh %s " % (kafka_dir(node), self.CONFIG_FILE) cmd += " & echo $! >&3 ) 1>> %s 2>> %s 3> %s" % (self.STDOUT_FILE, self.STDERR_FILE, self.PID_FILE) return cmd