def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context, num_nodes=1, kafka=self.kafka,
topic=self.topic, num_records=self.num_records,
record_size=self.record_size, throughput=-1, client_id=producer_id,
jmx_object_names=['kafka.producer:type=producer-metrics,client-id=%s' % producer_id],
jmx_attributes=['outgoing-byte-rate'])
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka, topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(core_test_action=
lambda: self.reassign_partitions(bounce_brokers, self.throttle))
def test_quota(self,
quota_type,
override_quota=True,
producer_num=1,
consumer_num=1,
old_broker_throttling_behavior=False,
old_client_throttling_behavior=False):
# Old (pre-2.0) throttling behavior for broker throttles before sending a response to the client.
if old_broker_throttling_behavior:
self.kafka.set_version(LATEST_1_1)
self.kafka.start()
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Old (pre-2.0) throttling behavior for client does not throttle upon receiving a response with a non-zero throttle time.
if old_client_throttling_behavior:
client_version = LATEST_1_1
else:
client_version = DEV_BRANCH
# Produce all messages
producer = ProducerPerformanceService(self.test_context,
producer_num,
self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_client_id,
version=client_version)
producer.run()
# Consume all messages
consumer = ConsoleConsumer(
self.test_context,
consumer_num,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
client_id=consumer_client_id,
jmx_object_names=[
'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s'
% consumer_client_id
],
jmx_attributes=['bytes-consumed-rate'],
version=client_version)
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(
messages
) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_quota(self,
quota_type,
override_quota=True,
producer_num=1,
consumer_num=1):
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Produce all messages
producer = ProducerPerformanceService(
self.test_context,
producer_num,
self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_client_id,
jmx_object_names=[
'kafka.producer:type=producer-metrics,client-id=%s' %
producer_client_id
],
jmx_attributes=['outgoing-byte-rate'])
producer.run()
# Consume all messages
consumer = ConsoleConsumer(
self.test_context,
consumer_num,
self.kafka,
self.topic,
new_consumer=True,
consumer_timeout_ms=60000,
client_id=consumer_client_id,
jmx_object_names=[
'kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s'
% consumer_client_id
],
jmx_attributes=['bytes-consumed-rate'])
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(
messages
) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_quota(self,
producer_id='default_id',
producer_num=1,
consumer_id='default_id',
consumer_num=1):
# Produce all messages
producer = ProducerPerformanceService(
self.test_context,
producer_num,
self.kafka,
security_protocol=self.security_protocol,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_id,
jmx_object_names=[
'kafka.producer:type=producer-metrics,client-id=%s' %
producer_id
],
jmx_attributes=['outgoing-byte-rate'])
producer.run()
# Consume all messages
consumer = ConsoleConsumer(
self.test_context,
consumer_num,
self.kafka,
self.topic,
security_protocol=self.security_protocol,
new_consumer=False,
consumer_timeout_ms=60000,
client_id=consumer_id,
jmx_object_names=[
'kafka.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=%s'
% consumer_id
],
jmx_attributes=['OneMinuteRate'])
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(
messages
) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_id)
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False,
wait_until_partitions_assigned=True)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(
core_test_action=lambda: self.reassign_partitions(
bounce_brokers, self.throttle))
self.logger.debug(
"Bulk producer outgoing-byte-rates: %s",
(metric.value
for k, metrics in bulk_producer.metrics(group='producer-metrics',
name='outgoing-byte-rate',
client_id=producer_id)
for metric in metrics))
def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=-1,
client_id=producer_id,
jmx_object_names=[
'kafka.producer:type=producer-metrics,client-id=%s' %
producer_id
],
jmx_attributes=['outgoing-byte-rate'])
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka,
topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
new_consumer=True,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(
core_test_action=lambda: self.reassign_partitions(
bounce_brokers, self.throttle))
def test_quota(self, producer_id='default_id', producer_num=1, consumer_id='default_id', consumer_num=1):
# Produce all messages
producer = ProducerPerformanceService(
self.test_context, producer_num, self.kafka,
topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_id,
jmx_object_names=['kafka.producer:type=producer-metrics,client-id=%s' % producer_id], jmx_attributes=['outgoing-byte-rate'])
producer.run()
# Consume all messages
consumer = ConsoleConsumer(self.test_context, consumer_num, self.kafka, self.topic,
new_consumer=False,
consumer_timeout_ms=60000, client_id=consumer_id,
jmx_object_names=['kafka.consumer:type=ConsumerTopicMetrics,name=BytesPerSec,clientId=%s' % consumer_id],
jmx_attributes=['OneMinuteRate'])
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(messages) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_quota(self, quota_type, override_quota=True, producer_num=1, consumer_num=1,
old_broker_throttling_behavior=False, old_client_throttling_behavior=False):
# Old (pre-2.0) throttling behavior for broker throttles before sending a response to the client.
if old_broker_throttling_behavior:
self.kafka.set_version(LATEST_1_1)
self.kafka.start()
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Old (pre-2.0) throttling behavior for client does not throttle upon receiving a response with a non-zero throttle time.
if old_client_throttling_behavior:
client_version = LATEST_1_1
else:
client_version = DEV_BRANCH
# Produce all messages
producer = ProducerPerformanceService(
self.test_context, producer_num, self.kafka,
topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1,
client_id=producer_client_id, version=client_version)
producer.run()
# Consume all messages
consumer = ConsoleConsumer(self.test_context, consumer_num, self.kafka, self.topic,
consumer_timeout_ms=60000, client_id=consumer_client_id,
jmx_object_names=['kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s' % consumer_client_id],
jmx_attributes=['bytes-consumed-rate'], version=client_version)
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(messages) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_quota(self, quota_type, override_quota=True, producer_num=1, consumer_num=1):
self.quota_config = QuotaConfig(quota_type, override_quota, self.kafka)
producer_client_id = self.quota_config.client_id
consumer_client_id = self.quota_config.client_id
# Produce all messages
producer = ProducerPerformanceService(
self.test_context, producer_num, self.kafka,
topic=self.topic, num_records=self.num_records, record_size=self.record_size, throughput=-1, client_id=producer_client_id)
producer.run()
# Consume all messages
consumer = ConsoleConsumer(self.test_context, consumer_num, self.kafka, self.topic,
consumer_timeout_ms=60000, client_id=consumer_client_id,
jmx_object_names=['kafka.consumer:type=consumer-fetch-manager-metrics,client-id=%s' % consumer_client_id],
jmx_attributes=['bytes-consumed-rate'])
consumer.run()
for idx, messages in consumer.messages_consumed.iteritems():
assert len(messages) > 0, "consumer %d didn't consume any message before timeout" % idx
success, msg = self.validate(self.kafka, producer, consumer)
assert success, msg
def test_throttled_reassignment(self, bounce_brokers):
security_protocol = 'PLAINTEXT'
self.kafka.security_protocol = security_protocol
self.kafka.interbroker_security_protocol = security_protocol
producer_id = 'bulk_producer'
bulk_producer = ProducerPerformanceService(
context=self.test_context, num_nodes=1, kafka=self.kafka,
topic=self.topic, num_records=self.num_records,
record_size=self.record_size, throughput=-1, client_id=producer_id)
self.producer = VerifiableProducer(context=self.test_context,
num_nodes=1,
kafka=self.kafka, topic=self.topic,
message_validator=is_int,
throughput=self.producer_throughput)
self.consumer = ConsoleConsumer(self.test_context,
self.num_consumers,
self.kafka,
self.topic,
consumer_timeout_ms=60000,
message_validator=is_int,
from_beginning=False)
self.kafka.start()
bulk_producer.run()
self.run_produce_consume_validate(core_test_action=
lambda: self.reassign_partitions(bounce_brokers, self.throttle))
self.logger.debug("Bulk producer outgoing-byte-rates: %s",
(metric.value for k, metrics in
bulk_producer.metrics(group='producer-metrics', name='outgoing-byte-rate', client_id=producer_id) for
metric in metrics)
)
class Benchmark(Test):
"""A benchmark of Kafka producer/consumer performance. This replicates the test
run here:
https://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines
"""
def __init__(self, test_context):
super(Benchmark, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 3
self.topics = {
TOPIC_REP_ONE: {
'partitions': 6,
'replication-factor': 1
},
TOPIC_REP_THREE: {
'partitions': 6,
'replication-factor': 3
}
}
self.zk = ZookeeperService(test_context, self.num_zk)
self.msgs_large = 10000000
self.batch_size = 8 * 1024
self.buffer_memory = 64 * 1024 * 1024
self.msg_sizes = [10, 100, 1000, 10000, 100000]
self.target_data_size = 128 * 1024 * 1024
self.target_data_size_gb = self.target_data_size / float(
1024 * 1024 * 1024)
def setUp(self):
self.zk.start()
def start_kafka(self, security_protocol, interbroker_security_protocol,
version):
self.kafka = KafkaService(
self.test_context,
self.num_brokers,
self.zk,
security_protocol=security_protocol,
interbroker_security_protocol=interbroker_security_protocol,
topics=self.topics,
version=version)
self.kafka.log_level = "INFO" # We don't DEBUG logging here
self.kafka.start()
@cluster(num_nodes=5)
@parametrize(acks=1, topic=TOPIC_REP_ONE)
@parametrize(acks=1, topic=TOPIC_REP_THREE)
@parametrize(acks=-1, topic=TOPIC_REP_THREE)
@matrix(acks=[1],
topic=[TOPIC_REP_THREE],
message_size=[10, 100, 1000, 10000, 100000],
compression_type=["none", "snappy"],
security_protocol=['PLAINTEXT', 'SSL'])
@cluster(num_nodes=7)
@parametrize(acks=1, topic=TOPIC_REP_THREE, num_producers=3)
def test_producer_throughput(self,
acks,
topic,
num_producers=1,
message_size=DEFAULT_RECORD_SIZE,
compression_type="none",
security_protocol='PLAINTEXT',
client_version=str(DEV_BRANCH),
broker_version=str(DEV_BRANCH)):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce ~128MB worth of messages to a topic with 6 partitions. Required acks, topic replication factor,
security protocol and message size are varied depending on arguments injected into this test.
Collect and return aggregate throughput statistics after all messages have been acknowledged.
(This runs ProducerPerformance.java under the hood)
"""
client_version = KafkaVersion(client_version)
broker_version = KafkaVersion(broker_version)
self.validate_versions(client_version, broker_version)
self.start_kafka(security_protocol, security_protocol, broker_version)
# Always generate the same total amount of data
nrecords = int(self.target_data_size / message_size)
self.producer = ProducerPerformanceService(self.test_context,
num_producers,
self.kafka,
topic=topic,
num_records=nrecords,
record_size=message_size,
throughput=-1,
version=client_version,
settings={
'acks':
acks,
'compression.type':
compression_type,
'batch.size':
self.batch_size,
'buffer.memory':
self.buffer_memory
})
self.producer.run()
return compute_aggregate_throughput(self.producer)
@cluster(num_nodes=5)
@parametrize(security_protocol='SSL',
interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'],
compression_type=["none", "snappy"])
def test_long_term_producer_throughput(self,
compression_type="none",
security_protocol='PLAINTEXT',
interbroker_security_protocol=None,
client_version=str(DEV_BRANCH),
broker_version=str(DEV_BRANCH)):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce 10e6 100 byte messages to a topic with 6 partitions, replication-factor 3, and acks=1.
Collect and return aggregate throughput statistics after all messages have been acknowledged.
(This runs ProducerPerformance.java under the hood)
"""
client_version = KafkaVersion(client_version)
broker_version = KafkaVersion(broker_version)
self.validate_versions(client_version, broker_version)
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol,
broker_version)
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=TOPIC_REP_THREE,
num_records=self.msgs_large,
record_size=DEFAULT_RECORD_SIZE,
throughput=-1,
version=client_version,
settings={
'acks': 1,
'compression.type': compression_type,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
},
intermediate_stats=True)
self.producer.run()
summary = ["Throughput over long run, data > memory:"]
data = {}
# FIXME we should be generating a graph too
# Try to break it into 5 blocks, but fall back to a smaller number if
# there aren't even 5 elements
block_size = max(len(self.producer.stats[0]) / 5, 1)
nblocks = len(self.producer.stats[0]) / block_size
for i in range(nblocks):
subset = self.producer.stats[0][i * block_size:min(
(i + 1) * block_size, len(self.producer.stats[0]))]
if len(subset) == 0:
summary.append(" Time block %d: (empty)" % i)
data[i] = None
else:
records_per_sec = sum(
[stat['records_per_sec']
for stat in subset]) / float(len(subset))
mb_per_sec = sum([stat['mbps']
for stat in subset]) / float(len(subset))
summary.append(" Time block %d: %f rec/sec (%f MB/s)" %
(i, records_per_sec, mb_per_sec))
data[i] = throughput(records_per_sec, mb_per_sec)
self.logger.info("\n".join(summary))
return data
@cluster(num_nodes=5)
@parametrize(security_protocol='SSL',
interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'],
compression_type=["none", "snappy"])
@cluster(num_nodes=6)
@matrix(security_protocol=['SASL_PLAINTEXT', 'SASL_SSL'],
compression_type=["none", "snappy"])
def test_end_to_end_latency(self,
compression_type="none",
security_protocol="PLAINTEXT",
interbroker_security_protocol=None,
client_version=str(DEV_BRANCH),
broker_version=str(DEV_BRANCH)):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce (acks = 1) and consume 10e3 messages to a topic with 6 partitions and replication-factor 3,
measuring the latency between production and consumption of each message.
Return aggregate latency statistics.
(Under the hood, this simply runs EndToEndLatency.scala)
"""
client_version = KafkaVersion(client_version)
broker_version = KafkaVersion(broker_version)
self.validate_versions(client_version, broker_version)
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol,
broker_version)
self.logger.info("BENCHMARK: End to end latency")
self.perf = EndToEndLatencyService(self.test_context,
1,
self.kafka,
topic=TOPIC_REP_THREE,
num_records=10000,
compression_type=compression_type,
version=client_version)
self.perf.run()
return latency(self.perf.results[0]['latency_50th_ms'],
self.perf.results[0]['latency_99th_ms'],
self.perf.results[0]['latency_999th_ms'])
@cluster(num_nodes=6)
@parametrize(security_protocol='PLAINTEXT', new_consumer=False)
@parametrize(security_protocol='SSL',
interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'],
compression_type=["none", "snappy"])
def test_producer_and_consumer(self,
compression_type="none",
security_protocol="PLAINTEXT",
interbroker_security_protocol=None,
new_consumer=True,
client_version=str(DEV_BRANCH),
broker_version=str(DEV_BRANCH)):
"""
Setup: 1 node zk + 3 node kafka cluster
Concurrently produce and consume 10e6 messages with a single producer and a single consumer,
using new consumer if new_consumer == True
Return aggregate throughput statistics for both producer and consumer.
(Under the hood, this runs ProducerPerformance.java, and ConsumerPerformance.scala)
"""
client_version = KafkaVersion(client_version)
broker_version = KafkaVersion(broker_version)
self.validate_versions(client_version, broker_version)
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol,
broker_version)
num_records = 10 * 1000 * 1000 # 10e6
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=TOPIC_REP_THREE,
num_records=num_records,
record_size=DEFAULT_RECORD_SIZE,
throughput=-1,
version=client_version,
settings={
'acks': 1,
'compression.type': compression_type,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.consumer = ConsumerPerformanceService(self.test_context,
1,
self.kafka,
topic=TOPIC_REP_THREE,
new_consumer=new_consumer,
messages=num_records)
Service.run_parallel(self.producer, self.consumer)
data = {
"producer": compute_aggregate_throughput(self.producer),
"consumer": compute_aggregate_throughput(self.consumer)
}
summary = ["Producer + consumer:", str(data)]
self.logger.info("\n".join(summary))
return data
@cluster(num_nodes=6)
@parametrize(security_protocol='PLAINTEXT', new_consumer=False)
@parametrize(security_protocol='SSL',
interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'],
compression_type=["none", "snappy"])
def test_consumer_throughput(self,
compression_type="none",
security_protocol="PLAINTEXT",
interbroker_security_protocol=None,
new_consumer=True,
num_consumers=1,
client_version=str(DEV_BRANCH),
broker_version=str(DEV_BRANCH)):
"""
Consume 10e6 100-byte messages with 1 or more consumers from a topic with 6 partitions
(using new consumer iff new_consumer == True), and report throughput.
"""
client_version = KafkaVersion(client_version)
broker_version = KafkaVersion(broker_version)
self.validate_versions(client_version, broker_version)
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol,
broker_version)
num_records = 10 * 1000 * 1000 # 10e6
# seed kafka w/messages
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=TOPIC_REP_THREE,
num_records=num_records,
record_size=DEFAULT_RECORD_SIZE,
throughput=-1,
version=client_version,
settings={
'acks': 1,
'compression.type': compression_type,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.producer.run()
# consume
self.consumer = ConsumerPerformanceService(self.test_context,
num_consumers,
self.kafka,
topic=TOPIC_REP_THREE,
new_consumer=new_consumer,
messages=num_records)
self.consumer.group = "test-consumer-group"
self.consumer.run()
return compute_aggregate_throughput(self.consumer)
def validate_versions(self, client_version, broker_version):
assert client_version <= broker_version, "Client version %s should be <= than broker version %s" (
client_version, broker_version)
class Benchmark(Test):
"""A benchmark of Kafka producer/consumer performance. This replicates the test
run here:
https://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines
"""
def __init__(self, test_context):
super(Benchmark, self).__init__(test_context)
self.num_zk = 1
self.num_brokers = 3
self.topics = {
TOPIC_REP_ONE: {'partitions': 6, 'replication-factor': 1},
TOPIC_REP_THREE: {'partitions': 6, 'replication-factor': 3}
}
self.zk = ZookeeperService(test_context, self.num_zk)
self.msgs_large = 10000000
self.batch_size = 8*1024
self.buffer_memory = 64*1024*1024
self.msg_sizes = [10, 100, 1000, 10000, 100000]
self.target_data_size = 128*1024*1024
self.target_data_size_gb = self.target_data_size/float(1024*1024*1024)
def setUp(self):
self.zk.start()
def start_kafka(self, security_protocol, interbroker_security_protocol):
self.kafka = KafkaService(
self.test_context, self.num_brokers,
self.zk, security_protocol=security_protocol,
interbroker_security_protocol=interbroker_security_protocol, topics=self.topics)
self.kafka.log_level = "INFO" # We don't DEBUG logging here
self.kafka.start()
@parametrize(acks=1, topic=TOPIC_REP_ONE)
@parametrize(acks=1, topic=TOPIC_REP_THREE)
@parametrize(acks=-1, topic=TOPIC_REP_THREE)
@parametrize(acks=1, topic=TOPIC_REP_THREE, num_producers=3)
@matrix(acks=[1], topic=[TOPIC_REP_THREE], message_size=[10, 100, 1000, 10000, 100000], security_protocol=['PLAINTEXT', 'SSL'])
def test_producer_throughput(self, acks, topic, num_producers=1, message_size=DEFAULT_RECORD_SIZE, security_protocol='PLAINTEXT'):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce ~128MB worth of messages to a topic with 6 partitions. Required acks, topic replication factor,
security protocol and message size are varied depending on arguments injected into this test.
Collect and return aggregate throughput statistics after all messages have been acknowledged.
(This runs ProducerPerformance.java under the hood)
"""
self.start_kafka(security_protocol, security_protocol)
# Always generate the same total amount of data
nrecords = int(self.target_data_size / message_size)
self.producer = ProducerPerformanceService(
self.test_context, num_producers, self.kafka, topic=topic,
num_records=nrecords, record_size=message_size, throughput=-1,
settings={
'acks': acks,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory})
self.producer.run()
return compute_aggregate_throughput(self.producer)
@parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'])
def test_long_term_producer_throughput(self, security_protocol, interbroker_security_protocol=None):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce 10e6 100 byte messages to a topic with 6 partitions, replication-factor 3, and acks=1.
Collect and return aggregate throughput statistics after all messages have been acknowledged.
(This runs ProducerPerformance.java under the hood)
"""
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol)
self.producer = ProducerPerformanceService(
self.test_context, 1, self.kafka,
topic=TOPIC_REP_THREE, num_records=self.msgs_large, record_size=DEFAULT_RECORD_SIZE,
throughput=-1, settings={'acks': 1, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory},
intermediate_stats=True
)
self.producer.run()
summary = ["Throughput over long run, data > memory:"]
data = {}
# FIXME we should be generating a graph too
# Try to break it into 5 blocks, but fall back to a smaller number if
# there aren't even 5 elements
block_size = max(len(self.producer.stats[0]) / 5, 1)
nblocks = len(self.producer.stats[0]) / block_size
for i in range(nblocks):
subset = self.producer.stats[0][i*block_size:min((i+1)*block_size, len(self.producer.stats[0]))]
if len(subset) == 0:
summary.append(" Time block %d: (empty)" % i)
data[i] = None
else:
records_per_sec = sum([stat['records_per_sec'] for stat in subset])/float(len(subset))
mb_per_sec = sum([stat['mbps'] for stat in subset])/float(len(subset))
summary.append(" Time block %d: %f rec/sec (%f MB/s)" % (i, records_per_sec, mb_per_sec))
data[i] = throughput(records_per_sec, mb_per_sec)
self.logger.info("\n".join(summary))
return data
@parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL', 'SASL_PLAINTEXT', 'SASL_SSL'])
def test_end_to_end_latency(self, security_protocol, interbroker_security_protocol=None):
"""
Setup: 1 node zk + 3 node kafka cluster
Produce (acks = 1) and consume 10e3 messages to a topic with 6 partitions and replication-factor 3,
measuring the latency between production and consumption of each message.
Return aggregate latency statistics.
(Under the hood, this simply runs EndToEndLatency.scala)
"""
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol)
self.logger.info("BENCHMARK: End to end latency")
self.perf = EndToEndLatencyService(
self.test_context, 1, self.kafka,
topic=TOPIC_REP_THREE, num_records=10000
)
self.perf.run()
return latency(self.perf.results[0]['latency_50th_ms'], self.perf.results[0]['latency_99th_ms'], self.perf.results[0]['latency_999th_ms'])
@parametrize(security_protocol='PLAINTEXT', new_consumer=False)
@parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'])
def test_producer_and_consumer(self, security_protocol, interbroker_security_protocol=None, new_consumer=True):
"""
Setup: 1 node zk + 3 node kafka cluster
Concurrently produce and consume 10e6 messages with a single producer and a single consumer,
using new consumer if new_consumer == True
Return aggregate throughput statistics for both producer and consumer.
(Under the hood, this runs ProducerPerformance.java, and ConsumerPerformance.scala)
"""
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol)
num_records = 10 * 1000 * 1000 # 10e6
self.producer = ProducerPerformanceService(
self.test_context, 1, self.kafka,
topic=TOPIC_REP_THREE,
num_records=num_records, record_size=DEFAULT_RECORD_SIZE, throughput=-1,
settings={'acks': 1, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory}
)
self.consumer = ConsumerPerformanceService(
self.test_context, 1, self.kafka, topic=TOPIC_REP_THREE, new_consumer=new_consumer, messages=num_records)
Service.run_parallel(self.producer, self.consumer)
data = {
"producer": compute_aggregate_throughput(self.producer),
"consumer": compute_aggregate_throughput(self.consumer)
}
summary = [
"Producer + consumer:",
str(data)]
self.logger.info("\n".join(summary))
return data
@parametrize(security_protocol='PLAINTEXT', new_consumer=False)
@parametrize(security_protocol='SSL', interbroker_security_protocol='PLAINTEXT')
@matrix(security_protocol=['PLAINTEXT', 'SSL'])
def test_consumer_throughput(self, security_protocol, interbroker_security_protocol=None, new_consumer=True, num_consumers=1):
"""
Consume 10e6 100-byte messages with 1 or more consumers from a topic with 6 partitions
(using new consumer iff new_consumer == True), and report throughput.
"""
if interbroker_security_protocol is None:
interbroker_security_protocol = security_protocol
self.start_kafka(security_protocol, interbroker_security_protocol)
num_records = 10 * 1000 * 1000 # 10e6
# seed kafka w/messages
self.producer = ProducerPerformanceService(
self.test_context, 1, self.kafka,
topic=TOPIC_REP_THREE,
num_records=num_records, record_size=DEFAULT_RECORD_SIZE, throughput=-1,
settings={'acks': 1, 'batch.size': self.batch_size, 'buffer.memory': self.buffer_memory}
)
self.producer.run()
# consume
self.consumer = ConsumerPerformanceService(
self.test_context, num_consumers, self.kafka,
topic=TOPIC_REP_THREE, new_consumer=new_consumer, messages=num_records)
self.consumer.group = "test-consumer-group"
self.consumer.run()
return compute_aggregate_throughput(self.consumer)
class Benchmark(KafkaTest):
'''A benchmark of Kafka producer/consumer performance. This replicates the test
run here:
https://engineering.linkedin.com/kafka/benchmarking-apache-kafka-2-million-writes-second-three-cheap-machines
'''
def __init__(self, test_context):
super(Benchmark, self).__init__(test_context,
num_zk=1,
num_brokers=3,
topics={
'test-rep-one': {
'partitions': 6,
'replication-factor': 1
},
'test-rep-three': {
'partitions': 6,
'replication-factor': 3
}
})
if True:
# Works on both aws and local
self.msgs = 1000000
self.msgs_default = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
self.msgs = 50000000
self.msgs_default = 50000000
self.msgs_large = 10000000
self.msg_size_default = 100
self.batch_size = 8 * 1024
self.buffer_memory = 64 * 1024 * 1024
self.msg_sizes = [10, 100, 1000, 10000, 100000]
self.target_data_size = 128 * 1024 * 1024
self.target_data_size_gb = self.target_data_size / float(
1024 * 1024 * 1024)
def test_single_producer_no_replication(self):
self.logger.info("BENCHMARK: Single producer, no replication")
self.perf = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-one",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Single producer, no replication: %s", str(data))
return data
def test_single_producer_replication(self):
self.logger.info("BENCHMARK: Single producer, async 3x replication")
self.perf = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Single producer, async 3x replication: %s" %
str(data))
return data
def test_single_producer_sync(self):
self.logger.info("BENCHMARK: Single producer, sync 3x replication")
self.perf = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': -1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Single producer, sync 3x replication: %s" % data)
return data
def test_three_producers_async(self):
self.logger.info("BENCHMARK: Three producers, async 3x replication")
self.perf = ProducerPerformanceService(
self.test_context,
3,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Three producers, async 3x replication: %s" % data)
return data
def test_multiple_message_size(self):
# TODO this would be a great place to use parametrization
self.perfs = {}
for msg_size in self.msg_sizes:
self.logger.info(
"BENCHMARK: Message size %d (%f GB total, single producer, async 3x replication)",
msg_size, self.target_data_size_gb)
# Always generate the same total amount of data
nrecords = int(self.target_data_size / msg_size)
self.perfs["perf-" + str(msg_size)] = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=nrecords,
record_size=msg_size,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.msg_size_perf = {}
for msg_size in self.msg_sizes:
perf = self.perfs["perf-" + str(msg_size)]
perf.run()
self.msg_size_perf[msg_size] = perf
summary = ["Message size:"]
data = {}
for msg_size in self.msg_sizes:
datum = compute_throughput(self.msg_size_perf[msg_size])
summary.append(" %d: %s" % (msg_size, datum))
data[msg_size] = datum
self.logger.info("\n".join(summary))
return data
def test_long_term_throughput(self):
self.logger.info("BENCHMARK: Long production")
self.perf = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_large,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
},
intermediate_stats=True)
self.perf.run()
summary = ["Throughput over long run, data > memory:"]
data = {}
# FIXME we should be generating a graph too
# Try to break it into 5 blocks, but fall back to a smaller number if
# there aren't even 5 elements
block_size = max(len(self.perf.stats[0]) / 5, 1)
nblocks = len(self.perf.stats[0]) / block_size
for i in range(nblocks):
subset = self.perf.stats[0][i * block_size:min(
(i + 1) * block_size, len(self.perf.stats[0]))]
if len(subset) == 0:
summary.append(" Time block %d: (empty)" % i)
data[i] = None
else:
records_per_sec = sum(
[stat['records_per_sec']
for stat in subset]) / float(len(subset))
mb_per_sec = sum([stat['mbps']
for stat in subset]) / float(len(subset))
summary.append(" Time block %d: %f rec/sec (%f MB/s)" %
(i, records_per_sec, mb_per_sec))
data[i] = throughput(records_per_sec, mb_per_sec)
self.logger.info("\n".join(summary))
return data
def test_end_to_end_latency(self):
self.logger.info("BENCHMARK: End to end latency")
self.perf = EndToEndLatencyService(self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=10000)
self.perf.run()
data = latency(self.perf.results[0]['latency_50th_ms'],
self.perf.results[0]['latency_99th_ms'],
self.perf.results[0]['latency_999th_ms'])
self.logger.info("End-to-end latency: %s" % str(data))
return data
def test_producer_and_consumer(self):
self.logger.info("BENCHMARK: Producer + Consumer")
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.consumer = ConsumerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
throughput=-1,
threads=1)
Service.run_parallel(self.producer, self.consumer)
data = {
"producer": compute_throughput(self.producer),
"consumer": compute_throughput(self.consumer)
}
summary = ["Producer + consumer:", str(data)]
self.logger.info("\n".join(summary))
return data
def test_single_consumer(self):
topic = "test-rep-three"
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=topic,
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.producer.run()
# All consumer tests use the messages from the first benchmark, so
# they'll get messages of the default message size
self.logger.info("BENCHMARK: Single consumer")
self.perf = ConsumerPerformanceService(self.test_context,
1,
self.kafka,
topic=topic,
num_records=self.msgs_default,
throughput=-1,
threads=1)
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Single consumer: %s" % data)
return data
def test_three_consumers(self):
topic = "test-rep-three"
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=topic,
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.producer.run()
self.logger.info("BENCHMARK: Three consumers")
self.perf = ConsumerPerformanceService(self.test_context,
3,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
throughput=-1,
threads=1)
self.perf.run()
data = compute_throughput(self.perf)
self.logger.info("Three consumers: %s", data)
return data
class PerformanceServiceTest(Test):
def __init__(self, test_context):
super(PerformanceServiceTest, self).__init__(test_context)
self.record_size = 100
self.num_records = 10000
self.topic = "topic"
self.zk = ZookeeperService(test_context, 1)
def setUp(self):
self.zk.start()
@cluster(num_nodes=5)
# We are keeping 0.8.2 here so that we don't inadvertently break support for it. Since this is just a sanity check,
# the overhead should be manageable.
@parametrize(version=str(LATEST_0_8_2), new_consumer=False)
@parametrize(version=str(LATEST_0_9), new_consumer=False)
@parametrize(version=str(LATEST_0_9))
@parametrize(version=str(DEV_BRANCH), new_consumer=False)
@parametrize(version=str(DEV_BRANCH))
def test_version(self, version=str(LATEST_0_9), new_consumer=True):
"""
Sanity check out producer performance service - verify that we can run the service with a small
number of messages. The actual stats here are pretty meaningless since the number of messages is quite small.
"""
version = KafkaVersion(version)
self.kafka = KafkaService(
self.test_context,
1,
self.zk,
topics={self.topic: {
'partitions': 1,
'replication-factor': 1
}},
version=version)
self.kafka.start()
# check basic run of producer performance
self.producer_perf = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic=self.topic,
num_records=self.num_records,
record_size=self.record_size,
throughput=
1000000000, # Set impossibly for no throttling for equivalent behavior between 0.8.X and 0.9.X
version=version,
settings={
'acks': 1,
'batch.size': 8 * 1024,
'buffer.memory': 64 * 1024 * 1024
})
self.producer_perf.run()
producer_perf_data = compute_aggregate_throughput(self.producer_perf)
# check basic run of end to end latency
self.end_to_end = EndToEndLatencyService(self.test_context,
1,
self.kafka,
topic=self.topic,
num_records=self.num_records,
version=version)
self.end_to_end.run()
end_to_end_data = latency(
self.end_to_end.results[0]['latency_50th_ms'],
self.end_to_end.results[0]['latency_99th_ms'],
self.end_to_end.results[0]['latency_999th_ms'])
# check basic run of consumer performance service
self.consumer_perf = ConsumerPerformanceService(
self.test_context,
1,
self.kafka,
new_consumer=new_consumer,
topic=self.topic,
version=version,
messages=self.num_records)
self.consumer_perf.group = "test-consumer-group"
self.consumer_perf.run()
consumer_perf_data = compute_aggregate_throughput(self.consumer_perf)
return {
"producer_performance": producer_perf_data,
"end_to_end_latency": end_to_end_data,
"consumer_performance": consumer_perf_data
}
class PerformanceServiceTest(Test):
def __init__(self, test_context):
super(PerformanceServiceTest, self).__init__(test_context)
self.record_size = 100
self.num_records = 10000
self.topic = "topic"
self.zk = ZookeeperService(test_context, 1)
def setUp(self):
self.zk.start()
# We are keeping 0.8.2 here so that we don't inadvertently break support for it. Since this is just a sanity check,
# the overhead should be manageable.
@parametrize(version=str(LATEST_0_8_2))
@parametrize(version=str(LATEST_0_9), new_consumer=False)
@parametrize(version=str(LATEST_0_9), new_consumer=True)
@parametrize(version=str(TRUNK), new_consumer=False)
@parametrize(version=str(TRUNK), new_consumer=True)
def test_version(self, version=str(LATEST_0_9), new_consumer=False):
"""
Sanity check out producer performance service - verify that we can run the service with a small
number of messages. The actual stats here are pretty meaningless since the number of messages is quite small.
"""
version = KafkaVersion(version)
self.kafka = KafkaService(
self.test_context, 1,
self.zk, topics={self.topic: {'partitions': 1, 'replication-factor': 1}}, version=version)
self.kafka.start()
# check basic run of producer performance
self.producer_perf = ProducerPerformanceService(
self.test_context, 1, self.kafka, topic=self.topic,
num_records=self.num_records, record_size=self.record_size,
throughput=1000000000, # Set impossibly for no throttling for equivalent behavior between 0.8.X and 0.9.X
version=version,
settings={
'acks': 1,
'batch.size': 8*1024,
'buffer.memory': 64*1024*1024})
self.producer_perf.run()
producer_perf_data = compute_aggregate_throughput(self.producer_perf)
# check basic run of end to end latency
self.end_to_end = EndToEndLatencyService(
self.test_context, 1, self.kafka,
topic=self.topic, num_records=self.num_records, version=version)
self.end_to_end.run()
end_to_end_data = latency(self.end_to_end.results[0]['latency_50th_ms'], self.end_to_end.results[0]['latency_99th_ms'], self.end_to_end.results[0]['latency_999th_ms'])
# check basic run of consumer performance service
self.consumer_perf = ConsumerPerformanceService(
self.test_context, 1, self.kafka, new_consumer=new_consumer,
topic=self.topic, version=version, messages=self.num_records)
self.consumer_perf.group = "test-consumer-group"
self.consumer_perf.run()
consumer_perf_data = compute_aggregate_throughput(self.consumer_perf)
return {
"producer_performance": producer_perf_data,
"end_to_end_latency": end_to_end_data,
"consumer_performance": consumer_perf_data
}