def test_producer_and_consumer(self):
self.logger.info("BENCHMARK: Producer + Consumer")
self.producer = ProducerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
record_size=self.msg_size_default,
throughput=-1,
settings={
'acks': 1,
'batch.size': self.batch_size,
'buffer.memory': self.buffer_memory
})
self.consumer = ConsumerPerformanceService(
self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
throughput=-1,
threads=1)
Service.run_parallel(self.producer, self.consumer)
summary = [
"Producer + consumer:",
" Producer: %s" % throughput(self.producer),
" Consumer: %s" % throughput(self.consumer)
]
self.logger.info("\n".join(summary))
def test_three_consumers(self):
self.logger.info("BENCHMARK: Three consumers")
self.perf = ConsumerPerformanceService(self.test_context,
3,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
throughput=-1,
threads=1)
self.perf.run()
self.logger.info("Three consumers: %s", throughput(self.perf))
def __init__(self, test_context):
super(NativeVsRestConsumerPerformance,
self).__init__(test_context,
num_zk=1,
num_brokers=1,
num_rest=1,
topics={
'test-rep-one': {
'partitions': 6,
'replication-factor': 1
}
})
if True:
# Works on both aws and local
msgs = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
msgs = 50000000
msg_size = 100
batch_size = 8196
acks = 1 # default for REST proxy, which isn't yet configurable
nthreads = 1 # not configurable for REST proxy
self.producer = ProducerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs + 1000,
record_size=msg_size,
throughput=-1,
settings={
'batch.size':
batch_size,
'acks': acks
})
self.consumer_perf = ConsumerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs,
throughput=-1,
threads=nthreads)
self.rest_consumer_perf = RestConsumerPerformanceService(
test_context,
1,
self.rest,
topic="test",
num_records=msgs,
throughput=-1)
def test_single_consumer(self):
# All consumer tests use the messages from the first benchmark, so
# they'll get messages of the default message size
self.logger.info("BENCHMARK: Single consumer")
self.perf = ConsumerPerformanceService(self.test_context,
1,
self.kafka,
topic="test-rep-three",
num_records=self.msgs_default,
throughput=-1,
threads=1)
self.perf.run()
self.logger.info("Single consumer: %s" % throughput(self.perf))
