def __init__(self, test_context):
super(NativeVsRestConsumerPerformance,
self).__init__(test_context,
num_zk=1,
num_brokers=1,
num_rest=1,
topics={
'test-rep-one': {
'partitions': 6,
'replication-factor': 1
}
})
if True:
# Works on both aws and local
msgs = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
msgs = 50000000
msg_size = 100
batch_size = 8196
acks = 1 # default for REST proxy, which isn't yet configurable
nthreads = 1 # not configurable for REST proxy
self.producer = ProducerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs + 1000,
record_size=msg_size,
throughput=-1,
settings={
'batch.size':
batch_size,
'acks': acks
})
self.consumer_perf = ConsumerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs,
throughput=-1,
threads=nthreads)
self.rest_consumer_perf = RestConsumerPerformanceService(
test_context,
1,
self.rest,
topic="test",
num_records=msgs,
throughput=-1)
def __init__(self, test_context):
super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={
'test-rep-one' : { 'partitions': 6, 'replication-factor': 1 }
})
if True:
# Works on both aws and local
msgs = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
msgs = 50000000
msg_size = 100
batch_size = 8196
acks = 1 # default for REST proxy, which isn't yet configurable
nthreads = 1 # not configurable for REST proxy
self.producer = ProducerPerformanceService(
test_context, 1, self.kafka,
topic="test", num_records=msgs+1000, record_size=msg_size, throughput=-1,
settings={'batch.size':batch_size, 'acks': acks}
)
self.consumer_perf = ConsumerPerformanceService(
test_context, 1, self.kafka,
topic="test", num_records=msgs, throughput=-1, threads=nthreads
)
self.rest_consumer_perf = RestConsumerPerformanceService(
test_context, 1, self.rest,
topic="test", num_records=msgs, throughput=-1
)
class NativeVsRestConsumerPerformance(RestProxyTest):
def __init__(self, test_context):
super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={
'test-rep-one' : { 'partitions': 6, 'replication-factor': 1 }
})
if True:
# Works on both aws and local
msgs = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
msgs = 50000000
msg_size = 100
batch_size = 8196
acks = 1 # default for REST proxy, which isn't yet configurable
nthreads = 1 # not configurable for REST proxy
self.producer = ProducerPerformanceService(
test_context, 1, self.kafka,
topic="test", num_records=msgs+1000, record_size=msg_size, throughput=-1,
settings={'batch.size':batch_size, 'acks': acks}
)
self.consumer_perf = ConsumerPerformanceService(
test_context, 1, self.kafka,
topic="test", num_records=msgs, throughput=-1, threads=nthreads
)
self.rest_consumer_perf = RestConsumerPerformanceService(
test_context, 1, self.rest,
topic="test", num_records=msgs, throughput=-1
)
def test(self):
# Seed data. FIXME currently the REST consumer isn't properly finishing
# unless we have some extra messages -- the last set isn't getting
# properly returned for some reason.
self.producer.run()
self.consumer_perf.run()
self.rest_consumer_perf.run()
self.logger.info("Consumer performance: %f MB/s, %f msg/sec", self.consumer_perf.results[0]['mbps'], self.consumer_perf.results[0]['records_per_sec'])
self.logger.info("REST Consumer performance: %f MB/s, %f msg/sec", self.rest_consumer_perf.results[0]['mbps'], self.rest_consumer_perf.results[0]['records_per_sec'])
class NativeVsRestConsumerPerformance(RestProxyTest):
def __init__(self, test_context):
super(NativeVsRestConsumerPerformance,
self).__init__(test_context,
num_zk=1,
num_brokers=1,
num_rest=1,
topics={
'test-rep-one': {
'partitions': 6,
'replication-factor': 1
}
})
if True:
# Works on both aws and local
msgs = 1000000
else:
# Can use locally on Vagrant VMs, but may use too much memory for aws
msgs = 50000000
msg_size = 100
batch_size = 8196
acks = 1 # default for REST proxy, which isn't yet configurable
nthreads = 1 # not configurable for REST proxy
self.producer = ProducerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs + 1000,
record_size=msg_size,
throughput=-1,
settings={
'batch.size':
batch_size,
'acks': acks
})
self.consumer_perf = ConsumerPerformanceService(test_context,
1,
self.kafka,
topic="test",
num_records=msgs,
throughput=-1,
threads=nthreads)
self.rest_consumer_perf = RestConsumerPerformanceService(
test_context,
1,
self.rest,
topic="test",
num_records=msgs,
throughput=-1)
def test(self):
# Seed data. FIXME currently the REST consumer isn't properly finishing
# unless we have some extra messages -- the last set isn't getting
# properly returned for some reason.
self.producer.run()
self.consumer_perf.run()
self.rest_consumer_perf.run()
self.logger.info("Consumer performance: %f MB/s, %f msg/sec",
self.consumer_perf.results[0]['mbps'],
self.consumer_perf.results[0]['records_per_sec'])
self.logger.info("REST Consumer performance: %f MB/s, %f msg/sec",
self.rest_consumer_perf.results[0]['mbps'],
self.rest_consumer_perf.results[0]['records_per_sec'])