def __init__(self, test_context): super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={ 'test-rep-one': { 'partitions': 6, 'replication-factor': 1 } }) if True: # Works on both aws and local msgs = 1000000 else: # Can use locally on Vagrant VMs, but may use too much memory for aws msgs = 50000000 msg_size = 100 batch_size = 8196 acks = 1 # default for REST proxy, which isn't yet configurable nthreads = 1 # not configurable for REST proxy self.producer = ProducerPerformanceService(test_context, 1, self.kafka, topic="test", num_records=msgs + 1000, record_size=msg_size, throughput=-1, settings={ 'batch.size': batch_size, 'acks': acks }) self.consumer_perf = ConsumerPerformanceService(test_context, 1, self.kafka, topic="test", num_records=msgs, throughput=-1, threads=nthreads) self.rest_consumer_perf = RestConsumerPerformanceService( test_context, 1, self.rest, topic="test", num_records=msgs, throughput=-1)
def __init__(self, test_context): super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={ 'test-rep-one' : { 'partitions': 6, 'replication-factor': 1 } }) if True: # Works on both aws and local msgs = 1000000 else: # Can use locally on Vagrant VMs, but may use too much memory for aws msgs = 50000000 msg_size = 100 batch_size = 8196 acks = 1 # default for REST proxy, which isn't yet configurable nthreads = 1 # not configurable for REST proxy self.producer = ProducerPerformanceService( test_context, 1, self.kafka, topic="test", num_records=msgs+1000, record_size=msg_size, throughput=-1, settings={'batch.size':batch_size, 'acks': acks} ) self.consumer_perf = ConsumerPerformanceService( test_context, 1, self.kafka, topic="test", num_records=msgs, throughput=-1, threads=nthreads ) self.rest_consumer_perf = RestConsumerPerformanceService( test_context, 1, self.rest, topic="test", num_records=msgs, throughput=-1 )
class NativeVsRestConsumerPerformance(RestProxyTest): def __init__(self, test_context): super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={ 'test-rep-one' : { 'partitions': 6, 'replication-factor': 1 } }) if True: # Works on both aws and local msgs = 1000000 else: # Can use locally on Vagrant VMs, but may use too much memory for aws msgs = 50000000 msg_size = 100 batch_size = 8196 acks = 1 # default for REST proxy, which isn't yet configurable nthreads = 1 # not configurable for REST proxy self.producer = ProducerPerformanceService( test_context, 1, self.kafka, topic="test", num_records=msgs+1000, record_size=msg_size, throughput=-1, settings={'batch.size':batch_size, 'acks': acks} ) self.consumer_perf = ConsumerPerformanceService( test_context, 1, self.kafka, topic="test", num_records=msgs, throughput=-1, threads=nthreads ) self.rest_consumer_perf = RestConsumerPerformanceService( test_context, 1, self.rest, topic="test", num_records=msgs, throughput=-1 ) def test(self): # Seed data. FIXME currently the REST consumer isn't properly finishing # unless we have some extra messages -- the last set isn't getting # properly returned for some reason. self.producer.run() self.consumer_perf.run() self.rest_consumer_perf.run() self.logger.info("Consumer performance: %f MB/s, %f msg/sec", self.consumer_perf.results[0]['mbps'], self.consumer_perf.results[0]['records_per_sec']) self.logger.info("REST Consumer performance: %f MB/s, %f msg/sec", self.rest_consumer_perf.results[0]['mbps'], self.rest_consumer_perf.results[0]['records_per_sec'])
class NativeVsRestConsumerPerformance(RestProxyTest): def __init__(self, test_context): super(NativeVsRestConsumerPerformance, self).__init__(test_context, num_zk=1, num_brokers=1, num_rest=1, topics={ 'test-rep-one': { 'partitions': 6, 'replication-factor': 1 } }) if True: # Works on both aws and local msgs = 1000000 else: # Can use locally on Vagrant VMs, but may use too much memory for aws msgs = 50000000 msg_size = 100 batch_size = 8196 acks = 1 # default for REST proxy, which isn't yet configurable nthreads = 1 # not configurable for REST proxy self.producer = ProducerPerformanceService(test_context, 1, self.kafka, topic="test", num_records=msgs + 1000, record_size=msg_size, throughput=-1, settings={ 'batch.size': batch_size, 'acks': acks }) self.consumer_perf = ConsumerPerformanceService(test_context, 1, self.kafka, topic="test", num_records=msgs, throughput=-1, threads=nthreads) self.rest_consumer_perf = RestConsumerPerformanceService( test_context, 1, self.rest, topic="test", num_records=msgs, throughput=-1) def test(self): # Seed data. FIXME currently the REST consumer isn't properly finishing # unless we have some extra messages -- the last set isn't getting # properly returned for some reason. self.producer.run() self.consumer_perf.run() self.rest_consumer_perf.run() self.logger.info("Consumer performance: %f MB/s, %f msg/sec", self.consumer_perf.results[0]['mbps'], self.consumer_perf.results[0]['records_per_sec']) self.logger.info("REST Consumer performance: %f MB/s, %f msg/sec", self.rest_consumer_perf.results[0]['mbps'], self.rest_consumer_perf.results[0]['records_per_sec'])