def run(): db = getdb() writer = WriterProcess() writer.start() client = KafkaClient(config.KAFKA_HOST) producer = SimpleProducer(client) timings = [] for i in range(1000, 2000): key = random_key() start = time.clock() message = IWAMessage(key, i) # print('Sending: %s' % message.dumps()) producer.send_messages(config.KAFKA_TOPIC, message.dumps()) # print('Queued: %s => %s' % (message.key, message.value)) while True: with db.begin() as txn: if txn.get(key): timings.append(time.clock() - start) break writer.terminate() print('Average write time: %s' % (sum(timings) / len(timings))) print('Total write time: %s' % (sum(timings)))
class KafkaClientPlugin(cherrypy.process.plugins.SimplePlugin): def start(self): self.client = KafkaClient(config.KAFKA_HOST) self.producer = SimpleProducer(self.client) self.writer = WriterProcess(config.KAFKA_HOST) self.writer.start() self.bus.subscribe("dbwrite", self.dbwrite) def stop(self): self.writer.terminate() self.client.close() def dbwrite(self, key, value): message = IWAMessage(key, value) self.producer.send_messages(config.KAFKA_TOPIC, message.dumps()) cherrypy.log("Queued: %s => %s" % (message.key, message.value))
def start(self): self.client = KafkaClient(config.KAFKA_HOST) self.producer = SimpleProducer(self.client) self.writer = WriterProcess(config.KAFKA_HOST) self.writer.start() self.bus.subscribe("dbwrite", self.dbwrite)