Beispiel #1
0
def run():
    db = getdb()

    writer = WriterProcess()
    writer.start()

    client = KafkaClient(config.KAFKA_HOST)
    producer = SimpleProducer(client)

    timings = []

    for i in range(1000, 2000):
        key = random_key()
        start = time.clock()
        message = IWAMessage(key, i)
        # print('Sending: %s' % message.dumps())
        producer.send_messages(config.KAFKA_TOPIC, message.dumps())
        # print('Queued: %s => %s' % (message.key, message.value))
        while True:
            with db.begin() as txn:
                if txn.get(key):
                    timings.append(time.clock() - start)
                    break

    writer.terminate()

    print('Average write time: %s' % (sum(timings) / len(timings)))
    print('Total write time: %s' % (sum(timings)))
Beispiel #2
0
class KafkaClientPlugin(cherrypy.process.plugins.SimplePlugin):
    def start(self):
        self.client = KafkaClient(config.KAFKA_HOST)
        self.producer = SimpleProducer(self.client)
        self.writer = WriterProcess(config.KAFKA_HOST)
        self.writer.start()
        self.bus.subscribe("dbwrite", self.dbwrite)

    def stop(self):
        self.writer.terminate()
        self.client.close()

    def dbwrite(self, key, value):
        message = IWAMessage(key, value)
        self.producer.send_messages(config.KAFKA_TOPIC, message.dumps())
        cherrypy.log("Queued: %s => %s" % (message.key, message.value))
Beispiel #3
0
 def start(self):
     self.client = KafkaClient(config.KAFKA_HOST)
     self.producer = SimpleProducer(self.client)
     self.writer = WriterProcess(config.KAFKA_HOST)
     self.writer.start()
     self.bus.subscribe("dbwrite", self.dbwrite)