Пример #1
0
def run():
    db = getdb()

    writer = WriterProcess()
    writer.start()

    client = KafkaClient(config.KAFKA_HOST)
    producer = SimpleProducer(client)

    timings = []

    for i in range(1000, 2000):
        key = random_key()
        start = time.clock()
        message = IWAMessage(key, i)
        # print('Sending: %s' % message.dumps())
        producer.send_messages(config.KAFKA_TOPIC, message.dumps())
        # print('Queued: %s => %s' % (message.key, message.value))
        while True:
            with db.begin() as txn:
                if txn.get(key):
                    timings.append(time.clock() - start)
                    break

    writer.terminate()

    print('Average write time: %s' % (sum(timings) / len(timings)))
    print('Total write time: %s' % (sum(timings)))
Пример #2
0
    def run(self):
        env = getdb()
        client = KafkaClient(self.kafka_host)
        consumer = self.consumer(client)

        for resp in consumer:
            if not resp.message.value:
                continue

            with env.begin(write=True) as txn:
                message = IWAMessage.loads(resp.message.value)
                if message:
                    txn.put(bytes(message.key), bytes(message.value))
Пример #3
0
 def dbwrite(self, key, value):
     message = IWAMessage(key, value)
     self.producer.send_messages(config.KAFKA_TOPIC, message.dumps())
     cherrypy.log("Queued: %s => %s" % (message.key, message.value))
Пример #4
0
 def test_dumps_then_loads(self):
     msg = IWAMessage('foo', 'bar')
     print(msg.dumps())
     assert msg == IWAMessage.loads(msg.dumps())