def run():
db = getdb()
writer = WriterProcess()
writer.start()
client = KafkaClient(config.KAFKA_HOST)
producer = SimpleProducer(client)
timings = []
for i in range(1000, 2000):
key = random_key()
start = time.clock()
message = IWAMessage(key, i)
# print('Sending: %s' % message.dumps())
producer.send_messages(config.KAFKA_TOPIC, message.dumps())
# print('Queued: %s => %s' % (message.key, message.value))
while True:
with db.begin() as txn:
if txn.get(key):
timings.append(time.clock() - start)
break
writer.terminate()
print('Average write time: %s' % (sum(timings) / len(timings)))
print('Total write time: %s' % (sum(timings)))
def run(self):
env = getdb()
client = KafkaClient(self.kafka_host)
consumer = self.consumer(client)
for resp in consumer:
if not resp.message.value:
continue
with env.begin(write=True) as txn:
message = IWAMessage.loads(resp.message.value)
if message:
txn.put(bytes(message.key), bytes(message.value))
def dbwrite(self, key, value):
message = IWAMessage(key, value)
self.producer.send_messages(config.KAFKA_TOPIC, message.dumps())
cherrypy.log("Queued: %s => %s" % (message.key, message.value))
def test_dumps_then_loads(self):
msg = IWAMessage('foo', 'bar')
print(msg.dumps())
assert msg == IWAMessage.loads(msg.dumps())