def writer(): QueueManager.register(WRITER_QUEUE) QueueManager.register(READER_QUEUE) m = QueueManager(address=('', CONTROL_PORT), authkey=CONTROL_KEY) m.connect() command_queue = getattr(m, WRITER_QUEUE)() consumer_queue = getattr(m, READER_QUEUE)() while True: command = command_queue.get() batch_size = command['batch_size'] mc = GetMemcacheClient(command['servers']) d = deque.bind(mc, command['memcache_queue_name']) value = 'x' * command['value_size'] print('writer received command') batch_time = time.time() for i in range(batch_size): op_time = time.time() d.appendleft(value) if TARGET_RATE: time.sleep(max(0, 1 / TARGET_RATE - (time.time() - op_time))) print('done: %.1f writes per second' % ( batch_size/(time.time() - batch_time))) consumer_queue.put({ 'servers': command['servers'], 'memcache_queue_name': command['memcache_queue_name'], 'batch_size': batch_size, })
def writer(): QueueManager.register(WRITER_QUEUE) QueueManager.register(READER_QUEUE) m = QueueManager(address=('', CONTROL_PORT), authkey=CONTROL_KEY) m.connect() command_queue = getattr(m, WRITER_QUEUE)() consumer_queue = getattr(m, READER_QUEUE)() while True: command = command_queue.get() batch_size = command['batch_size'] mc = GetMemcacheClient(command['servers']) d = deque.bind(mc, command['memcache_queue_name']) value = 'x' * command['value_size'] print('writer received command') batch_time = time.time() for i in range(batch_size): op_time = time.time() d.appendleft(value) if TARGET_RATE: time.sleep(max(0, 1 / TARGET_RATE - (time.time() - op_time))) print('done: %.1f writes per second' % (batch_size / (time.time() - batch_time))) consumer_queue.put({ 'servers': command['servers'], 'memcache_queue_name': command['memcache_queue_name'], 'batch_size': batch_size, })
def testDeque(self): # TODO: Write serious unit test which covers all concurrency # cases of the lock-free algorithm. Current test passes even when CAS is # ignored... mc = GetMemcacheClient() self.baseQueueTest(deque.create(mc)) # test create and bind d1 = deque.create(mc) d2 = deque.bind(mc, d1.name) d1.appendleft(5) self.assertEqual(5, d2.pop()) self.failUnlessRaises(IndexError, d1.popleft) # test named create and bind name = 'foo' d1 = deque.create(mc, name) self.failUnlessRaises(AddError, deque.create, mc, name) d2 = deque.bind(mc, name) self.assertEqual(name, d1.name) d1.appendleft(5) self.assertEqual(5, d2.pop()) self.failUnlessRaises(IndexError, d1.popleft)
def reader(): QueueManager.register(READER_QUEUE) m = QueueManager(address=('', CONTROL_PORT), authkey=CONTROL_KEY) m.connect() consumer_queue = getattr(m, READER_QUEUE)() while True: command = consumer_queue.get() mc = GetMemcacheClient(command['servers']) d = deque.bind(mc, command['memcache_queue_name']) print('reader received command') batch_time = time.time() for i in range(command['batch_size']): op_time = time.time() d.pop() if TARGET_RATE: time.sleep(max(0, 1 / TARGET_RATE - (time.time() - op_time))) print('done: %.1f reads per second' % ( command['batch_size']/(time.time() - batch_time)))
def reader(): QueueManager.register(READER_QUEUE) m = QueueManager(address=('', CONTROL_PORT), authkey=CONTROL_KEY) m.connect() consumer_queue = getattr(m, READER_QUEUE)() while True: command = consumer_queue.get() mc = GetMemcacheClient(command['servers']) d = deque.bind(mc, command['memcache_queue_name']) print('reader received command') batch_time = time.time() for i in range(command['batch_size']): op_time = time.time() d.pop() if TARGET_RATE: time.sleep(max(0, 1 / TARGET_RATE - (time.time() - op_time))) print('done: %.1f reads per second' % (command['batch_size'] / (time.time() - batch_time)))