def test_read_write(self):
byte_value = bytes('test message', 'utf-8')
total = 100
queue = ElasticQueue(queue_dir='/tmp', queue_id='test')
async def test_write():
for n in range(total):
await queue.write({
'v': 'hello world',
'n': n,
'b': byte_value
})
loop = asyncio.get_event_loop()
loop.run_until_complete(test_write())
for i in range(total):
s = queue.read()
self.assertIsNotNone(s)
self.assertTrue('n' in s)
self.assertTrue('v' in s)
self.assertTrue('b' in s)
self.assertEqual(s['n'], i)
self.assertEqual(s['b'], byte_value)
queue.close()
queue.destroy()
self.assertTrue(queue.is_closed())
def __init__(self, loop, executor, queue, route, user_function,
total_instances):
self.platform = Platform()
self.util = Utility()
self.log = self.platform.log
queue_dir = self.util.normalize_path(self.platform.work_dir +
"/queues/" +
self.platform.get_origin())
self.disk_queue = ElasticQueue(queue_dir=queue_dir, queue_id=route)
self._loop = loop
self._executor = executor
self.queue = queue
self.route = route
self.user_function = user_function
self.ready_queue = asyncio.Queue(loop=self._loop)
self.worker_list = dict()
self._peek_worker = None
self._buffering = True
self._interceptor = total_instances == 0
self._singleton = True if total_instances < 1 else False
self._loop.create_task(self.listen(total_instances))
class ServiceQueue:
def __init__(self, loop, executor, queue, route, user_function,
total_instances):
self.platform = Platform()
self.util = Utility()
self.log = self.platform.log
queue_dir = self.util.normalize_path(self.platform.work_dir +
"/queues/" +
self.platform.get_origin())
self.disk_queue = ElasticQueue(queue_dir=queue_dir, queue_id=route)
self._loop = loop
self._executor = executor
self.queue = queue
self.route = route
self.user_function = user_function
self.ready_queue = asyncio.Queue(loop=self._loop)
self.worker_list = dict()
self._peek_worker = None
self._buffering = True
self._interceptor = total_instances == 0
self._singleton = True if total_instances < 1 else False
self._loop.create_task(self.listen(total_instances))
def peek_next_worker(self):
if self._peek_worker is None:
self._peek_worker = self._fetch_next_worker()
return self._peek_worker
def get_next_worker(self):
if self._peek_worker is not None:
result = self._peek_worker
self._peek_worker = None
return result
return self._fetch_next_worker()
def _fetch_next_worker(self):
try:
worker_number = self.ready_queue.get_nowait()
if worker_number:
self.ready_queue.task_done()
return worker_number
except QueueEmpty:
return None
def send_to_worker(self, item):
worker_number = self.get_next_worker()
if worker_number:
wq = self.worker_list[worker_number]
if wq:
wq.put_nowait(item)
else:
self.log.error("Event for " + self.route +
" dropped because worker #" +
str(worker_number) + "not found")
else:
self.log.error("Event for " + self.route +
" dropped because there are no workers available")
async def listen(self, total_instances):
# create concurrent workers and
total = 1 if self._singleton else total_instances
for i in range(total):
instance_number = i + 1
worker_queue = asyncio.Queue(loop=self._loop)
self.worker_list[instance_number] = worker_queue
WorkerQueue(self._loop, self._executor, self.queue, worker_queue,
self.route, self.user_function, instance_number,
self._singleton, self._interceptor)
# populate the ready queue with an initial set of worker numbers
await self.queue.put(instance_number)
route_type = 'PRIVATE' if self.platform.route_is_private(
self.route) else 'PUBLIC'
# minimize logging for temporary inbox that starts with the "r" prefix
if self._interceptor and self.util.is_inbox(self.route):
self.log.debug(route_type + ' ' + self.route + " with " +
str(total) + " instance" +
('s' if total > 1 else '') + " started")
else:
self.log.info(route_type + ' ' + self.route + " with " +
str(total) + " instance" +
('s' if total > 1 else '') + " started")
# listen for incoming events
while True:
event = await self.queue.get()
self.queue.task_done()
if event is None:
break
else:
if isinstance(event, int):
# ready signal from a worker
await self.ready_queue.put(event)
if self._buffering:
buffered = self.disk_queue.read()
if buffered:
self.send_to_worker(buffered)
else:
# nothing buffered in disk queue
self._buffering = False
self.disk_queue.close()
if isinstance(event, dict):
# it is a data item
if self._buffering:
# Once buffering is started, continue to spool items to disk to guarantee items in order
await self.disk_queue.write(event)
else:
w = self.peek_next_worker()
if w:
# Nothing buffered in disk queue. Find a worker to receive the item.
self.send_to_worker(event)
else:
# start buffered because there are no available workers
self._buffering = True
await self.disk_queue.write(event)
# tell workers to stop
for i in self.worker_list:
wq = self.worker_list[i]
wq.put_nowait(None)
# destroy disk queue
self.disk_queue.destroy()
# minimize logging for temporary inbox that starts with the "r" prefix
if self._interceptor and self.util.is_inbox(self.route):
self.log.debug(self.route + " stopped")
else:
self.log.info(self.route + " stopped")