def __init__(self, backend=None, **kwargs):
self._loop = asyncio.get_event_loop()
if backend == 'mongo':
self.backend = MongoBackend(**kwargs)
elif backend == 'memory':
self.backend = MemoryBackend(**kwargs)
else:
raise PersistenceError
def __init__(self, backend=None, memory_size=None, loop=None, **kwargs):
"""
TODO: mongo is the only one yet, we should parse available modules
named `*_backend.py` and select after the given backend.
"""
self._loop = loop or asyncio.get_event_loop()
self._last_events = FIFOSizedQueue(memory_size or 10000)
self.backend = None
self._feed_future = None
if not backend:
log.info('No persistence backend selected, in-memory only')
return
if backend != 'mongo':
raise ValueError("'mongo' is the only available backend")
self.backend = MongoBackend(**kwargs)
if not isinstance(self.backend, PersistenceBackend):
raise PersistenceError(
'Wrong backend selected: {}'.format(backend))
self._feed_future = asyncio.ensure_future(self._feed_backend())
class BusPersistence(object):
"""
This module will enable local caching for bus events to replace the
current asyncio cache which is out of our control. (cf internal NYUKI-59)
"""
FEED_DELAY = 5
def __init__(self, backend=None, memory_size=None, loop=None, **kwargs):
"""
TODO: mongo is the only one yet, we should parse available modules
named `*_backend.py` and select after the given backend.
"""
self._loop = loop or asyncio.get_event_loop()
self._last_events = FIFOSizedQueue(memory_size or 10000)
self.backend = None
self._feed_future = None
if not backend:
log.info('No persistence backend selected, in-memory only')
return
if backend != 'mongo':
raise ValueError("'mongo' is the only available backend")
self.backend = MongoBackend(**kwargs)
if not isinstance(self.backend, PersistenceBackend):
raise PersistenceError(
'Wrong backend selected: {}'.format(backend))
self._feed_future = asyncio.ensure_future(self._feed_backend())
@property
def memory_buffer(self):
return self._last_events
async def close(self):
if self._feed_future:
self._feed_future.cancel()
await self._feed_future
async def _feed_backend(self):
"""
Periodically check connection to backend and dump in-memory events
into it
"""
while True:
try:
await asyncio.sleep(self.FEED_DELAY)
except asyncio.CancelledError:
log.debug('_feed_backend cancelled')
await self._empty_last_events()
break
if not self._last_events.list:
continue
await self._empty_last_events()
async def _empty_last_events(self):
if await self.backend.ping():
if self._last_events.list:
log.info('Dumping all event into backend')
try:
for event in self._last_events.empty():
await self.backend.store(event)
except Exception as exc:
reporting.exception(exc)
else:
log.warning('No connection to backend to empty in-memory events')
async def init(self):
"""
Init backend
"""
if self.backend:
try:
return await self.backend.init()
except Exception as exc:
raise PersistenceError from exc
async def ping(self):
"""
Connection check
"""
if self.backend:
try:
return await asyncio.wait_for(self.backend.ping(), 2.0)
except asyncio.TimeoutError:
log.error('Timeout pinging backend')
return False
async def store(self, event):
"""
Store a bus event from
{
"id": "uuid4",
"status": "EventStatus.value",
"topic": "muc",
"message": "json dump"
}
adding a 'created_at' key.
"""
log.debug("New event stored with uid '%s'", event['id'])
event['created_at'] = datetime.utcnow()
self._last_events.put(event)
async def update(self, uid, status):
"""
Update the status of a stored event
"""
log.debug("Updating status of event '%s' to '%s'", uid, status)
for event in self._last_events.list:
if event['id'] == uid:
event['status'] = status.value
return
log.debug('event not found in memory, checking backend')
if self.backend:
async def _ensure_status():
while True:
try:
return await self.backend.update(uid, status)
except Exception as exc:
reporting.exception(exc)
log.error('Backend not available, retrying update in 5')
await asyncio.sleep(5)
asyncio.ensure_future(_ensure_status())
async def retrieve(self, since=None, status=None):
"""
Return the list of events stored since the given datetime
"""
# Retrieve in-memory
def check_params(item):
since_check = True
status_check = True
if since:
since_check = item['created_at'] >= since
if status:
if isinstance(status, list):
status_check = EventStatus[item['status']] in status
else:
status_check = item['status'] == status.value
return since_check and status_check
in_backend = list()
if self.backend:
async def _ensure_backend():
while True:
try:
return await self.backend.retrieve(since=since,
status=status)
except Exception as exc:
reporting.exception(exc)
log.error('Backend not available, retrying retrieve in 5')
await asyncio.sleep(5)
in_backend = await _ensure_backend()
in_memory = list(filter(check_params, self._last_events.list))
return in_backend + in_memory