Exemple #1
0
    def __init__(self, backend=None, **kwargs):
        self._loop = asyncio.get_event_loop()

        if backend == 'mongo':
            self.backend = MongoBackend(**kwargs)
        elif backend == 'memory':
            self.backend = MemoryBackend(**kwargs)
        else:
            raise PersistenceError
Exemple #2
0
    def __init__(self, backend=None, memory_size=None, loop=None, **kwargs):
        """
        TODO: mongo is the only one yet, we should parse available modules
              named `*_backend.py` and select after the given backend.
        """
        self._loop = loop or asyncio.get_event_loop()
        self._last_events = FIFOSizedQueue(memory_size or 10000)
        self.backend = None
        self._feed_future = None

        if not backend:
            log.info('No persistence backend selected, in-memory only')
            return

        if backend != 'mongo':
            raise ValueError("'mongo' is the only available backend")

        self.backend = MongoBackend(**kwargs)
        if not isinstance(self.backend, PersistenceBackend):
            raise PersistenceError(
                'Wrong backend selected: {}'.format(backend))
        self._feed_future = asyncio.ensure_future(self._feed_backend())
Exemple #3
0
class BusPersistence(object):
    """
    This module will enable local caching for bus events to replace the
    current asyncio cache which is out of our control. (cf internal NYUKI-59)
    """

    FEED_DELAY = 5

    def __init__(self, backend=None, memory_size=None, loop=None, **kwargs):
        """
        TODO: mongo is the only one yet, we should parse available modules
              named `*_backend.py` and select after the given backend.
        """
        self._loop = loop or asyncio.get_event_loop()
        self._last_events = FIFOSizedQueue(memory_size or 10000)
        self.backend = None
        self._feed_future = None

        if not backend:
            log.info('No persistence backend selected, in-memory only')
            return

        if backend != 'mongo':
            raise ValueError("'mongo' is the only available backend")

        self.backend = MongoBackend(**kwargs)
        if not isinstance(self.backend, PersistenceBackend):
            raise PersistenceError(
                'Wrong backend selected: {}'.format(backend))
        self._feed_future = asyncio.ensure_future(self._feed_backend())

    @property
    def memory_buffer(self):
        return self._last_events

    async def close(self):
        if self._feed_future:
            self._feed_future.cancel()
            await self._feed_future

    async def _feed_backend(self):
        """
        Periodically check connection to backend and dump in-memory events
        into it
        """
        while True:
            try:
                await asyncio.sleep(self.FEED_DELAY)
            except asyncio.CancelledError:
                log.debug('_feed_backend cancelled')
                await self._empty_last_events()
                break

            if not self._last_events.list:
                continue

            await self._empty_last_events()

    async def _empty_last_events(self):
        if await self.backend.ping():
            if self._last_events.list:
                log.info('Dumping all event into backend')
            try:
                for event in self._last_events.empty():
                    await self.backend.store(event)
            except Exception as exc:
                reporting.exception(exc)
        else:
            log.warning('No connection to backend to empty in-memory events')

    async def init(self):
        """
        Init backend
        """
        if self.backend:
            try:
                return await self.backend.init()
            except Exception as exc:
                raise PersistenceError from exc

    async def ping(self):
        """
        Connection check
        """
        if self.backend:
            try:
                return await asyncio.wait_for(self.backend.ping(), 2.0)
            except asyncio.TimeoutError:
                log.error('Timeout pinging backend')
                return False

    async def store(self, event):
        """
        Store a bus event from
        {
            "id": "uuid4",
            "status": "EventStatus.value",
            "topic": "muc",
            "message": "json dump"
        }
        adding a 'created_at' key.
        """
        log.debug("New event stored with uid '%s'", event['id'])
        event['created_at'] = datetime.utcnow()
        self._last_events.put(event)

    async def update(self, uid, status):
        """
        Update the status of a stored event
        """
        log.debug("Updating status of event '%s' to '%s'", uid, status)
        for event in self._last_events.list:
            if event['id'] == uid:
                event['status'] = status.value
                return

        log.debug('event not found in memory, checking backend')

        if self.backend:

            async def _ensure_status():
                while True:
                    try:
                        return await self.backend.update(uid, status)
                    except Exception as exc:
                        reporting.exception(exc)
                    log.error('Backend not available, retrying update in 5')
                    await asyncio.sleep(5)

            asyncio.ensure_future(_ensure_status())

    async def retrieve(self, since=None, status=None):
        """
        Return the list of events stored since the given datetime
        """

        # Retrieve in-memory
        def check_params(item):
            since_check = True
            status_check = True

            if since:
                since_check = item['created_at'] >= since

            if status:
                if isinstance(status, list):
                    status_check = EventStatus[item['status']] in status
                else:
                    status_check = item['status'] == status.value

            return since_check and status_check

        in_backend = list()

        if self.backend:

            async def _ensure_backend():
                while True:
                    try:
                        return await self.backend.retrieve(since=since,
                                                           status=status)
                    except Exception as exc:
                        reporting.exception(exc)
                    log.error('Backend not available, retrying retrieve in 5')
                    await asyncio.sleep(5)

            in_backend = await _ensure_backend()

        in_memory = list(filter(check_params, self._last_events.list))
        return in_backend + in_memory