Пример #1
0
 def __init__(self, stream):
     """
     :param stream: The stream name.
     :type stream: str
     """
     self.stream = stream
     self.queue = Queue(maxsize=100)
     self.is_open = False
     self.sequential = Sequential()
     self.journal = {}
     self.thread = Thread(target=self._open)
     self.thread.setDaemon(True)
     self.thread.start()
Пример #2
0
 def run(self):
     while not Thread.aborted():
         sleep(self.INTERVAL)
         try:
             reply = protocol.Ping(self.pid)
             reply.send(self.pipe)
         except PipeBroken:
             sys.exit(1)
Пример #3
0
 def run(self):
     while not Thread.aborted():
         sleep(self.INTERVAL)
         try:
             reply = protocol.Ping(self.pid)
             reply.send(self.pipe)
         except PipeBroken:
             sys.exit(1)
Пример #4
0
 def _drain(self):
     """
     Drain the queue.
     """
     self.is_open = False
     while not Thread.aborted():
         try:
             request = self.queue.get(timeout=1)
             self.commit(request.sn)
         except Empty:
             break
Пример #5
0
 def _drain(self):
     """
     Drain the queue.
     """
     self.is_open = False
     while not Thread.aborted():
         try:
             request = self.queue.get(timeout=1)
             self.commit(request.sn)
         except Empty:
             break
Пример #6
0
 def get(self):
     """
     Get the next pending request to be dispatched.
     Blocks until a request is available.
     :return: The next pending request.
     :rtype: Document
     """
     while not Thread.aborted():
         try:
             return self.queue.get(timeout=10)
         except Empty:
             pass
Пример #7
0
 def get(self):
     """
     Get the next pending request to be dispatched.
     Blocks until a request is available.
     :return: The next pending request.
     :rtype: Document
     """
     while not Thread.aborted():
         try:
             return self.queue.get(timeout=10)
         except Empty:
             pass
Пример #8
0
 def __init__(self, stream):
     """
     :param stream: The stream name.
     :type stream: str
     """
     self.stream = stream
     self.queue = Queue(maxsize=100)
     self.is_open = False
     self.sequential = Sequential()
     self.journal = {}
     self.thread = Thread(target=self._open)
     self.thread.setDaemon(True)
     self.thread.start()
Пример #9
0
 def _fn(reader, timeout=None):
     delay = DELAY
     timer = float(timeout or 0)
     while not Thread.aborted():
         message = fn(reader, timer)
         if message:
             return message
         if timer > 0:
             sleep(delay)
             timer -= delay
             if delay < MAX_DELAY:
                 delay *= DELAY_MULTIPLIER
         else:
             break
Пример #10
0
 def _fn(reader, timeout=None):
     delay = DELAY
     timer = float(timeout or 0)
     while not Thread.aborted():
         message = fn(reader, timer)
         if message:
             return message
         if timer > 0:
             sleep(delay)
             timer -= delay
             if delay < MAX_DELAY:
                 delay *= DELAY_MULTIPLIER
         else:
             break
Пример #11
0
 def _fn(messenger, *args, **kwargs):
     repair = lambda: None
     while not Thread.aborted():
         try:
             repair()
             return fn(messenger, *args, **kwargs)
         except ChannelError, e:
             if e.code != 404:
                 sleep(DELAY)
                 repair = messenger.repair
             else:
                 raise NotFound(*e.args)
         except CONNECTION_EXCEPTIONS:
             sleep(DELAY)
             repair = messenger.repair
Пример #12
0
 def _fn(messenger, *args, **kwargs):
     repair = lambda: None
     while not Thread.aborted():
         try:
             repair()
             return fn(messenger, *args, **kwargs)
         except LinkDetached, le:
             if le.condition != NOT_FOUND:
                 sleep(DELAY)
                 repair = messenger.repair
             else:
                 raise NotFound(*le.args)
         except ConnectionException:
             sleep(DELAY)
             repair = messenger.repair
Пример #13
0
 def _fn(messenger, *args, **kwargs):
     repair = lambda: None
     while not Thread.aborted():
         try:
             repair()
             return fn(messenger, *args, **kwargs)
         except ChannelError, e:
             if e.code != 404:
                 sleep(DELAY)
                 repair = messenger.repair
             else:
                 raise NotFound(*e.args)
         except CONNECTION_EXCEPTIONS:
             sleep(DELAY)
             repair = messenger.repair
Пример #14
0
 def _fn(messenger, *args, **kwargs):
     repair = lambda: None
     while not Thread.aborted():
         try:
             repair()
             return fn(messenger, *args, **kwargs)
         except LinkDetached, le:
             if le.condition != NOT_FOUND:
                 sleep(DELAY)
                 repair = messenger.repair
             else:
                 raise NotFound(*le.args)
         except ConnectionException:
             sleep(DELAY)
             repair = messenger.repair
Пример #15
0
 def inner(connection):
     if connection.retry:
         retries = RETRIES
     else:
         retries = 0
     delay = DELAY
     url = connection.url
     while not Thread.aborted():
         try:
             log.info('connecting: %s', url)
             impl = fn(connection)
             log.info('connected: %s', url)
             return impl
         except exception, e:
             log.error('connect: %s, failed: %s', url, e)
             if retries > 0:
                 log.info('retry in %d seconds', delay)
                 sleep(delay)
                 if delay < MAX_DELAY:
                     delay *= DELAY_MULTIPLIER
                 retries -= 1
             else:
                 raise
Пример #16
0
 def inner(connection):
     if connection.retry:
         retries = RETRIES
     else:
         retries = 0
     delay = DELAY
     url = connection.url
     while not Thread.aborted():
         try:
             log.info('connecting: %s', url)
             impl = fn(connection)
             log.info('connected: %s', url)
             return impl
         except exception, e:
             log.error('connect: %s, failed: %s', url, e)
             if retries > 0:
                 log.info('retry in %d seconds', delay)
                 sleep(delay)
                 if delay < MAX_DELAY:
                     delay *= DELAY_MULTIPLIER
                 retries -= 1
             else:
                 raise
Пример #17
0
class Pending(object):
    """
    Persistent store and queuing for pending requests.
    """

    PENDING = '/var/lib/%s/messaging/pending' % NAME

    @staticmethod
    def _write(request, path):
        """
        Write a request to the journal.
        :param request: An AMQP request.
        :type request: Document
        :param path: The destination path.
        :type path: str
        """
        fp = open(path, 'w+')
        try:
            body = request.dump()
            fp.write(body)
            log.debug('wrote [%s]: %s', path, body)
        finally:
            fp.close()

    @staticmethod
    def _read(path):
        """
        Read a request.
        :param path: The path to the journal file.
        :type path: str
        :return: The read request.
        :rtype: Document
        """
        fp = open(path)
        try:
            try:
                request = Document()
                body = fp.read()
                request.load(body)
                log.debug('read [%s]: %s', path, body)
                return request
            except ValueError:
                log.error('%s corrupt (discarded)', path)
                os.unlink(path)
        finally:
            fp.close()

    def _list(self):
        """
        Directory listing sorted by when it was created.
        :return: A sorted directory listing (absolute paths).
        :rtype: list
        """
        path = os.path.join(Pending.PENDING, self.stream)
        paths = [os.path.join(path, name) for name in os.listdir(path)]
        return sorted(paths)

    def __init__(self, stream):
        """
        :param stream: The stream name.
        :type stream: str
        """
        self.stream = stream
        self.queue = Queue(maxsize=100)
        self.is_open = False
        self.sequential = Sequential()
        self.journal = {}
        self.thread = Thread(target=self._open)
        self.thread.setDaemon(True)
        self.thread.start()

    def _open(self):
        """
        Open for operations.
        Load journal(ed) requests. These are requests were in the queuing pipeline
        when the process was terminated. put() is blocked until this has completed.
        """
        path = os.path.join(Pending.PENDING, self.stream)
        mkdir(path)
        log.info('Using: %s', path)
        for path in self._list():
            log.info('Restoring: %s', path)
            request = Pending._read(path)
            if not request:
                # read failed
                continue
            self._put(request, path)
        self.is_open = True

    def put(self, request):
        """
        Enqueue a pending request.
        This is blocked until the _open() has re-queued journal(ed) entries.
        :param request: An AMQP request.
        :type request: Document
        """
        while not self.is_open:
            # block until opened
            sleep(1)
        fn = self.sequential.next()
        path = os.path.join(Pending.PENDING, self.stream, fn)
        Pending._write(request, path)
        self._put(request, path)

    def get(self):
        """
        Get the next pending request to be dispatched.
        Blocks until a request is available.
        :return: The next pending request.
        :rtype: Document
        """
        while not Thread.aborted():
            try:
                return self.queue.get(timeout=10)
            except Empty:
                pass

    def commit(self, sn):
        """
        The request referenced by the serial number has been completely
        processed and can be deleted from the journal.
        :param sn: A request serial number.
        :param sn: str
        """
        try:
            path = self.journal[sn]
            unlink(path)
            log.debug('%s committed', sn)
        except KeyError:
            log.warn('%s not found for commit', sn)

    def delete(self):
        """
        Drain the queue and delete the store.
        """
        self.is_open = False
        self.thread.abort()
        self.thread.join()
        self._drain()
        path = os.path.join(Pending.PENDING, self.stream)
        rmdir(path)
        log.info('%s, deleted', path)

    def _drain(self):
        """
        Drain the queue.
        """
        self.is_open = False
        while not Thread.aborted():
            try:
                request = self.queue.get(timeout=1)
                self.commit(request.sn)
            except Empty:
                break

    def _put(self, request, jnl_path):
        """
        Enqueue the request.
        :param request: An AMQP request.
        :type request: Document
        :param jnl_path: Path to the associated journal file.
        :type jnl_path: str
        """
        request.ts = time()
        tracker = Tracker()
        tracker.add(request.sn, request.data)
        self.journal[request.sn] = jnl_path
        self.queue.put(request)
Пример #18
0
class Pending(object):
    """
    Persistent store and queuing for pending requests.
    """

    PENDING = '/var/lib/%s/messaging/pending' % NAME

    @staticmethod
    def _write(request, path):
        """
        Write a request to the journal.
        :param request: An AMQP request.
        :type request: Document
        :param path: The destination path.
        :type path: str
        """
        with open(path, 'w+') as fp:
            body = request.dump()
            fp.write(body)
            log.debug('wrote [%s]: %s', path, body)

    @staticmethod
    def _read(path):
        """
        Read a request.
        :param path: The path to the journal file.
        :type path: str
        :return: The read request.
        :rtype: Document
        """
        with open(path) as fp:
            try:
                request = Document()
                body = fp.read()
                request.load(body)
                log.debug('read [%s]: %s', path, body)
                return request
            except ValueError:
                log.error('%s corrupt (discarded)', path)
                unlink(path)

    def _list(self):
        """
        Directory listing sorted by when it was created.
        :return: A sorted directory listing (absolute paths).
        :rtype: list
        """
        path = os.path.join(Pending.PENDING, self.stream)
        paths = [os.path.join(path, name) for name in os.listdir(path)]
        return sorted(paths)

    def __init__(self, stream):
        """
        :param stream: The stream name.
        :type stream: str
        """
        self.stream = stream
        self.queue = Queue(maxsize=100)
        self.is_open = False
        self.sequential = Sequential()
        self.journal = {}
        self.thread = Thread(target=self._open)
        self.thread.setDaemon(True)
        self.thread.start()

    def _open(self):
        """
        Open for operations.
        Load journal(ed) requests. These are requests were in the queuing pipeline
        when the process was terminated. put() is blocked until this has completed.
        """
        path = os.path.join(Pending.PENDING, self.stream)
        mkdir(path)
        log.info('Using: %s', path)
        for path in self._list():
            log.info('Restoring: %s', path)
            request = Pending._read(path)
            if not request:
                # read failed
                continue
            self._put(request, path)
        self.is_open = True

    def put(self, request):
        """
        Enqueue a pending request.
        This is blocked until the _open() has re-queued journal(ed) entries.
        :param request: An AMQP request.
        :type request: Document
        """
        while not self.is_open:
            # block until opened
            sleep(1)
        fn = self.sequential.next()
        path = os.path.join(Pending.PENDING, self.stream, fn)
        Pending._write(request, path)
        self._put(request, path)

    def get(self):
        """
        Get the next pending request to be dispatched.
        Blocks until a request is available.
        :return: The next pending request.
        :rtype: Document
        :raise Empty: on thread aborted.
        """
        while not Thread.aborted():
            try:
                return self.queue.get(timeout=3)
            except Empty:
                pass
        # aborted
        raise Empty()

    def commit(self, sn):
        """
        The request referenced by the serial number has been completely
        processed and can be deleted from the journal.
        :param sn: A request serial number.
        :param sn: str
        """
        try:
            path = self.journal.pop(sn)
            unlink(path)
            log.debug('%s committed', sn)
        except KeyError:
            log.warning('%s not found for commit', sn)

    def delete(self):
        """
        Drain the queue and delete the store.
        """
        self.is_open = False
        self.thread.abort()
        self.thread.join()
        self._drain()
        path = os.path.join(Pending.PENDING, self.stream)
        rmdir(path)
        log.info('%s, deleted', path)

    def _drain(self):
        """
        Drain the queue.
        """
        self.is_open = False
        while not Thread.aborted():
            try:
                request = self.queue.get(timeout=1)
                self.commit(request.sn)
            except Empty:
                break

    def _put(self, request, jnl_path):
        """
        Enqueue the request.
        :param request: An AMQP request.
        :type request: Document
        :param jnl_path: Path to the associated journal file.
        :type jnl_path: str
        """
        request.ts = time()
        tracker = Tracker()
        tracker.add(request.sn, request.data)
        self.journal[request.sn] = jnl_path
        self.queue.put(request)