Exemplo n.º 1
0
class PoolPublisher(object):
    def __init__(self, url):
        self.pool = Connection(url).Pool(limit=10)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        # pickling the payload for now. Better serialization mechanism is essential.
        with self.pool.acquire(block=True) as connection:
            with producers[connection].acquire(block=True) as producer:
                channel = None
                try:
                    # creating a new channel for every producer publish. This could be expensive
                    # and maybe there is a better way to do this by creating a ChannelPool etc.
                    channel = connection.channel()
                    # ain't clean but I am done trying to find out the right solution!
                    producer._set_channel(channel)
                    publish = connection.ensure(producer, producer.publish, errback=self.errback,
                                                max_retries=3)
                    publish(payload, exchange=exchange, routing_key=routing_key,
                            serializer='pickle')
                except Exception as e:
                    LOG.error('Connections to rabbitmq cannot be re-established: %s', e.message)
                finally:
                    if channel:
                        try:
                            channel.close()
                        except Exception:
                            LOG.warning('Error closing channel.', exc_info=True)
Exemplo n.º 2
0
class PoolPublisher(object):
    def __init__(self, url):
        self.pool = Connection(url).Pool(limit=10)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        # pickling the payload for now. Better serialization mechanism is essential.
        with self.pool.acquire(block=True) as connection:
            with producers[connection].acquire(block=True) as producer:
                try:
                    publish = connection.ensure(producer,
                                                producer.publish,
                                                errback=self.errback,
                                                max_retries=3)
                    publish(payload,
                            exchange=exchange,
                            routing_key=routing_key,
                            serializer='pickle')
                except Exception as e:
                    LOG.exception(
                        'Connections to rabbitmq cannot be re-established: %s',
                        e.message,
                        exc_info=False)
class PoolPublisher(object):
    def __init__(self, urls):
        self.pool = Connection(urls,
                               failover_strategy='round-robin').Pool(limit=10)
        self.cluster_size = len(urls)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        with self.pool.acquire(block=True) as connection:
            retry_wrapper = ConnectionRetryWrapper(
                cluster_size=self.cluster_size, logger=LOG)

            def do_publish(connection, channel):
                # ProducerPool ends up creating it own ConnectionPool which ends up completely
                # invalidating this ConnectionPool. Also, a ConnectionPool for producer does not
                # really solve any problems for us so better to create a Producer for each
                # publish.
                producer = Producer(channel)
                kwargs = {
                    'body': payload,
                    'exchange': exchange,
                    'routing_key': routing_key,
                    'serializer': 'pickle'
                }
                retry_wrapper.ensured(connection=connection,
                                      obj=producer,
                                      to_ensure_func=producer.publish,
                                      **kwargs)

            retry_wrapper.run(connection=connection,
                              wrapped_callback=do_publish)
Exemplo n.º 4
0
class PoolPublisher(object):
    def __init__(self, url):
        self.pool = Connection(url).Pool(limit=10)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        # pickling the payload for now. Better serialization mechanism is essential.
        with self.pool.acquire(block=True) as connection:
            with producers[connection].acquire(block=True) as producer:
                channel = None
                try:
                    # creating a new channel for every producer publish. This could be expensive
                    # and maybe there is a better way to do this by creating a ChannelPool etc.
                    channel = connection.channel()
                    # ain't clean but I am done trying to find out the right solution!
                    producer._set_channel(channel)
                    publish = connection.ensure(producer, producer.publish, errback=self.errback,
                                                max_retries=3)
                    publish(payload, exchange=exchange, routing_key=routing_key,
                            serializer='pickle')
                except Exception as e:
                    LOG.error('Connections to rabbitmq cannot be re-established: %s', e.message)
                finally:
                    if channel:
                        try:
                            channel.close()
                        except Exception:
                            LOG.warning('Error closing channel.', exc_info=True)
Exemplo n.º 5
0
class PoolPublisher(object):
    def __init__(self, urls):
        self.pool = Connection(urls, failover_strategy='round-robin').Pool(limit=10)
        self.cluster_size = len(urls)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        with self.pool.acquire(block=True) as connection:
            retry_wrapper = ConnectionRetryWrapper(cluster_size=self.cluster_size, logger=LOG)

            def do_publish(connection, channel):
                # ProducerPool ends up creating it own ConnectionPool which ends up completely
                # invalidating this ConnectionPool. Also, a ConnectionPool for producer does not
                # really solve any problems for us so better to create a Producer for each
                # publish.
                producer = Producer(channel)
                kwargs = {
                    'body': payload,
                    'exchange': exchange,
                    'routing_key': routing_key,
                    'serializer': 'pickle'
                }
                retry_wrapper.ensured(connection=connection,
                                      obj=producer,
                                      to_ensure_func=producer.publish,
                                      **kwargs)

            retry_wrapper.run(connection=connection, wrapped_callback=do_publish)
Exemplo n.º 6
0
class PoolPublisher(object):
    def __init__(self, url):
        self.pool = Connection(url).Pool(limit=10)

    def errback(self, exc, interval):
        LOG.error('Rabbitmq connection error: %s', exc.message, exc_info=False)

    def publish(self, payload, exchange, routing_key=''):
        # pickling the payload for now. Better serialization mechanism is essential.
        with self.pool.acquire(block=True) as connection:
            with producers[connection].acquire(block=True) as producer:
                try:
                    publish = connection.ensure(producer, producer.publish, errback=self.errback,
                                                max_retries=3)
                    publish(payload, exchange=exchange, routing_key=routing_key,
                            serializer='pickle')
                except:
                    LOG.exception('Connections to rabbitmq cannot be re-established.')