Beispiel #1
0
 def connect(self):
     if not self.connection:
         self.connection = amqp.Connection(self.conf.host,
                     userid=self.conf.username,
                     password=self.conf.passwd,
                     virtual_host=self.conf.vhost,
                     ssl=False)
         self.pool = ConnectionPool(None, self.conf.pools)
         for c in xrange(self.conf.pools):
             self.pool.put(self.connection.channel(c+1))
Beispiel #2
0
 def connect(self):
     if not self.pool:
         conn = beanstalk.serverconn.ServerConn(self.conf.host, self.conf.port)
         self.pool = ConnectionPool(conn, n_slots=self.conf.pools)
         log.info('BeanstalkEngine connected')
Beispiel #3
0
class BeanstalkEngine(EngineBase):
    def __init__(self, conf):
        """
        conf ==> ('host','port')
        """
        self.conf = conf
        self.conn = None
        self.count = 0
        self.pool = None
        log.info('init BeanstalkEngine')
        self.connect()
        
    def connect(self):
        if not self.pool:
            conn = beanstalk.serverconn.ServerConn(self.conf.host, self.conf.port)
            self.pool = ConnectionPool(conn, n_slots=self.conf.pools)
            log.info('BeanstalkEngine connected')
    
    def close(self):
        if self.pool:
            with self.pool.reserve() as conn:
                conn.close()
    
    def send(self, messages):
        self.connect()
        if not self.pool:
            raise Exception('can not connect beanstalk server.')
        if not isinstance(messages, list):
            messages = list(messages)
        total = 0
        with self.pool.reserve() as conn:
            for message in messages:
                try:
                    ttr = message.get('ttr', 60)
                    conn.use(message.queue.name)
                    conn.put(jsonfy.dumps(message), ttr=int(ttr))
                    self.count = self.count + 1
                    total = total + 1
                    log.debug('mq message count:%s' % self.count)
                except:
                    log.exception('unexpected error')
        return total
    
    def stat(self):
        pass
        
    def listen(self, consumer):
        self.consumer = consumer
        self.queue = consumer.queue
        def _executor(bs, msg):
            bs.touch(msg['jid'])
            content = msg['data']
            self.on_message_receive(content)
            try:
                bs.delete(jobdata['jid'])
            except:
                log.exception('unexpected error:')

        def _error_handler(e):
            log.error("Got an error:%s", e)

        def _executionGenerator(bs):
            while True:
                yield bs.reserve()\
                    .addCallback(lambda v: _executor(bs, v))\
                        .addErrback(_error_handler)

        def _worker(bs):
            bs.watch(self.consumer.qname)
            bs.ignore("default")
            
            log.info('Beanstalk Engine Start watching:%s' % self.queue.name)
            coop = task.Cooperator()
            coop.coiterate(_executionGenerator(bs))
            
        def _connect():
            d=protocol.ClientCreator(reactor,
                beanstalk.twisted_client.Beanstalk).connectTCP(self.conf.host, self.conf.port)
            return d.addCallback(_worker)
        
        return _connect()
Beispiel #4
0
class RabbitMQEngine(EngineBase):
    def __init__(self, conf):
        """
        conf => ('username','passwd','host','port','vhost')
        """
        self.conf = conf
        self.connection = None
        self.count = 0
        self._connstr = "amqp://%s:%s%s/%s" % (self.conf.host, self.conf.port, \
                                               self.conf.vhost, self.conf.exchange)
        
        self.channel = None
        self.cl = None

        self._disconnecting = False
        self._reconnect_delay = None
        self.spec_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),u'amqp0-8.xml')
        log.info('init RabbitMQEngine')
        self.connect()
        
    def connect(self):
        if not self.connection:
            self.connection = amqp.Connection(self.conf.host,
                        userid=self.conf.username,
                        password=self.conf.passwd,
                        virtual_host=self.conf.vhost,
                        ssl=False)
            self.pool = ConnectionPool(None, self.conf.pools)
            for c in xrange(self.conf.pools):
                self.pool.put(self.connection.channel(c+1))
            
    def close(self):
        if self.connection:
            try:
                self.connection.close()
                self.connection = None
            except:
                log.exception('unexpected error')
    
    def send(self, messages):
        self.connect()
        if not self.connection:
            raise Exception("RabbitMQ can't connect to server.")
        if not isinstance(messages, list):
            messages = list(messages)
        total = 0
        with self.pool.reserve() as c:
            for message in messages:
                try:
                    exchange_name = message.queue.exchange
                    routing_key = message.queue.routing_key
                    msg = amqp.Message(jsonfy.dumps(message))
                    msg.properties["delivery_mode"] = 2 # persistant
                    c.basic_publish(msg, exchange=exchange_name, routing_key=routing_key)
                    total = total + 1
                except:
                    log.exception('unexpected error')
        return total
    
    def listen(self, consumer):
        self._spec = txamqp.spec.load(self.spec_path)
        self.consumer = consumer
        self.queue = consumer.queue
        def gotConnection(conn):
            log.debug("Authenticating user %s..." % self.conf.username)
            d = conn.authenticate(self.conf.username, self.conf.passwd)
            d.addCallback(connectionAuthenticated, conn)
            return d

        def connectionAuthenticated(_, conn):
            log.debug("Allocating AMQP channel...")
            return conn.channel(self.queue.chan).addCallback(gotChannel, conn)

        def gotChannel(chan, conn):
            log.debug("Opening AMQP channel...")
            return chan.channel_open().addCallback(declareQueue, chan, conn)

        def declareQueue(_, chan, conn):
            log.debug("Declaring Queue %s..." % self.queue.name)
            d = chan.queue_declare(queue=self.queue.name, durable=True,
                                   exclusive=False, auto_delete=False)
            return d.addCallback(declareExchange, chan, conn)

        def declareExchange(_, chan, conn):
            log.debug("Declaring Exchange %s..." % self.queue.exchange)
            d = chan.exchange_declare(exchange=self.queue.exchange, durable=True,
                                      type="direct", auto_delete=False)
            return d.addCallback(bindQueue, chan, conn)

        def bindQueue(_, chan, conn):
            log.debug("Binding queue %s with exchange %s on key pattern %s..." % (self.queue.name, self.queue.exchange, self.queue.routing_key))
            d = chan.queue_bind(queue=self.queue.name, exchange=self.queue.exchange,
                                routing_key=self.queue.routing_key)
            return d.addCallback(startConsumer, chan, conn)

        def startConsumer(queue, chan, conn):
            log.debug("Start consuming messages...")
            d = chan.basic_consume(queue=self.queue.name, no_ack=True,
                                   consumer_tag=self.queue.consumer_tag)
            return d.addCallback(started, chan, conn)

        def started(_, chan, conn):
            log.debug("Connected to %s" % self._connstr)
            self.connection = conn
            self.channel = chan

        def gotDisconnected(reason):
            log.info("Disconnected:")
            log.info(reason)
            reconnect()

        def connectionFailed(failure):
            log.error("Connection Failed:")
            log.error(failure.getErrorMessage())
            reconnect()

        def reconnect():
            log.debug("Reconnecting in %s seconds" % RECONNECT_DELAY)
            if not self._disconnecting:
                dc = reactor.callLater(RECONNECT_DELAY, connect)
                self._reconnect_delay = dc

        def connect():
            delegate = ConsumerDelegate(gotDisconnected, self._processMessage)
            cli = protocol.ClientCreator(reactor, AMQClient, delegate=delegate,
                                         vhost=self.conf.vhost, spec=self._spec)
            log.debug("Connecting to %s:%s..." % (self.conf.host, self.conf.port))
            d = cli.connectTCP(self.conf.host, self.conf.port)

            return d.addCallbacks(gotConnection, connectionFailed)

        print "Starting AMQP consumer for %s" % self._connstr
        return connect()
    
    def stop(self):
        def cancelConsumer(_, channel):
            print "Stop consuming %s..." % self._constag
            return channel.basic_cancel(self.queue.consumer_tag)

        def closeChannel(_, channel):
            print "Closing channel..."
            return channel.channel_close()

        def getNewChannel(_, connection):
            print "Getting a new channel for closing connection..."
            return connection.channel(0)

        def closeConnection(channel):
            print "Closing connection..."
            return channel.connection_close()

        def connectorStopped(_):
            print "Connector stopped"
            return self

        print "Stopping connector to %s" % self._connstr
        d = defer.succeed(None)

        self._disconnecting = True
        if self._reconnect_delay:
            self._reconnect_delay.cancel()
            self._reconnect_delay = None

        if self.channel is not None:
            d.addCallback(closeChannel, self.channel)
            self.channel = None

        if self.connection is not None:
            d.addCallback(getNewChannel, self.connection)
            d.addCallback(closeConnection)
            self.connection = None

        return d.addCallback(connectorStopped)
    
    def stat(self):
        pass
    
    def _processMessage(self, msg):
        content = msg.content.body
        self.on_message_receive(content)