class QpidWrapper(object): __address_opt = '{create:always, node:{type:topic, durable:True}}' def __init__(self, broker, exchange, username, password): self.__conn = Connection(broker, username=username, password=password, tcp_nodelay=True, transport='tcp') self.__conn.open() self.__session = self.__conn.session() self.__exchange = exchange @property def session(self): return self.__session def create_address(self, routing_key=None): if routing_key: return '%s/%s;%s' % (self.__exchange, routing_key, self.__address_opt) else: return '%s;%s' % (self.__exchange, self.__address_opt) def close(self): self.__session.close() self.__conn.close()
class FreshDocWriter(PageWriterBase): def _initialize(self): self.set_name('FreshDocWriter') if get_project_settings()['DEBUG_MODE']: self._push_message = self._push_message_debug else: self._create_client() def _create_client(self): try: self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True, reconnect_limit=10, reconnect_interval=4) self.connection_.open() self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video') except: self.connection_ = None self.logger_.exception('failed to connect to message queue server.') def finalize(self): if self.connection_: self.connection_.close() PageWriterBase.finalize(self) def _push_message_debug(self, doc): pass def _push_message(self, doc): doc.video.doc_id = doc.id doc.video.id = str(doc.id) doc.video.crawl_time = doc.video.create_time = doc.crawl_time doc.video.discover_time = doc.discover_time doc.video.url = doc.url doc.video.domain = doc.domain doc.video.domain_id = doc.domain_id doc.video.in_links = doc.in_links msg_body = thrift_to_str(doc.video) if not msg_body: return self.sender_.send(Message(content=doc.url + '\t' + base64.b64encode(msg_body), durable=True)) self.logger_.info('send message successfully, %s', doc.url) def process_item(self, item): if not item: return doc = item.to_crawldoc() if doc.doc_type != CrawlDocType.PAGE_TIME: return try: self._push_message(doc) except: self.logger_.exception('failed to send message, %s', doc.url)
class FanoutPublisher(object): """ qpid fanout publisher """ def __init__(self, broker, address): self.broker = broker self.address = address self.connection = None self.session = None self.sender = None self.heartbeat_sender = None def __del__(self): self.destroy() def init(self): self.connection = Connection(self.broker, reconnect=True, reconnect_interval=3) try: self.connection.open() self.session = self.connection.session() self.sender = self.session.sender(self.address) except MessagingError: print(traceback.format_exc()) return False return True def destroy(self): if self.connection: self.connection.close() def publish(self, message): if self.sender is None: if not self.init(): return False try: message.properties = {'x-amqp-0-10.routing-key': self.address} self.sender.send(message) print('publish info. broker: {}, reply_to: {}'.format( self.broker, self.address)) except MessagingError: print(traceback.format_exc()) return False return True
class Qpid(MQ): _timeout = 1 def _conectar(self): try: logger.debug("Qpid: %s" % self._url.netloc) self._conn = Connection(self._url.netloc) if not self._conn: raise MQError(None, 2) self._conn.open() except ConnectError: raise MQError(cod=2) try: self._session = self._conn.session() self._sender = self._session.sender(self._url.path[1:]) self._receiver = self._session.receiver(self._url.path[1:]) logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc)) except ConnectError: raise MQError(cod=3) def _enviar(self, mensagem): logger.debug("Sending a message") m = Message(mensagem) self._sender.send(m, True, self._timeout) def _receber(self, timeout=None): logger.debug("Retrieving a message") self._mensagem = self._receiver.fetch(timeout) return self._mensagem.content def _tamanho(self): self._receiver.available() def _excluir(self): logger.debug("Ack last message") self._session.acknowledge() def _terminar(self): self._conn.close(self._timeout)
class QpidConnection(): port = None ip = None data_queue = None cmd_queue = None conn = None session = None sender = None receiver = None def __init__(self, ip, port, data_queue, cmd_queue): self.ip = ip; self.port = port self.data_queue = data_queue self.cmd_queue = cmd_queue def start(self): try: url = str(self.ip) url += str(':') url += str(self.port) self.conn = Connection(url) self.conn.open() self.session = self.conn.session() self.sender = self.session.sender(self.cmd_queue) self.receiver = self.session.receiver(self.data_queue) return 1 except MessagingError as m: print(m) return 0 def stop(self): try: self.conn.close() except MessagingError as m: print(m) return 0
class RpcConnectionQpidMQ(RpcConnection): def __init__(self, ep=None): RpcConnection.__init__(self, ep=ep) self.conn = None self.exitflag = False ep.impl = self self.mq_recv = '' RpcConnectionMQ_Collection.instance().add(self) @staticmethod def create(name, host, port, address, af=AF_WRITE): """ 创建MQ的连接对象 :param name: 必须是真实的 mq 名称 :param host: :param port: :param address: :param af: :return: """ ep = RpcEndPoint(name=name, host=host, port=port, addr=address, type_='qpid') if ep.open(af): return ep.impl return None # conn = RpcConnectionQpidMQ(ep) # conn.open(af) # return conn # @staticmethod # def createRpcInvocationPair(ep_read,ep_write): # conn_read = RpcConnectionQpidMQ(ep_read) # conn_read.open(AF_READ) # conn_write = RpcConnectionQpidMQ(ep_write) # conn_write.open(AF_WRITE) # conn_write.setLoopbackMQ(conn_read) # return conn_read,conn_write def open(self, af): ''' <ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> <ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> ''' from qpid.messaging import Connection from qpid.util import URL ep = self.ep self.af = af self.exitflag = False broker = "%s:%s" % (ep.host, ep.port) # log_debug('prepare mq : <%s %s>!'% (ep.name,broker)) try: self.conn = Connection(broker, reconnect=True, tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if af & AF_READ: self.rcv = self.ssn.receiver(self.ep.addr) # self.rcv.capacity = 4000 gevent.spawn(self.thread_recv) # self.thread = threading.Thread(target =self.thread_recv) # self.thread.start() # import gevent # gevent.spawn(self.thread_recv) if af & AF_WRITE: self.snd = self.ssn.sender(self.ep.addr) gevent.spawn(self.keepAliveWrite) except: log_error(traceback.format_exc()) return False # log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker)) return True def test(self): while True: print '*-' * 20 print 'qpid conn:', self gevent.sleep(3) def keepAliveWrite(self): """ 当mq为写入类型时,防止由于长时间没有消息进入,导致后继读取mq堵塞的情况,需要定时发送保活消息 这个问题目前没有应对方案,只能通过在写mq通道上频繁写入小数据,强迫mq接收者响应消息的读取。 :return: """ from qpid.messaging import Message while True: gevent.sleep(5) print 'keepalive write, target: ', self.ep.addr try: if self.exitflag: return True m = Message("keepalive") self.snd.send(m, False) except: log_error(traceback.format_exc()) def close(self): if self.conn: self.conn.close() self.conn = None self.exitflag = True def setLoopbackMQ(self, conn_recv): ''' 设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存 目前的回路mq名称取 队列名称,如果携带主机信息的话,回路可以从另外一台mq-server返回 ''' self.mq_recv = conn_recv.ep.getUnique() return self def sendMessage(self, m): if m.traits and m.traits.max_linger_time: value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME, '0') value = int(value) if not value: value += int(time.time()) m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value) #app制定了超时接收时间,这里调整为绝对时间,以便接收端进行判别,选择接受还是丢弃 if m.calltype & RpcMessage.RETURN: #--- Rpc的调用消息中包含接收消息的队列名称 --- mqname = m.callmsg.extra.props.get('__mq_return__') if mqname: mq = RpcConnectionMQ_Collection.instance().get(mqname) if mq: mq.sendDetail(m) return log_error('__mq_return__:<%s> is not in service mq-list!' % mqname) return False if self.mq_recv: m.extra.props['__mq_return__'] = self.mq_recv return RpcConnection.sendMessage(self, m) def sendDetail(self, m): from qpid.messaging import Message try: if self.exitflag: return True # if not self.conn: # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.snd = self.ssn.sender(self.ep.addr) d = m.marshall() m = Message(d) self.snd.send(m, False) except: log_error(traceback.format_exc()) # self.conn = None return False return True #接收消息 def __thread_recv(self): from communicator import RpcCommunicator # print 'qpid-mq recv thread start..' while not self.exitflag: try: # if not self.conn: # print 'try open mq:',self.ep.name # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.rcv = self.ssn.receiver(self.ep.addr) # self.rcv.capacity = 4000 m = self.rcv.fetch() # print '.'*10 d = m.content # print 'mq recv:',repr(d) print 'recved 1 msg from MQ..' self.ssn.acknowledge(sync=True) m = RpcMessage.unmarshall(d) if not m: log_error('decode mq-msg error!') continue m.conn = self value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME, '0') linger_time = int(value) if linger_time and time.time() > linger_time: # drop it continue #过期接收的消息直接丢弃 # self.dispatchMsg(m) RpcCommunicator.instance().dispatchMsg(m) except: log_error(traceback.format_exc()) gevent.sleep(1) if self.adapter: self.adapter.stopmtx.set() # log_info("qpid-mq thread exiting..") return False #接收消息 def thread_recv(self): # from communicator import RpcCommunicator # print 'qpid-mq recv thread start..' print 'qpid_connection:', self print 'mq_address:', self.ep.addr print 'receiver:', self.rcv while not self.exitflag: try: m = self.rcv.fetch() # print '.'*10 d = m.content # print 'mq recv:',repr(d) print 'recved 1 msg from MQ..' self.ssn.acknowledge(m, sync=True) m = RpcMessage.unmarshall(d) if not m: log_error('decode mq-msg error!') continue m.conn = self value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME, '0') linger_time = int(value) if linger_time and time.time() > linger_time: # drop it continue #过期接收的消息直接丢弃 self.dispatchMsg(m) # RpcCommunicator.instance().dispatchMsg(m) except: log_error(traceback.format_exc()) gevent.sleep(1) if self.adapter: self.adapter.stopmtx.set() # log_info("qpid-mq thread exiting..") return False
class RpcConnectionQpidMQ(RpcConnection): def __init__(self,ep=None): RpcConnection.__init__(self,ep=ep) self.conn = None self.exitflag = False ep.impl = self self.mq_recv ='' RpcConnectionMQ_Collection.instance().add(self) @staticmethod def create(name,host,port,address,af=AF_WRITE): ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type='qpid') conn = RpcConnectionQpidMQ(ep) conn.open(af) return conn def open(self,af): ''' <ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> <ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> ''' from qpid.messaging import Connection from qpid.util import URL ep = self.ep self.af = af self.exitflag = False broker = "%s:%s"%(ep.host,ep.port) # log_debug('prepare mq : <%s %s>!'% (ep.name,broker)) try: self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if af & AF_READ: self.rcv = self.ssn.receiver(self.ep.addr) self.rcv.capacity = 4000 # self.thread = threading.Thread(target =self.thread_recv) # self.thread.start() # import gevent gevent.spawn(self.thread_recv) if af & AF_WRITE: self.snd = self.ssn.sender(self.ep.addr) except: log_error(traceback.format_exc()) return False # log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker)) return True def close(self): if self.conn: self.conn.close() self.conn = None self.exitflag = True def setLoopbackMQ(self,conn): ''' 设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存 目前的回路mq名称取 队列名称,如果携带主机信息的话,回路可以从另外一台mq-server返回 ''' self.mq_recv = conn.ep.getUnique() return self def sendMessage(self,m): if m.calltype & RpcMessage.RETURN: #--- Rpc的调用消息中包含接收消息的队列名称 --- mqname = m.callmsg.extra.props.get('__mq_return__') if mqname: mq = RpcConnectionMQ_Collection.instance().get(mqname) if mq: mq.sendDetail(m) return log_error('__mq_return__:<%s> is not in service mq-list!'%mqname) return False if self.mq_recv: m.extra.props['__mq_return__'] = self.mq_recv return RpcConnection.sendMessage(self,m) def sendDetail(self,m): from qpid.messaging import Message try: if self.exitflag: return True # if not self.conn: # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.snd = self.ssn.sender(self.ep.addr) d = m.marshall() m = Message(d) self.snd.send(m,False) except: log_error(traceback.format_exc()) # self.conn = None return False return True #接收消息 def thread_recv(self): # print 'qpid-mq recv thread start..' while not self.exitflag: try: # if not self.conn: # print 'try open mq:',self.ep.name # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.rcv = self.ssn.receiver(self.ep.addr) # self.rcv.capacity = 4000 m = self.rcv.fetch() # print '.'*10 d = m.content # print 'mq recv:',repr(d) self.ssn.acknowledge(sync=False) m = RpcMessage.unmarshall(d) if not m: log_error('decode mq-msg error!') continue m.conn = self # self.dispatchMsg(m) RpcCommunicator.instance().dispatchMsg(m) except: log_error(traceback.format_exc()) gevent.sleep(1) if self.adapter: self.adapter.stopmtx.set() # log_info("qpid-mq thread exiting..") return False
class RpcConnectionQpidMQ(RpcConnection): def __init__(self,ep=None): RpcConnection.__init__(self,ep=ep) self.conn = None self.exitflag = False ep.impl = self self.mq_recv ='' RpcConnectionMQ_Collection.instance().add(self) @staticmethod def create(name,host,port,address,af=AF_WRITE): """ 创建MQ的连接对象 :param name: 必须是真实的 mq 名称 :param host: :param port: :param address: :param af: :return: """ ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type_='qpid') if ep.open(af): return ep.impl return None # conn = RpcConnectionQpidMQ(ep) # conn.open(af) # return conn # @staticmethod # def createRpcInvocationPair(ep_read,ep_write): # conn_read = RpcConnectionQpidMQ(ep_read) # conn_read.open(AF_READ) # conn_write = RpcConnectionQpidMQ(ep_write) # conn_write.open(AF_WRITE) # conn_write.setLoopbackMQ(conn_read) # return conn_read,conn_write def open(self,af): ''' <ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> <ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> ''' from qpid.messaging import Connection from qpid.util import URL ep = self.ep self.af = af self.exitflag = False broker = "%s:%s"%(ep.host,ep.port) # log_debug('prepare mq : <%s %s>!'% (ep.name,broker)) try: self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if af & AF_READ: self.rcv = self.ssn.receiver(self.ep.addr) self.rcv.capacity = 4000 # self.thread = threading.Thread(target =self.thread_recv) # self.thread.start() # import gevent gevent.spawn(self.thread_recv) if af & AF_WRITE: self.snd = self.ssn.sender(self.ep.addr) except: log_error(traceback.format_exc()) return False # log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker)) return True def close(self): if self.conn: self.conn.close() self.conn = None self.exitflag = True def setLoopbackMQ(self,conn_recv): ''' 设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存 目前的回路mq名称取 队列名称,如果携带主机信息的话,回路可以从另外一台mq-server返回 ''' self.mq_recv = conn_recv.ep.getUnique() return self def sendMessage(self,m): if m.traits and m.traits.max_linger_time: value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0') value = int(value) if not value: value += int(time.time()) m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value ) #app制定了超时接收时间,这里调整为绝对时间,以便接收端进行判别,选择接受还是丢弃 if m.calltype & RpcMessage.RETURN: #--- Rpc的调用消息中包含接收消息的队列名称 --- mqname = m.callmsg.extra.props.get('__mq_return__') if mqname: mq = RpcConnectionMQ_Collection.instance().get(mqname) if mq: mq.sendDetail(m) return log_error('__mq_return__:<%s> is not in service mq-list!'%mqname) return False if self.mq_recv: m.extra.props['__mq_return__'] = self.mq_recv return RpcConnection.sendMessage(self,m) def sendDetail(self,m): from qpid.messaging import Message try: if self.exitflag: return True # if not self.conn: # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.snd = self.ssn.sender(self.ep.addr) d = m.marshall() m = Message(d) self.snd.send(m,False) except: log_error(traceback.format_exc()) # self.conn = None return False return True #接收消息 def thread_recv(self): from communicator import RpcCommunicator # print 'qpid-mq recv thread start..' while not self.exitflag: try: # if not self.conn: # print 'try open mq:',self.ep.name # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.rcv = self.ssn.receiver(self.ep.addr) # self.rcv.capacity = 4000 m = self.rcv.fetch() # print '.'*10 d = m.content # print 'mq recv:',repr(d) print 'recved 1 msg from MQ..' self.ssn.acknowledge(sync=False) m = RpcMessage.unmarshall(d) if not m: log_error('decode mq-msg error!') continue m.conn = self value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0') linger_time = int(value) if linger_time and time.time() > linger_time: # drop it continue #过期接收的消息直接丢弃 # self.dispatchMsg(m) RpcCommunicator.instance().dispatchMsg(m) except: log_error(traceback.format_exc()) gevent.sleep(1) if self.adapter: self.adapter.stopmtx.set() # log_info("qpid-mq thread exiting..") return False
class NordicWayIC: def __init__(self, url, sender, receiver, username, password, options=None): self.options = options if options else {} self.url = url self._queue_sender = sender self._queue_receiver = receiver self._credentials = {"username": username, "password": password} self.log = logging.getLogger("geofencebroker") def __enter__(self): self.connect() return self def __exit__(self, type, value, traceback): self.close() def connect(self): self.connection = Connection(self.url, username=self._credentials.get("username"), password=self._credentials.get("password"), **self.options) self.connection.open() self.session = self.connection.session() self.sender = self.session.sender(self._queue_sender) self.receiver = self.session.receiver(self._queue_receiver) def send_messsage(self, msg): try: self.sender.send(msg) self.sender.check_error() except MessagingError: self.log.exception("Error sending message!") except Exception: self.log.exception("Exception occured while sending..") self.session.acknowledge() def send_obj(self, datex_obj): """ Use data from the 'datex2' object to construct a proper AMQP object with all the required properties set. """ tz = pytz.timezone("Europe/Oslo") now_iso_timestamp = datetime.datetime.now(tz).isoformat() centroid = datex_obj.centroid prop = { "who": "Norwegian Public Roads Administration", "how": "Datex2", "what": "PredefinedLocation", "lat": centroid[0], "lon": centroid[1], "where1": "no", "when": now_iso_timestamp } m = Message(user_id=self._credentials.get("username"), properties=prop, content=str(datex_obj)) self.log.debug(u"Sending message: version={}, name={}".format( datex_obj.version, datex_obj.name)) self.send_messsage(m) def close(self): self.connection.close() def recv(self, timeout=None): msg = self.receiver.fetch(timeout=timeout) return msg def __repr__(self): return "<{} url={}, sender={}, receiver={}, options={}>".format( self.__class__.__name__, self.url, self._queue_sender, self._queue_receiver, self.options)
class MQConnectionQpid(object): def __init__(self,cfg): self.type = MessageQueueType.QPID self.name = cfg.get('name') self.host = cfg.get('host') self.port = cfg.get('port') self.address = cfg.get('address') self.execthread_nr = cfg.get('exec_thread_nr',1) self.entry = cfg.get('entry') self.conn = None self.ssn = None self.producer = None self.consumer = None self.cond_readable = Condition() self.thread = None self.isclosed = True self.execthreads = [] self.message_pool = [] self.lock = Lock() self.func_list ={} #导入的函数列表 self.rw = 0 def open(self,access=AccessMode.READ): if access == 0: return self broker = "%s:%s" % (self.host, self.port) # if self.conn is not None: # return self if not self.conn: self.conn = Connection(broker, reconnect=True, tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if access & AccessMode.READ: if not self.consumer: self.consumer = self.ssn.receiver(self.address) self.consumer.capacity = 4000 func = import_function(self.entry) # importing functions dynamically self.func_list[self.entry] = func self.thread = Thread(target=self._messageRecieving) self.thread.start() if access & AccessMode.WRITE: if not self.producer: self.producer = self.ssn.sender(self.address) return self def close(self): self.isclosed = True with self.cond_readable: self.cond_readable.notify_all() if self.conn : self.conn.close() self.conn = None self.thread.join() def _executeThread(self): """多线程""" while not self.isclosed: with self.cond_readable: self.cond_readable.wait() if self.isclosed: break message = None self.lock.acquire() if len(self.message_pool): message = self.message_pool[0] del self.message_pool[0] self.lock.release() if message is not None: func = self.func_list[self.entry] func(message) # pass into user space print 'topic read thread is exiting..' def produce(self,message): message = Message(message) self.producer.send(message, False) def _messageRecieving(self): """消息接收线程,保证一个线程接收""" self.isclosed = False for nr in range(self.execthread_nr): thread = Thread(target=self._executeThread) self.execthreads.append(thread) thread.start() while not self.isclosed: try: message = self.consumer.fetch() message = message.content # self.ssn.acknowledge(sync=False) if message is not None: func = self.func_list[self.entry] try: func(message) self.ssn.acknowledge(sync=False) except: instance.getLogger().error(u'amqp::qpid message process error. detail:{}'.format(traceback.format_exc())) # self.lock.acquire() # self.message_pool.append(message) # self.lock.release() # with self.cond_readable: # # self.cond_readable.notify() # self.cond_readable.notify_all() except: instance.getLogger().warn(u'amqp::qpid fetch message failed. detail:{}'.format(traceback.format_exc())) gevent.sleep(5) # for thread in self.execthreads: # thread.join() instance.getLogger().info( 'amqp::qpid recieve-thread is exiting...')
class AMQPEventConsumer(object): """ The AMQPEventConsumer can be used to subscribe for events and process them thru a callback. The subscription is done thru *XQuery*, the callback can be a python method. Example listening for an event called *AsteriskNotification*:: >>> from gosa.common.components import AMQPEventConsumer >>> from lxml import etree >>> >>> # Event callback >>> def process(data): ... print(etree.tostring(data, pretty_print=True)) >>> >>> # Create event consumer >>> consumer = AMQPEventConsumer("amqps://*****:*****@localhost/org.gosa", ... xquery=\"\"\" ... declare namespace f='http://www.gonicus.de/Events'; ... let $e := ./f:Event ... return $e/f:AsteriskNotification ... \"\"\", ... callback=process) The consumer will start right away, listening for your events. =============== ============ Parameter Description =============== ============ url URL used to connect to the AMQP service broker domain If the domain is not already encoded in the URL, it can be specified here. xquery `XQuery <http://en.wikipedia.org/wiki/XQuery>`_ string to query for events. callback Python method to be called if the event happened. =============== ============ .. note:: The AMQP URL consists of these parts:: (amqp|amqps)://user:password@host:port/domain """ def __init__(self, url, domain="org.gosa", xquery=".", callback=None): # Build connection url = parseURL(url) self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True) self.__conn.open() # Assemble subscription query queue = 'event-listener-%s' % uuid4() address = """%s; { create: always, delete:always, node: { durable: False, x-declare: { exclusive: True, auto-delete: True } }, link: { x-bindings: [ { exchange: '%s', queue: %s, key: event, arguments: { xquery: %r} } ] } }""" % (queue, domain, queue, xquery) # Add processor for core.event queue self.__callback = callback self.__eventWorker = AMQPStandaloneWorker( self.__conn, r_address=address, workers=1, callback=self.__eventProcessor) def __del__(self): self.__eventWorker.join() self.__conn.close() #pylint: disable=W0613 def __eventProcessor(self, ssn, data): # Call callback, let exceptions pass to the caller xml = objectify.fromstring(data.content) self.__callback(xml) def join(self): self.__eventWorker.join()
class QpidConnection(object): """ This class represents a connection to a Qpid broker. A QpidConnection must be created in order to send or receive AMQP messages using a Qpid broker. """ def __init__(self, url, username, password, transport='tcp', reconnection_interval=60, reconnect_handler=None, context=None, log=None): """ Create a new connection to a Qpid message broker in order to send or receive AMQP messages. :param: url URL for the Qpid connection, e.g. 9.10.49.164:5672 :param: username Qpid username :param: password Qpid password :param: transport Transport mechanism, one of tcp, tcp+tls, or ssl (alias for tcp+tls). :param: reconnection_interval Interval in seconds between reconnect attempts. :param: reconnect_handler The function to call upon reconnecting to the Qpid broker after connection was lost and then reestablished. This function will be called after the connections is reestablished but before the listeners are started up again. It is not passed any parameters. :param: context The security context :param: log The logging module used for logging messages. If not provided then no logging will be done. """ self.url = url self.username = username self.password = password self.context = context self.log = log.getLogger(__name__) if log else None self.transport = transport self.reconnection_interval = reconnection_interval self.reconnect_handler = reconnect_handler self._listeners = [] self._is_connected = False def create_listener(self, exchange, topic): """ Create a new listener on the given exchange for the given topic. :param: exchange The name of the Qpid exchange, e.g. 'nova' :param: topic The topic to listen for, e.g. 'notifications.info' :returns: A new QpidListener that will listen for messages on the given exchange and topic. """ listener = QpidListener(self, exchange, topic) self._listeners.append(listener) return listener def start(self, is_reconnect=False): """ Initiate the Qpid connection and start up any listeners. :param: is_reconnect True if this method is called as part of a reconnect attempt, False otherwise :raise: ConnectionError if a connection cannot be established """ # If the Qpid broker URL is not specified (or just the hostname is not # specified) then we can't make a connection. if not self.url or self.url.startswith(':'): log(self.log, 'warn', _('Qpid broker not specified, cannot start ' 'connection.')) return if not self._is_connected: self.conn = Connection(self.url, username=self.username, password=self.password, transport=self.transport) try: self.conn.open() except ConnectionError as e: log(self.log, 'critical', _('Cannot connect to Qpid message ' 'broker: %s') % (e.message)) # close this connection when encounter connection error # otherwise, it will leave an ESTABLISHED connection # to qpid server forever. if self.conn is not None: self.conn.close() raise e self._is_connected = True if is_reconnect and self.reconnect_handler: self.reconnect_handler() for listener in self._listeners: listener._start(self.conn) log(self.log, 'info', _('Connected to Qpid message broker: ' '%s@%s') % (self.username, self.url)) def _reconnect(self): """ Attempt to reconnect to the Qpid message broker in intervals until the connection comes back. """ self.conn = None class ReconnectionThread(threading.Thread): def __init__(self, qpid_connection): super(ReconnectionThread, self).__init__( name='ReconnectionThread') self.qpid_connection = qpid_connection def run(self): while not self.qpid_connection._is_connected: try: self.qpid_connection.start(is_reconnect=True) except ConnectionError: sleep(self.qpid_connection.reconnection_interval) pass reconnection_thread = ReconnectionThread(self) reconnection_thread.start() def set_reconnect_handler(self, reconnect_handler): """ Set the function to call upon reconnecting to the Qpid broker after connection is lost and then reestablished. :param: reconnect_handler The function to call upon reconnecting. """ self.reconnect_handler = reconnect_handler