def get_qpid_connection(self):
"""Connect to a broker, set and return the connection object
Authenticates (if necessary), and sets bkr.common._connection and
returns it. This method is thread safe.
"""
self.connection_lock.acquire()
try:
global can_use_qpid, _connection
if not can_use_qpid:
global qpid_import_error
raise ImportError(str(qpid_import_error))
if _connection is None or _connection.get_error():
connection_params = [[self._broker],
{'reconnect': self._reconnect,
'heartbeat': self._heartbeat_timeout}]
if self.krb_auth:
connection_params[1].update({'sasl_mechanisms' : 'GSSAPI'})
# As connections can recover from krb errors, we don't need
# to worry about doing this manually.
self.do_krb_auth()
_connection = Connection(*connection_params[0], **connection_params[1])
_connection.open()
return _connection
finally:
self.connection_lock.release()
class QpidWrapper(object):
__address_opt = '{create:always, node:{type:topic, durable:True}}'
def __init__(self, broker, exchange, username, password):
self.__conn = Connection(broker,
username=username,
password=password,
tcp_nodelay=True,
transport='tcp')
self.__conn.open()
self.__session = self.__conn.session()
self.__exchange = exchange
@property
def session(self):
return self.__session
def create_address(self, routing_key=None):
if routing_key:
return '%s/%s;%s' % (self.__exchange, routing_key,
self.__address_opt)
else:
return '%s;%s' % (self.__exchange, self.__address_opt)
def close(self):
self.__session.close()
self.__conn.close()
def get_qpid_connection(self):
"""Connect to a broker, set and return the connection object
Authenticates (if necessary), and sets bkr.common._connection and
returns it. This method is thread safe.
"""
self.connection_lock.acquire()
try:
global can_use_qpid, _connection
if not can_use_qpid:
global qpid_import_error
raise ImportError(str(qpid_import_error))
if _connection is None or _connection.get_error():
connection_params = [[self._broker],
{'reconnect': self._reconnect,
'heartbeat': self._heartbeat_timeout}]
if self.krb_auth:
connection_params[1].update({'sasl_mechanisms' : 'GSSAPI'})
# As connections can recover from krb errors, we don't need
# to worry about doing this manually.
self.do_krb_auth()
_connection = Connection(*connection_params[0], **connection_params[1])
_connection.open()
return _connection
finally:
self.connection_lock.release()
class FreshDocWriter(PageWriterBase):
def _initialize(self):
self.set_name('FreshDocWriter')
if get_project_settings()['DEBUG_MODE']:
self._push_message = self._push_message_debug
else:
self._create_client()
def _create_client(self):
try:
self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True,
reconnect_limit=10, reconnect_interval=4)
self.connection_.open()
self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video')
except:
self.connection_ = None
self.logger_.exception('failed to connect to message queue server.')
def finalize(self):
if self.connection_:
self.connection_.close()
PageWriterBase.finalize(self)
def _push_message_debug(self, doc):
pass
def _push_message(self, doc):
doc.video.doc_id = doc.id
doc.video.id = str(doc.id)
doc.video.crawl_time = doc.video.create_time = doc.crawl_time
doc.video.discover_time = doc.discover_time
doc.video.url = doc.url
doc.video.domain = doc.domain
doc.video.domain_id = doc.domain_id
doc.video.in_links = doc.in_links
msg_body = thrift_to_str(doc.video)
if not msg_body:
return
self.sender_.send(Message(content=doc.url + '\t' + base64.b64encode(msg_body), durable=True))
self.logger_.info('send message successfully, %s', doc.url)
def process_item(self, item):
if not item:
return
doc = item.to_crawldoc()
if doc.doc_type != CrawlDocType.PAGE_TIME:
return
try:
self._push_message(doc)
except:
self.logger_.exception('failed to send message, %s', doc.url)
def testConnectError(self):
try:
self.conn = Connection.open("localhost", 0)
assert False, "connect succeeded"
except ConnectError, e:
# XXX: should verify that e includes appropriate diagnostic info
pass
class FanoutPublisher(object):
"""
qpid fanout publisher
"""
def __init__(self, broker, address):
self.broker = broker
self.address = address
self.connection = None
self.session = None
self.sender = None
self.heartbeat_sender = None
def __del__(self):
self.destroy()
def init(self):
self.connection = Connection(self.broker,
reconnect=True,
reconnect_interval=3)
try:
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self.address)
except MessagingError:
print(traceback.format_exc())
return False
return True
def destroy(self):
if self.connection:
self.connection.close()
def publish(self, message):
if self.sender is None:
if not self.init():
return False
try:
message.properties = {'x-amqp-0-10.routing-key': self.address}
self.sender.send(message)
print('publish info. broker: {}, reply_to: {}'.format(
self.broker, self.address))
except MessagingError:
print(traceback.format_exc())
return False
return True
class Qpid(MQ):
_timeout = 1
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" %
(self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def _enviar(self, mensagem):
logger.debug("Sending a message")
m = Message(mensagem)
self._sender.send(m, True, self._timeout)
def _receber(self, timeout=None):
logger.debug("Retrieving a message")
self._mensagem = self._receiver.fetch(timeout)
return self._mensagem.content
def _tamanho(self):
self._receiver.available()
def _excluir(self):
logger.debug("Ack last message")
self._session.acknowledge()
def _terminar(self):
self._conn.close(self._timeout)
class Qpid(MQ):
_timeout = 1
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def _enviar(self, mensagem):
logger.debug("Sending a message")
m = Message(mensagem)
self._sender.send(m, True, self._timeout)
def _receber(self, timeout=None):
logger.debug("Retrieving a message")
self._mensagem = self._receiver.fetch(timeout)
return self._mensagem.content
def _tamanho(self):
self._receiver.available()
def _excluir(self):
logger.debug("Ack last message")
self._session.acknowledge()
def _terminar(self):
self._conn.close(self._timeout)
def open(self):
"""
Open a connection to the broker.
"""
if self.is_open():
# already open
return
connector = Connector.find(self.url)
Connection.add_transports()
domain = self.ssl_domain(connector)
log.info('open: %s', connector)
impl = RealConnection(host=connector.host,
port=connector.port,
tcp_nodelay=True,
transport=connector.url.scheme,
username=connector.userid,
password=connector.password,
heartbeat=10,
**domain)
impl.open()
self._impl = impl
log.info('opened: %s', self.url)
class QpidConnection():
port = None
ip = None
data_queue = None
cmd_queue = None
conn = None
session = None
sender = None
receiver = None
def __init__(self, ip, port, data_queue, cmd_queue):
self.ip = ip;
self.port = port
self.data_queue = data_queue
self.cmd_queue = cmd_queue
def start(self):
try:
url = str(self.ip)
url += str(':')
url += str(self.port)
self.conn = Connection(url)
self.conn.open()
self.session = self.conn.session()
self.sender = self.session.sender(self.cmd_queue)
self.receiver = self.session.receiver(self.data_queue)
return 1
except MessagingError as m:
print(m)
return 0
def stop(self):
try:
self.conn.close()
except MessagingError as m:
print(m)
return 0
def open(self):
"""
Open a connection to the broker.
"""
if self.is_open():
# already open
return
connector = Connector.find(self.url)
Connection.add_transports()
domain = self.ssl_domain(connector)
log.info('open: %s', connector)
impl = RealConnection(
host=connector.host,
port=connector.port,
tcp_nodelay=True,
transport=connector.url.scheme,
username=connector.userid,
password=connector.password,
heartbeat=10,
**domain)
impl.open()
self._impl = impl
log.info('opened: %s', self.url)
class NordicWayIC:
def __init__(self, url, sender, receiver, username, password, options=None):
self.options = options if options else {}
self.url = url
self._queue_sender = sender
self._queue_receiver = receiver
self._credentials = {"username": username, "password": password}
self.log = logging.getLogger("geofencebroker")
def __enter__(self):
self.connect()
return self
def __exit__(self, type, value, traceback):
self.close()
def connect(self):
self.connection = Connection(self.url,
username=self._credentials.get("username"),
password=self._credentials.get("password"),
**self.options)
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self._queue_sender)
self.receiver = self.session.receiver(self._queue_receiver)
def send_messsage(self, msg):
try:
self.sender.send(msg)
self.sender.check_error()
except MessagingError:
self.log.exception("Error sending message!")
except Exception:
self.log.exception("Exception occured while sending..")
self.session.acknowledge()
def send_obj(self, datex_obj):
"""
Use data from the 'datex2' object to construct a proper
AMQP object with all the required properties set.
"""
tz = pytz.timezone("Europe/Oslo")
now_iso_timestamp = datetime.datetime.now(tz).isoformat()
centroid = datex_obj.centroid
prop = {
"who": "Norwegian Public Roads Administration",
"how": "Datex2",
"what": "PredefinedLocation",
"lat": centroid[0],
"lon": centroid[1],
"where1": "no",
"when": now_iso_timestamp
}
m = Message(user_id=self._credentials.get("username"),
properties=prop,
content=str(datex_obj))
self.log.debug(u"Sending message: version={}, name={}".format(
datex_obj.version, datex_obj.name))
self.send_messsage(m)
def close(self):
self.connection.close()
def recv(self, timeout=None):
msg = self.receiver.fetch(timeout=timeout)
return msg
def __repr__(self):
return "<{} url={}, sender={}, receiver={}, options={}>".format(
self.__class__.__name__, self.url,
self._queue_sender, self._queue_receiver, self.options)
# import camel.biz.application.use_gevent
import gevent
from gevent import monkey
# monkey.patch_all()
from camel.koala.tcelib.conn_qpid import RpcConnectionQpidMQ, AF_READ
# mq = RpcConnectionQpidMQ.create('abc',host="rhinoceros/jiangxiaoyu@ytodev2",port="5672",
# address="mq_messageserver;{create:always,node:{type:queue,durable:true}}",af=AF_READ)
from qpid.messaging import Connection
from qpid.util import URL
host = "rhinoceros/jiangxiaoyu@ytodev2"
host = "rhinoceros/[email protected]"
host = "118.190.89.205"
port = "5672"
addr = "mq_messageserver;{create:always,node:{type:queue,durable:true}}"
broker = "%s:%s" % (host, port)
conn = Connection(broker, reconnect=True, tcp_nodelay=True)
conn.open()
ssn = conn.session()
rcv = ssn.receiver(addr)
rcv.capacity = 4000
while True:
m = rcv.fetch()
ssn.acknowledge()
print m
gevent.sleep(1000000)
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self, ep=None):
RpcConnection.__init__(self, ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv = ''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name, host, port, address, af=AF_WRITE):
"""
创建MQ的连接对象
:param name: 必须是真实的 mq 名称
:param host:
:param port:
:param address:
:param af:
:return:
"""
ep = RpcEndPoint(name=name,
host=host,
port=port,
addr=address,
type_='qpid')
if ep.open(af):
return ep.impl
return None
# conn = RpcConnectionQpidMQ(ep)
# conn.open(af)
# return conn
# @staticmethod
# def createRpcInvocationPair(ep_read,ep_write):
# conn_read = RpcConnectionQpidMQ(ep_read)
# conn_read.open(AF_READ)
# conn_write = RpcConnectionQpidMQ(ep_write)
# conn_write.open(AF_WRITE)
# conn_write.setLoopbackMQ(conn_read)
# return conn_read,conn_write
def open(self, af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s" % (ep.host, ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection(broker, reconnect=True, tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
gevent.spawn(self.thread_recv)
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
# gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
gevent.spawn(self.keepAliveWrite)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def test(self):
while True:
print '*-' * 20
print 'qpid conn:', self
gevent.sleep(3)
def keepAliveWrite(self):
"""
当mq为写入类型时，防止由于长时间没有消息进入，导致后继读取mq堵塞的情况，需要定时发送保活消息
这个问题目前没有应对方案，只能通过在写mq通道上频繁写入小数据，强迫mq接收者响应消息的读取。
:return:
"""
from qpid.messaging import Message
while True:
gevent.sleep(5)
print 'keepalive write, target: ', self.ep.addr
try:
if self.exitflag:
return True
m = Message("keepalive")
self.snd.send(m, False)
except:
log_error(traceback.format_exc())
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self, conn_recv):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn_recv.ep.getUnique()
return self
def sendMessage(self, m):
if m.traits and m.traits.max_linger_time:
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
value = int(value)
if not value:
value += int(time.time())
m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME,
value)
#app制定了超时接收时间，这里调整为绝对时间，以便接收端进行判别，选择接受还是丢弃
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!' % mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self, m)
def sendDetail(self, m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m, False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def __thread_recv(self):
from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(sync=True)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
#接收消息
def thread_recv(self):
# from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
print 'qpid_connection:', self
print 'mq_address:', self.ep.addr
print 'receiver:', self.rcv
while not self.exitflag:
try:
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(m, sync=True)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
self.dispatchMsg(m)
# RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self,ep=None):
RpcConnection.__init__(self,ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv =''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name,host,port,address,af=AF_WRITE):
"""
创建MQ的连接对象
:param name: 必须是真实的 mq 名称
:param host:
:param port:
:param address:
:param af:
:return:
"""
ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type_='qpid')
if ep.open(af):
return ep.impl
return None
# conn = RpcConnectionQpidMQ(ep)
# conn.open(af)
# return conn
# @staticmethod
# def createRpcInvocationPair(ep_read,ep_write):
# conn_read = RpcConnectionQpidMQ(ep_read)
# conn_read.open(AF_READ)
# conn_write = RpcConnectionQpidMQ(ep_write)
# conn_write.open(AF_WRITE)
# conn_write.setLoopbackMQ(conn_read)
# return conn_read,conn_write
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self,conn_recv):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn_recv.ep.getUnique()
return self
def sendMessage(self,m):
if m.traits and m.traits.max_linger_time:
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
value = int(value)
if not value:
value += int(time.time())
m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value )
#app制定了超时接收时间，这里调整为绝对时间，以便接收端进行判别，选择接受还是丢弃
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!'%mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self,m)
def sendDetail(self,m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m,False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def thread_recv(self):
from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(sync=False)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
def testOpen(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection.open(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.ping(self.conn.session())
def setup_connection(self):
return Connection.open(self.broker.host, self.broker.port,
reconnect=self.reconnect())
class AMQPEventConsumer(object):
"""
The AMQPEventConsumer can be used to subscribe for events
and process them thru a callback. The subscription is done
thru *XQuery*, the callback can be a python method.
Example listening for an event called *AsteriskNotification*::
>>> from gosa.common.components import AMQPEventConsumer
>>> from lxml import etree
>>>
>>> # Event callback
>>> def process(data):
... print(etree.tostring(data, pretty_print=True))
>>>
>>> # Create event consumer
>>> consumer = AMQPEventConsumer("amqps://*****:*****@localhost/org.gosa",
... xquery=\"\"\"
... declare namespace f='http://www.gonicus.de/Events';
... let $e := ./f:Event
... return $e/f:AsteriskNotification
... \"\"\",
... callback=process)
The consumer will start right away, listening for your events.
=============== ============
Parameter Description
=============== ============
url URL used to connect to the AMQP service broker
domain If the domain is not already encoded in the URL, it can be specified here.
xquery `XQuery <http://en.wikipedia.org/wiki/XQuery>`_ string to query for events.
callback Python method to be called if the event happened.
=============== ============
.. note::
The AMQP URL consists of these parts::
(amqp|amqps)://user:password@host:port/domain
"""
def __init__(self, url, domain="org.gosa", xquery=".", callback=None):
# Build connection
url = parseURL(url)
self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True)
self.__conn.open()
# Assemble subscription query
queue = 'event-listener-%s' % uuid4()
address = """%s; {
create: always,
delete:always,
node: {
durable: False,
x-declare: {
exclusive: True,
auto-delete: True }
},
link: {
x-bindings: [
{
exchange: '%s',
queue: %s,
key: event,
arguments: { xquery: %r}
}
]
}
}""" % (queue, domain, queue, xquery)
# Add processor for core.event queue
self.__callback = callback
self.__eventWorker = AMQPStandaloneWorker(
self.__conn,
r_address=address,
workers=1,
callback=self.__eventProcessor)
def __del__(self):
self.__eventWorker.join()
self.__conn.close()
#pylint: disable=W0613
def __eventProcessor(self, ssn, data):
# Call callback, let exceptions pass to the caller
xml = objectify.fromstring(data.content)
self.__callback(xml)
def join(self):
self.__eventWorker.join()
def __init__(self, serviceURL, serviceAddress=None, serviceName=None,
conn=None, workers=3):
self.__URL = url = parseURL(serviceURL)
self.__serviceURL = serviceURL
self.__serviceName = serviceName
self.__serviceAddress = serviceAddress
self.__workers = workers
domain = url['path']
# Prepare AMQP connection if not already there
if not conn:
conn = Connection(url['url'], transport=url['transport'], reconnect=True)
conn.open()
AMQPServiceProxy.domain= domain
# Prefill __serviceAddress correctly if domain is given
if AMQPServiceProxy.domain:
self.__serviceAddress = '%s.command.core' % AMQPServiceProxy.domain
if not self.__serviceAddress:
raise AMQPException("no serviceAddress or domain specified")
try:
AMQPServiceProxy.worker[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.worker[self.__serviceAddress] = {}
# Pre instanciate core sessions
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[self.__serviceAddress][i] = {
'ssn': ssn,
'sender': ssn.sender(self.__serviceAddress),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
# Store connection
self.__conn = conn
self.__ssn = None
self.__sender = None
self.__receiver = None
self.__worker = None
# Retrieve methods
try:
AMQPServiceProxy.methods
except AttributeError:
AMQPServiceProxy.methods = None
AMQPServiceProxy.methods = {}
# Retrieve methods
try:
AMQPServiceProxy.methods[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.methods[self.__serviceAddress] = None
AMQPServiceProxy.methods[self.__serviceAddress] = self.getMethods()
# If we've no direct queue, we need to push to different queues
if AMQPServiceProxy.domain:
queues = set([
x['target'] for x in AMQPServiceProxy.methods[self.__serviceAddress].itervalues()
if x['target'] != 'core'
])
# Pre instanciate queue sessions
for queue in queues:
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[queue] = {}
AMQPServiceProxy.worker[queue][i] = {
'ssn': ssn,
'sender': ssn.sender("%s.command.%s" %
(AMQPServiceProxy.domain, queue)),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self,ep=None):
RpcConnection.__init__(self,ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv =''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name,host,port,address,af=AF_WRITE):
ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type='qpid')
conn = RpcConnectionQpidMQ(ep)
conn.open(af)
return conn
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self,conn):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn.ep.getUnique()
return self
def sendMessage(self,m):
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!'%mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self,m)
def sendDetail(self,m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m,False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def thread_recv(self):
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
self.ssn.acknowledge(sync=False)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
class MQConnectionQpid(object):
def __init__(self,cfg):
self.type = MessageQueueType.QPID
self.name = cfg.get('name')
self.host = cfg.get('host')
self.port = cfg.get('port')
self.address = cfg.get('address')
self.execthread_nr = cfg.get('exec_thread_nr',1)
self.entry = cfg.get('entry')
self.conn = None
self.ssn = None
self.producer = None
self.consumer = None
self.cond_readable = Condition()
self.thread = None
self.isclosed = True
self.execthreads = []
self.message_pool = []
self.lock = Lock()
self.func_list ={} #导入的函数列表
self.rw = 0
def open(self,access=AccessMode.READ):
if access == 0:
return self
broker = "%s:%s" % (self.host, self.port)
# if self.conn is not None:
# return self
if not self.conn:
self.conn = Connection(broker, reconnect=True, tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if access & AccessMode.READ:
if not self.consumer:
self.consumer = self.ssn.receiver(self.address)
self.consumer.capacity = 4000
func = import_function(self.entry) # importing functions dynamically
self.func_list[self.entry] = func
self.thread = Thread(target=self._messageRecieving)
self.thread.start()
if access & AccessMode.WRITE:
if not self.producer:
self.producer = self.ssn.sender(self.address)
return self
def close(self):
self.isclosed = True
with self.cond_readable:
self.cond_readable.notify_all()
if self.conn :
self.conn.close()
self.conn = None
self.thread.join()
def _executeThread(self):
"""多线程"""
while not self.isclosed:
with self.cond_readable:
self.cond_readable.wait()
if self.isclosed:
break
message = None
self.lock.acquire()
if len(self.message_pool):
message = self.message_pool[0]
del self.message_pool[0]
self.lock.release()
if message is not None:
func = self.func_list[self.entry]
func(message) # pass into user space
print 'topic read thread is exiting..'
def produce(self,message):
message = Message(message)
self.producer.send(message, False)
def _messageRecieving(self):
"""消息接收线程,保证一个线程接收"""
self.isclosed = False
for nr in range(self.execthread_nr):
thread = Thread(target=self._executeThread)
self.execthreads.append(thread)
thread.start()
while not self.isclosed:
try:
message = self.consumer.fetch()
message = message.content
# self.ssn.acknowledge(sync=False)
if message is not None:
func = self.func_list[self.entry]
try:
func(message)
self.ssn.acknowledge(sync=False)
except:
instance.getLogger().error(u'amqp::qpid message process error. detail:{}'.format(traceback.format_exc()))
# self.lock.acquire()
# self.message_pool.append(message)
# self.lock.release()
# with self.cond_readable:
# # self.cond_readable.notify()
# self.cond_readable.notify_all()
except:
instance.getLogger().warn(u'amqp::qpid fetch message failed. detail:{}'.format(traceback.format_exc()))
gevent.sleep(5)
# for thread in self.execthreads:
# thread.join()
instance.getLogger().info( 'amqp::qpid recieve-thread is exiting...')
from qpid.messaging import Connection, Message
from time import sleep
import sys
host = "localhost:10009"
if len(sys.argv) > 1:
host = sys.argv[1]
conn = Connection(host)
try:
conn.open()
sess = conn.session()
tx = sess.sender("spfdemo.com/mobile.45")
count = 0
while(True):
tx.send("Seq: %d" % count)
print "Origin: %s Seq: %d" % (host, count)
count += 1
sleep(1)
except Exception, e:
print "Exception: %r" % e
except KeyboardInterrupt:
print
class QpidConnection(object):
"""
This class represents a connection to a Qpid broker. A QpidConnection must
be created in order to send or receive AMQP messages using a Qpid broker.
"""
def __init__(self, url, username, password, transport='tcp',
reconnection_interval=60, reconnect_handler=None,
context=None, log=None):
"""
Create a new connection to a Qpid message broker in order to send or
receive AMQP messages.
:param: url URL for the Qpid connection, e.g. 9.10.49.164:5672
:param: username Qpid username
:param: password Qpid password
:param: transport Transport mechanism, one of tcp, tcp+tls,
or ssl (alias for tcp+tls).
:param: reconnection_interval Interval in seconds between reconnect
attempts.
:param: reconnect_handler The function to call upon reconnecting to
the Qpid broker after connection was lost and
then reestablished. This function will be called after the
connections is reestablished but before the listeners are
started up again. It is not passed any parameters.
:param: context The security context
:param: log The logging module used for logging messages. If not
provided then no logging will be done.
"""
self.url = url
self.username = username
self.password = password
self.context = context
self.log = log.getLogger(__name__) if log else None
self.transport = transport
self.reconnection_interval = reconnection_interval
self.reconnect_handler = reconnect_handler
self._listeners = []
self._is_connected = False
def create_listener(self, exchange, topic):
"""
Create a new listener on the given exchange for the given topic.
:param: exchange The name of the Qpid exchange, e.g. 'nova'
:param: topic The topic to listen for, e.g. 'notifications.info'
:returns: A new QpidListener that will listen for messages on the
given exchange and topic.
"""
listener = QpidListener(self, exchange, topic)
self._listeners.append(listener)
return listener
def start(self, is_reconnect=False):
"""
Initiate the Qpid connection and start up any listeners.
:param: is_reconnect True if this method is called as part of a
reconnect attempt, False otherwise
:raise: ConnectionError if a connection cannot be established
"""
# If the Qpid broker URL is not specified (or just the hostname is not
# specified) then we can't make a connection.
if not self.url or self.url.startswith(':'):
log(self.log, 'warn', _('Qpid broker not specified, cannot start '
'connection.'))
return
if not self._is_connected:
self.conn = Connection(self.url, username=self.username,
password=self.password,
transport=self.transport)
try:
self.conn.open()
except ConnectionError as e:
log(self.log, 'critical', _('Cannot connect to Qpid message '
'broker: %s') % (e.message))
# close this connection when encounter connection error
# otherwise, it will leave an ESTABLISHED connection
# to qpid server forever.
if self.conn is not None:
self.conn.close()
raise e
self._is_connected = True
if is_reconnect and self.reconnect_handler:
self.reconnect_handler()
for listener in self._listeners:
listener._start(self.conn)
log(self.log, 'info', _('Connected to Qpid message broker: '
'%s@%s') % (self.username, self.url))
def _reconnect(self):
"""
Attempt to reconnect to the Qpid message broker in intervals until the
connection comes back.
"""
self.conn = None
class ReconnectionThread(threading.Thread):
def __init__(self, qpid_connection):
super(ReconnectionThread, self).__init__(
name='ReconnectionThread')
self.qpid_connection = qpid_connection
def run(self):
while not self.qpid_connection._is_connected:
try:
self.qpid_connection.start(is_reconnect=True)
except ConnectionError:
sleep(self.qpid_connection.reconnection_interval)
pass
reconnection_thread = ReconnectionThread(self)
reconnection_thread.start()
def set_reconnect_handler(self, reconnect_handler):
"""
Set the function to call upon reconnecting to the Qpid broker after
connection is lost and then reestablished.
:param: reconnect_handler The function to call upon reconnecting.
"""
self.reconnect_handler = reconnect_handler
