class QpidWrapper(object):
__address_opt = '{create:always, node:{type:topic, durable:True}}'
def __init__(self, broker, exchange, username, password):
self.__conn = Connection(broker,
username=username,
password=password,
tcp_nodelay=True,
transport='tcp')
self.__conn.open()
self.__session = self.__conn.session()
self.__exchange = exchange
@property
def session(self):
return self.__session
def create_address(self, routing_key=None):
if routing_key:
return '%s/%s;%s' % (self.__exchange, routing_key,
self.__address_opt)
else:
return '%s;%s' % (self.__exchange, self.__address_opt)
def close(self):
self.__session.close()
self.__conn.close()
class FreshDocWriter(PageWriterBase):
def _initialize(self):
self.set_name('FreshDocWriter')
if get_project_settings()['DEBUG_MODE']:
self._push_message = self._push_message_debug
else:
self._create_client()
def _create_client(self):
try:
self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True,
reconnect_limit=10, reconnect_interval=4)
self.connection_.open()
self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video')
except:
self.connection_ = None
self.logger_.exception('failed to connect to message queue server.')
def finalize(self):
if self.connection_:
self.connection_.close()
PageWriterBase.finalize(self)
def _push_message_debug(self, doc):
pass
def _push_message(self, doc):
doc.video.doc_id = doc.id
doc.video.id = str(doc.id)
doc.video.crawl_time = doc.video.create_time = doc.crawl_time
doc.video.discover_time = doc.discover_time
doc.video.url = doc.url
doc.video.domain = doc.domain
doc.video.domain_id = doc.domain_id
doc.video.in_links = doc.in_links
msg_body = thrift_to_str(doc.video)
if not msg_body:
return
self.sender_.send(Message(content=doc.url + '\t' + base64.b64encode(msg_body), durable=True))
self.logger_.info('send message successfully, %s', doc.url)
def process_item(self, item):
if not item:
return
doc = item.to_crawldoc()
if doc.doc_type != CrawlDocType.PAGE_TIME:
return
try:
self._push_message(doc)
except:
self.logger_.exception('failed to send message, %s', doc.url)
class FanoutPublisher(object):
"""
qpid fanout publisher
"""
def __init__(self, broker, address):
self.broker = broker
self.address = address
self.connection = None
self.session = None
self.sender = None
self.heartbeat_sender = None
def __del__(self):
self.destroy()
def init(self):
self.connection = Connection(self.broker,
reconnect=True,
reconnect_interval=3)
try:
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self.address)
except MessagingError:
print(traceback.format_exc())
return False
return True
def destroy(self):
if self.connection:
self.connection.close()
def publish(self, message):
if self.sender is None:
if not self.init():
return False
try:
message.properties = {'x-amqp-0-10.routing-key': self.address}
self.sender.send(message)
print('publish info. broker: {}, reply_to: {}'.format(
self.broker, self.address))
except MessagingError:
print(traceback.format_exc())
return False
return True
class Qpid(MQ):
_timeout = 1
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" %
(self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def _enviar(self, mensagem):
logger.debug("Sending a message")
m = Message(mensagem)
self._sender.send(m, True, self._timeout)
def _receber(self, timeout=None):
logger.debug("Retrieving a message")
self._mensagem = self._receiver.fetch(timeout)
return self._mensagem.content
def _tamanho(self):
self._receiver.available()
def _excluir(self):
logger.debug("Ack last message")
self._session.acknowledge()
def _terminar(self):
self._conn.close(self._timeout)
class SetupTests(Base):
def testOpen(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection.open(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.ping(self.conn.session())
def testConnect(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.conn.connect()
self.ping(self.conn.session())
def testConnectError(self):
try:
self.conn = Connection.open("localhost", 0)
assert False, "connect succeeded"
except ConnectError, e:
# XXX: should verify that e includes appropriate diagnostic info
pass
class Qpid(MQ):
_timeout = 1
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def _enviar(self, mensagem):
logger.debug("Sending a message")
m = Message(mensagem)
self._sender.send(m, True, self._timeout)
def _receber(self, timeout=None):
logger.debug("Retrieving a message")
self._mensagem = self._receiver.fetch(timeout)
return self._mensagem.content
def _tamanho(self):
self._receiver.available()
def _excluir(self):
logger.debug("Ack last message")
self._session.acknowledge()
def _terminar(self):
self._conn.close(self._timeout)
class QpidConnection():
port = None
ip = None
data_queue = None
cmd_queue = None
conn = None
session = None
sender = None
receiver = None
def __init__(self, ip, port, data_queue, cmd_queue):
self.ip = ip;
self.port = port
self.data_queue = data_queue
self.cmd_queue = cmd_queue
def start(self):
try:
url = str(self.ip)
url += str(':')
url += str(self.port)
self.conn = Connection(url)
self.conn.open()
self.session = self.conn.session()
self.sender = self.session.sender(self.cmd_queue)
self.receiver = self.session.receiver(self.data_queue)
return 1
except MessagingError as m:
print(m)
return 0
def stop(self):
try:
self.conn.close()
except MessagingError as m:
print(m)
return 0
from qpid.messaging import Connection, Message
from time import sleep
import sys
host = "localhost:10009"
if len(sys.argv) > 1:
host = sys.argv[1]
conn = Connection(host)
try:
conn.open()
sess = conn.session()
tx = sess.sender("spfdemo.com/mobile.45")
count = 0
while(True):
tx.send("Seq: %d" % count)
print "Origin: %s Seq: %d" % (host, count)
count += 1
sleep(1)
except Exception, e:
print "Exception: %r" % e
except KeyboardInterrupt:
print
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self, ep=None):
RpcConnection.__init__(self, ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv = ''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name, host, port, address, af=AF_WRITE):
"""
创建MQ的连接对象
:param name: 必须是真实的 mq 名称
:param host:
:param port:
:param address:
:param af:
:return:
"""
ep = RpcEndPoint(name=name,
host=host,
port=port,
addr=address,
type_='qpid')
if ep.open(af):
return ep.impl
return None
# conn = RpcConnectionQpidMQ(ep)
# conn.open(af)
# return conn
# @staticmethod
# def createRpcInvocationPair(ep_read,ep_write):
# conn_read = RpcConnectionQpidMQ(ep_read)
# conn_read.open(AF_READ)
# conn_write = RpcConnectionQpidMQ(ep_write)
# conn_write.open(AF_WRITE)
# conn_write.setLoopbackMQ(conn_read)
# return conn_read,conn_write
def open(self, af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s" % (ep.host, ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection(broker, reconnect=True, tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
gevent.spawn(self.thread_recv)
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
# gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
gevent.spawn(self.keepAliveWrite)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def test(self):
while True:
print '*-' * 20
print 'qpid conn:', self
gevent.sleep(3)
def keepAliveWrite(self):
"""
当mq为写入类型时，防止由于长时间没有消息进入，导致后继读取mq堵塞的情况，需要定时发送保活消息
这个问题目前没有应对方案，只能通过在写mq通道上频繁写入小数据，强迫mq接收者响应消息的读取。
:return:
"""
from qpid.messaging import Message
while True:
gevent.sleep(5)
print 'keepalive write, target: ', self.ep.addr
try:
if self.exitflag:
return True
m = Message("keepalive")
self.snd.send(m, False)
except:
log_error(traceback.format_exc())
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self, conn_recv):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn_recv.ep.getUnique()
return self
def sendMessage(self, m):
if m.traits and m.traits.max_linger_time:
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
value = int(value)
if not value:
value += int(time.time())
m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME,
value)
#app制定了超时接收时间，这里调整为绝对时间，以便接收端进行判别，选择接受还是丢弃
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!' % mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self, m)
def sendDetail(self, m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m, False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def __thread_recv(self):
from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(sync=True)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
#接收消息
def thread_recv(self):
# from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
print 'qpid_connection:', self
print 'mq_address:', self.ep.addr
print 'receiver:', self.rcv
while not self.exitflag:
try:
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(m, sync=True)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,
'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
self.dispatchMsg(m)
# RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self,ep=None):
RpcConnection.__init__(self,ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv =''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name,host,port,address,af=AF_WRITE):
ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type='qpid')
conn = RpcConnectionQpidMQ(ep)
conn.open(af)
return conn
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self,conn):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn.ep.getUnique()
return self
def sendMessage(self,m):
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!'%mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self,m)
def sendDetail(self,m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m,False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def thread_recv(self):
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
self.ssn.acknowledge(sync=False)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self,ep=None):
RpcConnection.__init__(self,ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv =''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name,host,port,address,af=AF_WRITE):
"""
创建MQ的连接对象
:param name: 必须是真实的 mq 名称
:param host:
:param port:
:param address:
:param af:
:return:
"""
ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type_='qpid')
if ep.open(af):
return ep.impl
return None
# conn = RpcConnectionQpidMQ(ep)
# conn.open(af)
# return conn
# @staticmethod
# def createRpcInvocationPair(ep_read,ep_write):
# conn_read = RpcConnectionQpidMQ(ep_read)
# conn_read.open(AF_READ)
# conn_write = RpcConnectionQpidMQ(ep_write)
# conn_write.open(AF_WRITE)
# conn_write.setLoopbackMQ(conn_read)
# return conn_read,conn_write
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self,conn_recv):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn_recv.ep.getUnique()
return self
def sendMessage(self,m):
if m.traits and m.traits.max_linger_time:
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
value = int(value)
if not value:
value += int(time.time())
m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value )
#app制定了超时接收时间，这里调整为绝对时间，以便接收端进行判别，选择接受还是丢弃
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!'%mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self,m)
def sendDetail(self,m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m,False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def thread_recv(self):
from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(sync=False)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
class NordicWayIC:
def __init__(self, url, sender, receiver, username, password, options=None):
self.options = options if options else {}
self.url = url
self._queue_sender = sender
self._queue_receiver = receiver
self._credentials = {"username": username, "password": password}
self.log = logging.getLogger("geofencebroker")
def __enter__(self):
self.connect()
return self
def __exit__(self, type, value, traceback):
self.close()
def connect(self):
self.connection = Connection(self.url,
username=self._credentials.get("username"),
password=self._credentials.get("password"),
**self.options)
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self._queue_sender)
self.receiver = self.session.receiver(self._queue_receiver)
def send_messsage(self, msg):
try:
self.sender.send(msg)
self.sender.check_error()
except MessagingError:
self.log.exception("Error sending message!")
except Exception:
self.log.exception("Exception occured while sending..")
self.session.acknowledge()
def send_obj(self, datex_obj):
"""
Use data from the 'datex2' object to construct a proper
AMQP object with all the required properties set.
"""
tz = pytz.timezone("Europe/Oslo")
now_iso_timestamp = datetime.datetime.now(tz).isoformat()
centroid = datex_obj.centroid
prop = {
"who": "Norwegian Public Roads Administration",
"how": "Datex2",
"what": "PredefinedLocation",
"lat": centroid[0],
"lon": centroid[1],
"where1": "no",
"when": now_iso_timestamp
}
m = Message(user_id=self._credentials.get("username"),
properties=prop,
content=str(datex_obj))
self.log.debug(u"Sending message: version={}, name={}".format(
datex_obj.version, datex_obj.name))
self.send_messsage(m)
def close(self):
self.connection.close()
def recv(self, timeout=None):
msg = self.receiver.fetch(timeout=timeout)
return msg
def __repr__(self):
return "<{} url={}, sender={}, receiver={}, options={}>".format(
self.__class__.__name__, self.url,
self._queue_sender, self._queue_receiver, self.options)
class MQConnectionQpid(object):
def __init__(self,cfg):
self.type = MessageQueueType.QPID
self.name = cfg.get('name')
self.host = cfg.get('host')
self.port = cfg.get('port')
self.address = cfg.get('address')
self.execthread_nr = cfg.get('exec_thread_nr',1)
self.entry = cfg.get('entry')
self.conn = None
self.ssn = None
self.producer = None
self.consumer = None
self.cond_readable = Condition()
self.thread = None
self.isclosed = True
self.execthreads = []
self.message_pool = []
self.lock = Lock()
self.func_list ={} #导入的函数列表
self.rw = 0
def open(self,access=AccessMode.READ):
if access == 0:
return self
broker = "%s:%s" % (self.host, self.port)
# if self.conn is not None:
# return self
if not self.conn:
self.conn = Connection(broker, reconnect=True, tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if access & AccessMode.READ:
if not self.consumer:
self.consumer = self.ssn.receiver(self.address)
self.consumer.capacity = 4000
func = import_function(self.entry) # importing functions dynamically
self.func_list[self.entry] = func
self.thread = Thread(target=self._messageRecieving)
self.thread.start()
if access & AccessMode.WRITE:
if not self.producer:
self.producer = self.ssn.sender(self.address)
return self
def close(self):
self.isclosed = True
with self.cond_readable:
self.cond_readable.notify_all()
if self.conn :
self.conn.close()
self.conn = None
self.thread.join()
def _executeThread(self):
"""多线程"""
while not self.isclosed:
with self.cond_readable:
self.cond_readable.wait()
if self.isclosed:
break
message = None
self.lock.acquire()
if len(self.message_pool):
message = self.message_pool[0]
del self.message_pool[0]
self.lock.release()
if message is not None:
func = self.func_list[self.entry]
func(message) # pass into user space
print 'topic read thread is exiting..'
def produce(self,message):
message = Message(message)
self.producer.send(message, False)
def _messageRecieving(self):
"""消息接收线程,保证一个线程接收"""
self.isclosed = False
for nr in range(self.execthread_nr):
thread = Thread(target=self._executeThread)
self.execthreads.append(thread)
thread.start()
while not self.isclosed:
try:
message = self.consumer.fetch()
message = message.content
# self.ssn.acknowledge(sync=False)
if message is not None:
func = self.func_list[self.entry]
try:
func(message)
self.ssn.acknowledge(sync=False)
except:
instance.getLogger().error(u'amqp::qpid message process error. detail:{}'.format(traceback.format_exc()))
# self.lock.acquire()
# self.message_pool.append(message)
# self.lock.release()
# with self.cond_readable:
# # self.cond_readable.notify()
# self.cond_readable.notify_all()
except:
instance.getLogger().warn(u'amqp::qpid fetch message failed. detail:{}'.format(traceback.format_exc()))
gevent.sleep(5)
# for thread in self.execthreads:
# thread.join()
instance.getLogger().info( 'amqp::qpid recieve-thread is exiting...')
def __init__(self, serviceURL, serviceAddress=None, serviceName=None,
conn=None, workers=3):
self.__URL = url = parseURL(serviceURL)
self.__serviceURL = serviceURL
self.__serviceName = serviceName
self.__serviceAddress = serviceAddress
self.__workers = workers
domain = url['path']
# Prepare AMQP connection if not already there
if not conn:
conn = Connection(url['url'], transport=url['transport'], reconnect=True)
conn.open()
AMQPServiceProxy.domain= domain
# Prefill __serviceAddress correctly if domain is given
if AMQPServiceProxy.domain:
self.__serviceAddress = '%s.command.core' % AMQPServiceProxy.domain
if not self.__serviceAddress:
raise AMQPException("no serviceAddress or domain specified")
try:
AMQPServiceProxy.worker[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.worker[self.__serviceAddress] = {}
# Pre instanciate core sessions
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[self.__serviceAddress][i] = {
'ssn': ssn,
'sender': ssn.sender(self.__serviceAddress),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
# Store connection
self.__conn = conn
self.__ssn = None
self.__sender = None
self.__receiver = None
self.__worker = None
# Retrieve methods
try:
AMQPServiceProxy.methods
except AttributeError:
AMQPServiceProxy.methods = None
AMQPServiceProxy.methods = {}
# Retrieve methods
try:
AMQPServiceProxy.methods[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.methods[self.__serviceAddress] = None
AMQPServiceProxy.methods[self.__serviceAddress] = self.getMethods()
# If we've no direct queue, we need to push to different queues
if AMQPServiceProxy.domain:
queues = set([
x['target'] for x in AMQPServiceProxy.methods[self.__serviceAddress].itervalues()
if x['target'] != 'core'
])
# Pre instanciate queue sessions
for queue in queues:
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[queue] = {}
AMQPServiceProxy.worker[queue][i] = {
'ssn': ssn,
'sender': ssn.sender("%s.command.%s" %
(AMQPServiceProxy.domain, queue)),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
# import camel.biz.application.use_gevent
import gevent
from gevent import monkey
# monkey.patch_all()
from camel.koala.tcelib.conn_qpid import RpcConnectionQpidMQ, AF_READ
# mq = RpcConnectionQpidMQ.create('abc',host="rhinoceros/jiangxiaoyu@ytodev2",port="5672",
# address="mq_messageserver;{create:always,node:{type:queue,durable:true}}",af=AF_READ)
from qpid.messaging import Connection
from qpid.util import URL
host = "rhinoceros/jiangxiaoyu@ytodev2"
host = "rhinoceros/[email protected]"
host = "118.190.89.205"
port = "5672"
addr = "mq_messageserver;{create:always,node:{type:queue,durable:true}}"
broker = "%s:%s" % (host, port)
conn = Connection(broker, reconnect=True, tcp_nodelay=True)
conn.open()
ssn = conn.session()
rcv = ssn.receiver(addr)
rcv.capacity = 4000
while True:
m = rcv.fetch()
ssn.acknowledge()
print m
gevent.sleep(1000000)
