def connect(self):
"""
Connect to the broker.
@return: The AMQP connection object.
@rtype: I{Connection}
"""
self.__lock()
try:
if self.connection is None:
url = self.url.simple()
transport = self.url.transport
log.info('connecting:\n%s', self)
con = Connection(
url=url,
tcp_nodelay=True,
reconnect=True,
transport=transport)
con.attach()
log.info('{%s} connected to AMQP', self.id())
self.connection = con
else:
con = self.connection
return con
finally:
self.__unlock()
def get_qpid_connection(self):
"""Connect to a broker, set and return the connection object
Authenticates (if necessary), and sets bkr.common._connection and
returns it. This method is thread safe.
"""
self.connection_lock.acquire()
try:
global can_use_qpid, _connection
if not can_use_qpid:
global qpid_import_error
raise ImportError(str(qpid_import_error))
if _connection is None or _connection.get_error():
connection_params = [[self._broker],
{'reconnect': self._reconnect,
'heartbeat': self._heartbeat_timeout}]
if self.krb_auth:
connection_params[1].update({'sasl_mechanisms' : 'GSSAPI'})
# As connections can recover from krb errors, we don't need
# to worry about doing this manually.
self.do_krb_auth()
_connection = Connection(*connection_params[0], **connection_params[1])
_connection.open()
return _connection
finally:
self.connection_lock.release()
def init(self):
self.connection = Connection(self.broker,
reconnect=True,
reconnect_interval=3)
try:
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self.address)
except MessagingError:
print(traceback.format_exc())
return False
return True
class FreshDocWriter(PageWriterBase):
def _initialize(self):
self.set_name('FreshDocWriter')
if get_project_settings()['DEBUG_MODE']:
self._push_message = self._push_message_debug
else:
self._create_client()
def _create_client(self):
try:
self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True,
reconnect_limit=10, reconnect_interval=4)
self.connection_.open()
self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video')
except:
self.connection_ = None
self.logger_.exception('failed to connect to message queue server.')
def finalize(self):
if self.connection_:
self.connection_.close()
PageWriterBase.finalize(self)
def _push_message_debug(self, doc):
pass
def _push_message(self, doc):
doc.video.doc_id = doc.id
doc.video.id = str(doc.id)
doc.video.crawl_time = doc.video.create_time = doc.crawl_time
doc.video.discover_time = doc.discover_time
doc.video.url = doc.url
doc.video.domain = doc.domain
doc.video.domain_id = doc.domain_id
doc.video.in_links = doc.in_links
msg_body = thrift_to_str(doc.video)
if not msg_body:
return
self.sender_.send(Message(content=doc.url + '\t' + base64.b64encode(msg_body), durable=True))
self.logger_.info('send message successfully, %s', doc.url)
def process_item(self, item):
if not item:
return
doc = item.to_crawldoc()
if doc.doc_type != CrawlDocType.PAGE_TIME:
return
try:
self._push_message(doc)
except:
self.logger_.exception('failed to send message, %s', doc.url)
def start(self):
"""
Enable AMQP queueing. This method puts up the event processor and
sets it to "active".
"""
self.log.debug("enabling AMQP queueing")
# Evaluate username
user = self.env.config.get("amqp.id", default=None)
if not user:
user = self.env.uuid
# Create initial broker connection
url = "%s:%s" % (self.url['host'], self.url['port'])
self._conn = Connection.establish(url, reconnect=self.reconnect,
username=user,
password=self.env.config.get("amqp.key"),
transport=self.url['transport'],
reconnect_interval=self.reconnect_interval,
reconnect_limit=self.reconnect_limit)
# Do automatic broker failover if requested
if self.env.config.get('amqp.failover', default=False):
auto_fetch_reconnect_urls(self._conn)
# Create event provider
self._eventProvider = EventProvider(self.env, self._conn)
def checkAuth(self, user, password):
"""
This function checks a username / password combination using
the AMQP service' SASL configuration.
=============== ============
Parameter Description
=============== ============
user Username
password Password
=============== ============
``Return:`` Bool, success or failure
"""
# Strip username/password parts of url
url = "%s:%s" % (self.url['host'], self.url['port'])
# Don't allow blank authentication
if user == "" or password == "":
return False
try:
conn = Connection.establish(url, transport=self.url['transport'], username=user, password=password)
conn.close()
except ConnectionError as e:
self.log.debug("AMQP service authentication reports: %s" % str(e))
return False
except Exception as e:
self.log.critical("cannot proceed with authentication")
self.log.exception(e)
return False
return True
def testConnectError(self):
try:
self.conn = Connection.open("localhost", 0)
assert False, "connect succeeded"
except ConnectError, e:
# XXX: should verify that e includes appropriate diagnostic info
pass
class FanoutPublisher(object):
"""
qpid fanout publisher
"""
def __init__(self, broker, address):
self.broker = broker
self.address = address
self.connection = None
self.session = None
self.sender = None
self.heartbeat_sender = None
def __del__(self):
self.destroy()
def init(self):
self.connection = Connection(self.broker,
reconnect=True,
reconnect_interval=3)
try:
self.connection.open()
self.session = self.connection.session()
self.sender = self.session.sender(self.address)
except MessagingError:
print(traceback.format_exc())
return False
return True
def destroy(self):
if self.connection:
self.connection.close()
def publish(self, message):
if self.sender is None:
if not self.init():
return False
try:
message.properties = {'x-amqp-0-10.routing-key': self.address}
self.sender.send(message)
print('publish info. broker: {}, reply_to: {}'.format(
self.broker, self.address))
except MessagingError:
print(traceback.format_exc())
return False
return True
def _create_client(self):
try:
self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True,
reconnect_limit=10, reconnect_interval=4)
self.connection_.open()
self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video')
except:
self.connection_ = None
self.logger_.exception('failed to connect to message queue server.')
def checkOverStockedQueues(host):
connection = Connection.establish(host)
broker = BrokerAgent(connection)
queues = broker.getAllQueues()
result = list()
for q in queues:
if (q.msgDepth != 0):
print(q.name + " " + str(q.msgDepth))
result.append([q.name, q.msgDepth])
return result
def migrate(*args, **kwargs):
"""
Migrate qpid queues:
- Ensure pulp.task is no longer *exclusive*.
- Rename agent queues: consumer_id> => pulp.agent.<consumer_id>
"""
transport = pulp_conf.get('messaging', 'transport')
if transport != 'qpid':
# not using qpid
return
if not QPID_MESSAGING_AVAILABLE:
msg = _('Migration 0009 did not run because the python package qpid.messaging is not '
'installed. Please install qpid.messaging and rerun the migrations. See %s'
'for more information.')
msg = msg % QPID_MESSAGING_URL
_logger.error(msg)
raise Exception(msg)
if not QPIDTOOLLIBS_AVAILABLE:
msg = _('Migration 0009 did not run because the python package qpidtoollibs is not '
'installed. Please install qpidtoollibs and rerun the migrations. See %s for more '
'information.')
msg = msg % QPIDTOOLLIBS_URL
_logger.error(msg)
raise Exception(msg)
url = urlparse(pulp_conf.get('messaging', 'url'))
connection = Connection(
host=url.hostname,
port=url.port,
transport=url.scheme,
reconnect=False,
ssl_certfile=pulp_conf.get('messaging', 'clientcert'),
ssl_skip_hostname_check=True)
connection.attach()
broker = BrokerAgent(connection)
_migrate_reply_queue(broker)
_migrate_agent_queues(broker)
connection.detach()
def migrate(*args, **kwargs):
"""
Migrate qpid queues:
- Ensure pulp.task is no longer *exclusive*.
- Rename agent queues: consumer_id> => pulp.agent.<consumer_id>
"""
transport = pulp_conf.get('messaging', 'transport')
if transport != 'qpid':
# not using qpid
return
if not QPID_MESSAGING_AVAILABLE:
msg = _('Migration 0009 did not run because the python package qpid.messaging is not '
'installed. Pulp\'s Qpid client dependencies can be installed with the '
'\"pulp-server-qpid\" package group. See the installation docs for more '
'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.')
_logger.error(msg)
raise Exception(msg)
if not QPIDTOOLLIBS_AVAILABLE:
msg = _('Migration 0009 did not run because the python package qpidtoollibs is not '
'installed. Pulp\'s Qpid client dependencies can be installed with the '
'\"pulp-server-qpid\" package group. See the installation docs for more '
'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.')
_logger.error(msg)
raise Exception(msg)
url = urlparse(pulp_conf.get('messaging', 'url'))
connection = Connection(
host=url.hostname,
port=url.port,
transport=url.scheme,
reconnect=False,
ssl_certfile=pulp_conf.get('messaging', 'clientcert'),
ssl_skip_hostname_check=True)
connection.attach()
broker = BrokerAgent(connection)
_migrate_reply_queue(broker)
_migrate_agent_queues(broker)
connection.detach()
def __init__(self, url, receiver_name, sender_name='pulp.task', asserting=False, **options):
'''establishes a connection to given url; initializes session, sender and receiver'''
self.url = url
self.receiver_name = receiver_name
self.sender_name = sender_name
self._asserting = asserting
self.last_sent = None
self.last_fetched = None
self.session = Connection.establish(self.url, **options).session()
self.receiver = self.session.receiver("%s; {create: always}" % self.receiver_name)
self.sender = self.session.sender(self.sender_name)
self._timeout = None
def open(self):
"""
Open a connection to the broker.
"""
if self.is_open():
# already open
return
connector = Connector.find(self.url)
Connection.add_transports()
domain = self.ssl_domain(connector)
log.info('open: %s', connector)
impl = RealConnection(host=connector.host,
port=connector.port,
tcp_nodelay=True,
transport=connector.url.scheme,
username=connector.userid,
password=connector.password,
heartbeat=10,
**domain)
impl.open()
self._impl = impl
log.info('opened: %s', self.url)
class SetupTests(Base):
def testOpen(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection.open(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.ping(self.conn.session())
def testConnect(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.conn.connect()
self.ping(self.conn.session())
def testConnectError(self):
try:
self.conn = Connection.open("localhost", 0)
assert False, "connect succeeded"
except ConnectError, e:
# XXX: should verify that e includes appropriate diagnostic info
pass
class Qpid(MQ):
_timeout = 1
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def _enviar(self, mensagem):
logger.debug("Sending a message")
m = Message(mensagem)
self._sender.send(m, True, self._timeout)
def _receber(self, timeout=None):
logger.debug("Retrieving a message")
self._mensagem = self._receiver.fetch(timeout)
return self._mensagem.content
def _tamanho(self):
self._receiver.available()
def _excluir(self):
logger.debug("Ack last message")
self._session.acknowledge()
def _terminar(self):
self._conn.close(self._timeout)
class QpidConnection():
port = None
ip = None
data_queue = None
cmd_queue = None
conn = None
session = None
sender = None
receiver = None
def __init__(self, ip, port, data_queue, cmd_queue):
self.ip = ip;
self.port = port
self.data_queue = data_queue
self.cmd_queue = cmd_queue
def start(self):
try:
url = str(self.ip)
url += str(':')
url += str(self.port)
self.conn = Connection(url)
self.conn.open()
self.session = self.conn.session()
self.sender = self.session.sender(self.cmd_queue)
self.receiver = self.session.receiver(self.data_queue)
return 1
except MessagingError as m:
print(m)
return 0
def stop(self):
try:
self.conn.close()
except MessagingError as m:
print(m)
return 0
def start(self):
try:
url = str(self.ip)
url += str(':')
url += str(self.port)
self.conn = Connection(url)
self.conn.open()
self.session = self.conn.session()
self.sender = self.session.sender(self.cmd_queue)
self.receiver = self.session.receiver(self.data_queue)
return 1
except MessagingError as m:
print(m)
return 0
def open(self):
"""
Open a connection to the broker.
"""
if self.is_open():
# already open
return
connector = Connector.find(self.url)
Connection.add_transports()
domain = self.ssl_domain(connector)
log.info('open: %s', connector)
impl = RealConnection(
host=connector.host,
port=connector.port,
tcp_nodelay=True,
transport=connector.url.scheme,
username=connector.userid,
password=connector.password,
heartbeat=10,
**domain)
impl.open()
self._impl = impl
log.info('opened: %s', self.url)
def _conectar(self):
try:
logger.debug("Qpid: %s" % self._url.netloc)
self._conn = Connection(self._url.netloc)
if not self._conn:
raise MQError(None, 2)
self._conn.open()
except ConnectError:
raise MQError(cod=2)
try:
self._session = self._conn.session()
self._sender = self._session.sender(self._url.path[1:])
self._receiver = self._session.receiver(self._url.path[1:])
logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc))
except ConnectError:
raise MQError(cod=3)
def __init__(self, url, domain="org.clacks", xquery=".", callback=None):
# Build connection
url = parseURL(url)
_url = "%s:%s" % (url['host'], url['port'])
self.__conn = Connection.establish(_url, reconnect=True,
username=url['user'],
password=url['password'],
transport=url['transport'],
reconnect_interval=3,
reconnect_limit=0)
# Do automatic broker failover if requested
#TODO: configure reconnect
#auto_fetch_reconnect_urls(self.__conn)
# Assemble subscription query
queue = 'event-listener-%s' % uuid4()
address = """%s; {
create: always,
delete:always,
node: {
durable: False,
x-declare: {
exclusive: True,
auto-delete: True }
},
link: {
x-bindings: [
{
exchange: '%s',
queue: %s,
key: event,
arguments: { xquery: %r}
}
]
}
}""" % (queue, domain, queue, xquery)
# Add processor for core.event queue
self.__callback = callback
self.__eventWorker = AMQPStandaloneWorker(
self.__conn,
r_address=address,
workers=1,
callback=self.__eventProcessor)
def start(self):
"""
Enable AMQP queueing. This method puts up the event processor and
sets it to "active".
"""
self.log.debug("enabling AMQP queueing")
# Evaluate username
user = self.config.get("amqp.id", default=None)
if not user:
user = self.env.uuid
password = self.config.get("amqp.key")
# Create initial broker connection
url = "%s:%s" % (self.url['host'], self.url['port'])
self._conn = Connection.establish(url, reconnect=self.reconnect,
username=user,
password=password,
transport=self.url['transport'],
reconnect_interval=self.reconnect_interval,
reconnect_limit=self.reconnect_limit)
# Do automatic broker failover if requested
if self.config.get('amqp.failover', False):
auto_fetch_reconnect_urls(self._conn)
# Create event exchange
socket = connect(self.url['host'], self.url['port'])
if self.url['scheme'][-1] == 's':
socket = ssl(socket)
user = self.config.get("amqp.id", default=None)
if not user:
user = self.env.uuid
connection = DirectConnection(sock=socket,
username=user,
password=self.config.get("amqp.key"))
connection.start()
session = connection.session(str(uuid4()))
# pylint: disable=E1103
session.exchange_declare(exchange=self.env.domain, type="xml")
connection.close()
# Create event provider
self._eventProvider = EventProvider(self.env, self.getConnection())
def __init__(self,
url,
receiver_name,
sender_name='pulp.task',
asserting=False,
auth=Authenticator(),
**options):
'''establishes a connection to given url; initializes session, sender and receiver'''
self.url = url
self.receiver_name = receiver_name
self.sender_name = sender_name
self._asserting = asserting
self.last_sent = None
self.last_fetched = None
self.session = Connection.establish(self.url, **options).session()
self.receiver = self.session.receiver(
"pulp.agent.%s; {create: always}" % self.receiver_name)
self.sender = self.session.sender(self.sender_name)
self._timeout = None
self.auth = auth
def start(self, is_reconnect=False):
"""
Initiate the Qpid connection and start up any listeners.
:param: is_reconnect True if this method is called as part of a
reconnect attempt, False otherwise
:raise: ConnectionError if a connection cannot be established
"""
# If the Qpid broker URL is not specified (or just the hostname is not
# specified) then we can't make a connection.
if not self.url or self.url.startswith(':'):
log(self.log, 'warn', _('Qpid broker not specified, cannot start '
'connection.'))
return
if not self._is_connected:
self.conn = Connection(self.url, username=self.username,
password=self.password,
transport=self.transport)
try:
self.conn.open()
except ConnectionError as e:
log(self.log, 'critical', _('Cannot connect to Qpid message '
'broker: %s') % (e.message))
# close this connection when encounter connection error
# otherwise, it will leave an ESTABLISHED connection
# to qpid server forever.
if self.conn is not None:
self.conn.close()
raise e
self._is_connected = True
if is_reconnect and self.reconnect_handler:
self.reconnect_handler()
for listener in self._listeners:
listener._start(self.conn)
log(self.log, 'info', _('Connected to Qpid message broker: '
'%s@%s') % (self.username, self.url))
def __init__(self, url, domain="org.gosa", xquery=".", callback=None):
# Build connection
url = parseURL(url)
self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True)
self.__conn.open()
# Assemble subscription query
queue = 'event-listener-%s' % uuid4()
address = """%s; {
create: always,
delete:always,
node: {
durable: False,
x-declare: {
exclusive: True,
auto-delete: True }
},
link: {
x-bindings: [
{
exchange: '%s',
queue: %s,
key: event,
arguments: { xquery: %r}
}
]
}
}""" % (queue, domain, queue, xquery)
# Add processor for core.event queue
self.__callback = callback
self.__eventWorker = AMQPStandaloneWorker(
self.__conn,
r_address=address,
workers=1,
callback=self.__eventProcessor)
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def migrate(*args, **kwargs):
"""
Migrate qpid queues:
- Ensure pulp.task is no longer *exclusive*.
- Rename agent queues: consumer_id> => pulp.agent.<consumer_id>
"""
transport = pulp_conf.get('messaging', 'transport')
if transport != 'qpid':
# not using qpid
return
if not QPID_MESSAGING_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpid.messaging is not '
'installed. Please install qpid.messaging and rerun the migrations. See %s'
'for more information.')
msg = msg % QPID_MESSAGING_URL
_logger.error(msg)
raise Exception(msg)
if not QPIDTOOLLIBS_AVAILABLE:
msg = _(
'Migration 0009 did not run because the python package qpidtoollibs is not '
'installed. Please install qpidtoollibs and rerun the migrations. See %s for more '
'information.')
msg = msg % QPIDTOOLLIBS_URL
_logger.error(msg)
raise Exception(msg)
url = urlparse(pulp_conf.get('messaging', 'url'))
connection = Connection(host=url.hostname,
port=url.port,
transport=url.scheme,
reconnect=False,
ssl_certfile=pulp_conf.get('messaging',
'clientcert'),
ssl_skip_hostname_check=True)
connection.attach()
broker = BrokerAgent(connection)
_migrate_reply_queue(broker)
_migrate_agent_queues(broker)
connection.detach()
def __init__(self, serviceURL, serviceAddress=None, serviceName=None,
conn=None, workers=3):
self.__URL = url = parseURL(serviceURL)
self.__serviceURL = serviceURL
self.__serviceName = serviceName
self.__serviceAddress = serviceAddress
self.__workers = workers
domain = url['path']
# Prepare AMQP connection if not already there
if not conn:
conn = Connection(url['url'], transport=url['transport'], reconnect=True)
conn.open()
AMQPServiceProxy.domain= domain
# Prefill __serviceAddress correctly if domain is given
if AMQPServiceProxy.domain:
self.__serviceAddress = '%s.command.core' % AMQPServiceProxy.domain
if not self.__serviceAddress:
raise AMQPException("no serviceAddress or domain specified")
try:
AMQPServiceProxy.worker[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.worker[self.__serviceAddress] = {}
# Pre instanciate core sessions
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[self.__serviceAddress][i] = {
'ssn': ssn,
'sender': ssn.sender(self.__serviceAddress),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
# Store connection
self.__conn = conn
self.__ssn = None
self.__sender = None
self.__receiver = None
self.__worker = None
# Retrieve methods
try:
AMQPServiceProxy.methods
except AttributeError:
AMQPServiceProxy.methods = None
AMQPServiceProxy.methods = {}
# Retrieve methods
try:
AMQPServiceProxy.methods[self.__serviceAddress]
except KeyError:
AMQPServiceProxy.methods[self.__serviceAddress] = None
AMQPServiceProxy.methods[self.__serviceAddress] = self.getMethods()
# If we've no direct queue, we need to push to different queues
if AMQPServiceProxy.domain:
queues = set([
x['target'] for x in AMQPServiceProxy.methods[self.__serviceAddress].itervalues()
if x['target'] != 'core'
])
# Pre instanciate queue sessions
for queue in queues:
for i in range(0, workers):
ssn = conn.session(str(uuid4()))
AMQPServiceProxy.worker[queue] = {}
AMQPServiceProxy.worker[queue][i] = {
'ssn': ssn,
'sender': ssn.sender("%s.command.%s" %
(AMQPServiceProxy.domain, queue)),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
class RpcConnectionQpidMQ(RpcConnection):
def __init__(self,ep=None):
RpcConnection.__init__(self,ep=ep)
self.conn = None
self.exitflag = False
ep.impl = self
self.mq_recv =''
RpcConnectionMQ_Collection.instance().add(self)
@staticmethod
def create(name,host,port,address,af=AF_WRITE):
"""
创建MQ的连接对象
:param name: 必须是真实的 mq 名称
:param host:
:param port:
:param address:
:param af:
:return:
"""
ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type_='qpid')
if ep.open(af):
return ep.impl
return None
# conn = RpcConnectionQpidMQ(ep)
# conn.open(af)
# return conn
# @staticmethod
# def createRpcInvocationPair(ep_read,ep_write):
# conn_read = RpcConnectionQpidMQ(ep_read)
# conn_read.open(AF_READ)
# conn_write = RpcConnectionQpidMQ(ep_write)
# conn_write.open(AF_WRITE)
# conn_write.setLoopbackMQ(conn_read)
# return conn_read,conn_write
def open(self,af):
'''
<ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
<ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/>
'''
from qpid.messaging import Connection
from qpid.util import URL
ep = self.ep
self.af = af
self.exitflag = False
broker = "%s:%s"%(ep.host,ep.port)
# log_debug('prepare mq : <%s %s>!'% (ep.name,broker))
try:
self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
self.conn.open()
self.ssn = self.conn.session()
if af & AF_READ:
self.rcv = self.ssn.receiver(self.ep.addr)
self.rcv.capacity = 4000
# self.thread = threading.Thread(target =self.thread_recv)
# self.thread.start()
# import gevent
gevent.spawn(self.thread_recv)
if af & AF_WRITE:
self.snd = self.ssn.sender(self.ep.addr)
except:
log_error(traceback.format_exc())
return False
# log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker))
return True
def close(self):
if self.conn:
self.conn.close()
self.conn = None
self.exitflag = True
def setLoopbackMQ(self,conn_recv):
'''
设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存
目前的回路mq名称取 队列名称，如果携带主机信息的话，回路可以从另外一台mq-server返回
'''
self.mq_recv = conn_recv.ep.getUnique()
return self
def sendMessage(self,m):
if m.traits and m.traits.max_linger_time:
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
value = int(value)
if not value:
value += int(time.time())
m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value )
#app制定了超时接收时间，这里调整为绝对时间，以便接收端进行判别，选择接受还是丢弃
if m.calltype & RpcMessage.RETURN:
#--- Rpc的调用消息中包含接收消息的队列名称 ---
mqname = m.callmsg.extra.props.get('__mq_return__')
if mqname:
mq = RpcConnectionMQ_Collection.instance().get(mqname)
if mq:
mq.sendDetail(m)
return
log_error('__mq_return__:<%s> is not in service mq-list!'%mqname)
return False
if self.mq_recv:
m.extra.props['__mq_return__'] = self.mq_recv
return RpcConnection.sendMessage(self,m)
def sendDetail(self,m):
from qpid.messaging import Message
try:
if self.exitflag:
return True
# if not self.conn:
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.snd = self.ssn.sender(self.ep.addr)
d = m.marshall()
m = Message(d)
self.snd.send(m,False)
except:
log_error(traceback.format_exc())
# self.conn = None
return False
return True
#接收消息
def thread_recv(self):
from communicator import RpcCommunicator
# print 'qpid-mq recv thread start..'
while not self.exitflag:
try:
# if not self.conn:
# print 'try open mq:',self.ep.name
# broker = "%s:%s"%(self.ep.host,self.ep.port)
# self.conn = Connection( broker,reconnect= True,tcp_nodelay=True)
# self.conn.open()
# self.ssn = self.conn.session()
# self.rcv = self.ssn.receiver(self.ep.addr)
# self.rcv.capacity = 4000
m = self.rcv.fetch()
# print '.'*10
d = m.content
# print 'mq recv:',repr(d)
print 'recved 1 msg from MQ..'
self.ssn.acknowledge(sync=False)
m = RpcMessage.unmarshall(d)
if not m:
log_error('decode mq-msg error!')
continue
m.conn = self
value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0')
linger_time = int(value)
if linger_time and time.time() > linger_time:
# drop it
continue #过期接收的消息直接丢弃
# self.dispatchMsg(m)
RpcCommunicator.instance().dispatchMsg(m)
except:
log_error(traceback.format_exc())
gevent.sleep(1)
if self.adapter:
self.adapter.stopmtx.set()
# log_info("qpid-mq thread exiting..")
return False
# import camel.biz.application.use_gevent
import gevent
from gevent import monkey
# monkey.patch_all()
from camel.koala.tcelib.conn_qpid import RpcConnectionQpidMQ, AF_READ
# mq = RpcConnectionQpidMQ.create('abc',host="rhinoceros/jiangxiaoyu@ytodev2",port="5672",
# address="mq_messageserver;{create:always,node:{type:queue,durable:true}}",af=AF_READ)
from qpid.messaging import Connection
from qpid.util import URL
host = "rhinoceros/jiangxiaoyu@ytodev2"
host = "rhinoceros/[email protected]"
host = "118.190.89.205"
port = "5672"
addr = "mq_messageserver;{create:always,node:{type:queue,durable:true}}"
broker = "%s:%s" % (host, port)
conn = Connection(broker, reconnect=True, tcp_nodelay=True)
conn.open()
ssn = conn.session()
rcv = ssn.receiver(addr)
rcv.capacity = 4000
while True:
m = rcv.fetch()
ssn.acknowledge()
print m
gevent.sleep(1000000)
def __init__(self, address, **kwargs):
self._connection = Connection.establish(
address, client_properties={"qpid.ha-admin": 1}, **kwargs)
self._agent = BrokerAgent(self._connection)
assert self._agent.getHaBroker(
), "HA module not loaded in broker at: %s" % (address)
def get_args():
parser = argparse.ArgumentParser(description="print messages sent to a qpid exchange")
parser.add_argument("-e", "--exchange", help="name of a qpid exchange (default: amq.topic)", default="amq.topic")
parser.add_argument("-s", "--subject", help="message subject to bind to", required=False)
parser.add_argument("-a", "--address", help="hostname to connect to (default:localhost)", default="localhost")
parser.add_argument("-p", "--port", help="port to connect to (default: 5672)", default="5672")
parser.add_argument("-q", "--quiet", help="show message subject only", default=False, action="store_true")
return parser.parse_args()
args = get_args()
source = args.exchange
if args.subject:
source = "%s/%s" % (source, args.subject)
receiver = Connection.establish("%s:%s" % (args.address, args.port)).session().receiver(source)
try:
while True:
message = receiver.fetch()
if args.quiet:
print message.subject
else:
print "------------------"
print message
except KeyboardInterrupt:
print ""
'--port',
help='port to connect to (default: 5672)',
default='5672')
parser.add_argument('-q',
'--quiet',
help='show message subject only',
default=False,
action='store_true')
return parser.parse_args()
args = get_args()
source = args.exchange
if args.subject:
source = '%s/%s' % (source, args.subject)
receiver = Connection.establish(
'%s:%s' % (args.address, args.port)).session().receiver(source)
try:
while True:
message = receiver.fetch()
if args.quiet:
print message.subject
else:
print '------------------'
print message
except KeyboardInterrupt:
print ''
def testConnect(self):
# XXX: need to flesh out URL support/syntax
self.conn = Connection(self.broker.host, self.broker.port,
reconnect=self.reconnect())
self.conn.connect()
self.ping(self.conn.session())
def setup_connection(self):
return Connection.open(self.broker.host, self.broker.port,
reconnect=self.reconnect())
consumers = get('/consumers/')
return [ c['id'] for c in consumers if c['notes'].get('_child-node', False) ]
def get_nodes_repos(node):
" Returns a child node's 'bound' repos."
bindings = get("/consumers/%s/bindings/" % (node))
return [ repo['repo_id'] for repo in bindings ]
if __name__ == '__main__':
args = get_args()
# qpid connection
address = "%s:%s" % (args.host, args.port)
receiver = Connection.establish(address).session().receiver(args.exchange)
try:
while True:
message = receiver.fetch()
json_message = json.loads(message.content)
if json_message['payload']['result'] == 'success':
repo_id = json_message['payload']['repo_id']
else:
continue
nodes = get_nodes()
for node in nodes:
repos = get_nodes_repos(node)
if repo_id in repos:
def test_08_integration_test_with_messagebus(self):
""" Full blown integration test listening for notifications on the bus,
and checking which dir is up for a visit next.
Needs a working local qpid broker. Test is skipped if qpid not available.
"""
try:
broker = None
connection = None
import uuid
from threading import Event
from qpid.messaging import Connection, ConnectError
from qpidtoollibs import BrokerAgent
from lofar.messaging.messagebus import ToBus
from lofar.messaging.messages import EventMessage
from lofar.lta.ingest.common.config import DEFAULT_INGEST_NOTIFICATION_PREFIX
# setup broker connection
connection = Connection.establish('127.0.0.1')
broker = BrokerAgent(connection)
# add test service bus
busname = 'test-LTASOIngestEventHandler-%s' % (uuid.uuid1())
broker.addExchange('topic', busname)
sync_event = Event()
class SyncedLTASOIngestEventHandler(LTASOIngestEventHandler):
"""This derived LTASOIngestEventHandler behaves exactly like the normal
object under test LTASOIngestEventHandler, but it also sets a sync_event
to sync between the listener thread and this main test thread"""
def _handleMessage(self, msg):
super(SyncedLTASOIngestEventHandler,
self)._handleMessage(msg)
sync_event.set()
with SyncedLTASOIngestEventHandler(self.dbcreds, busname=busname):
for site in self.db.sites():
for root_dir in self.db.rootDirectoriesForSite(site['id']):
self._markAllDirectoriesRecentlyVisited()
# create the subdir surl
sub_dir_name = '/foo'
sub_dir_path = root_dir['dir_name'] + sub_dir_name
surl = site['url'] + sub_dir_path
with ToBus(busname) as sender:
msg = EventMessage(
subject=DEFAULT_INGEST_NOTIFICATION_PREFIX +
"TaskFinished",
content={'srm_url': surl})
sender.send(msg)
# wait for the handler to have processed the message
self.assertTrue(sync_event.wait(2))
sync_event.clear()
# surl should have been scheduled for a visit, all other dir's were marked as visited already...
# so there should be a new dir for this surl, and it should be the least_recent_visited_dir
site_visit_stats = self.db.visitStats(
datetime.utcnow())[site['name']]
least_recent_visited_dir_id = site_visit_stats.get(
'least_recent_visited_dir_id')
self.assertIsNotNone(least_recent_visited_dir_id)
least_recent_visited_dir = self.db.directory(
least_recent_visited_dir_id)
self.assertEqual(sub_dir_path,
least_recent_visited_dir['dir_name'])
except ImportError as e:
logger.warning("skipping test due to: %s", e)
except ConnectError as e:
logger.warning("skipping test due to: %s", e)
finally:
# cleanup test bus and exit
if broker:
broker.delExchange(busname)
if connection:
connection.close()
class AMQPEventConsumer(object):
"""
The AMQPEventConsumer can be used to subscribe for events
and process them thru a callback. The subscription is done
thru *XQuery*, the callback can be a python method.
Example listening for an event called *AsteriskNotification*::
>>> from gosa.common.components import AMQPEventConsumer
>>> from lxml import etree
>>>
>>> # Event callback
>>> def process(data):
... print(etree.tostring(data, pretty_print=True))
>>>
>>> # Create event consumer
>>> consumer = AMQPEventConsumer("amqps://*****:*****@localhost/org.gosa",
... xquery=\"\"\"
... declare namespace f='http://www.gonicus.de/Events';
... let $e := ./f:Event
... return $e/f:AsteriskNotification
... \"\"\",
... callback=process)
The consumer will start right away, listening for your events.
=============== ============
Parameter Description
=============== ============
url URL used to connect to the AMQP service broker
domain If the domain is not already encoded in the URL, it can be specified here.
xquery `XQuery <http://en.wikipedia.org/wiki/XQuery>`_ string to query for events.
callback Python method to be called if the event happened.
=============== ============
.. note::
The AMQP URL consists of these parts::
(amqp|amqps)://user:password@host:port/domain
"""
def __init__(self, url, domain="org.gosa", xquery=".", callback=None):
# Build connection
url = parseURL(url)
self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True)
self.__conn.open()
# Assemble subscription query
queue = 'event-listener-%s' % uuid4()
address = """%s; {
create: always,
delete:always,
node: {
durable: False,
x-declare: {
exclusive: True,
auto-delete: True }
},
link: {
x-bindings: [
{
exchange: '%s',
queue: %s,
key: event,
arguments: { xquery: %r}
}
]
}
}""" % (queue, domain, queue, xquery)
# Add processor for core.event queue
self.__callback = callback
self.__eventWorker = AMQPStandaloneWorker(
self.__conn,
r_address=address,
workers=1,
callback=self.__eventProcessor)
def __del__(self):
self.__eventWorker.join()
self.__conn.close()
#pylint: disable=W0613
def __eventProcessor(self, ssn, data):
# Call callback, let exceptions pass to the caller
xml = objectify.fromstring(data.content)
self.__callback(xml)
def join(self):
self.__eventWorker.join()
def __init__(self, address, **kwargs):
self._connection = Connection.establish(
address, client_properties={"qpid.ha-admin": 1}, **kwargs)
self._agent = BrokerAgent(self._connection)
def __init__(self, serviceURL, serviceAddress=None, serviceName=None,
conn=None, worker=None, methods=None):
self.__URL = url = parseURL(serviceURL)
self.__serviceURL = serviceURL
self.__serviceName = serviceName
self.__serviceAddress = serviceAddress
self.__worker = worker
self.__domain = url['path']
self.__methods = methods
# Prepare AMQP connection if not already there
if not conn:
_url = "%s:%s" % (url['host'], url['port'])
conn = Connection.establish(_url, reconnect=True,
username=url['user'],
password=url['password'],
transport=url['transport'],
reconnect_interval=3,
reconnect_limit=0)
#TODO: configure reconnect
#auto_fetch_reconnect_urls(conn)
# Prefill __serviceAddress correctly if domain is given
if self.__domain:
self.__serviceAddress = '%s.command.core' % self.__domain
if not self.__serviceAddress:
raise AMQPException("no serviceAddress or domain specified")
if not self.__worker:
self.__worker = {self.__serviceAddress: {}}
# Pre instanciate core sessions
for i in range(0, WORKERS):
ssn = conn.session(str(uuid4()))
self.__worker[self.__serviceAddress][i] = {
'ssn': ssn,
'sender': ssn.sender(self.__serviceAddress),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
# Store connection
self.__conn = conn
self.__ssn = None
self.__sender = None
self.__receiver = None
self.__sess = None
# Retrieve methods
if not self.__methods:
self.__serviceName = "getMethods"
self.__methods = self.__call__()
self.__serviceName = None
# If we've no direct queue, we need to push to different queues
if self.__domain:
queues = set([
x['target'] for x in self.__methods.itervalues()
if x['target'] != 'core'
])
# Pre instanciate queue sessions
for queue in queues:
for i in range(0, WORKERS):
ssn = conn.session(str(uuid4()))
self.__worker[queue] = {}
self.__worker[queue][i] = {
'ssn': ssn,
'sender': ssn.sender("%s.command.%s" %
(self.__domain, queue)),
'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name),
'locked': False}
from qpid.messaging import Connection, Message
from time import sleep
import sys
host = "localhost:10009"
if len(sys.argv) > 1:
host = sys.argv[1]
conn = Connection(host)
try:
conn.open()
sess = conn.session()
tx = sess.sender("spfdemo.com/mobile.45")
count = 0
while(True):
tx.send("Seq: %d" % count)
print "Origin: %s Seq: %d" % (host, count)
count += 1
sleep(1)
except Exception, e:
print "Exception: %r" % e
except KeyboardInterrupt:
print
from mock import patch
except ImportError:
print 'Cannot run test without python MagicMock'
print 'Please install MagicMock: pip install mock'
exit(3)
connection = None
broker = None
try:
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
logger = logging.getLogger(__name__)
# setup broker connection
connection = Connection.establish('127.0.0.1')
broker = BrokerAgent(connection)
# add test service busname
busname = 'test-lofarbus-%s' % (uuid.uuid1())
broker.addExchange('topic', busname)
# the system under test is the service and the rpc, not the RADatabase
# so, patch (mock) the RADatabase class during these tests.
# when the service instantiates an RADatabase it will get the mocked class.
with patch('lofar.sas.resourceassignment.database.radb.RADatabase',
autospec=True) as MockRADatabase:
mock = MockRADatabase.return_value
# modify the return values of the various RADatabase methods with pre-cooked answers
mock.getTaskStatuses.return_value = [{
'id': 1,
