def connect(self): """ Connect to the broker. @return: The AMQP connection object. @rtype: I{Connection} """ self.__lock() try: if self.connection is None: url = self.url.simple() transport = self.url.transport log.info('connecting:\n%s', self) con = Connection( url=url, tcp_nodelay=True, reconnect=True, transport=transport) con.attach() log.info('{%s} connected to AMQP', self.id()) self.connection = con else: con = self.connection return con finally: self.__unlock()
def get_qpid_connection(self): """Connect to a broker, set and return the connection object Authenticates (if necessary), and sets bkr.common._connection and returns it. This method is thread safe. """ self.connection_lock.acquire() try: global can_use_qpid, _connection if not can_use_qpid: global qpid_import_error raise ImportError(str(qpid_import_error)) if _connection is None or _connection.get_error(): connection_params = [[self._broker], {'reconnect': self._reconnect, 'heartbeat': self._heartbeat_timeout}] if self.krb_auth: connection_params[1].update({'sasl_mechanisms' : 'GSSAPI'}) # As connections can recover from krb errors, we don't need # to worry about doing this manually. self.do_krb_auth() _connection = Connection(*connection_params[0], **connection_params[1]) _connection.open() return _connection finally: self.connection_lock.release()
def init(self): self.connection = Connection(self.broker, reconnect=True, reconnect_interval=3) try: self.connection.open() self.session = self.connection.session() self.sender = self.session.sender(self.address) except MessagingError: print(traceback.format_exc()) return False return True
class FreshDocWriter(PageWriterBase): def _initialize(self): self.set_name('FreshDocWriter') if get_project_settings()['DEBUG_MODE']: self._push_message = self._push_message_debug else: self._create_client() def _create_client(self): try: self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True, reconnect_limit=10, reconnect_interval=4) self.connection_.open() self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video') except: self.connection_ = None self.logger_.exception('failed to connect to message queue server.') def finalize(self): if self.connection_: self.connection_.close() PageWriterBase.finalize(self) def _push_message_debug(self, doc): pass def _push_message(self, doc): doc.video.doc_id = doc.id doc.video.id = str(doc.id) doc.video.crawl_time = doc.video.create_time = doc.crawl_time doc.video.discover_time = doc.discover_time doc.video.url = doc.url doc.video.domain = doc.domain doc.video.domain_id = doc.domain_id doc.video.in_links = doc.in_links msg_body = thrift_to_str(doc.video) if not msg_body: return self.sender_.send(Message(content=doc.url + '\t' + base64.b64encode(msg_body), durable=True)) self.logger_.info('send message successfully, %s', doc.url) def process_item(self, item): if not item: return doc = item.to_crawldoc() if doc.doc_type != CrawlDocType.PAGE_TIME: return try: self._push_message(doc) except: self.logger_.exception('failed to send message, %s', doc.url)
def start(self): """ Enable AMQP queueing. This method puts up the event processor and sets it to "active". """ self.log.debug("enabling AMQP queueing") # Evaluate username user = self.env.config.get("amqp.id", default=None) if not user: user = self.env.uuid # Create initial broker connection url = "%s:%s" % (self.url['host'], self.url['port']) self._conn = Connection.establish(url, reconnect=self.reconnect, username=user, password=self.env.config.get("amqp.key"), transport=self.url['transport'], reconnect_interval=self.reconnect_interval, reconnect_limit=self.reconnect_limit) # Do automatic broker failover if requested if self.env.config.get('amqp.failover', default=False): auto_fetch_reconnect_urls(self._conn) # Create event provider self._eventProvider = EventProvider(self.env, self._conn)
def checkAuth(self, user, password): """ This function checks a username / password combination using the AMQP service' SASL configuration. =============== ============ Parameter Description =============== ============ user Username password Password =============== ============ ``Return:`` Bool, success or failure """ # Strip username/password parts of url url = "%s:%s" % (self.url['host'], self.url['port']) # Don't allow blank authentication if user == "" or password == "": return False try: conn = Connection.establish(url, transport=self.url['transport'], username=user, password=password) conn.close() except ConnectionError as e: self.log.debug("AMQP service authentication reports: %s" % str(e)) return False except Exception as e: self.log.critical("cannot proceed with authentication") self.log.exception(e) return False return True
def testConnectError(self): try: self.conn = Connection.open("localhost", 0) assert False, "connect succeeded" except ConnectError, e: # XXX: should verify that e includes appropriate diagnostic info pass
class FanoutPublisher(object): """ qpid fanout publisher """ def __init__(self, broker, address): self.broker = broker self.address = address self.connection = None self.session = None self.sender = None self.heartbeat_sender = None def __del__(self): self.destroy() def init(self): self.connection = Connection(self.broker, reconnect=True, reconnect_interval=3) try: self.connection.open() self.session = self.connection.session() self.sender = self.session.sender(self.address) except MessagingError: print(traceback.format_exc()) return False return True def destroy(self): if self.connection: self.connection.close() def publish(self, message): if self.sender is None: if not self.init(): return False try: message.properties = {'x-amqp-0-10.routing-key': self.address} self.sender.send(message) print('publish info. broker: {}, reply_to: {}'.format( self.broker, self.address)) except MessagingError: print(traceback.format_exc()) return False return True
def _create_client(self): try: self.connection_ = Connection(url='10.100.151.13:5672', heartbeat=4, reconnect=True, reconnect_limit=10, reconnect_interval=4) self.connection_.open() self.sender_ = self.connection_.session().sender('leso.exchange.fresh_video') except: self.connection_ = None self.logger_.exception('failed to connect to message queue server.')
def checkOverStockedQueues(host): connection = Connection.establish(host) broker = BrokerAgent(connection) queues = broker.getAllQueues() result = list() for q in queues: if (q.msgDepth != 0): print(q.name + " " + str(q.msgDepth)) result.append([q.name, q.msgDepth]) return result
def migrate(*args, **kwargs): """ Migrate qpid queues: - Ensure pulp.task is no longer *exclusive*. - Rename agent queues: consumer_id> => pulp.agent.<consumer_id> """ transport = pulp_conf.get('messaging', 'transport') if transport != 'qpid': # not using qpid return if not QPID_MESSAGING_AVAILABLE: msg = _('Migration 0009 did not run because the python package qpid.messaging is not ' 'installed. Please install qpid.messaging and rerun the migrations. See %s' 'for more information.') msg = msg % QPID_MESSAGING_URL _logger.error(msg) raise Exception(msg) if not QPIDTOOLLIBS_AVAILABLE: msg = _('Migration 0009 did not run because the python package qpidtoollibs is not ' 'installed. Please install qpidtoollibs and rerun the migrations. See %s for more ' 'information.') msg = msg % QPIDTOOLLIBS_URL _logger.error(msg) raise Exception(msg) url = urlparse(pulp_conf.get('messaging', 'url')) connection = Connection( host=url.hostname, port=url.port, transport=url.scheme, reconnect=False, ssl_certfile=pulp_conf.get('messaging', 'clientcert'), ssl_skip_hostname_check=True) connection.attach() broker = BrokerAgent(connection) _migrate_reply_queue(broker) _migrate_agent_queues(broker) connection.detach()
def migrate(*args, **kwargs): """ Migrate qpid queues: - Ensure pulp.task is no longer *exclusive*. - Rename agent queues: consumer_id> => pulp.agent.<consumer_id> """ transport = pulp_conf.get('messaging', 'transport') if transport != 'qpid': # not using qpid return if not QPID_MESSAGING_AVAILABLE: msg = _('Migration 0009 did not run because the python package qpid.messaging is not ' 'installed. Pulp\'s Qpid client dependencies can be installed with the ' '\"pulp-server-qpid\" package group. See the installation docs for more ' 'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.') _logger.error(msg) raise Exception(msg) if not QPIDTOOLLIBS_AVAILABLE: msg = _('Migration 0009 did not run because the python package qpidtoollibs is not ' 'installed. Pulp\'s Qpid client dependencies can be installed with the ' '\"pulp-server-qpid\" package group. See the installation docs for more ' 'information. Alternatively, you may reconfigure Pulp to use RabbitMQ.') _logger.error(msg) raise Exception(msg) url = urlparse(pulp_conf.get('messaging', 'url')) connection = Connection( host=url.hostname, port=url.port, transport=url.scheme, reconnect=False, ssl_certfile=pulp_conf.get('messaging', 'clientcert'), ssl_skip_hostname_check=True) connection.attach() broker = BrokerAgent(connection) _migrate_reply_queue(broker) _migrate_agent_queues(broker) connection.detach()
def __init__(self, url, receiver_name, sender_name='pulp.task', asserting=False, **options): '''establishes a connection to given url; initializes session, sender and receiver''' self.url = url self.receiver_name = receiver_name self.sender_name = sender_name self._asserting = asserting self.last_sent = None self.last_fetched = None self.session = Connection.establish(self.url, **options).session() self.receiver = self.session.receiver("%s; {create: always}" % self.receiver_name) self.sender = self.session.sender(self.sender_name) self._timeout = None
def open(self): """ Open a connection to the broker. """ if self.is_open(): # already open return connector = Connector.find(self.url) Connection.add_transports() domain = self.ssl_domain(connector) log.info('open: %s', connector) impl = RealConnection(host=connector.host, port=connector.port, tcp_nodelay=True, transport=connector.url.scheme, username=connector.userid, password=connector.password, heartbeat=10, **domain) impl.open() self._impl = impl log.info('opened: %s', self.url)
class SetupTests(Base): def testOpen(self): # XXX: need to flesh out URL support/syntax self.conn = Connection.open(self.broker.host, self.broker.port, reconnect=self.reconnect()) self.ping(self.conn.session()) def testConnect(self): # XXX: need to flesh out URL support/syntax self.conn = Connection(self.broker.host, self.broker.port, reconnect=self.reconnect()) self.conn.connect() self.ping(self.conn.session()) def testConnectError(self): try: self.conn = Connection.open("localhost", 0) assert False, "connect succeeded" except ConnectError, e: # XXX: should verify that e includes appropriate diagnostic info pass
class Qpid(MQ): _timeout = 1 def _conectar(self): try: logger.debug("Qpid: %s" % self._url.netloc) self._conn = Connection(self._url.netloc) if not self._conn: raise MQError(None, 2) self._conn.open() except ConnectError: raise MQError(cod=2) try: self._session = self._conn.session() self._sender = self._session.sender(self._url.path[1:]) self._receiver = self._session.receiver(self._url.path[1:]) logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc)) except ConnectError: raise MQError(cod=3) def _enviar(self, mensagem): logger.debug("Sending a message") m = Message(mensagem) self._sender.send(m, True, self._timeout) def _receber(self, timeout=None): logger.debug("Retrieving a message") self._mensagem = self._receiver.fetch(timeout) return self._mensagem.content def _tamanho(self): self._receiver.available() def _excluir(self): logger.debug("Ack last message") self._session.acknowledge() def _terminar(self): self._conn.close(self._timeout)
class QpidConnection(): port = None ip = None data_queue = None cmd_queue = None conn = None session = None sender = None receiver = None def __init__(self, ip, port, data_queue, cmd_queue): self.ip = ip; self.port = port self.data_queue = data_queue self.cmd_queue = cmd_queue def start(self): try: url = str(self.ip) url += str(':') url += str(self.port) self.conn = Connection(url) self.conn.open() self.session = self.conn.session() self.sender = self.session.sender(self.cmd_queue) self.receiver = self.session.receiver(self.data_queue) return 1 except MessagingError as m: print(m) return 0 def stop(self): try: self.conn.close() except MessagingError as m: print(m) return 0
def start(self): try: url = str(self.ip) url += str(':') url += str(self.port) self.conn = Connection(url) self.conn.open() self.session = self.conn.session() self.sender = self.session.sender(self.cmd_queue) self.receiver = self.session.receiver(self.data_queue) return 1 except MessagingError as m: print(m) return 0
def open(self): """ Open a connection to the broker. """ if self.is_open(): # already open return connector = Connector.find(self.url) Connection.add_transports() domain = self.ssl_domain(connector) log.info('open: %s', connector) impl = RealConnection( host=connector.host, port=connector.port, tcp_nodelay=True, transport=connector.url.scheme, username=connector.userid, password=connector.password, heartbeat=10, **domain) impl.open() self._impl = impl log.info('opened: %s', self.url)
def _conectar(self): try: logger.debug("Qpid: %s" % self._url.netloc) self._conn = Connection(self._url.netloc) if not self._conn: raise MQError(None, 2) self._conn.open() except ConnectError: raise MQError(cod=2) try: self._session = self._conn.session() self._sender = self._session.sender(self._url.path[1:]) self._receiver = self._session.receiver(self._url.path[1:]) logger.info("Connected on queue %s on %s" % (self._url.path[1:], self._url.netloc)) except ConnectError: raise MQError(cod=3)
def __init__(self, url, domain="org.clacks", xquery=".", callback=None): # Build connection url = parseURL(url) _url = "%s:%s" % (url['host'], url['port']) self.__conn = Connection.establish(_url, reconnect=True, username=url['user'], password=url['password'], transport=url['transport'], reconnect_interval=3, reconnect_limit=0) # Do automatic broker failover if requested #TODO: configure reconnect #auto_fetch_reconnect_urls(self.__conn) # Assemble subscription query queue = 'event-listener-%s' % uuid4() address = """%s; { create: always, delete:always, node: { durable: False, x-declare: { exclusive: True, auto-delete: True } }, link: { x-bindings: [ { exchange: '%s', queue: %s, key: event, arguments: { xquery: %r} } ] } }""" % (queue, domain, queue, xquery) # Add processor for core.event queue self.__callback = callback self.__eventWorker = AMQPStandaloneWorker( self.__conn, r_address=address, workers=1, callback=self.__eventProcessor)
def start(self): """ Enable AMQP queueing. This method puts up the event processor and sets it to "active". """ self.log.debug("enabling AMQP queueing") # Evaluate username user = self.config.get("amqp.id", default=None) if not user: user = self.env.uuid password = self.config.get("amqp.key") # Create initial broker connection url = "%s:%s" % (self.url['host'], self.url['port']) self._conn = Connection.establish(url, reconnect=self.reconnect, username=user, password=password, transport=self.url['transport'], reconnect_interval=self.reconnect_interval, reconnect_limit=self.reconnect_limit) # Do automatic broker failover if requested if self.config.get('amqp.failover', False): auto_fetch_reconnect_urls(self._conn) # Create event exchange socket = connect(self.url['host'], self.url['port']) if self.url['scheme'][-1] == 's': socket = ssl(socket) user = self.config.get("amqp.id", default=None) if not user: user = self.env.uuid connection = DirectConnection(sock=socket, username=user, password=self.config.get("amqp.key")) connection.start() session = connection.session(str(uuid4())) # pylint: disable=E1103 session.exchange_declare(exchange=self.env.domain, type="xml") connection.close() # Create event provider self._eventProvider = EventProvider(self.env, self.getConnection())
def __init__(self, url, receiver_name, sender_name='pulp.task', asserting=False, auth=Authenticator(), **options): '''establishes a connection to given url; initializes session, sender and receiver''' self.url = url self.receiver_name = receiver_name self.sender_name = sender_name self._asserting = asserting self.last_sent = None self.last_fetched = None self.session = Connection.establish(self.url, **options).session() self.receiver = self.session.receiver( "pulp.agent.%s; {create: always}" % self.receiver_name) self.sender = self.session.sender(self.sender_name) self._timeout = None self.auth = auth
def start(self, is_reconnect=False): """ Initiate the Qpid connection and start up any listeners. :param: is_reconnect True if this method is called as part of a reconnect attempt, False otherwise :raise: ConnectionError if a connection cannot be established """ # If the Qpid broker URL is not specified (or just the hostname is not # specified) then we can't make a connection. if not self.url or self.url.startswith(':'): log(self.log, 'warn', _('Qpid broker not specified, cannot start ' 'connection.')) return if not self._is_connected: self.conn = Connection(self.url, username=self.username, password=self.password, transport=self.transport) try: self.conn.open() except ConnectionError as e: log(self.log, 'critical', _('Cannot connect to Qpid message ' 'broker: %s') % (e.message)) # close this connection when encounter connection error # otherwise, it will leave an ESTABLISHED connection # to qpid server forever. if self.conn is not None: self.conn.close() raise e self._is_connected = True if is_reconnect and self.reconnect_handler: self.reconnect_handler() for listener in self._listeners: listener._start(self.conn) log(self.log, 'info', _('Connected to Qpid message broker: ' '%s@%s') % (self.username, self.url))
def __init__(self, url, domain="org.gosa", xquery=".", callback=None): # Build connection url = parseURL(url) self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True) self.__conn.open() # Assemble subscription query queue = 'event-listener-%s' % uuid4() address = """%s; { create: always, delete:always, node: { durable: False, x-declare: { exclusive: True, auto-delete: True } }, link: { x-bindings: [ { exchange: '%s', queue: %s, key: event, arguments: { xquery: %r} } ] } }""" % (queue, domain, queue, xquery) # Add processor for core.event queue self.__callback = callback self.__eventWorker = AMQPStandaloneWorker( self.__conn, r_address=address, workers=1, callback=self.__eventProcessor)
def open(self,af): ''' <ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> <ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> ''' from qpid.messaging import Connection from qpid.util import URL ep = self.ep self.af = af self.exitflag = False broker = "%s:%s"%(ep.host,ep.port) # log_debug('prepare mq : <%s %s>!'% (ep.name,broker)) try: self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if af & AF_READ: self.rcv = self.ssn.receiver(self.ep.addr) self.rcv.capacity = 4000 # self.thread = threading.Thread(target =self.thread_recv) # self.thread.start() # import gevent gevent.spawn(self.thread_recv) if af & AF_WRITE: self.snd = self.ssn.sender(self.ep.addr) except: log_error(traceback.format_exc()) return False # log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker)) return True
def migrate(*args, **kwargs): """ Migrate qpid queues: - Ensure pulp.task is no longer *exclusive*. - Rename agent queues: consumer_id> => pulp.agent.<consumer_id> """ transport = pulp_conf.get('messaging', 'transport') if transport != 'qpid': # not using qpid return if not QPID_MESSAGING_AVAILABLE: msg = _( 'Migration 0009 did not run because the python package qpid.messaging is not ' 'installed. Please install qpid.messaging and rerun the migrations. See %s' 'for more information.') msg = msg % QPID_MESSAGING_URL _logger.error(msg) raise Exception(msg) if not QPIDTOOLLIBS_AVAILABLE: msg = _( 'Migration 0009 did not run because the python package qpidtoollibs is not ' 'installed. Please install qpidtoollibs and rerun the migrations. See %s for more ' 'information.') msg = msg % QPIDTOOLLIBS_URL _logger.error(msg) raise Exception(msg) url = urlparse(pulp_conf.get('messaging', 'url')) connection = Connection(host=url.hostname, port=url.port, transport=url.scheme, reconnect=False, ssl_certfile=pulp_conf.get('messaging', 'clientcert'), ssl_skip_hostname_check=True) connection.attach() broker = BrokerAgent(connection) _migrate_reply_queue(broker) _migrate_agent_queues(broker) connection.detach()
def __init__(self, serviceURL, serviceAddress=None, serviceName=None, conn=None, workers=3): self.__URL = url = parseURL(serviceURL) self.__serviceURL = serviceURL self.__serviceName = serviceName self.__serviceAddress = serviceAddress self.__workers = workers domain = url['path'] # Prepare AMQP connection if not already there if not conn: conn = Connection(url['url'], transport=url['transport'], reconnect=True) conn.open() AMQPServiceProxy.domain= domain # Prefill __serviceAddress correctly if domain is given if AMQPServiceProxy.domain: self.__serviceAddress = '%s.command.core' % AMQPServiceProxy.domain if not self.__serviceAddress: raise AMQPException("no serviceAddress or domain specified") try: AMQPServiceProxy.worker[self.__serviceAddress] except KeyError: AMQPServiceProxy.worker[self.__serviceAddress] = {} # Pre instanciate core sessions for i in range(0, workers): ssn = conn.session(str(uuid4())) AMQPServiceProxy.worker[self.__serviceAddress][i] = { 'ssn': ssn, 'sender': ssn.sender(self.__serviceAddress), 'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name), 'locked': False} # Store connection self.__conn = conn self.__ssn = None self.__sender = None self.__receiver = None self.__worker = None # Retrieve methods try: AMQPServiceProxy.methods except AttributeError: AMQPServiceProxy.methods = None AMQPServiceProxy.methods = {} # Retrieve methods try: AMQPServiceProxy.methods[self.__serviceAddress] except KeyError: AMQPServiceProxy.methods[self.__serviceAddress] = None AMQPServiceProxy.methods[self.__serviceAddress] = self.getMethods() # If we've no direct queue, we need to push to different queues if AMQPServiceProxy.domain: queues = set([ x['target'] for x in AMQPServiceProxy.methods[self.__serviceAddress].itervalues() if x['target'] != 'core' ]) # Pre instanciate queue sessions for queue in queues: for i in range(0, workers): ssn = conn.session(str(uuid4())) AMQPServiceProxy.worker[queue] = {} AMQPServiceProxy.worker[queue][i] = { 'ssn': ssn, 'sender': ssn.sender("%s.command.%s" % (AMQPServiceProxy.domain, queue)), 'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name), 'locked': False}
class RpcConnectionQpidMQ(RpcConnection): def __init__(self,ep=None): RpcConnection.__init__(self,ep=ep) self.conn = None self.exitflag = False ep.impl = self self.mq_recv ='' RpcConnectionMQ_Collection.instance().add(self) @staticmethod def create(name,host,port,address,af=AF_WRITE): """ 创建MQ的连接对象 :param name: 必须是真实的 mq 名称 :param host: :param port: :param address: :param af: :return: """ ep = RpcEndPoint(name=name,host=host,port=port,addr=address,type_='qpid') if ep.open(af): return ep.impl return None # conn = RpcConnectionQpidMQ(ep) # conn.open(af) # return conn # @staticmethod # def createRpcInvocationPair(ep_read,ep_write): # conn_read = RpcConnectionQpidMQ(ep_read) # conn_read.open(AF_READ) # conn_write = RpcConnectionQpidMQ(ep_write) # conn_write.open(AF_WRITE) # conn_write.setLoopbackMQ(conn_read) # return conn_read,conn_write def open(self,af): ''' <ep name="mq_gwa_2" address="mq_gwa_2;{create:always,node:{type:queue,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> <ep name="mq_gwa_broadcast" address="mq_gwa_broadcast;{create:always,node:{type:topic,durable:true}}" type="mq" host="127.0.0.1" port="5672"/> ''' from qpid.messaging import Connection from qpid.util import URL ep = self.ep self.af = af self.exitflag = False broker = "%s:%s"%(ep.host,ep.port) # log_debug('prepare mq : <%s %s>!'% (ep.name,broker)) try: self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) self.conn.open() self.ssn = self.conn.session() if af & AF_READ: self.rcv = self.ssn.receiver(self.ep.addr) self.rcv.capacity = 4000 # self.thread = threading.Thread(target =self.thread_recv) # self.thread.start() # import gevent gevent.spawn(self.thread_recv) if af & AF_WRITE: self.snd = self.ssn.sender(self.ep.addr) except: log_error(traceback.format_exc()) return False # log_debug('prepare mq : <%s %s> okay!'% (ep.name,broker)) return True def close(self): if self.conn: self.conn.close() self.conn = None self.exitflag = True def setLoopbackMQ(self,conn_recv): ''' 设置rpc调用的回路连接, mq_recv为回路mq的名称, mq在EasyMQ_Collection中被缓存 目前的回路mq名称取 队列名称,如果携带主机信息的话,回路可以从另外一台mq-server返回 ''' self.mq_recv = conn_recv.ep.getUnique() return self def sendMessage(self,m): if m.traits and m.traits.max_linger_time: value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0') value = int(value) if not value: value += int(time.time()) m.extra.setPropertyValue(RpcMessageTraits.MAX_MSG_LINGER_TIME, value ) #app制定了超时接收时间,这里调整为绝对时间,以便接收端进行判别,选择接受还是丢弃 if m.calltype & RpcMessage.RETURN: #--- Rpc的调用消息中包含接收消息的队列名称 --- mqname = m.callmsg.extra.props.get('__mq_return__') if mqname: mq = RpcConnectionMQ_Collection.instance().get(mqname) if mq: mq.sendDetail(m) return log_error('__mq_return__:<%s> is not in service mq-list!'%mqname) return False if self.mq_recv: m.extra.props['__mq_return__'] = self.mq_recv return RpcConnection.sendMessage(self,m) def sendDetail(self,m): from qpid.messaging import Message try: if self.exitflag: return True # if not self.conn: # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.snd = self.ssn.sender(self.ep.addr) d = m.marshall() m = Message(d) self.snd.send(m,False) except: log_error(traceback.format_exc()) # self.conn = None return False return True #接收消息 def thread_recv(self): from communicator import RpcCommunicator # print 'qpid-mq recv thread start..' while not self.exitflag: try: # if not self.conn: # print 'try open mq:',self.ep.name # broker = "%s:%s"%(self.ep.host,self.ep.port) # self.conn = Connection( broker,reconnect= True,tcp_nodelay=True) # self.conn.open() # self.ssn = self.conn.session() # self.rcv = self.ssn.receiver(self.ep.addr) # self.rcv.capacity = 4000 m = self.rcv.fetch() # print '.'*10 d = m.content # print 'mq recv:',repr(d) print 'recved 1 msg from MQ..' self.ssn.acknowledge(sync=False) m = RpcMessage.unmarshall(d) if not m: log_error('decode mq-msg error!') continue m.conn = self value = m.extra.props.get(RpcMessageTraits.MAX_MSG_LINGER_TIME,'0') linger_time = int(value) if linger_time and time.time() > linger_time: # drop it continue #过期接收的消息直接丢弃 # self.dispatchMsg(m) RpcCommunicator.instance().dispatchMsg(m) except: log_error(traceback.format_exc()) gevent.sleep(1) if self.adapter: self.adapter.stopmtx.set() # log_info("qpid-mq thread exiting..") return False
# import camel.biz.application.use_gevent import gevent from gevent import monkey # monkey.patch_all() from camel.koala.tcelib.conn_qpid import RpcConnectionQpidMQ, AF_READ # mq = RpcConnectionQpidMQ.create('abc',host="rhinoceros/jiangxiaoyu@ytodev2",port="5672", # address="mq_messageserver;{create:always,node:{type:queue,durable:true}}",af=AF_READ) from qpid.messaging import Connection from qpid.util import URL host = "rhinoceros/jiangxiaoyu@ytodev2" host = "rhinoceros/[email protected]" host = "118.190.89.205" port = "5672" addr = "mq_messageserver;{create:always,node:{type:queue,durable:true}}" broker = "%s:%s" % (host, port) conn = Connection(broker, reconnect=True, tcp_nodelay=True) conn.open() ssn = conn.session() rcv = ssn.receiver(addr) rcv.capacity = 4000 while True: m = rcv.fetch() ssn.acknowledge() print m gevent.sleep(1000000)
def __init__(self, address, **kwargs): self._connection = Connection.establish( address, client_properties={"qpid.ha-admin": 1}, **kwargs) self._agent = BrokerAgent(self._connection) assert self._agent.getHaBroker( ), "HA module not loaded in broker at: %s" % (address)
def get_args(): parser = argparse.ArgumentParser(description="print messages sent to a qpid exchange") parser.add_argument("-e", "--exchange", help="name of a qpid exchange (default: amq.topic)", default="amq.topic") parser.add_argument("-s", "--subject", help="message subject to bind to", required=False) parser.add_argument("-a", "--address", help="hostname to connect to (default:localhost)", default="localhost") parser.add_argument("-p", "--port", help="port to connect to (default: 5672)", default="5672") parser.add_argument("-q", "--quiet", help="show message subject only", default=False, action="store_true") return parser.parse_args() args = get_args() source = args.exchange if args.subject: source = "%s/%s" % (source, args.subject) receiver = Connection.establish("%s:%s" % (args.address, args.port)).session().receiver(source) try: while True: message = receiver.fetch() if args.quiet: print message.subject else: print "------------------" print message except KeyboardInterrupt: print ""
'--port', help='port to connect to (default: 5672)', default='5672') parser.add_argument('-q', '--quiet', help='show message subject only', default=False, action='store_true') return parser.parse_args() args = get_args() source = args.exchange if args.subject: source = '%s/%s' % (source, args.subject) receiver = Connection.establish( '%s:%s' % (args.address, args.port)).session().receiver(source) try: while True: message = receiver.fetch() if args.quiet: print message.subject else: print '------------------' print message except KeyboardInterrupt: print ''
def testConnect(self): # XXX: need to flesh out URL support/syntax self.conn = Connection(self.broker.host, self.broker.port, reconnect=self.reconnect()) self.conn.connect() self.ping(self.conn.session())
def setup_connection(self): return Connection.open(self.broker.host, self.broker.port, reconnect=self.reconnect())
consumers = get('/consumers/') return [ c['id'] for c in consumers if c['notes'].get('_child-node', False) ] def get_nodes_repos(node): " Returns a child node's 'bound' repos." bindings = get("/consumers/%s/bindings/" % (node)) return [ repo['repo_id'] for repo in bindings ] if __name__ == '__main__': args = get_args() # qpid connection address = "%s:%s" % (args.host, args.port) receiver = Connection.establish(address).session().receiver(args.exchange) try: while True: message = receiver.fetch() json_message = json.loads(message.content) if json_message['payload']['result'] == 'success': repo_id = json_message['payload']['repo_id'] else: continue nodes = get_nodes() for node in nodes: repos = get_nodes_repos(node) if repo_id in repos:
def test_08_integration_test_with_messagebus(self): """ Full blown integration test listening for notifications on the bus, and checking which dir is up for a visit next. Needs a working local qpid broker. Test is skipped if qpid not available. """ try: broker = None connection = None import uuid from threading import Event from qpid.messaging import Connection, ConnectError from qpidtoollibs import BrokerAgent from lofar.messaging.messagebus import ToBus from lofar.messaging.messages import EventMessage from lofar.lta.ingest.common.config import DEFAULT_INGEST_NOTIFICATION_PREFIX # setup broker connection connection = Connection.establish('127.0.0.1') broker = BrokerAgent(connection) # add test service bus busname = 'test-LTASOIngestEventHandler-%s' % (uuid.uuid1()) broker.addExchange('topic', busname) sync_event = Event() class SyncedLTASOIngestEventHandler(LTASOIngestEventHandler): """This derived LTASOIngestEventHandler behaves exactly like the normal object under test LTASOIngestEventHandler, but it also sets a sync_event to sync between the listener thread and this main test thread""" def _handleMessage(self, msg): super(SyncedLTASOIngestEventHandler, self)._handleMessage(msg) sync_event.set() with SyncedLTASOIngestEventHandler(self.dbcreds, busname=busname): for site in self.db.sites(): for root_dir in self.db.rootDirectoriesForSite(site['id']): self._markAllDirectoriesRecentlyVisited() # create the subdir surl sub_dir_name = '/foo' sub_dir_path = root_dir['dir_name'] + sub_dir_name surl = site['url'] + sub_dir_path with ToBus(busname) as sender: msg = EventMessage( subject=DEFAULT_INGEST_NOTIFICATION_PREFIX + "TaskFinished", content={'srm_url': surl}) sender.send(msg) # wait for the handler to have processed the message self.assertTrue(sync_event.wait(2)) sync_event.clear() # surl should have been scheduled for a visit, all other dir's were marked as visited already... # so there should be a new dir for this surl, and it should be the least_recent_visited_dir site_visit_stats = self.db.visitStats( datetime.utcnow())[site['name']] least_recent_visited_dir_id = site_visit_stats.get( 'least_recent_visited_dir_id') self.assertIsNotNone(least_recent_visited_dir_id) least_recent_visited_dir = self.db.directory( least_recent_visited_dir_id) self.assertEqual(sub_dir_path, least_recent_visited_dir['dir_name']) except ImportError as e: logger.warning("skipping test due to: %s", e) except ConnectError as e: logger.warning("skipping test due to: %s", e) finally: # cleanup test bus and exit if broker: broker.delExchange(busname) if connection: connection.close()
class AMQPEventConsumer(object): """ The AMQPEventConsumer can be used to subscribe for events and process them thru a callback. The subscription is done thru *XQuery*, the callback can be a python method. Example listening for an event called *AsteriskNotification*:: >>> from gosa.common.components import AMQPEventConsumer >>> from lxml import etree >>> >>> # Event callback >>> def process(data): ... print(etree.tostring(data, pretty_print=True)) >>> >>> # Create event consumer >>> consumer = AMQPEventConsumer("amqps://*****:*****@localhost/org.gosa", ... xquery=\"\"\" ... declare namespace f='http://www.gonicus.de/Events'; ... let $e := ./f:Event ... return $e/f:AsteriskNotification ... \"\"\", ... callback=process) The consumer will start right away, listening for your events. =============== ============ Parameter Description =============== ============ url URL used to connect to the AMQP service broker domain If the domain is not already encoded in the URL, it can be specified here. xquery `XQuery <http://en.wikipedia.org/wiki/XQuery>`_ string to query for events. callback Python method to be called if the event happened. =============== ============ .. note:: The AMQP URL consists of these parts:: (amqp|amqps)://user:password@host:port/domain """ def __init__(self, url, domain="org.gosa", xquery=".", callback=None): # Build connection url = parseURL(url) self.__conn = Connection(url['url'], transport=url['transport'], reconnect=True) self.__conn.open() # Assemble subscription query queue = 'event-listener-%s' % uuid4() address = """%s; { create: always, delete:always, node: { durable: False, x-declare: { exclusive: True, auto-delete: True } }, link: { x-bindings: [ { exchange: '%s', queue: %s, key: event, arguments: { xquery: %r} } ] } }""" % (queue, domain, queue, xquery) # Add processor for core.event queue self.__callback = callback self.__eventWorker = AMQPStandaloneWorker( self.__conn, r_address=address, workers=1, callback=self.__eventProcessor) def __del__(self): self.__eventWorker.join() self.__conn.close() #pylint: disable=W0613 def __eventProcessor(self, ssn, data): # Call callback, let exceptions pass to the caller xml = objectify.fromstring(data.content) self.__callback(xml) def join(self): self.__eventWorker.join()
def __init__(self, address, **kwargs): self._connection = Connection.establish( address, client_properties={"qpid.ha-admin": 1}, **kwargs) self._agent = BrokerAgent(self._connection)
def __init__(self, serviceURL, serviceAddress=None, serviceName=None, conn=None, worker=None, methods=None): self.__URL = url = parseURL(serviceURL) self.__serviceURL = serviceURL self.__serviceName = serviceName self.__serviceAddress = serviceAddress self.__worker = worker self.__domain = url['path'] self.__methods = methods # Prepare AMQP connection if not already there if not conn: _url = "%s:%s" % (url['host'], url['port']) conn = Connection.establish(_url, reconnect=True, username=url['user'], password=url['password'], transport=url['transport'], reconnect_interval=3, reconnect_limit=0) #TODO: configure reconnect #auto_fetch_reconnect_urls(conn) # Prefill __serviceAddress correctly if domain is given if self.__domain: self.__serviceAddress = '%s.command.core' % self.__domain if not self.__serviceAddress: raise AMQPException("no serviceAddress or domain specified") if not self.__worker: self.__worker = {self.__serviceAddress: {}} # Pre instanciate core sessions for i in range(0, WORKERS): ssn = conn.session(str(uuid4())) self.__worker[self.__serviceAddress][i] = { 'ssn': ssn, 'sender': ssn.sender(self.__serviceAddress), 'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name), 'locked': False} # Store connection self.__conn = conn self.__ssn = None self.__sender = None self.__receiver = None self.__sess = None # Retrieve methods if not self.__methods: self.__serviceName = "getMethods" self.__methods = self.__call__() self.__serviceName = None # If we've no direct queue, we need to push to different queues if self.__domain: queues = set([ x['target'] for x in self.__methods.itervalues() if x['target'] != 'core' ]) # Pre instanciate queue sessions for queue in queues: for i in range(0, WORKERS): ssn = conn.session(str(uuid4())) self.__worker[queue] = {} self.__worker[queue][i] = { 'ssn': ssn, 'sender': ssn.sender("%s.command.%s" % (self.__domain, queue)), 'receiver': ssn.receiver('reply-%s; {create:always, delete:always, node: { type: queue, durable: False, x-declare: { exclusive: False, auto-delete: True } }}' % ssn.name), 'locked': False}
from qpid.messaging import Connection, Message from time import sleep import sys host = "localhost:10009" if len(sys.argv) > 1: host = sys.argv[1] conn = Connection(host) try: conn.open() sess = conn.session() tx = sess.sender("spfdemo.com/mobile.45") count = 0 while(True): tx.send("Seq: %d" % count) print "Origin: %s Seq: %d" % (host, count) count += 1 sleep(1) except Exception, e: print "Exception: %r" % e except KeyboardInterrupt: print
from mock import patch except ImportError: print 'Cannot run test without python MagicMock' print 'Please install MagicMock: pip install mock' exit(3) connection = None broker = None try: logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO) logger = logging.getLogger(__name__) # setup broker connection connection = Connection.establish('127.0.0.1') broker = BrokerAgent(connection) # add test service busname busname = 'test-lofarbus-%s' % (uuid.uuid1()) broker.addExchange('topic', busname) # the system under test is the service and the rpc, not the RADatabase # so, patch (mock) the RADatabase class during these tests. # when the service instantiates an RADatabase it will get the mocked class. with patch('lofar.sas.resourceassignment.database.radb.RADatabase', autospec=True) as MockRADatabase: mock = MockRADatabase.return_value # modify the return values of the various RADatabase methods with pre-cooked answers mock.getTaskStatuses.return_value = [{ 'id': 1,