def on_queue_declared(self, frame): self._queue = frame.method.queue # get the real queue name logger.default(u"Consumer declared queue '%s' on connection '%s'", self._queue, self.connection_id) if self.auto_declare and self.exchange: self.bind_queue() else: self.on_ready_to_consume()
def on_message_received(self, message): logger.default( u"Received message '%s' sent to exchange '%s' with " u"routing key '%s'", message.method_frame.delivery_tag, message.method_frame.exchange, message.method_frame.routing_key) req, zreq, resp = self.get_requests_and_response(message) self.zhandler('Zope2', zreq, resp)
def on_exchange_declared(self, frame): logger.default(u"Consumer declared exchange '%s' on connection '%s'", self.exchange, self.connection_id) if self.queue_auto_declare and self.queue is not None\ and not self.queue.startswith('amq.'): self.declare_queue() else: self.on_ready_to_consume()
def on_message_received(self, message): logger.default(u"Received message '%s' sent to exchange '%s' with " u"routing key '%s'", message.method_frame.delivery_tag, message.method_frame.exchange, message.method_frame.routing_key) req, zreq, resp = self.get_requests_and_response(message) self.zhandler('Zope2', zreq, resp)
def on_exchange_declared(self, frame): logger.default(u"Producer declared exchange '%s' on connection '%s'", self.exchange, self.connection_id) if self.queue_auto_declare and self.queue is not None\ and not self.queue.startswith('amq.'): self.declare_queue() else: self.on_ready_to_publish()
def __call__(self): message = self.request.environ.get("AMQP_MESSAGE") user_id = self.request.environ.get("AMQP_USER_ID") exchange = message.method_frame.exchange routing_key = message.method_frame.routing_key delivery_tag = message.method_frame.delivery_tag age = unicode(datetime.datetime.utcnow() - message.created_datetime) logger.default( u"Worker started processing message '%s' " u"(status = '%s', age = '%s')", delivery_tag, message.state, age ) message._register() event = createObject("AMQPMessageArrivedEvent", message) with security_manager(self.request, user_id): try: notify(event) except ConflictError: logger.error(u"Conflict while working on message '%s' " u"(status = '%s')", delivery_tag, message.state) message.state = "ERROR" raise except: exc_type, exc_value, exc_traceback = sys.exc_info() err_handler = queryUtility(IErrorHandler, name=exchange) if err_handler is not None: err_handler(message, exc_value, exc_traceback) else: logger.error( u"Error while handling message '%s' sent to " u"exchange '%s' with routing key '%s'", delivery_tag, exchange, routing_key, ) message.state = "ERROR" raise age = unicode(datetime.datetime.utcnow() - message.created_datetime) if not (message.acknowledged or message.rejected): logger.warning( u"Nobody acknowledged or rejected message '%s' " u"sent to exchange exchange '%s' " u"with routing key '%s'", delivery_tag, exchange, routing_key, ) else: logger.default( u"Letting Zope to commit database transaction for " u"message '%s' (status = '%s', age = '%s')", delivery_tag, message.state, age, ) return u"" # 200 No Content
def _ack(self): self.acknowledged = True if self.channel: self.channel.basic_ack(delivery_tag=self.method_frame.delivery_tag) self.state = 'ACK' if self.channel and self.tx_select: self.channel.tx_commit() # min support for transactional channel age = unicode(datetime.datetime.utcnow() - self.created_datetime) logger.default(u"Handled message '%s' (status = '%s', age = '%s')", self.method_frame.delivery_tag, self.state, age)
def on_channel_open(self, channel): logger.default(u"Channel for connection '%s' opened", self.connection_id) self._channel = channel self._channel.add_on_close_callback(self.on_channel_closed) if self.prefetch_count: from pika import spec self._channel.transport.rpc( spec.Basic.Qos(0, self.prefetch_count, False), self.on_qos_ok, [spec.Basic.QosOk]) else: self.on_qos_ok(self._channel)
def _ack(self): self.acknowledged = True if self.channel: self.channel.basic_ack(delivery_tag=self.method_frame.delivery_tag) self.state = "ACK" if self.channel and self.tx_select: self.channel.tx_commit() # min support for transactional channel age = unicode(datetime.datetime.utcnow() - self.created_datetime) logger.default( u"Handled message '%s' (status = '%s', age = '%s')", self.method_frame.delivery_tag, self.state, age )
def on_queue_bound(self, frame, index=0): logger.default(u"Consumer bound queue '%s' to exchange '%s' " u"on connection '%s'", self._queue, self.exchange, self.connection_id) if type(self.routing_key) not in (tuple, list): self.on_ready_to_consume() elif len(self.routing_key) <= index + 1: self.on_ready_to_consume() else: index = index + 1 cb = lambda f, s=self, index=index: self.on_queue_bound(f, index) self._channel.queue_bind(exchange=self.exchange, queue=self._queue, routing_key=self.routing_key[index], callback=cb)
def __call__(self): message = self.request.environ.get('AMQP_MESSAGE') user_id = self.request.environ.get('AMQP_USER_ID') exchange = message.method_frame.exchange routing_key = message.method_frame.routing_key delivery_tag = message.method_frame.delivery_tag age = unicode(datetime.datetime.utcnow() - message.created_datetime) logger.default(u"Worker started processing message '%s' " u"(status = '%s', age = '%s')", delivery_tag, message.state, age) message._register() event = createObject('AMQPMessageArrivedEvent', message) with security_manager(self.request, user_id): try: notify(event) except ConflictError: logger.error(u"Conflict while working on message '%s' " u"(status = '%s')", delivery_tag, message.state) message.state = "ERROR" raise except: exc_type, exc_value, exc_traceback = sys.exc_info() err_handler = queryUtility(IErrorHandler, name=exchange) if err_handler is not None: err_handler(message, exc_value, exc_traceback) else: logger.error(u"Error while handling message '%s' sent to " u"exchange '%s' with routing key '%s'", delivery_tag, exchange, routing_key) message.state = "ERROR" raise age = unicode(datetime.datetime.utcnow() - message.created_datetime) if not (message.acknowledged or message.rejected): logger.warning(u"Nobody acknowledged or rejected message '%s' " u"sent to exchange exchange '%s' " u"with routing key '%s'", delivery_tag, exchange, routing_key) else: logger.default(u"Letting Zope to commit database transaction for " u"message '%s' (status = '%s', age = '%s')", delivery_tag, message.state, age) return u'' # 200 No Content
def on_channel_open(self, channel): logger.default(u"Channel for connection '%s' opened", self.connection_id) self._channel = channel self._channel.add_on_close_callback(self.on_channel_closed) if self.prefetch_count: from pika import spec self._channel.transport.rpc( spec.Basic.Qos(0, self.prefetch_count, False), self.on_qos_ok, [spec.Basic.QosOk] ) else: self.on_qos_ok(self._channel)
def __init__(self, connection_id, site_id, user_id='Anonymous User', scheme='http', hostname=None, port=80, use_vhm=True, vhm_method_prefix='', logger=None, handler=None): self.logger = AMQPMedusaLogger(logger) from collective.zamqp.utils import logger logger.default( u"AMQP Consuming Server for connection '%s' started " u"(site '%s' user: '******')", connection_id, site_id, user_id) self._USE_VHM = use_vhm h = self.headers = [] h.append('User-Agent: AMQP Consuming Server') h.append('Accept: text/html,text/plain') if not hostname: hostname = socket.gethostname() if use_vhm or ':' in hostname: h.append('Host: {0:s}'.format(hostname)) else: h.append('Host: {0:s}:{1:d}'.format(hostname, port)) self.hostname = hostname self.connection_id = connection_id self.site_id = site_id self.user_id = user_id self.port = port self.scheme = scheme self.vhm_method_prefix = vhm_method_prefix if handler is None: # for unit testing handler = handle self.zhandler = handler self.consumers = [] provideHandler(self.on_before_broker_connect, [IBeforeBrokerConnectEvent])
def on_queue_bound(self, frame, index=0): logger.default( u"Consumer bound queue '%s' to exchange '%s' " u"on connection '%s'", self._queue, self.exchange, self.connection_id, ) if type(self.routing_key) not in (tuple, list): self.on_ready_to_consume() elif len(self.routing_key) <= index + 1: self.on_ready_to_consume() else: index = index + 1 cb = lambda f, s=self, index=index: self.on_queue_bound(f, index) self._channel.queue_bind( exchange=self.exchange, queue=self._queue, routing_key=self.routing_key[index], callback=cb )
def _reject(self, requeue=True): self.rejected = True self.requeued = requeue if self.channel: self.channel.basic_reject( delivery_tag=self.method_frame.delivery_tag, requeue=requeue) if requeue: self.state = 'REQUEUED' else: self.state = 'REJECTED' if self.channel and self.tx_select: self.channel.tx_commit() # min support for transactional channel age = unicode(datetime.datetime.utcnow() - self.created_datetime) logger.default(u"Rejected message '%s' (status = '%s', age = '%s')", self.method_frame.delivery_tag, self.state, age)
def connect(self): logger.default(u"Connection '%s' connecting", self.connection_id) credentials = PlainCredentials( self.username, self.password, erase_on_connect=False) parameters = ConnectionParameters( self.hostname, self.port, self.virtual_host, credentials=credentials, heartbeat=self.heartbeat and True or False) # AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5: if parameters.heartbeat: parameters.heartbeat = int(self.heartbeat) self._connection = AsyncoreConnection( parameters=parameters, on_open_callback=self.on_connect) self._reconnection_timeout = None self._connection.add_on_close_callback(self.reconnect)
def _reject(self, requeue=True): self.rejected = True self.requeued = requeue if self.channel: self.channel.basic_reject(delivery_tag=self.method_frame.delivery_tag, requeue=requeue) if requeue: self.state = "REQUEUED" else: self.state = "REJECTED" if self.channel and self.tx_select: self.channel.tx_commit() # min support for transactional channel age = unicode(datetime.datetime.utcnow() - self.created_datetime) logger.default( u"Rejected message '%s' (status = '%s', age = '%s')", self.method_frame.delivery_tag, self.state, age )
def connect(self): logger.default(u"Connection '%s' connecting", self.connection_id) credentials = PlainCredentials(self.username, self.password, erase_on_connect=False) parameters = ConnectionParameters(self.hostname, self.port, self.virtual_host, credentials=credentials, heartbeat=self.heartbeat and True or False) # AMQP-heartbeat timeout must be set manually due to bug in pika 0.9.5: if parameters.heartbeat: parameters.heartbeat = int(self.heartbeat) self._connection = AsyncoreConnection(parameters=parameters, on_open_callback=self.on_connect) self._reconnection_timeout = None self._connection.add_on_close_callback(self.reconnect)
def _abort(self): # collect execution info for guessing the reason for abort exc_type, exc_value, exc_traceback = sys.exc_info() if self.state != "ACK": self.acknowledged = False if self.state == "ACK" and issubclass(exc_type, ConflictError): if not getattr(self, "_aborted", False): logger.warning( u"Transaction aborted due to database conflict. " u"Message '%s' was acked before commit and could " u"not be requeued (status = '%s')", self.method_frame.delivery_tag, self.state, ) self._aborted = True elif self.state not in ("FAILED", "REQUEUED"): # on transactional channel, rollback on abort if self.channel and self.tx_select: self.channel.tx_rollback() # min support for transactional # channel # ^ XXX: Because the same channel may be shared by multiple # threads, tx_rollback may be far from safe. It's supported # only to make single-threaded AMQP-consuming ZEO-clients # support transactional channel. DO NOT run multi-threaded # consuming-server with transactional channel. # reject messages with requeue when ConflictError in ZPublisher if self.state != "ERROR" and issubclass(exc_type, ConflictError): # reject message with requeue self.channel.basic_reject(delivery_tag=self.method_frame.delivery_tag, requeue=True) self.state = "REQUEUED" logger.default( u"Transaction aborted due to database conflict. " u"Requeued message '%s' (status = '%s')", self.method_frame.delivery_tag, self.state, ) # otherwise, message handling has failed and un-acknowledged else: self.state = "FAILED"
def _abort(self): # collect execution info for guessing the reason for abort exc_type, exc_value, exc_traceback = sys.exc_info() if self.state != 'ACK': self.acknowledged = False if self.state == 'ACK' and issubclass(exc_type, ConflictError): if not getattr(self, '_aborted', False): logger.warning( u"Transaction aborted due to database conflict. " u"Message '%s' was acked before commit and could " u"not be requeued (status = '%s')", self.method_frame.delivery_tag, self.state) self._aborted = True elif self.state not in ('FAILED', 'REQUEUED'): # on transactional channel, rollback on abort if self.channel and self.tx_select: self.channel.tx_rollback() # min support for transactional # channel # ^ XXX: Because the same channel may be shared by multiple # threads, tx_rollback may be far from safe. It's supported # only to make single-threaded AMQP-consuming ZEO-clients # support transactional channel. DO NOT run multi-threaded # consuming-server with transactional channel. # reject messages with requeue when ConflictError in ZPublisher if self.state != 'ERROR' and issubclass(exc_type, ConflictError): # reject message with requeue self.channel.basic_reject( delivery_tag=self.method_frame.delivery_tag, requeue=True) self.state = "REQUEUED" logger.default( u"Transaction aborted due to database conflict. " u"Requeued message '%s' (status = '%s')", self.method_frame.delivery_tag, self.state) # otherwise, message handling has failed and un-acknowledged else: self.state = 'FAILED'
def on_exchange_declared(self, frame): logger.default(u"Consumer declared exchange '%s' on connection '%s'", self.exchange, self.connection_id) if self.queue_auto_declare and self.queue is not None and not self.queue.startswith("amq."): self.declare_queue() else: self.on_ready_to_consume()
def create(self): from zope.component import provideUtility from collective.zamqp.interfaces import IBrokerConnection from collective.zamqp.connection import BrokerConnection connection = BrokerConnection(connection_id=self.connection_id, hostname=self.hostname, port=self.port, virtual_host=self.virtual_host, username=self.username, password=self.password, heartbeat=self.heartbeat, prefetch_count=self.prefetch_count, tx_select=self.tx_select) # set expected ZServer-properties to support debugtoolbar connection.server_name = "ZAMQP Broker Connection" connection.ip = None provideUtility(connection, IBrokerConnection, name=self.connection_id) if self.keepalive: from collective.zamqp.utils import logger logger.default(u"Setting up keepalive (%s s) for connection '%s'", self.keepalive, self.connection_id) # register a ping producer, a ping consumer, a ping view and a ping # clock-server to keep the connection alive from collective.zamqp.interfaces import IProducer, IConsumer from collective.zamqp import keepalive name = "%s.ping" % self.connection_id # the producer producer = keepalive.PingProducer(self.connection_id) provideUtility(producer, IProducer, name=name) # the consumer consumer = keepalive.PingConsumer(self.connection_id) provideUtility(consumer, IConsumer, name=name) from zope.interface import Interface from zope.component import provideAdapter from OFS.interfaces import IApplication # the view ping = lambda context, request: lambda: keepalive.ping(name) provideAdapter(ping, adapts=(IApplication, Interface), provides=Interface, name=name) # the clock-server from ZServer.AccessLogger import access_logger from ZServer.ClockServer import ClockServer clock = ClockServer(method="/%s" % name, period=self.keepalive, host="localhost", logger=access_logger) # just in case, store the created utilities, view and server connection._keepalive = {"producer": producer, "consumer": consumer, "view": ping, "clock": clock} if self.producer: # generate default producer with the name of the connection from collective.zamqp.interfaces import IProducer from collective.zamqp.producer import Producer producer = Producer(self.connection_id, exchange="", routing_key="", durable=False, auto_declare=False) provideUtility(producer, IProducer, name=self.connection_id) # set expected ZServer-properties to support debugtoolbar connection.server_name = "ZAMQP Broker Connection" connection.ip = None return connection
def on_queue_bound(self, frame): logger.default(u"Producer bound queue '%s' to exchange '%s' " u"on connection '%s'", self._queue, self.exchange, self.connection_id) self.on_ready_to_publish()
def on_connect(self, connection): logger.default(u"Connection '%s' connected", self.connection_id) self._connection = connection self._connection.channel(self.on_channel_open) self._reconnection_time = time.time()
def on_ready_to_publish(self): logger.default(u"Producer ready to publish to exchange '%s' " u"with routing key '%s' on connection '%s'", self.exchange, self.routing_key, self.connection_id) self._callbacks.process(0, "_on_ready_to_publish", self)
def __init__(self, connection_id=None, hostname=None, port=None, virtual_host=None, username=None, password=None, heartbeat=None, prefetch_count=None, tx_select=None): # Allow class variables to provide defaults for: # connection_id if self.connection_id is None and connection_id is None: connection_id =\ getattr(self, 'grokcore.component.directive.name', 'default') if connection_id is not None: self.connection_id = connection_id # hostname if hostname is not None: self.hostname = hostname assert self.hostname is not None,\ u"Connection configuration is missing hostname." # port if port is not None: self.port = port assert self.port is not None,\ u"Connection configuration is missing port." # virtual_host if virtual_host is not None: self.virtual_host = virtual_host assert self.virtual_host is not None,\ u"Connection configuration is missing virtual_host." # username if username is not None: self.username = username assert self.username is not None,\ u"Connection configuration is missing username." # password if password is not None: self.password = password assert self.password is not None,\ u"Connection configuration is missing password." # heartbeat if heartbeat is not None: self.heartbeat = heartbeat # prefetch_count if prefetch_count is not None: self.prefetch_count = prefetch_count # tx_select if tx_select is not None: self.tx_select = tx_select self._callbacks = CallbackManager() # callbacks are NOT thread-safe self._reconnection_time = time.time() self._reconnection_delay = 1.0 # BBB for affinitic.zamqp if getattr(self, 'userid', None): from zope.deprecation import deprecated self.username = self.userid self.userid =\ deprecated(self.userid, ('Connection.userid is no more. ' 'Please, use Connection.username instead.')) logger.default( u"AMQP Broker connection '%s' created. " u"hostname: '%s', " u"port: '%s', " u"virtual_host: '%s', " u"username: '******', " u"heartbeat: '%s', " u"prefetch_count: '%s', " u"tx_select: '%s'", self.connection_id, self.hostname, self.port, self.virtual_host, self.username, self.heartbeat, self.prefetch_count, self.tx_select)
def on_ready_to_publish(self): logger.default( u"Producer ready to publish to exchange '%s' " u"with routing key '%s' on connection '%s'", self.exchange, self.routing_key, self.connection_id) self._callbacks.process(0, "_on_ready_to_publish", self)
def on_queue_bound(self, frame): logger.default( u"Producer bound queue '%s' to exchange '%s' " u"on connection '%s'", self._queue, self.exchange, self.connection_id) self.on_ready_to_publish()
def on_connect(self, connection): logger.default(u"Connection '%s' connected", self.connection_id) self._connection = connection self._connection.channel(self.on_channel_open) self._reconnection_delay = 1.0
def on_ready_to_consume(self): if self.marker is not None: # False is allowed to trigger consuming logger.default(u"Consumer ready to consume queue '%s' on " u"connection '%s'", self._queue, self.connection_id) self._channel.basic_consume(self.on_message_received, queue=self._queue)
def __init__(self, connection_id=None, hostname=None, port=None, virtual_host=None, username=None, password=None, heartbeat=None, prefetch_count=None, tx_select=None): # Allow class variables to provide defaults for: # connection_id if self.connection_id is None and connection_id is None: connection_id =\ getattr(self, 'grokcore.component.directive.name', 'default') if connection_id is not None: self.connection_id = connection_id # hostname if hostname is not None: self.hostname = hostname assert self.hostname is not None,\ u"Connection configuration is missing hostname." # port if port is not None: self.port = port assert self.port is not None,\ u"Connection configuration is missing port." # virtual_host if virtual_host is not None: self.virtual_host = virtual_host assert self.virtual_host is not None,\ u"Connection configuration is missing virtual_host." # username if username is not None: self.username = username assert self.username is not None,\ u"Connection configuration is missing username." # password if password is not None: self.password = password assert self.password is not None,\ u"Connection configuration is missing password." # heartbeat if heartbeat is not None: self.heartbeat = heartbeat # prefetch_count if prefetch_count is not None: self.prefetch_count = prefetch_count # tx_select if tx_select is not None: self.tx_select = tx_select self._callbacks = CallbackManager() # callbacks are NOT thread-safe self._reconnection_delay = 1.0 # BBB for affinitic.zamqp if getattr(self, 'userid', None): from zope.deprecation import deprecated self.username = self.userid self.userid =\ deprecated(self.userid, ('Connection.userid is no more. ' 'Please, use Connection.username instead.')) logger.default(u"AMQP Broker connection '%s' created. " u"hostname: '%s', " u"port: '%s', " u"virtual_host: '%s', " u"username: '******', " u"heartbeat: '%s', " u"prefetch_count: '%s', " u"tx_select: '%s'", self.connection_id, self.hostname, self.port, self.virtual_host, self.username, self.heartbeat, self.prefetch_count, self.tx_select)
def create(self): from zope.component import provideUtility from collective.zamqp.interfaces import IBrokerConnection from collective.zamqp.connection import BrokerConnection connection = BrokerConnection(connection_id=self.connection_id, hostname=self.hostname, port=self.port, virtual_host=self.virtual_host, username=self.username, password=self.password, heartbeat=self.heartbeat, prefetch_count=self.prefetch_count, tx_select=self.tx_select) # set expected ZServer-properties to support debugtoolbar connection.server_name = "ZAMQP Broker Connection" connection.ip = None provideUtility(connection, IBrokerConnection, name=self.connection_id) if self.keepalive: from collective.zamqp.utils import logger logger.default(u"Setting up keepalive (%s s) for connection '%s'", self.keepalive, self.connection_id) # register a ping producer, a ping consumer, a ping view and a ping # clock-server to keep the connection alive from collective.zamqp.interfaces import IProducer, IConsumer from collective.zamqp import keepalive name = "%s.ping" % self.connection_id # the producer producer = keepalive.PingProducer(self.connection_id) provideUtility(producer, IProducer, name=name) # the consumer consumer = keepalive.PingConsumer(self.connection_id) provideUtility(consumer, IConsumer, name=name) from zope.interface import Interface from zope.component import provideAdapter from OFS.interfaces import IApplication # the view ping = lambda context, request: lambda: keepalive.ping(name) provideAdapter(ping, adapts=(IApplication, Interface), provides=Interface, name=name) # the clock-server from ZServer.AccessLogger import access_logger from ZServer.ClockServer import ClockServer clock = ClockServer(method="/%s" % name, period=self.keepalive, host="localhost", logger=access_logger) # just in case, store the created utilities, view and server connection._keepalive = { "producer": producer, "consumer": consumer, "view": ping, "clock": clock } if self.producer: # generate default producer with the name of the connection from collective.zamqp.interfaces import IProducer from collective.zamqp.producer import Producer producer = Producer(self.connection_id, exchange="", routing_key="", durable=False, auto_declare=False) provideUtility(producer, IProducer, name=self.connection_id) # set expected ZServer-properties to support debugtoolbar connection.server_name = "ZAMQP Broker Connection" connection.ip = None return connection
def on_ready_to_consume(self): if self.marker is not None: # False is allowed to trigger consuming logger.default( u"Consumer ready to consume queue '%s' on " u"connection '%s'", self._queue, self.connection_id ) self._channel.basic_consume(self.on_message_received, queue=self._queue)