def on_ready_to_consume(self):
if self.marker is not None: # False is allowed to trigger consuming
logger.default(
u"Consumer ready to consume queue '%s' on "
u"connection '%s'", self._queue, self.connection_id)
self._channel.basic_consume(self.on_message_received,
queue=self._queue)
def on_message_received(self, message):
logger.default(
u"Received message '%s' sent to exchange '%s' with "
u"routing key '%s'", message.method_frame.delivery_tag,
message.method_frame.exchange, message.method_frame.routing_key)
req, zreq, resp = self.get_requests_and_response(message)
self.zhandler('Zope2', zreq, resp)
def on_message_received(self, message):
logger.default(u"Received message '%s' sent to exchange '%s' with "
u"routing key '%s'",
message.method_frame.delivery_tag,
message.method_frame.exchange,
message.method_frame.routing_key)
req, zreq, resp = self.get_requests_and_response(message)
self.zhandler('Zope2', zreq, resp)
def on_exchange_declared(self, frame):
logger.default(u"Consumer declared exchange '%s' on connection '%s'",
self.exchange, self.connection_id)
if self.queue_auto_declare and self.queue is not None\
and not self.queue.startswith('amq.'):
self.declare_queue()
else:
self.on_ready_to_consume()
def on_queue_declared(self, frame):
self._queue = frame.method.queue # get the real queue name
logger.default(u"Consumer declared queue '%s' on connection '%s'",
self._queue, self.connection_id)
if self.auto_declare and self.exchange:
self.bind_queue()
else:
self.on_ready_to_consume()
def on_queue_declared(self, frame):
self._queue = frame.method.queue # get the real queue name
logger.default(u"Producer declared queue '%s' on connection '%s'",
self._queue, self.connection_id)
if self.auto_declare and self.exchange:
self.bind_queue()
else:
self.on_ready_to_publish()
def on_exchange_declared(self, frame):
logger.default(u"Producer declared exchange '%s' on connection '%s'",
self.exchange, self.connection_id)
if self.queue_auto_declare and self.queue is not None\
and not self.queue.startswith('amq.'):
self.declare_queue()
else:
self.on_ready_to_publish()
def _ack(self):
self.acknowledged = True
if self.channel:
self.channel.basic_ack(delivery_tag=self.method_frame.delivery_tag)
self.state = 'ACK'
if self.channel and self.tx_select:
self.channel.tx_commit() # min support for transactional channel
age = unicode(datetime.datetime.utcnow() - self.created_datetime)
logger.default(u"Handled message '%s' (status = '%s', age = '%s')",
self.method_frame.delivery_tag, self.state, age)
def _ack(self):
self.acknowledged = True
if self.channel:
self.channel.basic_ack(
delivery_tag=self.method_frame.delivery_tag)
self.state = 'ACK'
if self.channel and self.tx_select:
self.channel.tx_commit() # min support for transactional channel
age = unicode(datetime.datetime.utcnow() - self.created_datetime)
logger.default(u"Handled message '%s' (status = '%s', age = '%s')",
self.method_frame.delivery_tag, self.state, age)
def __call__(self):
message = self.request.environ.get('AMQP_MESSAGE')
user_id = self.request.environ.get('AMQP_USER_ID')
exchange = message.method_frame.exchange
routing_key = message.method_frame.routing_key
delivery_tag = message.method_frame.delivery_tag
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
logger.default(
u"Worker started processing message '%s' "
u"(status = '%s', age = '%s')", delivery_tag, message.state, age)
message._register()
event = createObject('AMQPMessageArrivedEvent', message)
with security_manager(self.request, user_id):
try:
notify(event)
except ConflictError:
logger.error(
u"Conflict while working on message '%s' "
u"(status = '%s')", delivery_tag, message.state)
message.state = "ERROR"
raise
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
err_handler = queryUtility(IErrorHandler, name=exchange)
if err_handler is not None:
err_handler(message, exc_value, exc_traceback)
else:
logger.error(
u"Error while handling message '%s' sent to "
u"exchange '%s' with routing key '%s'", delivery_tag,
exchange, routing_key)
message.state = "ERROR"
raise
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
if not (message.acknowledged or message.rejected):
logger.warning(
u"Nobody acknowledged or rejected message '%s' "
u"sent to exchange exchange '%s' "
u"with routing key '%s'", delivery_tag, exchange, routing_key)
else:
logger.default(
u"Letting Zope to commit database transaction for "
u"message '%s' (status = '%s', age = '%s')", delivery_tag,
message.state, age)
return u'' # 200 No Content
def on_queue_bound(self, frame, index=0):
logger.default(u"Consumer bound queue '%s' to exchange '%s' "
u"on connection '%s'",
self._queue, self.exchange, self.connection_id)
if type(self.routing_key) not in (tuple, list):
self.on_ready_to_consume()
elif len(self.routing_key) <= index + 1:
self.on_ready_to_consume()
else:
index = index + 1
cb = lambda f, s=self, index=index: self.on_queue_bound(f, index)
self._channel.queue_bind(exchange=self.exchange, queue=self._queue,
routing_key=self.routing_key[index],
callback=cb)
def __call__(self):
message = self.request.environ.get('AMQP_MESSAGE')
user_id = self.request.environ.get('AMQP_USER_ID')
exchange = message.method_frame.exchange
routing_key = message.method_frame.routing_key
delivery_tag = message.method_frame.delivery_tag
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
logger.default(u"Worker started processing message '%s' "
u"(status = '%s', age = '%s')",
delivery_tag, message.state, age)
message._register()
event = createObject('AMQPMessageArrivedEvent', message)
with security_manager(self.request, user_id):
try:
notify(event)
except ConflictError:
logger.error(u"Conflict while working on message '%s' "
u"(status = '%s')",
delivery_tag, message.state)
message.state = "ERROR"
raise
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
err_handler = queryUtility(IErrorHandler, name=exchange)
if err_handler is not None:
err_handler(message, exc_value, exc_traceback)
else:
logger.error(u"Error while handling message '%s' sent to "
u"exchange '%s' with routing key '%s'",
delivery_tag, exchange, routing_key)
message.state = "ERROR"
raise
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
if not (message.acknowledged or message.rejected):
logger.warning(u"Nobody acknowledged or rejected message '%s' "
u"sent to exchange exchange '%s' "
u"with routing key '%s'",
delivery_tag, exchange, routing_key)
else:
logger.default(u"Letting Zope to commit database transaction for "
u"message '%s' (status = '%s', age = '%s')",
delivery_tag, message.state, age)
return u'' # 200 No Content
def on_queue_bound(self, frame, index=0):
logger.default(
u"Consumer bound queue '%s' to exchange '%s' "
u"on connection '%s'", self._queue, self.exchange,
self.connection_id)
if type(self.routing_key) not in (tuple, list):
self.on_ready_to_consume()
elif len(self.routing_key) <= index + 1:
self.on_ready_to_consume()
else:
index = index + 1
cb = lambda f, s=self, index=index: self.on_queue_bound(f, index)
self._channel.queue_bind(exchange=self.exchange,
queue=self._queue,
routing_key=self.routing_key[index],
callback=cb)
def _reject(self, requeue=True):
self.rejected = True
self.requeued = requeue
if self.channel:
self.channel.basic_reject(
delivery_tag=self.method_frame.delivery_tag, requeue=requeue)
if requeue:
self.state = 'REQUEUED'
else:
self.state = 'REJECTED'
if self.channel and self.tx_select:
self.channel.tx_commit() # min support for transactional channel
age = unicode(datetime.datetime.utcnow() - self.created_datetime)
logger.default(u"Rejected message '%s' (status = '%s', age = '%s')",
self.method_frame.delivery_tag, self.state)
def _reject(self, requeue=True):
self.rejected = True
self.requeued = requeue
if self.channel:
self.channel.basic_reject(
delivery_tag=self.method_frame.delivery_tag, requeue=requeue)
if requeue:
self.state = 'REQUEUED'
else:
self.state = 'REJECTED'
if self.channel and self.tx_select:
self.channel.tx_commit() # min support for transactional channel
age = unicode(datetime.datetime.utcnow() - self.created_datetime)
logger.default(u"Rejected message '%s' (status = '%s', age = '%s')",
self.method_frame.delivery_tag, self.state)
def __init__(self,
connection_id,
site_id,
user_id='Anonymous User',
scheme='http',
hostname=None,
port=80,
use_vhm=True,
logger=None,
handler=None):
self.logger = AMQPMedusaLogger(logger)
from collective.zamqp import logger
logger.default(
u"AMQP Consuming Server for connection '%s' started "
u"(site '%s' user: '******')", connection_id, site_id, user_id)
self._USE_VHM = use_vhm
h = self.headers = []
h.append('User-Agent: AMQP Consuming Server')
h.append('Accept: text/html,text/plain')
if not hostname:
hostname = socket.gethostname()
if use_vhm or ':' in hostname:
h.append('Host: {0:s}'.format(hostname))
else:
h.append('Host: {0:s}:{1:d}'.format(hostname, port))
self.hostname = hostname
self.connection_id = connection_id
self.site_id = site_id
self.user_id = user_id
self.port = port
self.scheme = scheme
if handler is None:
# for unit testing
handler = handle
self.zhandler = handler
self.consumers = []
provideHandler(self.on_before_broker_connect,
[IBeforeBrokerConnectEvent])
def __init__(self, connection_id, site_id, user_id='Anonymous User',
scheme='http', hostname=None, port=80, use_vhm=True,
vhm_method_prefix='', logger=None, handler=None):
self.logger = AMQPMedusaLogger(logger)
from collective.zamqp import logger
logger.default(
u"AMQP Consuming Server for connection '%s' started "
u"(site '%s' user: '******')",
connection_id, site_id, user_id)
self._USE_VHM = use_vhm
h = self.headers = []
h.append('User-Agent: AMQP Consuming Server')
h.append('Accept: text/html,text/plain')
if not hostname:
hostname = socket.gethostname()
if use_vhm or ':' in hostname:
h.append('Host: {0:s}'.format(hostname))
else:
h.append('Host: {0:s}:{1:d}'.format(hostname, port))
self.hostname = hostname
self.connection_id = connection_id
self.site_id = site_id
self.user_id = user_id
self.port = port
self.scheme = scheme
self.vhm_method_prefix = vhm_method_prefix
if handler is None:
# for unit testing
handler = handle
self.zhandler = handler
self.consumers = []
provideHandler(self.on_before_broker_connect,
[IBeforeBrokerConnectEvent])
def _abort(self):
# collect execution info for guessing the reason for abort
exc_type, exc_value, exc_traceback = sys.exc_info()
if self.state != 'ACK':
self.acknowledged = False
if self.state == 'ACK' and issubclass(exc_type, ConflictError):
if not getattr(self, '_aborted', False):
logger.warning(
u"Transaction aborted due to database conflict. "
u"Message '%s' was acked before commit and could "
u"not be requeued (status = '%s')",
self.method_frame.delivery_tag, self.state)
self._aborted = True
elif self.state not in ('FAILED', 'REQUEUED'):
# on transactional channel, rollback on abort
if self.channel and self.tx_select:
self.channel.tx_rollback() # min support for transactional
# channel
# ^ XXX: Because the same channel may be shared by multiple
# threads, tx_rollback may be far from safe. It's supported
# only to make single-threaded AMQP-consuming ZEO-clients
# support transactional channel. DO NOT run multi-threaded
# consuming-server with transactional channel.
# reject messages with requeue when ConflictError in ZPublisher
if self.state != 'ERROR' and issubclass(exc_type, ConflictError):
# reject message with requeue
self.channel.basic_reject(
delivery_tag=self.method_frame.delivery_tag, requeue=True)
self.state = "REQUEUED"
logger.default(
u"Transaction aborted due to database conflict. "
u"Requeued message '%s' (status = '%s')",
self.method_frame.delivery_tag, self.state)
# otherwise, message handling has failed and un-acknowledged
else:
self.state = 'FAILED'
def _abort(self):
# collect execution info for guessing the reason for abort
exc_type, exc_value, exc_traceback = sys.exc_info()
if self.state != 'ACK':
self.acknowledged = False
if self.state == 'ACK' and issubclass(exc_type, ConflictError):
if not getattr(self, '_aborted', False):
logger.warning(
u"Transaction aborted due to database conflict. "
u"Message '%s' was acked before commit and could "
u"not be requeued (status = '%s')",
self.method_frame.delivery_tag, self.state)
self._aborted = True
elif self.state not in ('FAILED', 'REQUEUED'):
# on transactional channel, rollback on abort
if self.channel and self.tx_select:
self.channel.tx_rollback() # min support for transactional
# channel
# ^ XXX: Because the same channel may be shared by multiple
# threads, tx_rollback may be far from safe. It's supported
# only to make single-threaded AMQP-consuming ZEO-clients
# support transactional channel. DO NOT run multi-threaded
# consuming-server with transactional channel.
# reject messages with requeue when ConflictError in ZPublisher
if self.state != 'ERROR' and issubclass(exc_type, ConflictError):
# reject message with requeue
self.channel.basic_reject(
delivery_tag=self.method_frame.delivery_tag, requeue=True)
self.state = "REQUEUED"
logger.default(
u"Transaction aborted due to database conflict. "
u"Requeued message '%s' (status = '%s')",
self.method_frame.delivery_tag, self.state)
# otherwise, message handling has failed and un-acknowledged
else:
self.state = 'FAILED'
def on_queue_bound(self, frame):
logger.default(u"Producer bound queue '%s' to exchange '%s' "
u"on connection '%s'",
self._queue, self.exchange, self.connection_id)
self.on_ready_to_publish()
def on_ready_to_publish(self):
logger.default(u"Producer ready to publish to exchange '%s' "
u"with routing key '%s' on connection '%s'",
self.exchange, self.routing_key, self.connection_id)
self._callbacks.process(0, "_on_ready_to_publish", self)
def on_ready_to_consume(self):
if self.marker is not None: # False is allowed to trigger consuming
logger.default(u"Consumer ready to consume queue '%s' on "
u"connection '%s'", self._queue, self.connection_id)
self._channel.basic_consume(self.on_message_received,
queue=self._queue)
