def __call__(self):
message = self.request.environ.get("AMQP_MESSAGE")
user_id = self.request.environ.get("AMQP_USER_ID")
exchange = message.method_frame.exchange
routing_key = message.method_frame.routing_key
delivery_tag = message.method_frame.delivery_tag
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
logger.default(
u"Worker started processing message '%s' " u"(status = '%s', age = '%s')", delivery_tag, message.state, age
)
message._register()
event = createObject("AMQPMessageArrivedEvent", message)
with security_manager(self.request, user_id):
try:
notify(event)
except ConflictError:
logger.error(u"Conflict while working on message '%s' " u"(status = '%s')", delivery_tag, message.state)
message.state = "ERROR"
raise
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
err_handler = queryUtility(IErrorHandler, name=exchange)
if err_handler is not None:
err_handler(message, exc_value, exc_traceback)
else:
logger.error(
u"Error while handling message '%s' sent to " u"exchange '%s' with routing key '%s'",
delivery_tag,
exchange,
routing_key,
)
message.state = "ERROR"
raise
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
if not (message.acknowledged or message.rejected):
logger.warning(
u"Nobody acknowledged or rejected message '%s' "
u"sent to exchange exchange '%s' "
u"with routing key '%s'",
delivery_tag,
exchange,
routing_key,
)
else:
logger.default(
u"Letting Zope to commit database transaction for " u"message '%s' (status = '%s', age = '%s')",
delivery_tag,
message.state,
age,
)
return u"" # 200 No Content
def on_before_broker_connect(self, event=None):
self._connection = queryUtility(IBrokerConnection,
name=self.connection_id)
if self._connection:
self._connection.add_on_channel_open_callback(self.on_channel_open)
else:
logger.warning(u"Connection '%s' was not registered. "
u"Producer '%s' cannot be connected.",
self.connection_id, self.routing_key)
def on_channel_closed(self, code, text):
logger.warning(u"Channel closed with reason '%s %s'", code, text)
# With ZAMQP channel should only be closed when
# 1) connection is closed due ot any reason
# 2) permission or binding error from producer or consumer
self._connection.close(code, text) # safe to call also during closing
# Enforce connection close because _connection.close() does not
# close socket when connection has been closed by RabbitMQ or network
self._connection._adapter_disconnect()
def on_before_broker_connect(self, event=None):
self._connection = queryUtility(IBrokerConnection,
name=self.connection_id)
if self._connection:
self._connection.add_on_channel_open_callback(self.on_channel_open)
else:
logger.warning(
u"Connection '%s' was not registered. "
u"Producer '%s' cannot be connected.", self.connection_id,
self.routing_key)
def on_channel_closed(self, code, text):
logger.warning(u"Channel closed with reason '%s %s'",
code, text)
# With ZAMQP channel should only be closed when
# 1) connection is closed due ot any reason
# 2) permission or binding error from producer or consumer
self._connection.close(code, text) # safe to call also during closing
# Enforce connection close because _connection.close() does not
# close socket when connection has been closed by RabbitMQ or network
self._connection._adapter_disconnect()
def _finish(self):
if self.acknowledged and not self.state == 'ACK':
self._ack()
if self.rejected:
if self.state == 'ACK':
logger.warning(
u"Message '%s' was both acknowledged and rejected "
u"(status = '%s'). Rejection was cancelled and "
u"message was acknowledged",
self.method_frame.delivery_tag, self.state)
elif self.state not in ['REJECTED', 'REQUEUED']:
self._reject(self.requeued)
def _finish(self):
if self.acknowledged and not self.state == "ACK":
self._ack()
if self.rejected:
if self.state == "ACK":
logger.warning(
u"Message '%s' was both acknowledged and rejected "
u"(status = '%s'). Rejection was cancelled and "
u"message was acknowledged",
self.method_frame.delivery_tag,
self.state,
)
elif self.state not in ["REJECTED", "REQUEUED"]:
self._reject(self.requeued)
def __call__(self):
message = self.request.environ.get('AMQP_MESSAGE')
user_id = self.request.environ.get('AMQP_USER_ID')
exchange = message.method_frame.exchange
routing_key = message.method_frame.routing_key
delivery_tag = message.method_frame.delivery_tag
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
logger.default(u"Worker started processing message '%s' "
u"(status = '%s', age = '%s')",
delivery_tag, message.state, age)
message._register()
event = createObject('AMQPMessageArrivedEvent', message)
with security_manager(self.request, user_id):
try:
notify(event)
except ConflictError:
logger.error(u"Conflict while working on message '%s' "
u"(status = '%s')",
delivery_tag, message.state)
message.state = "ERROR"
raise
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
err_handler = queryUtility(IErrorHandler, name=exchange)
if err_handler is not None:
err_handler(message, exc_value, exc_traceback)
else:
logger.error(u"Error while handling message '%s' sent to "
u"exchange '%s' with routing key '%s'",
delivery_tag, exchange, routing_key)
message.state = "ERROR"
raise
age = unicode(datetime.datetime.utcnow() - message.created_datetime)
if not (message.acknowledged or message.rejected):
logger.warning(u"Nobody acknowledged or rejected message '%s' "
u"sent to exchange exchange '%s' "
u"with routing key '%s'",
delivery_tag, exchange, routing_key)
else:
logger.default(u"Letting Zope to commit database transaction for "
u"message '%s' (status = '%s', age = '%s')",
delivery_tag, message.state, age)
return u'' # 200 No Content
def _basic_publish(self, **kwargs):
retry_constructor = lambda func, kwargs: lambda: func(**kwargs)
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.basic_publish(**kwargs)
return True
elif getattr(kwargs.get("properties"), "delivery_mode", 1) == 2:
logger.warning(u"No connection. Durable message was left into "
u"volatile memory to wait for a new connection "
u"'%s'", kwargs)
retry_callback = retry_constructor(self._basic_publish, kwargs)
with self._lock:
self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
# XXX: ^^^ When connection is down, durable messages should be
# stored in ZODB to prevent losing them, e.g. because of restart.
return False
def _basic_publish(self, **kwargs):
retry_constructor = lambda func, kwargs: lambda: func(**kwargs)
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.basic_publish(**kwargs)
return True
elif getattr(kwargs.get("properties"), "delivery_mode", 1) == 2:
logger.warning(
u"No connection. Durable message was left into "
u"volatile memory to wait for a new connection "
u"'%s'", kwargs)
retry_callback = retry_constructor(self._basic_publish, kwargs)
with self._lock:
self._callbacks.add(0, '_on_ready_to_publish', retry_callback)
# XXX: ^^^ When connection is down, durable messages should be
# stored in ZODB to prevent losing them, e.g. because of restart.
return False
def _abort(self):
# collect execution info for guessing the reason for abort
exc_type, exc_value, exc_traceback = sys.exc_info()
if self.state != "ACK":
self.acknowledged = False
if self.state == "ACK" and issubclass(exc_type, ConflictError):
if not getattr(self, "_aborted", False):
logger.warning(
u"Transaction aborted due to database conflict. "
u"Message '%s' was acked before commit and could "
u"not be requeued (status = '%s')",
self.method_frame.delivery_tag,
self.state,
)
self._aborted = True
elif self.state not in ("FAILED", "REQUEUED"):
# on transactional channel, rollback on abort
if self.channel and self.tx_select:
self.channel.tx_rollback() # min support for transactional
# channel
# ^ XXX: Because the same channel may be shared by multiple
# threads, tx_rollback may be far from safe. It's supported
# only to make single-threaded AMQP-consuming ZEO-clients
# support transactional channel. DO NOT run multi-threaded
# consuming-server with transactional channel.
# reject messages with requeue when ConflictError in ZPublisher
if self.state != "ERROR" and issubclass(exc_type, ConflictError):
# reject message with requeue
self.channel.basic_reject(delivery_tag=self.method_frame.delivery_tag, requeue=True)
self.state = "REQUEUED"
logger.default(
u"Transaction aborted due to database conflict. " u"Requeued message '%s' (status = '%s')",
self.method_frame.delivery_tag,
self.state,
)
# otherwise, message handling has failed and un-acknowledged
else:
self.state = "FAILED"
def _abort(self):
# collect execution info for guessing the reason for abort
exc_type, exc_value, exc_traceback = sys.exc_info()
if self.state != 'ACK':
self.acknowledged = False
if self.state == 'ACK' and issubclass(exc_type, ConflictError):
if not getattr(self, '_aborted', False):
logger.warning(
u"Transaction aborted due to database conflict. "
u"Message '%s' was acked before commit and could "
u"not be requeued (status = '%s')",
self.method_frame.delivery_tag, self.state)
self._aborted = True
elif self.state not in ('FAILED', 'REQUEUED'):
# on transactional channel, rollback on abort
if self.channel and self.tx_select:
self.channel.tx_rollback() # min support for transactional
# channel
# ^ XXX: Because the same channel may be shared by multiple
# threads, tx_rollback may be far from safe. It's supported
# only to make single-threaded AMQP-consuming ZEO-clients
# support transactional channel. DO NOT run multi-threaded
# consuming-server with transactional channel.
# reject messages with requeue when ConflictError in ZPublisher
if self.state != 'ERROR' and issubclass(exc_type, ConflictError):
# reject message with requeue
self.channel.basic_reject(
delivery_tag=self.method_frame.delivery_tag, requeue=True)
self.state = "REQUEUED"
logger.default(
u"Transaction aborted due to database conflict. "
u"Requeued message '%s' (status = '%s')",
self.method_frame.delivery_tag, self.state)
# otherwise, message handling has failed and un-acknowledged
else:
self.state = 'FAILED'
def on_channel_closed(self, code, text):
logger.warning(u"Channel closed with reason '%s %s'",
code, text)
self._connection.close(code, text)
self.reconnect()
def _tx_commit(self):
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.tx_commit()
else:
logger.warning(u'No connection. Tx.Commit was not sent')
def _tx_commit(self):
if getattr(self._connection, "is_open", False)\
and getattr(self, '_channel', None):
self._channel.tx_commit()
else:
logger.warning(u'No connection. Tx.Commit was not sent')