def archive(self, body, message, reason=''):
"""
Put the message onto the archive queue
"""
_logger.warning(reason)
try:
self.archive_producer.publish(
body,
headers=message.headers,
retry=True,
declares=[self.archive_queue],
)
except Exception as e:
message.requeue()
_logger.error("Archive failure: retry-reason='{reason}' "
"exception='{cls}, {error}'\n"
"{traceback}".format(
reason=reason,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug("Archive: {reason}".format(reason=reason))
def retry(self, body, message, reason=''):
"""
Put the message onto the retry queue
"""
_logger.warning(reason)
try:
retry_count = self.retry_count(message)
headers = message.headers.copy()
headers.update({settings.RETRY_HEADER: retry_count + 1})
self.retry_producer.publish(
body,
headers=headers,
retry=True,
declares=[self.retry_queue],
expiration=self.backoff_func(retry_count))
except Exception as e:
message.requeue()
_logger.error("Retry failure: retry-reason='{reason}' "
"exception='{cls}, {error}'\n"
"{traceback}".format(
reason=reason,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug("Retry: {reason}".format(reason=reason))
def archive(self, body, message, reason=''):
"""
Put the message onto the archive queue
"""
try:
self.archive_producer.publish(
body,
headers=message.headers,
retry=True,
declares=[self.archive_queue],
)
except Exception as e:
message.requeue()
_logger.error(
"Archive failure: retry-reason='{}' exception='{}, {}'\n{}".format(
reason,
e.__class__.__name__,
e,
traceback.format_exc(),
)
)
else:
message.ack()
_logger.debug("Archive: {}".format(reason))
def archive(self, body, message, reason=''):
"""
Put the message onto the archive queue
"""
_logger.warning(reason)
try:
kombu.Producer(
self.channel,
exchange=self.archive_queue.exchange,
routing_key=self.archive_queue.routing_key,
serializer=settings.SERIALIZER,
).publish(
body,
headers=message.headers,
retry=True,
declares=[self.archive_queue],
)
except Exception as e:
message.requeue()
_logger.error("Archive failure: retry-reason='{reason}' "
"exception='{cls}, {error}'\n"
"{traceback}".format(
reason=reason,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug("Archive: {reason}".format(reason=reason))
def __call__(self, body, message):
"""
Handle a vanilla AMQP message, called by the Celery framework.
Raising an exception in this method will crash the Celery worker. Ensure
that all Exceptions are caught and messages acknowledged or rejected
as they are processed.
Args:
body (Any): the message content, which has been deserialized by Kombu
message (kombu.message.Message)
Returns:
None
"""
retry_count = self.retry_count(message)
try:
_logger.debug(
'Received: (key={routing_key}, retry_count={retry_count})'.
format(
routing_key=self.routing_key,
retry_count=retry_count,
))
self.func(body)
except Exception as e:
if isinstance(e, PermanentFailure):
self.archive(
body, message,
"Task '{routing_key}' raised '{cls}, {error}'\n"
"{traceback}".format(
routing_key=self.routing_key,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
elif retry_count >= settings.MAX_RETRIES:
self.archive(
body, message,
"Task '{routing_key}' ran out of retries ({retries}) on exception "
"'{cls}, {error}'\n"
"{traceback}".format(
routing_key=self.routing_key,
retries=retry_count,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
self.retry(
body, message,
"Task '{routing_key}' raised the exception '{cls}, {error}', but there are "
"{retries} retries left\n"
"{traceback}".format(
routing_key=self.routing_key,
retries=settings.MAX_RETRIES - retry_count,
cls=e.__class__.__name__,
error=e,
traceback=traceback.format_exc(),
))
else:
message.ack()
_logger.debug(
"Task '{routing_key}' processed and ack() sent".format(
routing_key=self.routing_key))
finally:
if settings.USE_DJANGO:
# avoid various problems with db connections, due to long-lived
# worker not automatically participating in Django request lifecycle
request_finished.send(sender="AMQPRetryHandler")
if not message.acknowledged:
message.requeue()
_logger.critical(
"Messages for task '{routing_key}' are not sending an ack() or a reject(). "
"This needs attention. Assuming some kind of error and requeueing the "
"message.".format(routing_key=self.routing_key))
