async def route_message(self, message:aio_pika.IncomingMessage):
handler = self.routes.get(message.routing_key)
if not handler:
message.reject()
return
await handler(message)
async def process_message(self, message: aio_pika.IncomingMessage) -> None:
if self.closing:
await message.nack(requeue=True)
return
task_message = TaskMessage.parse_raw(message.body,
content_type=message.content_type)
task = self.registry.get_task(task_message.name)
if not task:
self.logger.error(
"task_not_found",
extra={
"task_name": task_message.name,
},
)
message.reject(requeue=False)
return
async with message.process(reject_on_redelivered=True, requeue=True):
logging_extra = {
"name": task_message.name,
"uid": task_message.uid
}
self.logger.info(
"task_processing_start",
extra=logging_extra,
)
try:
await self.processing_lock.acquire()
await asyncio.wait_for(task.callback(**task_message.kwargs),
timeout=task_message.timeout)
except RejectMessage as e:
await message.reject(requeue=e.requeue)
self.logger.error("task_rejected",
extra=dict(logging_extra, requeue=e.requeue))
except NackMessage as e:
await message.nack(requeue=e.requeue)
self.logger.error("task_nack",
extra=dict(logging_extra, requeue=e.requeue))
except asyncio.TimeoutError:
self.logger.error(
"task_timeout",
extra=dict(logging_extra, timeout=task_message.timeout),
)
raise
except Exception:
self.logger.exception("error_while_processing_task",
extra=logging_extra)
raise
finally:
self.processing_lock.release()
def __call__(self, message: IncomingMessage):
value = int(message.body.decode())
if value == 3 and self.rejected < 3:
print('[blue]Nacking[/blue] 3')
message.nack()
self.rejected += 1
elif value == 5:
print('[red]Rejecting[/red] 5')
message.reject()
else:
print(f'[green]Processed[/green] {value}')
message.ack()
async def handle_msg(msg: IncomingMessage):
print(f"dcmq: got message with routing key {msg.routing_key}")
ds = datasetFromBinary(msg.body)
uri = ""
if "uri" in msg.headers:
uri = msg.headers["uri"]
if ds != None:
try:
await dcmhandler(channel, ds, uri, msg.routing_key, *additional_args)
msg.ack()
except Exception as e:
msg.reject(requeue=True)
raise(e)
async def _process_message_context(
self, settings: ProcessorSettings,
message: aio_pika.IncomingMessage) -> None:
"""
Processes an incoming message, passing in incoming and outgoing objects, and
handling any outgoing passing to the output message queue.
"""
# Context block so the consumer can set ignore processed. NOTE: aio-pika has
# a bas return type, so we need to manually set the more accurate type rather
# than the generic 'Async Manager' it uses.
message_context: aio_pika.message.ProcessContext = (
message.process(**
settings.processing_options.kwargs) # type: ignore
)
async with message_context:
reject = False
incoming: Incoming = Incoming(
message=message,
schema=settings.in_schema,
_decoders=self.lifecycle.scribe._decoders,
)
try:
await self._process_message(settings, incoming)
except BaseException as error:
# Handle rejecting quorum queue messages that have hit the max error
# count.
max_count = settings.processing_options.max_delivery_count
is_final_delivery = (message.headers.get(
"x-delivery-count",
0) >= settings.processing_options.max_delivery_count)
# If we are using quorum queues hand have a max retry value, reject the
# the message instead of re-queuing it.
if is_final_delivery and max_count != -1:
reject = True
raise error
finally:
if reject or incoming.reject:
# Tell the context the message gas been processed so it doesn't
# throw an error.
message.reject()
message_context.ignore_processed = True
async def on_rabbitmq_message(self, message: aio_pika.IncomingMessage):
self.concurrent_requests += 1
if self.concurrent_requests >= self.max_concurrent_requests:
self.logger.warning('TOO MANY CONCURRENT REQUESTS... requeuing')
message.reject(requeue=True)
self.concurrent_requests -= 1
return
if not self.status.is_active():
message.reject(requeue=True)
self.concurrent_requests -= 1
return
callback_url = self.get_callback_for_message(message)
if not callback_url:
self.logger.error('WTF? unknown routing key: {}'.format(
message.routing_key))
message.reject(requeue=True)
self.concurrent_requests -= 1
return
try:
msg = json.dumps({
'headers':
message.headers,
'content_encoding':
message.content_encoding,
'message_id':
message.message_id,
'type':
message.type,
'routing_key':
message.routing_key,
'body':
message.body.decode(message.content_encoding or 'utf8')
})
await self.kiss_api.send_msg(msg, callback_url)
message.ack()
except KissApiException as e:
message.reject(requeue=True)
raise e
except KissOfflineException as e:
message.reject(requeue=True)
self.root_service.loop.create_task(
self.take_a_break(
60, 'Kiss is Offline. Stopping {} for 1 min. Details: {}'.
format(self, e)))
finally:
self.concurrent_requests -= 1
async def received_job(message: IncomingMessage) -> None:
""" AMQP job hook """
log.info("Received new job", extra=message.info())
try:
job = _validate_message(message)
except ValidationError as e:
log.error(f"{e}. Rejecting without requeuing", extra=message.info())
message.reject(requeue=False)
return
result = await _process_job(job)
message.ack()
log.info("Job completed",
extra={
"job_id": result.job.id,
"url": result.job.url,
"status": result.status,
"size": result.size
})
