async def nacker(subscription: str, nack_queue: 'asyncio.Queue[str]',
subscriber_client: 'SubscriberClient', nack_window: float,
metrics_client: MetricsAgent) -> None:
ack_ids: List[str] = []
while True:
if not ack_ids:
ack_ids.append(await nack_queue.get())
nack_queue.task_done()
ack_ids += await _budgeted_queue_get(nack_queue, nack_window)
# modifyAckDeadline endpoint limit is 524288 bytes
# which is ~2744 ack_ids
if len(ack_ids) > 2500:
log.error(
'nacker is falling behind, dropping %d unacked messages',
len(ack_ids) - 2500)
ack_ids = ack_ids[-2500:]
try:
await subscriber_client.modify_ack_deadline(
subscription, ack_ids=ack_ids, ack_deadline_seconds=0)
except asyncio.CancelledError: # pylint: disable=try-except-raise
raise
except aiohttp.client_exceptions.ClientResponseError as e:
if e.status == 400:
log.error(
'Nack error is unrecoverable, '
'one or more messages may be dropped',
exc_info=e)
async def maybe_nack(ack_id: str) -> None:
try:
await subscriber_client.modify_ack_deadline(
subscription,
ack_ids=[ack_id],
ack_deadline_seconds=0)
except Exception as e:
log.warning('Nack failed for ack_id=%s',
ack_id,
exc_info=e)
for ack_id in ack_ids:
asyncio.ensure_future(maybe_nack(ack_id))
ack_ids = []
log.warning('Nack request failed, better luck next batch',
exc_info=e)
metrics_client.increment('pubsub.nacker.batch.failed')
continue
except Exception as e:
log.warning('Nack request failed, better luck next batch',
exc_info=e)
metrics_client.increment('pubsub.nacker.batch.failed')
continue
metrics_client.histogram('pubsub.nacker.batch', len(ack_ids))
ack_ids = []
async def _execute_callback(message: SubscriberMessage,
callback: ApplicationHandler,
ack_queue: 'asyncio.Queue[str]',
nack_queue: 'Optional[asyncio.Queue[str]]',
metrics_client: MetricsAgent) -> None:
try:
start = time.perf_counter()
await callback(message)
await ack_queue.put(message.ack_id)
metrics_client.increment('pubsub.consumer.succeeded')
metrics_client.histogram('pubsub.consumer.latency.runtime',
time.perf_counter() - start)
except Exception:
if nack_queue:
await nack_queue.put(message.ack_id)
log.exception('Application callback raised an exception')
metrics_client.increment('pubsub.consumer.failed')