async def nacker(subscription: str, nack_queue: 'asyncio.Queue[str]',
subscriber_client: 'SubscriberClient', nack_window: float,
metrics_client: MetricsAgent) -> None:
ack_ids: List[str] = []
while True:
if not ack_ids:
ack_ids.append(await nack_queue.get())
nack_queue.task_done()
ack_ids += await _budgeted_queue_get(nack_queue, nack_window)
# modifyAckDeadline endpoint limit is 524288 bytes
# which is ~2744 ack_ids
if len(ack_ids) > 2500:
log.error(
'nacker is falling behind, dropping %d unacked messages',
len(ack_ids) - 2500)
ack_ids = ack_ids[-2500:]
try:
await subscriber_client.modify_ack_deadline(
subscription, ack_ids=ack_ids, ack_deadline_seconds=0)
except asyncio.CancelledError: # pylint: disable=try-except-raise
raise
except aiohttp.client_exceptions.ClientResponseError as e:
if e.status == 400:
log.error(
'Nack error is unrecoverable, '
'one or more messages may be dropped',
exc_info=e)
async def maybe_nack(ack_id: str) -> None:
try:
await subscriber_client.modify_ack_deadline(
subscription,
ack_ids=[ack_id],
ack_deadline_seconds=0)
except Exception as e:
log.warning('Nack failed for ack_id=%s',
ack_id,
exc_info=e)
for ack_id in ack_ids:
asyncio.ensure_future(maybe_nack(ack_id))
ack_ids = []
log.warning('Nack request failed, better luck next batch',
exc_info=e)
metrics_client.increment('pubsub.nacker.batch.failed')
continue
except Exception as e:
log.warning('Nack request failed, better luck next batch',
exc_info=e)
metrics_client.increment('pubsub.nacker.batch.failed')
continue
metrics_client.histogram('pubsub.nacker.batch', len(ack_ids))
ack_ids = []
async def producer(
subscription: str,
message_queue: MessageQueue,
subscriber_client: 'SubscriberClient',
max_messages: int,
metrics_client: MetricsAgent) -> None:
try:
while True:
new_messages = []
try:
pull_task = asyncio.ensure_future(
subscriber_client.pull(
subscription=subscription,
max_messages=max_messages,
# it is important to have this value reasonably
# high as long lived connections may be left
# hanging on a server which will cause delay in
# message delivery or even false deadlettering if
# it is enabled
timeout=30))
new_messages = await asyncio.shield(pull_task)
except (asyncio.TimeoutError, KeyError):
continue
metrics_client.histogram(
'pubsub.producer.batch', len(new_messages))
pulled_at = time.perf_counter()
while new_messages:
await message_queue.put((new_messages[-1], pulled_at))
new_messages.pop()
await message_queue.join()
except asyncio.CancelledError:
log.info('Producer worker cancelled. Gracefully terminating...')
if not pull_task.done():
# Leaving the connection hanging can result in redelivered
# messages, so try to finish before shutting down
try:
new_messages += await asyncio.wait_for(pull_task, 5)
except (asyncio.TimeoutError, KeyError):
pass
pulled_at = time.perf_counter()
for m in new_messages:
await message_queue.put((m, pulled_at))
await message_queue.join()
log.info('Producer terminated gracefully.')
raise
async def _execute_callback(message: SubscriberMessage,
callback: ApplicationHandler,
ack_queue: 'asyncio.Queue[str]',
nack_queue: 'Optional[asyncio.Queue[str]]',
metrics_client: MetricsAgent) -> None:
try:
start = time.perf_counter()
await callback(message)
await ack_queue.put(message.ack_id)
metrics_client.increment('pubsub.consumer.succeeded')
metrics_client.histogram('pubsub.consumer.latency.runtime',
time.perf_counter() - start)
except Exception:
if nack_queue:
await nack_queue.put(message.ack_id)
log.exception('Application callback raised an exception')
metrics_client.increment('pubsub.consumer.failed')
async def producer(subscription: str, message_queue: MessageQueue,
subscriber_client: 'SubscriberClient',
max_messages: int,
metrics_client: MetricsAgent) -> None:
try:
while True:
new_messages = []
try:
new_messages = await subscriber_client.pull(
subscription=subscription,
max_messages=max_messages,
# it is important to have this value reasonably high
# as long lived connections may be left hanging
# on a server which will cause delay in message
# delivery or even false deadlettering if it is enabled
timeout=30)
except (asyncio.TimeoutError, KeyError):
continue
metrics_client.histogram('pubsub.producer.batch',
len(new_messages))
pulled_at = time.perf_counter()
while new_messages:
await message_queue.put((new_messages[-1], pulled_at))
new_messages.pop()
await message_queue.join()
except asyncio.CancelledError:
log.info('Producer worker cancelled. Gracefully terminating...')
pulled_at = time.perf_counter()
for m in new_messages:
await message_queue.put((m, pulled_at))
await message_queue.join()
log.info('Producer terminated gracefully.')
raise
