async def nacker(subscription: str, nack_queue: 'asyncio.Queue[str]', subscriber_client: 'SubscriberClient', nack_window: float, metrics_client: MetricsAgent) -> None: ack_ids: List[str] = [] while True: if not ack_ids: ack_ids.append(await nack_queue.get()) nack_queue.task_done() ack_ids += await _budgeted_queue_get(nack_queue, nack_window) # modifyAckDeadline endpoint limit is 524288 bytes # which is ~2744 ack_ids if len(ack_ids) > 2500: log.error( 'nacker is falling behind, dropping %d unacked messages', len(ack_ids) - 2500) ack_ids = ack_ids[-2500:] try: await subscriber_client.modify_ack_deadline( subscription, ack_ids=ack_ids, ack_deadline_seconds=0) except asyncio.CancelledError: # pylint: disable=try-except-raise raise except aiohttp.client_exceptions.ClientResponseError as e: if e.status == 400: log.error( 'Nack error is unrecoverable, ' 'one or more messages may be dropped', exc_info=e) async def maybe_nack(ack_id: str) -> None: try: await subscriber_client.modify_ack_deadline( subscription, ack_ids=[ack_id], ack_deadline_seconds=0) except Exception as e: log.warning('Nack failed for ack_id=%s', ack_id, exc_info=e) for ack_id in ack_ids: asyncio.ensure_future(maybe_nack(ack_id)) ack_ids = [] log.warning('Nack request failed, better luck next batch', exc_info=e) metrics_client.increment('pubsub.nacker.batch.failed') continue except Exception as e: log.warning('Nack request failed, better luck next batch', exc_info=e) metrics_client.increment('pubsub.nacker.batch.failed') continue metrics_client.histogram('pubsub.nacker.batch', len(ack_ids)) ack_ids = []
async def producer( subscription: str, message_queue: MessageQueue, subscriber_client: 'SubscriberClient', max_messages: int, metrics_client: MetricsAgent) -> None: try: while True: new_messages = [] try: pull_task = asyncio.ensure_future( subscriber_client.pull( subscription=subscription, max_messages=max_messages, # it is important to have this value reasonably # high as long lived connections may be left # hanging on a server which will cause delay in # message delivery or even false deadlettering if # it is enabled timeout=30)) new_messages = await asyncio.shield(pull_task) except (asyncio.TimeoutError, KeyError): continue metrics_client.histogram( 'pubsub.producer.batch', len(new_messages)) pulled_at = time.perf_counter() while new_messages: await message_queue.put((new_messages[-1], pulled_at)) new_messages.pop() await message_queue.join() except asyncio.CancelledError: log.info('Producer worker cancelled. Gracefully terminating...') if not pull_task.done(): # Leaving the connection hanging can result in redelivered # messages, so try to finish before shutting down try: new_messages += await asyncio.wait_for(pull_task, 5) except (asyncio.TimeoutError, KeyError): pass pulled_at = time.perf_counter() for m in new_messages: await message_queue.put((m, pulled_at)) await message_queue.join() log.info('Producer terminated gracefully.') raise
async def _execute_callback(message: SubscriberMessage, callback: ApplicationHandler, ack_queue: 'asyncio.Queue[str]', nack_queue: 'Optional[asyncio.Queue[str]]', metrics_client: MetricsAgent) -> None: try: start = time.perf_counter() await callback(message) await ack_queue.put(message.ack_id) metrics_client.increment('pubsub.consumer.succeeded') metrics_client.histogram('pubsub.consumer.latency.runtime', time.perf_counter() - start) except Exception: if nack_queue: await nack_queue.put(message.ack_id) log.exception('Application callback raised an exception') metrics_client.increment('pubsub.consumer.failed')
async def producer(subscription: str, message_queue: MessageQueue, subscriber_client: 'SubscriberClient', max_messages: int, metrics_client: MetricsAgent) -> None: try: while True: new_messages = [] try: new_messages = await subscriber_client.pull( subscription=subscription, max_messages=max_messages, # it is important to have this value reasonably high # as long lived connections may be left hanging # on a server which will cause delay in message # delivery or even false deadlettering if it is enabled timeout=30) except (asyncio.TimeoutError, KeyError): continue metrics_client.histogram('pubsub.producer.batch', len(new_messages)) pulled_at = time.perf_counter() while new_messages: await message_queue.put((new_messages[-1], pulled_at)) new_messages.pop() await message_queue.join() except asyncio.CancelledError: log.info('Producer worker cancelled. Gracefully terminating...') pulled_at = time.perf_counter() for m in new_messages: await message_queue.put((m, pulled_at)) await message_queue.join() log.info('Producer terminated gracefully.') raise