def test_ack_splitting_large_payload(): manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager, instance=True) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [ # use realistic lengths for ACK IDs (max 176 bytes) requests.AckRequest(ack_id=str(i).zfill(176), byte_size=0, time_to_ack=20, ordering_key="") for i in range(5001) ] dispatcher_.ack(items) calls = manager.send.call_args_list assert len(calls) == 3 all_ack_ids = {item.ack_id for item in items} sent_ack_ids = collections.Counter() for call in calls: message = call.args[0] assert message._pb.ByteSize() <= 524288 # server-side limit (2**19) sent_ack_ids.update(message.ack_ids) assert set( sent_ack_ids) == all_ack_ids # all messages should have been ACK-ed assert sent_ack_ids.most_common( 1)[0][1] == 1 # each message ACK-ed exactly once
def ack(self): """Acknowledge the given message. Acknowledging a message in Pub/Sub means that you are done with it, and it will not be delivered to this subscription again. You should avoid acknowledging messages until you have *finished* processing them, so that in the event of a failure, you receive the message again. .. warning:: Acks in Pub/Sub are best effort. You should always ensure that your processing code is idempotent, as you may receive any given message more than once. """ try: message_data = json.loads(self.data) task_db_id = message_data.get("task_db_id") call_stack = "\n".join(traceback.format_stack()) log_message = f'ACKING MESSAGE. TASK DB ID: {task_db_id} | MESSAGE: {self._data} Call Stack: {call_stack}' logger.info(log_message) except Exception as e: pass time_to_ack = math.ceil(time.time() - self._received_timestamp) self._request_queue.put( requests.AckRequest( ack_id=self._ack_id, byte_size=self.size, time_to_ack=time_to_ack, ordering_key=self.ordering_key, ))
def test_dispatch_callback_inactive(): subscriber_ = mock.create_autospec(subscriber.Subscriber, instance=True) subscriber_.is_active = False dispatcher_ = dispatcher.Dispatcher(mock.sentinel.queue, subscriber_) dispatcher_.dispatch_callback([requests.AckRequest(0, 0, 0)]) subscriber_.ack.assert_not_called()
def test_dispatch_callback_inactive(): manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager, instance=True) manager.is_active = False dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) dispatcher_.dispatch_callback([requests.AckRequest(0, 0, 0, "")]) manager.send.assert_not_called()
def test_ack(): msg = create_message(b"foo", ack_id="bogus_ack_id") with mock.patch.object(msg._request_queue, "put") as put: msg.ack() put.assert_called_once_with( requests.AckRequest(ack_id="bogus_ack_id", byte_size=30, time_to_ack=mock.ANY)) check_call_types(put, requests.AckRequest)
def test_ack(): msg = create_message(b'foo', ack_id='bogus_ack_id') with mock.patch.object(msg._request_queue, 'put') as put: msg.ack() put.assert_called_once_with( requests.AckRequest( ack_id='bogus_ack_id', byte_size=30, time_to_ack=mock.ANY, )) check_call_types(put, requests.AckRequest)
def test_ack_no_time(): manager = mock.create_autospec( streaming_pull_manager.StreamingPullManager, instance=True) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [requests.AckRequest( ack_id='ack_id_string', byte_size=0, time_to_ack=None)] dispatcher_.ack(items) manager.send.assert_called_once_with(types.StreamingPullRequest( ack_ids=['ack_id_string'], )) manager.ack_histogram.add.assert_not_called()
def _handle_nack(self, message: requests.NackRequest): sized_message = self._messages_by_ack_id[message.ack_id] try: # Put the ack request back into the queue since the callback may be called from another thread. self._nack_handler.on_nack( sized_message.message, lambda: self._queue.put( requests.AckRequest( ack_id=message.ack_id, byte_size=0, # Ignored time_to_ack=0, # Ignored ordering_key="", # Ignored )), ) except GoogleAPICallError as e: self.fail(e)
def test_ack(): manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager, instance=True) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [ requests.AckRequest(ack_id="ack_id_string", byte_size=0, time_to_ack=20) ] dispatcher_.ack(items) manager.send.assert_called_once_with( types.StreamingPullRequest(ack_ids=["ack_id_string"])) manager.leaser.remove.assert_called_once_with(items) manager.maybe_resume_consumer.assert_called_once() manager.ack_histogram.add.assert_called_once_with(20)
def ack(self): """Acknowledge the given message. Acknowledging a message in Pub/Sub means that you are done with it, and it will not be delivered to this subscription again. You should avoid acknowledging messages until you have *finished* processing them, so that in the event of a failure, you receive the message again. .. warning:: Acks in Pub/Sub are best effort. You should always ensure that your processing code is idempotent, as you may receive any given message more than once. """ time_to_ack = math.ceil(time.time() - self._received_timestamp) self._request_queue.put( requests.AckRequest(ack_id=self._ack_id, byte_size=self.size, time_to_ack=time_to_ack))
import threading from google.cloud.pubsub_v1.subscriber._protocol import dispatcher from google.cloud.pubsub_v1.subscriber._protocol import helper_threads from google.cloud.pubsub_v1.subscriber._protocol import requests from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager from google.pubsub_v1 import types as gapic_types import mock import pytest @pytest.mark.parametrize( "item,method_name", [ (requests.AckRequest(0, 0, 0, ""), "ack"), (requests.DropRequest(0, 0, ""), "drop"), (requests.LeaseRequest(0, 0, ""), "lease"), (requests.ModAckRequest(0, 0), "modify_ack_deadline"), (requests.NackRequest(0, 0, ""), "nack"), ], ) def test_dispatch_callback(item, method_name): manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager, instance=True) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [item] with mock.patch.object(dispatcher_, method_name) as method: dispatcher_.dispatch_callback(items)
import threading from google.cloud.pubsub_v1.subscriber._protocol import dispatcher from google.cloud.pubsub_v1.subscriber._protocol import helper_threads from google.cloud.pubsub_v1.subscriber._protocol import requests from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager from google.pubsub_v1 import types as gapic_types import mock import pytest @pytest.mark.parametrize( "item,method_name", [ (requests.AckRequest("0", 0, 0, ""), "ack"), (requests.DropRequest("0", 0, ""), "drop"), (requests.LeaseRequest("0", 0, ""), "lease"), (requests.ModAckRequest("0", 0), "modify_ack_deadline"), (requests.NackRequest("0", 0, ""), "nack"), ], ) def test_dispatch_callback_active_manager(item, method_name): manager = mock.create_autospec( streaming_pull_manager.StreamingPullManager, instance=True ) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [item] with mock.patch.object(dispatcher_, method_name) as method:
import threading from google.cloud.pubsub_v1 import types from google.cloud.pubsub_v1.subscriber._protocol import dispatcher from google.cloud.pubsub_v1.subscriber._protocol import helper_threads from google.cloud.pubsub_v1.subscriber._protocol import requests from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager import mock from six.moves import queue import pytest @pytest.mark.parametrize('item,method_name', [ (requests.AckRequest(0, 0, 0), 'ack'), (requests.DropRequest(0, 0), 'drop'), (requests.LeaseRequest(0, 0), 'lease'), (requests.ModAckRequest(0, 0), 'modify_ack_deadline'), (requests.NackRequest(0, 0), 'nack') ]) def test_dispatch_callback(item, method_name): manager = mock.create_autospec( streaming_pull_manager.StreamingPullManager, instance=True) dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue) items = [item] with mock.patch.object(dispatcher_, method_name) as method: dispatcher_.dispatch_callback(items)