def test_ack_splitting_large_payload():
manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager,
instance=True)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [
# use realistic lengths for ACK IDs (max 176 bytes)
requests.AckRequest(ack_id=str(i).zfill(176),
byte_size=0,
time_to_ack=20,
ordering_key="") for i in range(5001)
]
dispatcher_.ack(items)
calls = manager.send.call_args_list
assert len(calls) == 3
all_ack_ids = {item.ack_id for item in items}
sent_ack_ids = collections.Counter()
for call in calls:
message = call.args[0]
assert message._pb.ByteSize() <= 524288 # server-side limit (2**19)
sent_ack_ids.update(message.ack_ids)
assert set(
sent_ack_ids) == all_ack_ids # all messages should have been ACK-ed
assert sent_ack_ids.most_common(
1)[0][1] == 1 # each message ACK-ed exactly once
def ack(self):
"""Acknowledge the given message.
Acknowledging a message in Pub/Sub means that you are done
with it, and it will not be delivered to this subscription again.
You should avoid acknowledging messages until you have
*finished* processing them, so that in the event of a failure,
you receive the message again.
.. warning::
Acks in Pub/Sub are best effort. You should always
ensure that your processing code is idempotent, as you may
receive any given message more than once.
"""
try:
message_data = json.loads(self.data)
task_db_id = message_data.get("task_db_id")
call_stack = "\n".join(traceback.format_stack())
log_message = f'ACKING MESSAGE. TASK DB ID: {task_db_id} | MESSAGE: {self._data} Call Stack: {call_stack}'
logger.info(log_message)
except Exception as e:
pass
time_to_ack = math.ceil(time.time() - self._received_timestamp)
self._request_queue.put(
requests.AckRequest(
ack_id=self._ack_id,
byte_size=self.size,
time_to_ack=time_to_ack,
ordering_key=self.ordering_key,
))
def test_dispatch_callback_inactive():
subscriber_ = mock.create_autospec(subscriber.Subscriber, instance=True)
subscriber_.is_active = False
dispatcher_ = dispatcher.Dispatcher(mock.sentinel.queue, subscriber_)
dispatcher_.dispatch_callback([requests.AckRequest(0, 0, 0)])
subscriber_.ack.assert_not_called()
def test_dispatch_callback_inactive():
manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager,
instance=True)
manager.is_active = False
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
dispatcher_.dispatch_callback([requests.AckRequest(0, 0, 0, "")])
manager.send.assert_not_called()
def test_ack():
msg = create_message(b"foo", ack_id="bogus_ack_id")
with mock.patch.object(msg._request_queue, "put") as put:
msg.ack()
put.assert_called_once_with(
requests.AckRequest(ack_id="bogus_ack_id",
byte_size=30,
time_to_ack=mock.ANY))
check_call_types(put, requests.AckRequest)
def test_ack():
msg = create_message(b'foo', ack_id='bogus_ack_id')
with mock.patch.object(msg._request_queue, 'put') as put:
msg.ack()
put.assert_called_once_with(
requests.AckRequest(
ack_id='bogus_ack_id',
byte_size=30,
time_to_ack=mock.ANY,
))
check_call_types(put, requests.AckRequest)
def test_ack_no_time():
manager = mock.create_autospec(
streaming_pull_manager.StreamingPullManager, instance=True)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [requests.AckRequest(
ack_id='ack_id_string', byte_size=0, time_to_ack=None)]
dispatcher_.ack(items)
manager.send.assert_called_once_with(types.StreamingPullRequest(
ack_ids=['ack_id_string'],
))
manager.ack_histogram.add.assert_not_called()
def _handle_nack(self, message: requests.NackRequest):
sized_message = self._messages_by_ack_id[message.ack_id]
try:
# Put the ack request back into the queue since the callback may be called from another thread.
self._nack_handler.on_nack(
sized_message.message,
lambda: self._queue.put(
requests.AckRequest(
ack_id=message.ack_id,
byte_size=0, # Ignored
time_to_ack=0, # Ignored
ordering_key="", # Ignored
)),
)
except GoogleAPICallError as e:
self.fail(e)
def test_ack():
manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager,
instance=True)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [
requests.AckRequest(ack_id="ack_id_string",
byte_size=0,
time_to_ack=20)
]
dispatcher_.ack(items)
manager.send.assert_called_once_with(
types.StreamingPullRequest(ack_ids=["ack_id_string"]))
manager.leaser.remove.assert_called_once_with(items)
manager.maybe_resume_consumer.assert_called_once()
manager.ack_histogram.add.assert_called_once_with(20)
def ack(self):
"""Acknowledge the given message.
Acknowledging a message in Pub/Sub means that you are done
with it, and it will not be delivered to this subscription again.
You should avoid acknowledging messages until you have
*finished* processing them, so that in the event of a failure,
you receive the message again.
.. warning::
Acks in Pub/Sub are best effort. You should always
ensure that your processing code is idempotent, as you may
receive any given message more than once.
"""
time_to_ack = math.ceil(time.time() - self._received_timestamp)
self._request_queue.put(
requests.AckRequest(ack_id=self._ack_id,
byte_size=self.size,
time_to_ack=time_to_ack))
import threading
from google.cloud.pubsub_v1.subscriber._protocol import dispatcher
from google.cloud.pubsub_v1.subscriber._protocol import helper_threads
from google.cloud.pubsub_v1.subscriber._protocol import requests
from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager
from google.pubsub_v1 import types as gapic_types
import mock
import pytest
@pytest.mark.parametrize(
"item,method_name",
[
(requests.AckRequest(0, 0, 0, ""), "ack"),
(requests.DropRequest(0, 0, ""), "drop"),
(requests.LeaseRequest(0, 0, ""), "lease"),
(requests.ModAckRequest(0, 0), "modify_ack_deadline"),
(requests.NackRequest(0, 0, ""), "nack"),
],
)
def test_dispatch_callback(item, method_name):
manager = mock.create_autospec(streaming_pull_manager.StreamingPullManager,
instance=True)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [item]
with mock.patch.object(dispatcher_, method_name) as method:
dispatcher_.dispatch_callback(items)
import threading
from google.cloud.pubsub_v1.subscriber._protocol import dispatcher
from google.cloud.pubsub_v1.subscriber._protocol import helper_threads
from google.cloud.pubsub_v1.subscriber._protocol import requests
from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager
from google.pubsub_v1 import types as gapic_types
import mock
import pytest
@pytest.mark.parametrize(
"item,method_name",
[
(requests.AckRequest("0", 0, 0, ""), "ack"),
(requests.DropRequest("0", 0, ""), "drop"),
(requests.LeaseRequest("0", 0, ""), "lease"),
(requests.ModAckRequest("0", 0), "modify_ack_deadline"),
(requests.NackRequest("0", 0, ""), "nack"),
],
)
def test_dispatch_callback_active_manager(item, method_name):
manager = mock.create_autospec(
streaming_pull_manager.StreamingPullManager, instance=True
)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [item]
with mock.patch.object(dispatcher_, method_name) as method:
import threading
from google.cloud.pubsub_v1 import types
from google.cloud.pubsub_v1.subscriber._protocol import dispatcher
from google.cloud.pubsub_v1.subscriber._protocol import helper_threads
from google.cloud.pubsub_v1.subscriber._protocol import requests
from google.cloud.pubsub_v1.subscriber._protocol import streaming_pull_manager
import mock
from six.moves import queue
import pytest
@pytest.mark.parametrize('item,method_name', [
(requests.AckRequest(0, 0, 0), 'ack'),
(requests.DropRequest(0, 0), 'drop'),
(requests.LeaseRequest(0, 0), 'lease'),
(requests.ModAckRequest(0, 0), 'modify_ack_deadline'),
(requests.NackRequest(0, 0), 'nack')
])
def test_dispatch_callback(item, method_name):
manager = mock.create_autospec(
streaming_pull_manager.StreamingPullManager, instance=True)
dispatcher_ = dispatcher.Dispatcher(manager, mock.sentinel.queue)
items = [item]
with mock.patch.object(dispatcher_, method_name) as method:
dispatcher_.dispatch_callback(items)