def test_streaming_pull_ack_deadline(self, publisher, subscriber, project,
topic_path, subscription_path,
cleanup):
# Make sure the topic and subscription get deleted.
cleanup.append((publisher.delete_topic, topic_path))
cleanup.append((subscriber.delete_subscription, subscription_path))
# Create a topic and a subscription, then subscribe to the topic. This
# must happen before the messages are published.
publisher.create_topic(topic_path)
# Subscribe to the topic. This must happen before the messages
# are published.
subscriber.create_subscription(subscription_path,
topic_path,
ack_deadline_seconds=45)
# publish some messages and wait for completion
self._publish_messages(publisher, topic_path, batch_sizes=[2])
# subscribe to the topic
callback = StreamingPullCallback(
processing_time=
13, # more than the default stream ACK deadline (10s)
resolve_at_msg_count=3, # one more than the published messages count
)
flow_control = types.FlowControl(max_messages=1)
subscription_future = subscriber.subscribe(subscription_path,
callback,
flow_control=flow_control)
# We expect to process the first two messages in 2 * 13 seconds, and
# any duplicate message that is re-sent by the backend in additional
# 13 seconds, totalling 39 seconds (+ overhead) --> if there have been
# no duplicates in 60 seconds, we can reasonably assume that there
# won't be any.
try:
callback.done_future.result(timeout=60)
except exceptions.TimeoutError:
# future timed out, because we received no excessive messages
assert sorted(callback.seen_message_ids) == [1, 2]
else:
pytest.fail(
"Expected to receive 2 messages, but got at least {}.".format(
len(callback.seen_message_ids)))
finally:
subscription_future.cancel()
def test__maybe_release_messages_below_overload():
manager = make_manager(
flow_control=types.FlowControl(max_messages=10, max_bytes=1000))
manager._callback = mock.sentinel.callback
# init leaser message count to 8 to leave room for 2 more messages
_leaser = manager._leaser = mock.create_autospec(leaser.Leaser)
fake_leaser_add(_leaser, init_msg_count=8, assumed_msg_size=25)
_leaser.add = mock.Mock(wraps=_leaser.add) # to spy on calls
messages = [
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_foo",
size=11),
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_bar",
size=22),
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_baz",
size=33),
]
for msg in messages:
manager._messages_on_hold.put(msg)
# the actual call of MUT
manager._maybe_release_messages()
assert manager._messages_on_hold.qsize() == 1
msg = manager._messages_on_hold.get_nowait()
assert msg.ack_id == "ack_baz"
assert len(_leaser.add.mock_calls) == 2
expected_calls = [
mock.call([requests.LeaseRequest(ack_id="ack_foo", byte_size=11)]),
mock.call([requests.LeaseRequest(ack_id="ack_bar", byte_size=22)]),
]
_leaser.add.assert_has_calls(expected_calls)
schedule_calls = manager._scheduler.schedule.mock_calls
assert len(schedule_calls) == 2
for _, call_args, _ in schedule_calls:
assert call_args[0] == mock.sentinel.callback
assert isinstance(call_args[1], message.Message)
assert call_args[1].ack_id in ("ack_foo", "ack_bar")
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
executor=None,
queue=None):
"""Instantiate the policy.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow
control settings.
executor (~concurrent.futures.ThreadPoolExecutor): (Optional.) A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
queue (~queue.Queue): (Optional.) A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
"""
# Default the callback to a no-op; it is provided by `.open`.
self._callback = lambda message: None
# Create a queue for keeping track of shared state.
if queue is None:
queue = Queue()
self._request_queue = queue
# Call the superclass constructor.
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# Also maintain a request queue and an executor.
logger.debug('Creating callback requests thread (not starting).')
if executor is None:
executor = futures.ThreadPoolExecutor(max_workers=10)
self._executor = executor
self._callback_requests = _helper_threads.QueueCallbackThread(
self._request_queue,
self.on_callback_request,
)
def __init__(self, client, subscription, flow_control=types.FlowControl(),
executor=None, queue=None):
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# Default the callback to a no-op; the **actual** callback is
# provided by ``.open()``.
self._callback = _do_nothing_callback
# Create a queue for keeping track of shared state.
self._request_queue = self._get_queue(queue)
# Also maintain an executor.
self._executor = self._get_executor(executor)
# The threads created in ``.open()``.
self._dispatch_thread = None
self._leases_thread = None
def test__maybe_release_messages_on_overload():
manager = make_manager(
flow_control=types.FlowControl(max_messages=10, max_bytes=1000)
)
# Ensure load is exactly 1.0 (to verify that >= condition is used)
_leaser = manager._leaser = mock.create_autospec(leaser.Leaser)
_leaser.message_count = 10
_leaser.bytes = 1000
msg = mock.create_autospec(message.Message, instance=True, ack_id="ack", size=11)
manager._messages_on_hold.put(msg)
manager._maybe_release_messages()
assert manager._messages_on_hold.qsize() == 1
manager._leaser.add.assert_not_called()
manager._scheduler.schedule.assert_not_called()
def test_load():
flow_control = types.FlowControl(max_messages=10, max_bytes=1000)
policy = create_policy(flow_control=flow_control)
# This should mean that our messages count is at 10%, and our bytes
# are at 15%; the ._load property should return the higher (0.15).
policy.lease(ack_id='one', byte_size=150)
assert policy._load == 0.15
# After this message is added, the messages should be higher at 20%
# (versus 16% for bytes).
policy.lease(ack_id='two', byte_size=10)
assert policy._load == 0.2
# Returning a number above 100% is fine.
policy.lease(ack_id='three', byte_size=1000)
assert policy._load == 1.16
def test_constructor_and_default_state():
manager = streaming_pull_manager.StreamingPullManager(
mock.sentinel.client, mock.sentinel.subscription)
# Public state
assert manager.is_active is False
assert manager.flow_control == types.FlowControl()
assert manager.dispatcher is None
assert manager.leaser is None
assert manager.ack_histogram is not None
assert manager.ack_deadline == 10
assert manager.load == 0
# Private state
assert manager._client == mock.sentinel.client
assert manager._subscription == mock.sentinel.subscription
assert manager._scheduler is not None
def test_subscribe_options(manager_open):
creds = mock.Mock(spec=credentials.Credentials)
client = subscriber.Client(credentials=creds)
flow_control = types.FlowControl(max_bytes=42)
scheduler = mock.sentinel.scheduler
future = client.subscribe(
"sub_name_a",
callback=mock.sentinel.callback,
flow_control=flow_control,
scheduler=scheduler,
)
assert isinstance(future, futures.StreamingPullFuture)
assert future._manager._subscription == "sub_name_a"
assert future._manager.flow_control == flow_control
assert future._manager._scheduler == scheduler
manager_open.assert_called_once_with(mock.ANY, mock.sentinel.callback)
def test_load_w_requests():
flow_control = types.FlowControl(max_bytes=100, max_requests=100)
policy = create_policy(flow_control=flow_control)
consumer = policy._consumer
pending_requests_patch = mock.patch.object(consumer.__class__,
'pending_requests',
new_callable=mock.PropertyMock)
with pending_requests_patch as pending_requests:
pending_requests.return_value = 0
assert policy._load == 0
pending_requests.return_value = 100
assert policy._load == 1
# If bytes count is higher, it should return that.
policy._bytes = 110
assert policy._load == 1.1
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
histogram_data=None):
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer()
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
scheduler=None):
self._client = client
self._subscription = subscription
self._flow_control = flow_control
self._ack_histogram = histogram.Histogram()
self._last_histogram_size = 0
self._ack_deadline = 10
self._rpc = None
self._callback = None
self._closing = threading.Lock()
self._closed = False
self._close_callbacks = []
if scheduler is None:
self._scheduler = (
google.cloud.pubsub_v1.subscriber.scheduler.ThreadScheduler())
else:
self._scheduler = scheduler
# A FIFO queue for the messages that have been received from the server,
# but not yet added to the lease management (and not sent to user callback),
# because the FlowControl limits have been hit.
self._messages_on_hold = queue.Queue()
# the total number of bytes consumed by the messages currently on hold
self._on_hold_bytes = 0
# A lock ensuring that pausing / resuming the consumer are both atomic
# operations that cannot be executed concurrently. Needed for properly
# syncing these operations with the current leaser load. Additionally,
# the lock is used to protect modifications of internal data that
# affects the load computation, i.e. the count and size of the messages
# currently on hold.
self._pause_resume_lock = threading.Lock()
# The threads created in ``.open()``.
self._dispatcher = None
self._leaser = None
self._consumer = None
self._heartbeater = None
def subscribe(self, subscription, callback=None, flow_control=()):
"""Return a representation of an individual subscription.
This method creates and returns a ``Consumer`` object (that is, a
:class:`~.pubsub_v1.subscriber._consumer.Consumer`)
subclass) bound to the topic. It does `not` create the subcription
on the backend (or do any API call at all); it simply returns an
object capable of doing these things.
If the ``callback`` argument is provided, then the :meth:`open` method
is automatically called on the returned object. If ``callback`` is
not provided, the subscription is returned unopened.
.. note::
It only makes sense to provide ``callback`` here if you have
already created the subscription manually in the API.
Args:
subscription (str): The name of the subscription. The
subscription should have already been created (for example,
by using :meth:`create_subscription`).
callback (function): The callback function. This function receives
the :class:`~.pubsub_v1.types.PubsubMessage` as its only
argument.
flow_control (~.pubsub_v1.types.FlowControl): The flow control
settings. Use this to prevent situations where you are
inundated with too many messages at once.
Returns:
~.pubsub_v1.subscriber._consumer.Consumer: An instance
of the defined ``consumer_class`` on the client.
Raises:
TypeError: If ``callback`` is not callable.
"""
flow_control = types.FlowControl(*flow_control)
subscr = self._policy_class(self, subscription, flow_control)
if callable(callback):
subscr.open(callback)
elif callback is not None:
error = '{!r} is not callable, please check input'.format(callback)
raise TypeError(error)
return subscr
def __init__(self, client, subscription,
flow_control=types.FlowControl(), histogram_data=None):
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer()
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
""".Histogram: the histogram tracking ack latency."""
self.leased_messages = {}
"""dict[str, float]: A mapping of ack IDs to the local time when the
ack ID was initially leased in seconds since the epoch."""
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
def test__maybe_release_messages_below_overload():
manager = make_manager(
flow_control=types.FlowControl(max_messages=10, max_bytes=1000))
manager._callback = mock.sentinel.callback
# Init leaser message count to 11, so that when subtracting the 3 messages
# that are on hold, there is still room for another 2 messages before the
# max load is hit.
_leaser = manager._leaser = mock.create_autospec(leaser.Leaser)
fake_leaser_add(_leaser, init_msg_count=11, assumed_msg_size=10)
messages = [
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_foo",
size=10),
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_bar",
size=10),
mock.create_autospec(message.Message,
instance=True,
ack_id="ack_baz",
size=10),
]
for msg in messages:
manager._messages_on_hold.put(msg)
manager._on_hold_bytes = 3 * 10
# the actual call of MUT
manager._maybe_release_messages()
assert manager._messages_on_hold.size == 1
msg = manager._messages_on_hold.get()
assert msg.ack_id == "ack_baz"
schedule_calls = manager._scheduler.schedule.mock_calls
assert len(schedule_calls) == 2
for _, call_args, _ in schedule_calls:
assert call_args[0] == mock.sentinel.callback
assert isinstance(call_args[1], message.Message)
assert call_args[1].ack_id in ("ack_foo", "ack_bar")
def test_load():
flow_control = types.FlowControl(max_messages=10, max_bytes=1000)
policy = create_policy(flow_control=flow_control)
consumer = policy._consumer
with mock.patch.object(consumer, 'pause') as pause:
# This should mean that our messages count is at 10%, and our bytes
# are at 15%; the ._load property should return the higher (0.15).
policy.lease(ack_id='one', byte_size=150)
assert policy._load == 0.15
pause.assert_not_called()
# After this message is added, the messages should be higher at 20%
# (versus 16% for bytes).
policy.lease(ack_id='two', byte_size=10)
assert policy._load == 0.2
pause.assert_not_called()
# Returning a number above 100% is fine.
policy.lease(ack_id='three', byte_size=1000)
assert policy._load == 1.16
pause.assert_called_once_with()
def main():
# Do set-up.
logger = utils.setup_logging(CURR_DIR)
thread_names.monkey_patch()
utils.make_lease_deterministic()
# Get clients and resource paths.
topic_name = 't-repro-{}'.format(int(1000 * time.time()))
subscription_name = 's-repro-{}'.format(int(1000 * time.time()))
client_info = utils.get_client_info(topic_name,
subscription_name,
policy_class=utils.FlowControlPolicy)
publisher, topic_path, subscriber, subscription_path = client_info
# Create a topic.
publisher.create_topic(topic_path)
# Subscribe to the topic. We do this before the messages are
# published so that we'll receive them as they come in.
subscriber.create_subscription(subscription_path, topic_path)
logger.info('Listening for messages on %s', subscription_path)
flow_control = types.FlowControl(max_messages=8)
subscription = subscriber.subscribe(subscription_path,
flow_control=flow_control)
sub_future = subscription.open(utils.AckCallback(logger))
consumer = subscription._consumer
# Set off async job to publish some messages.
publish_async(publisher, consumer, topic_path, logger)
# The subscriber is non-blocking, so we must keep the main thread from
# exiting to allow it to process messages in the background.
utils.heartbeats_block(logger, sub_future, max_time=MAX_TIME)
# Do clean-up.
subscription.close()
publisher.delete_topic(topic_path)
subscriber.delete_subscription(subscription_path)
thread_names.save_tree(CURR_DIR, logger)
thread_names.restore()
utils.restore()
def test__obtain_ack_deadline_no_custom_flow_control_setting():
from google.cloud.pubsub_v1.subscriber._protocol import histogram
manager = make_manager()
# Make sure that max_duration_per_lease_extension is disabled.
manager._flow_control = types.FlowControl(
max_duration_per_lease_extension=0)
deadline = manager._obtain_ack_deadline(maybe_update=True)
assert deadline == histogram.MIN_ACK_DEADLINE
# When we get some historical data, the deadline is adjusted.
manager.ack_histogram.add(histogram.MIN_ACK_DEADLINE * 2)
deadline = manager._obtain_ack_deadline(maybe_update=True)
assert deadline == histogram.MIN_ACK_DEADLINE * 2
# Adding just a single additional data point does not yet change the deadline.
manager.ack_histogram.add(histogram.MIN_ACK_DEADLINE)
deadline = manager._obtain_ack_deadline(maybe_update=True)
assert deadline == histogram.MIN_ACK_DEADLINE * 2
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
histogram_data=None):
"""Instantiate the policy.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~.pubsub_v1.types.FlowControl): The flow control
settings.
histogram_data (dict): Optional: A structure to store the histogram
data for predicting appropriate ack times. If set, this should
be a dictionary-like object.
.. note::
Additionally, the histogram relies on the assumption
that the dictionary will properly sort keys provided
that all keys are positive integers. If you are sending
your own dictionary class, ensure this assumption holds
or you will get strange behavior.
"""
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer(self)
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
self._paused = False
def test_on_request_below_threshold():
"""Establish that we resume a paused subscription when the pending
requests count is below threshold."""
flow_control = types.FlowControl(max_requests=100)
policy = create_policy(flow_control=flow_control)
consumer = policy._consumer
assert consumer.paused is True
pending_requests_patch = mock.patch.object(consumer.__class__,
'pending_requests',
new_callable=mock.PropertyMock)
with pending_requests_patch as pending_requests:
# should still be paused, not under the threshold.
pending_requests.return_value = 90
policy.on_request(None)
assert consumer.paused is True
# should unpause, we're under the resume threshold
pending_requests.return_value = 50
policy.on_request(None)
assert consumer.paused is False
def test_subscribe_options(manager_open, creds):
client = subscriber.Client(credentials=creds)
flow_control = types.FlowControl(max_bytes=42)
scheduler = mock.sentinel.scheduler
future = client.subscribe(
"sub_name_a",
callback=mock.sentinel.callback,
flow_control=flow_control,
scheduler=scheduler,
await_callbacks_on_shutdown=mock.sentinel.await_callbacks,
)
assert isinstance(future, futures.StreamingPullFuture)
assert future._manager._subscription == "sub_name_a"
assert future._manager.flow_control == flow_control
assert future._manager._scheduler == scheduler
assert future._manager._await_callbacks_on_shutdown is mock.sentinel.await_callbacks
manager_open.assert_called_once_with(
mock.ANY,
callback=mock.sentinel.callback,
on_callback_error=future.set_exception,
)
def test__obtain_ack_deadline_no_value_update():
manager = make_manager()
# Make sure that max_duration_per_lease_extension is disabled.
manager._flow_control = types.FlowControl(
max_duration_per_lease_extension=0)
manager.ack_histogram.add(21)
deadline = manager._obtain_ack_deadline(maybe_update=True)
assert deadline == 21
for _ in range(5):
manager.ack_histogram.add(35) # Gather some new ACK data.
deadline = manager._obtain_ack_deadline(maybe_update=False)
assert deadline == 21 # still the same
# Accessing the value through the ack_deadline property has no side effects either.
assert manager.ack_deadline == 21
# Updating the ack deadline is reflected on ack_deadline wrapper, too.
deadline = manager._obtain_ack_deadline(maybe_update=True)
assert manager.ack_deadline == deadline == 35
def __init__(self, client, subscription, flow_control=types.FlowControl(),
scheduler=None):
self._client = client
self._subscription = subscription
self._flow_control = flow_control
self._ack_histogram = histogram.Histogram()
self._last_histogram_size = 0
self._ack_deadline = 10
self._rpc = None
self._callback = None
self._closing = threading.Lock()
self._closed = False
self._close_callbacks = []
if scheduler is None:
self._scheduler = (
google.cloud.pubsub_v1.subscriber.scheduler.ThreadScheduler())
else:
self._scheduler = scheduler
# The threads created in ``.open()``.
self._dispatcher = None
self._leaser = None
self._consumer = None
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
scheduler_cls=None):
raise NotImplementedError
def subscribe_experimental(
self, subscription, callback, flow_control=(),
scheduler_=None):
"""Asynchronously start receiving messages on a given subscription.
This method starts a background thread to begin pulling messages from
a Pub/Sub subscription and scheduling them to be processed using the
provided ``callback``.
The ``callback`` will be called with an individual
:class:`google.cloud.pubsub_v1.subscriber.message.Message`. It is the
responsibility of the callback to either call ``ack()`` or ``nack()``
on the message when it finished processing. If an exception occurs in
the callback during processing, the exception is logged and the message
is ``nack()`` ed.
The ``flow_control`` argument can be used to control the rate of at
which messages are pulled. The settings are relatively conservative by
default to prevent "message hoarding" - a situation where the client
pulls a large number of messages but can not process them fast enough
leading it to "starve" other clients of messages. Increasing these
settings may lead to faster throughput for messages that do not take
a long time to process.
This method starts the receiver in the background and returns a
*Future* representing its execution. Waiting on the future (calling
``result()``) will block forever or until a non-recoverable error
is encountered (such as loss of network connectivity). Cancelling the
future will signal the process to shutdown gracefully and exit.
Example
.. code-block:: python
from google.cloud.pubsub_v1 import subscriber
subscriber_client = pubsub.SubscriberClient()
# existing subscription
subscription = subscriber_client.subscription_path(
'my-project-id', 'my-subscription')
def callback(message):
print(message)
message.ack()
future = subscriber.subscribe_experimental(
subscription, callback)
try:
future.result()
except KeyboardInterrupt:
future.cancel()
Args:
subscription (str): The name of the subscription. The
subscription should have already been created (for example,
by using :meth:`create_subscription`).
callback (Callable[~.pubsub_v1.subscriber.message.Message]):
The callback function. This function receives the message as
its only argument and will be called from a different thread/
process depending on the scheduling strategy.
flow_control (~.pubsub_v1.types.FlowControl): The flow control
settings. Use this to prevent situations where you are
inundated with too many messages at once.
Returns:
google.cloud.pubsub_v1.futures.StreamingPullFuture: A Future object
that can be used to manage the background stream.
"""
flow_control = types.FlowControl(*flow_control)
manager = streaming_pull_manager.StreamingPullManager(
self, subscription, flow_control)
future = futures.StreamingPullFuture(manager)
manager.open(callback)
return future
def subscribe(
self,
subscription,
callback,
flow_control=(),
scheduler=None,
use_legacy_flow_control=False,
):
"""Asynchronously start receiving messages on a given subscription.
This method starts a background thread to begin pulling messages from
a Pub/Sub subscription and scheduling them to be processed using the
provided ``callback``.
The ``callback`` will be called with an individual
:class:`google.cloud.pubsub_v1.subscriber.message.Message`. It is the
responsibility of the callback to either call ``ack()`` or ``nack()``
on the message when it finished processing. If an exception occurs in
the callback during processing, the exception is logged and the message
is ``nack()`` ed.
The ``flow_control`` argument can be used to control the rate of at
which messages are pulled. The settings are relatively conservative by
default to prevent "message hoarding" - a situation where the client
pulls a large number of messages but can not process them fast enough
leading it to "starve" other clients of messages. Increasing these
settings may lead to faster throughput for messages that do not take
a long time to process.
The ``use_legacy_flow_control`` argument disables enforcing flow control
settings at the Cloud PubSub server and uses the less accurate method of
only enforcing flow control at the client side.
This method starts the receiver in the background and returns a
*Future* representing its execution. Waiting on the future (calling
``result()``) will block forever or until a non-recoverable error
is encountered (such as loss of network connectivity). Cancelling the
future will signal the process to shutdown gracefully and exit.
.. note:: This uses Pub/Sub's *streaming pull* feature. This feature
properties that may be surprising. Please take a look at
https://cloud.google.com/pubsub/docs/pull#streamingpull for
more details on how streaming pull behaves compared to the
synchronous pull method.
Example:
.. code-block:: python
from google.cloud import pubsub_v1
subscriber_client = pubsub_v1.SubscriberClient()
# existing subscription
subscription = subscriber_client.subscription_path(
'my-project-id', 'my-subscription')
def callback(message):
print(message)
message.ack()
future = subscriber_client.subscribe(
subscription, callback)
try:
future.result()
except KeyboardInterrupt:
future.cancel()
Args:
subscription (str): The name of the subscription. The
subscription should have already been created (for example,
by using :meth:`create_subscription`).
callback (Callable[~google.cloud.pubsub_v1.subscriber.message.Message]):
The callback function. This function receives the message as
its only argument and will be called from a different thread/
process depending on the scheduling strategy.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow control
settings. Use this to prevent situations where you are
inundated with too many messages at once.
scheduler (~google.cloud.pubsub_v1.subscriber.scheduler.Scheduler): An optional
*scheduler* to use when executing the callback. This controls
how callbacks are executed concurrently. This object must not be shared
across multiple SubscriberClients.
Returns:
A :class:`~google.cloud.pubsub_v1.subscriber.futures.StreamingPullFuture`
instance that can be used to manage the background stream.
"""
flow_control = types.FlowControl(*flow_control)
manager = streaming_pull_manager.StreamingPullManager(
self,
subscription,
flow_control=flow_control,
scheduler=scheduler,
use_legacy_flow_control=use_legacy_flow_control,
)
future = futures.StreamingPullFuture(manager)
manager.open(callback=callback, on_callback_error=future.set_exception)
return future
def test_maybe_resume_consumer_wo_consumer_set():
manager = make_manager(
flow_control=types.FlowControl(max_messages=10, max_bytes=1000))
manager.maybe_resume_consumer() # no raise
def create_policy(flow_control=types.FlowControl()):
creds = mock.Mock(spec=credentials.Credentials)
client = subscriber.Client(credentials=creds)
return thread.Policy(client, 'sub_name_d', flow_control=flow_control)
def __init__(
self,
client,
subscription,
flow_control=types.FlowControl(),
scheduler=None,
use_legacy_flow_control=False,
await_callbacks_on_shutdown=False,
):
self._client = client
self._subscription = subscription
self._flow_control = flow_control
self._use_legacy_flow_control = use_legacy_flow_control
self._await_callbacks_on_shutdown = await_callbacks_on_shutdown
self._ack_histogram = histogram.Histogram()
self._last_histogram_size = 0
self._ack_deadline = histogram.MIN_ACK_DEADLINE
self._rpc = None
self._callback = None
self._closing = threading.Lock()
self._closed = False
self._close_callbacks = []
self._regular_shutdown_thread = None # Created on intentional shutdown.
# Generate a random client id tied to this object. All streaming pull
# connections (initial and re-connects) will then use the same client
# id. Doing so lets the server establish affinity even across stream
# disconncetions.
self._client_id = str(uuid.uuid4())
if scheduler is None:
self._scheduler = (
google.cloud.pubsub_v1.subscriber.scheduler.ThreadScheduler())
else:
self._scheduler = scheduler
# A collection for the messages that have been received from the server,
# but not yet sent to the user callback.
self._messages_on_hold = messages_on_hold.MessagesOnHold()
# The total number of bytes consumed by the messages currently on hold
self._on_hold_bytes = 0
# A lock ensuring that pausing / resuming the consumer are both atomic
# operations that cannot be executed concurrently. Needed for properly
# syncing these operations with the current leaser load. Additionally,
# the lock is used to protect modifications of internal data that
# affects the load computation, i.e. the count and size of the messages
# currently on hold.
self._pause_resume_lock = threading.Lock()
# A lock protecting the current ACK deadline used in the lease management. This
# value can be potentially updated both by the leaser thread and by the message
# consumer thread when invoking the internal _on_response() callback.
self._ack_deadline_lock = threading.Lock()
# The threads created in ``.open()``.
self._dispatcher = None
self._leaser = None
self._consumer = None
self._heartbeater = None
def create_subscriber(flow_control=types.FlowControl()):
subscriber_ = mock.create_autospec(subscriber.Subscriber, instance=True)
subscriber_.is_active = True
subscriber_.flow_control = flow_control
subscriber_.ack_histogram = _histogram.Histogram()
return subscriber_
def test_streaming_flow_control():
manager = make_manager(
flow_control=types.FlowControl(max_messages=10, max_bytes=1000))
request = manager._get_initial_request(stream_ack_deadline_seconds=10)
assert request.max_outstanding_messages == 10
assert request.max_outstanding_bytes == 1000