def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
"""
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
self._callback = callback
self._start_dispatch()
# Actually start consuming messages.
self._consumer.start_consuming(self)
self._start_lease_worker()
# Return the future.
return self._future
def test_on_exception_other():
policy = create_policy()
policy._future = Future(policy=policy)
exc = TypeError('wahhhhhh')
assert policy.on_exception(exc) is False
with pytest.raises(TypeError):
policy.future.result()
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
"""
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
_LOGGER.debug('Starting callback requests worker.')
self._callback = callback
self._consumer.helper_threads.start(
_CALLBACK_WORKER_NAME,
self._request_queue,
self._callback_requests,
)
# Actually start consuming messages.
self._consumer.start_consuming()
# Spawn a helper thread that maintains all of the leases for
# this policy.
_LOGGER.debug('Starting lease maintenance worker.')
self._leaser = threading.Thread(
name='Thread-LeaseMaintenance',
target=self.maintain_leases,
)
self._leaser.daemon = True
self._leaser.start()
# Return the future.
return self._future
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`close`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
Raises:
ValueError: If the policy has already been opened.
"""
if self._future is not None:
raise ValueError('This policy has already been opened.')
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
self._callback = callback
self._start_dispatch()
# Actually start consuming messages.
self._consumer.start_consuming(self)
self._start_lease_worker()
# Return the future.
return self._future
def test_close_with_future():
policy = create_policy()
policy._future = Future(policy=policy)
consumer = policy._consumer
with mock.patch.object(consumer, 'stop_consuming') as stop_consuming:
future = policy.future
policy.close()
stop_consuming.assert_called_once_with()
assert policy.future != future
assert future.result() is None
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`close`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
Raises:
ValueError: If the policy has already been opened.
"""
if self._future is not None:
raise ValueError('This policy has already been opened.')
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self, completed=threading.Event())
# Start the thread to pass the requests.
self._callback = functools.partial(_wrap_callback_errors, callback)
self._start_dispatch()
# Actually start consuming messages.
self._consumer.start_consuming(self)
self._start_lease_worker()
# Return the future.
return self._future
def test_close_with_future():
dispatch_thread = mock.Mock(spec=threading.Thread)
leases_thread = mock.Mock(spec=threading.Thread)
policy = create_policy()
policy._dispatch_thread = dispatch_thread
policy._leases_thread = leases_thread
policy._future = Future(policy=policy)
consumer = policy._consumer
with mock.patch.object(consumer, 'stop_consuming') as stop_consuming:
future = policy.future
policy.close()
stop_consuming.assert_called_once_with()
assert policy._dispatch_thread is None
dispatch_thread.join.assert_called_once_with()
assert policy._leases_thread is None
leases_thread.join.assert_called_once_with()
assert policy.future != future
assert future.result() is None
class Policy(base.BasePolicy):
"""A consumer class based on :class:`threading.Thread`.
This consumer handles the connection to the Pub/Sub service and all of
the concurrency needs.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow
control settings.
executor (~concurrent.futures.ThreadPoolExecutor): (Optional.) A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
queue (~queue.Queue): (Optional.) A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
"""
def __init__(self, client, subscription, flow_control=types.FlowControl(),
executor=None, queue=None):
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# Default the callback to a no-op; the **actual** callback is
# provided by ``.open()``.
self._callback = _do_nothing_callback
# Create a queue for keeping track of shared state.
self._request_queue = self._get_queue(queue)
# Also maintain an executor.
self._executor = self._get_executor(executor)
# The threads created in ``.open()``.
self._dispatch_thread = None
self._leases_thread = None
@staticmethod
def _get_queue(queue):
"""Gets a queue for the constructor.
Args:
queue (Optional[~queue.Queue]): A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
Returns:
~queue.Queue: Either ``queue`` if not :data:`None` or a default
queue.
"""
if queue is None:
return queue_mod.Queue()
else:
return queue
@staticmethod
def _get_executor(executor):
"""Gets an executor for the constructor.
Args:
executor (Optional[~concurrent.futures.ThreadPoolExecutor]): A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
Returns:
~concurrent.futures.ThreadPoolExecutor: Either ``executor`` if not
:data:`None` or a default thread pool executor with 10 workers
and a prefix (if supported).
"""
if executor is None:
executor_kwargs = {}
if sys.version_info[:2] == (2, 7) or sys.version_info >= (3, 6):
executor_kwargs['thread_name_prefix'] = (
'ThreadPoolExecutor-SubscriberPolicy')
return futures.ThreadPoolExecutor(
max_workers=multiprocessing.cpu_count(),
**executor_kwargs
)
else:
return executor
def close(self):
"""Close the existing connection.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`open`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
Returns:
~google.api_core.future.Future: The future that **was** attached
to the subscription.
Raises:
ValueError: If the policy has not been opened yet.
"""
if self._future is None:
raise ValueError('This policy has not been opened yet.')
# Stop consuming messages.
self._request_queue.put(_helper_threads.STOP)
self._dispatch_thread.join() # Wait until stopped.
self._dispatch_thread = None
self._consumer.stop_consuming()
self._leases_thread.join()
self._leases_thread = None
self._executor.shutdown()
# The subscription is closing cleanly; resolve the future if it is not
# resolved already.
if not self._future.done():
self._future.set_result(None)
future = self._future
self._future = None
return future
def _start_dispatch(self):
"""Start a thread to dispatch requests queued up by callbacks.
.. note::
This assumes, but does not check, that ``_dispatch_thread``
is :data:`None`.
Spawns a thread to run :meth:`dispatch_callback` and sets the
"dispatch thread" member on the current policy.
"""
_LOGGER.debug('Starting callback requests worker.')
dispatch_worker = _helper_threads.QueueCallbackWorker(
self._request_queue,
self.dispatch_callback,
)
# Create and start the helper thread.
thread = threading.Thread(
name=_CALLBACK_WORKER_NAME,
target=dispatch_worker,
)
thread.daemon = True
thread.start()
_LOGGER.debug('Started helper thread %s', thread.name)
self._dispatch_thread = thread
def _start_lease_worker(self):
"""Spawn a helper thread that maintains all of leases for this policy.
.. note::
This assumes, but does not check, that ``_leases_thread`` is
:data:`None`.
Spawns a thread to run :meth:`maintain_leases` and sets the
"leases thread" member on the current policy.
"""
_LOGGER.debug('Starting lease maintenance worker.')
thread = threading.Thread(
name='Thread-LeaseMaintenance',
target=self.maintain_leases,
)
thread.daemon = True
thread.start()
self._leases_thread = thread
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`close`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
Raises:
ValueError: If the policy has already been opened.
"""
if self._future is not None:
raise ValueError('This policy has already been opened.')
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
self._callback = callback
self._start_dispatch()
# Actually start consuming messages.
self._consumer.start_consuming(self)
self._start_lease_worker()
# Return the future.
return self._future
def dispatch_callback(self, action, kwargs):
"""Map the callback request to the appropriate gRPC request.
Args:
action (str): The method to be invoked.
kwargs (Dict[str, Any]): The keyword arguments for the method
specified by ``action``.
Raises:
ValueError: If ``action`` isn't one of the expected actions
"ack", "drop", "lease", "modify_ack_deadline" or "nack".
"""
if action == 'ack':
self.ack(**kwargs)
elif action == 'drop':
self.drop(**kwargs)
elif action == 'lease':
self.lease(**kwargs)
elif action == 'modify_ack_deadline':
self.modify_ack_deadline(**kwargs)
elif action == 'nack':
self.nack(**kwargs)
else:
raise ValueError(
'Unexpected action', action,
'Must be one of "ack", "drop", "lease", '
'"modify_ack_deadline" or "nack".')
def on_exception(self, exception):
"""Handle the exception.
If the exception is one of the retryable exceptions, this will signal
to the consumer thread that it should "recover" from the failure.
This will cause the stream to exit when it returns :data:`False`.
Returns:
bool: Indicates if the caller should recover or shut down.
Will be :data:`True` if the ``exception`` is "acceptable", i.e.
in a list of retryable / idempotent exceptions.
"""
# If this is in the list of idempotent exceptions, then we want to
# retry. That entails just returning None.
if isinstance(exception, self._RETRYABLE_STREAM_ERRORS):
return True
# Set any other exception on the future.
self._future.set_exception(exception)
return False
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
_LOGGER.debug(
'Using %s to process new message received:\n%r',
self._callback, msg)
message = Message(msg.message, msg.ack_id, self._request_queue)
future = self._executor.submit(self._callback, message)
future.add_done_callback(_callback_completed)
class Policy(base.BasePolicy):
"""A consumer class based on :class:`threading.Thread`.
This consumer handles the connection to the Pub/Sub service and all of
the concurrency needs.
"""
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
executor=None,
queue=None):
"""Instantiate the policy.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow
control settings.
executor (~concurrent.futures.ThreadPoolExecutor): (Optional.) A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
queue (~queue.Queue): (Optional.) A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
"""
# Default the callback to a no-op; it is provided by `.open`.
self._callback = _do_nothing_callback
# Default the future to None; it is provided by `.open`.
self._future = None
# Create a queue for keeping track of shared state.
if queue is None:
queue = queue_mod.Queue()
self._request_queue = queue
# Call the superclass constructor.
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# Also maintain a request queue and an executor.
if executor is None:
executor_kwargs = {}
if sys.version_info >= (3, 6):
executor_kwargs['thread_name_prefix'] = (
'ThreadPoolExecutor-SubscriberPolicy')
executor = futures.ThreadPoolExecutor(max_workers=10,
**executor_kwargs)
self._executor = executor
_LOGGER.debug('Creating callback requests thread (not starting).')
self._callback_requests = _helper_threads.QueueCallbackThread(
self._request_queue,
self.on_callback_request,
)
def close(self):
"""Close the existing connection."""
# Stop consuming messages.
self._consumer.helper_threads.stop(_CALLBACK_WORKER_NAME)
self._consumer.stop_consuming()
# The subscription is closing cleanly; resolve the future if it is not
# resolved already.
if self._future is not None and not self._future.done():
self._future.set_result(None)
self._future = None
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
"""
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
_LOGGER.debug('Starting callback requests worker.')
self._callback = callback
self._consumer.helper_threads.start(
_CALLBACK_WORKER_NAME,
self._request_queue,
self._callback_requests,
)
# Actually start consuming messages.
self._consumer.start_consuming()
# Spawn a helper thread that maintains all of the leases for
# this policy.
_LOGGER.debug('Starting lease maintenance worker.')
self._leaser = threading.Thread(
name='Thread-LeaseMaintenance',
target=self.maintain_leases,
)
self._leaser.daemon = True
self._leaser.start()
# Return the future.
return self._future
def on_callback_request(self, callback_request):
"""Map the callback request to the appropriate gRPC request."""
action, kwargs = callback_request[0], callback_request[1]
getattr(self, action)(**kwargs)
def on_exception(self, exception):
"""Handle the exception.
If the exception is one of the retryable exceptions, this will signal
to the consumer thread that it should "recover" from the failure.
This will cause the stream to exit when it returns :data:`False`.
Returns:
bool: Indicates if the caller should recover or shut down.
Will be :data:`True` if the ``exception`` is "acceptable", i.e.
in a list of retryable / idempotent exceptions.
"""
# If this is in the list of idempotent exceptions, then we want to
# retry. That entails just returning None.
if isinstance(exception, self._RETRYABLE_STREAM_ERRORS):
return True
# Set any other exception on the future.
self._future.set_exception(exception)
return False
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
_LOGGER.debug('New message received from Pub/Sub:\n%r', msg)
_LOGGER.debug(self._callback)
message = Message(msg.message, msg.ack_id, self._request_queue)
future = self._executor.submit(self._callback, message)
future.add_done_callback(_callback_completed)
class Policy(base.BasePolicy):
"""A consumer class based on :class:`threading.Thread`.
This consumer handles the connection to the Pub/Sub service and all of
the concurrency needs.
"""
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
executor=None,
queue=None):
"""Instantiate the policy.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow
control settings.
executor (~concurrent.futures.ThreadPoolExecutor): (Optional.) A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
queue (~queue.Queue): (Optional.) A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
"""
# Default the callback to a no-op; it is provided by `.open`.
self._callback = lambda message: None
# Default the future to None; it is provided by `.open`.
self._future = None
# Create a queue for keeping track of shared state.
if queue is None:
queue = queue_mod.Queue()
self._request_queue = queue
# Call the superclass constructor.
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# Also maintain a request queue and an executor.
logger.debug('Creating callback requests thread (not starting).')
if executor is None:
executor = futures.ThreadPoolExecutor(max_workers=10)
self._executor = executor
self._callback_requests = _helper_threads.QueueCallbackThread(
self._request_queue,
self.on_callback_request,
)
def close(self):
"""Close the existing connection."""
# Stop consuming messages.
self._consumer.helper_threads.stop('callback requests worker')
self._consumer.stop_consuming()
# The subscription is closing cleanly; resolve the future if it is not
# resolved already.
if self._future and not self._future.done():
self._future.set_result(None)
self._future = None
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
"""
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self)
# Start the thread to pass the requests.
logger.debug('Starting callback requests worker.')
self._callback = callback
self._consumer.helper_threads.start(
'callback requests worker',
self._request_queue,
self._callback_requests,
)
# Actually start consuming messages.
self._consumer.start_consuming()
# Spawn a helper thread that maintains all of the leases for
# this policy.
logger.debug('Spawning lease maintenance worker.')
self._leaser = threading.Thread(target=self.maintain_leases)
self._leaser.daemon = True
self._leaser.start()
# Return the future.
return self._future
def on_callback_request(self, callback_request):
"""Map the callback request to the appropriate GRPC request."""
action, kwargs = callback_request[0], callback_request[1]
getattr(self, action)(**kwargs)
def on_exception(self, exception):
"""Bubble the exception.
This will cause the stream to exit loudly.
"""
# If this is DEADLINE_EXCEEDED, then we want to retry.
# That entails just returning None.
deadline_exceeded = grpc.StatusCode.DEADLINE_EXCEEDED
if getattr(exception, 'code', lambda: None)() == deadline_exceeded:
return
# Set any other exception on the future.
self._future.set_exception(exception)
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
logger.debug('New message received from Pub/Sub: %r', msg)
logger.debug(self._callback)
message = Message(msg.message, msg.ack_id, self._request_queue)
future = self._executor.submit(self._callback, message)
future.add_done_callback(_callback_completed)
class Policy(base.BasePolicy):
"""A consumer class based on :class:`threading.Thread`.
This consumer handles the connection to the Pub/Sub service and all of
the concurrency needs.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~google.cloud.pubsub_v1.types.FlowControl): The flow
control settings.
executor (~concurrent.futures.ThreadPoolExecutor): (Optional.) A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
queue (~queue.Queue): (Optional.) A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
"""
def __init__(self, client, subscription, flow_control=types.FlowControl(),
executor=None, queue=None):
super(Policy, self).__init__(
client=client,
flow_control=flow_control,
subscription=subscription,
)
# The **actual** callback is provided by ``.open()``.
self._callback = None
# Create a queue for keeping track of shared state.
self._request_queue = self._get_queue(queue)
# Also maintain an executor.
self._executor = self._get_executor(executor)
# The threads created in ``.open()``.
self._dispatch_thread = None
self._leases_thread = None
@staticmethod
def _get_queue(queue):
"""Gets a queue for the constructor.
Args:
queue (Optional[~queue.Queue]): A Queue instance, appropriate
for crossing the concurrency boundary implemented by
``executor``.
Returns:
~queue.Queue: Either ``queue`` if not :data:`None` or a default
queue.
"""
if queue is None:
return queue_mod.Queue()
else:
return queue
@staticmethod
def _get_executor(executor):
"""Gets an executor for the constructor.
Args:
executor (Optional[~concurrent.futures.ThreadPoolExecutor]): A
ThreadPoolExecutor instance, or anything duck-type compatible
with it.
Returns:
~concurrent.futures.ThreadPoolExecutor: Either ``executor`` if not
:data:`None` or a default thread pool executor with 10 workers
and a prefix (if supported).
"""
if executor is None:
executor_kwargs = {}
if sys.version_info[:2] == (2, 7) or sys.version_info >= (3, 6):
executor_kwargs['thread_name_prefix'] = (
'ThreadPoolExecutor-SubscriberPolicy')
return futures.ThreadPoolExecutor(
max_workers=10,
**executor_kwargs
)
else:
return executor
def close(self):
"""Close the existing connection.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`open`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
Returns:
~google.api_core.future.Future: The future that **was** attached
to the subscription.
Raises:
ValueError: If the policy has not been opened yet.
"""
if self._future is None:
raise ValueError('This policy has not been opened yet.')
# Stop consuming messages.
self._request_queue.put(helper_threads.STOP)
self._dispatch_thread.join() # Wait until stopped.
self._dispatch_thread = None
self._consumer.stop_consuming()
self._leases_thread.join()
self._leases_thread = None
self._executor.shutdown()
# The subscription is closing cleanly; resolve the future if it is not
# resolved already.
if not self._future.done():
self._future.set_result(None)
future = self._future
self._future = None
return future
def _start_dispatch(self):
"""Start a thread to dispatch requests queued up by callbacks.
.. note::
This assumes, but does not check, that ``_dispatch_thread``
is :data:`None`.
Spawns a thread to run :meth:`dispatch_callback` and sets the
"dispatch thread" member on the current policy.
"""
_LOGGER.debug('Starting callback requests worker.')
dispatch_worker = helper_threads.QueueCallbackWorker(
self._request_queue,
self.dispatch_callback,
max_items=self.flow_control.max_request_batch_size,
max_latency=self.flow_control.max_request_batch_latency
)
# Create and start the helper thread.
thread = threading.Thread(
name=_CALLBACK_WORKER_NAME,
target=dispatch_worker,
)
thread.daemon = True
thread.start()
_LOGGER.debug('Started helper thread %s', thread.name)
self._dispatch_thread = thread
def _start_lease_worker(self):
"""Spawn a helper thread that maintains all of leases for this policy.
.. note::
This assumes, but does not check, that ``_leases_thread`` is
:data:`None`.
Spawns a thread to run :meth:`maintain_leases` and sets the
"leases thread" member on the current policy.
"""
_LOGGER.debug('Starting lease maintenance worker.')
thread = threading.Thread(
name='Thread-LeaseMaintenance',
target=self.maintain_leases,
)
thread.daemon = True
thread.start()
self._leases_thread = thread
def open(self, callback):
"""Open a streaming pull connection and begin receiving messages.
.. warning::
This method is not thread-safe. For example, if this method is
called while another thread is executing :meth:`close`, then the
policy could end up in an undefined state. The **same** policy
instance is not intended to be used by multiple workers (though
each policy instance **does** have a thread-safe private queue).
For each message received, the ``callback`` function is fired with
a :class:`~.pubsub_v1.subscriber.message.Message` as its only
argument.
Args:
callback (Callable): The callback function.
Returns:
~google.api_core.future.Future: A future that provides
an interface to block on the subscription if desired, and
handle errors.
Raises:
ValueError: If the policy has already been opened.
"""
if self._future is not None:
raise ValueError('This policy has already been opened.')
# Create the Future that this method will return.
# This future is the main thread's interface to handle exceptions,
# block on the subscription, etc.
self._future = Future(policy=self, completed=threading.Event())
# Start the thread to pass the requests.
self._callback = functools.partial(_wrap_callback_errors, callback)
self._start_dispatch()
# Actually start consuming messages.
self._consumer.start_consuming(self)
self._start_lease_worker()
# Return the future.
return self._future
def dispatch_callback(self, items):
"""Map the callback request to the appropriate gRPC request.
Args:
action (str): The method to be invoked.
kwargs (Dict[str, Any]): The keyword arguments for the method
specified by ``action``.
Raises:
ValueError: If ``action`` isn't one of the expected actions
"ack", "drop", "lease", "modify_ack_deadline" or "nack".
"""
batched_commands = collections.defaultdict(list)
for item in items:
batched_commands[item.__class__].append(item)
_LOGGER.debug('Handling %d batched requests', len(items))
if batched_commands[base.LeaseRequest]:
self.lease(batched_commands.pop(base.LeaseRequest))
if batched_commands[base.ModAckRequest]:
self.modify_ack_deadline(
batched_commands.pop(base.ModAckRequest))
# Note: Drop and ack *must* be after lease. It's possible to get both
# the lease the and ack/drop request in the same batch.
if batched_commands[base.AckRequest]:
self.ack(batched_commands.pop(base.AckRequest))
if batched_commands[base.NackRequest]:
self.nack(batched_commands.pop(base.NackRequest))
if batched_commands[base.DropRequest]:
self.drop(batched_commands.pop(base.DropRequest))
def on_exception(self, exception):
"""Handle the exception.
If the exception is one of the retryable exceptions, this will signal
to the consumer thread that it should "recover" from the failure.
This will cause the stream to exit when it returns :data:`False`.
Returns:
bool: Indicates if the caller should recover or shut down.
Will be :data:`True` if the ``exception`` is "acceptable", i.e.
in a list of retryable / idempotent exceptions.
"""
# If this is in the list of idempotent exceptions, then we want to
# retry. That entails just returning None.
if isinstance(exception, self._RETRYABLE_STREAM_ERRORS):
return True
# Set any other exception on the future.
self._future.set_exception(exception)
return False
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, send a modified acknowledgement request to the
server. This prevents expiration of the message due to buffering by
gRPC or proxy/firewall. This makes the server and client expiration
timer closer to each other thus preventing the message being
redelivered multiple times.
After the messages have all had their ack deadline updated, execute
the callback for each message using the executor.
"""
items = [
base.ModAckRequest(message.ack_id, self.histogram.percentile(99))
for message in response.received_messages
]
self.modify_ack_deadline(items)
for msg in response.received_messages:
_LOGGER.debug(
'Using %s to process message with ack_id %s.',
self._callback, msg.ack_id)
message = Message(msg.message, msg.ack_id, self._request_queue)
self._executor.submit(self._callback, message)