def test_pause(_LOGGER):
consumer = _consumer.Consumer()
consumer._can_consume.set()
assert consumer.pause() is None
assert not consumer._can_consume.is_set()
_LOGGER.debug.assert_called_once_with('Pausing consumer')
def test_blocking_consume_two_exceptions():
policy = mock.create_autospec(base.BasePolicy, instance=True)
exc1 = NameError('Oh noes.')
exc2 = ValueError('Something grumble.')
policy.on_exception.side_effect = OnException(acceptable=exc1)
response_generator1 = RaisingResponseGenerator(exc1)
response_generator2 = RaisingResponseGenerator(exc2)
policy.call_rpc.side_effect = (response_generator1, response_generator2)
consumer = _consumer.Consumer()
consumer.resume()
consumer._consumer_thread = mock.create_autospec(threading.Thread,
instance=True)
# Establish that we get responses until we are sent the exiting event.
assert consumer._blocking_consume(policy) is None
assert consumer._consumer_thread is None
# Check mocks.
assert policy.call_rpc.call_count == 2
assert response_generator1.next_calls == 1
assert response_generator2.next_calls == 1
policy.on_exception.assert_has_calls([mock.call(exc1), mock.call(exc2)])
def test_resume(_LOGGER):
consumer = _consumer.Consumer()
consumer._can_consume.clear()
assert consumer.resume() is None
assert consumer._can_consume.is_set()
_LOGGER.debug.assert_called_once_with('Resuming consumer')
def test_blocking_consume_when_exiting(_LOGGER):
consumer = _consumer.Consumer()
assert consumer._stopped.is_set() is False
consumer._stopped.set()
# Make sure method cleanly exits.
assert consumer._blocking_consume(None) is None
_LOGGER.debug.assert_called_once_with('Event signaled consumer exit.')
def test_paused():
consumer = _consumer.Consumer()
assert consumer.paused is True
consumer._can_consume.set()
assert consumer.paused is False
consumer._can_consume.clear()
assert consumer.paused is True
def test_stop_request_generator_response_not_done():
consumer = _consumer.Consumer()
response_generator = mock.Mock(spec=('done', ))
response_generator.done.return_value = False
stopped = consumer._stop_request_generator(None, response_generator)
assert stopped is False
# Check mocks.
response_generator.done.assert_called_once_with()
def test_blocking_consume():
policy = mock.Mock(spec=('call_rpc', 'on_response'))
policy.call_rpc.return_value = iter((mock.sentinel.A, mock.sentinel.B))
consumer = _consumer.Consumer()
consumer.resume()
assert consumer._blocking_consume(policy) is None
policy.call_rpc.assert_called_once()
policy.on_response.assert_has_calls(
[mock.call(mock.sentinel.A), mock.call(mock.sentinel.B)])
def test_blocking_consume():
policy = mock.create_autospec(base.BasePolicy, instance=True)
policy.call_rpc.return_value = iter((mock.sentinel.A, mock.sentinel.B))
consumer = _consumer.Consumer()
consumer.resume()
assert consumer._blocking_consume(policy) is None
policy.call_rpc.assert_called_once()
policy.on_response.assert_has_calls(
[mock.call(mock.sentinel.A),
mock.call(mock.sentinel.B)])
def test_stop_consuming():
consumer = _consumer.Consumer()
assert consumer._stopped.is_set() is False
thread = mock.Mock(spec=threading.Thread)
consumer._consumer_thread = thread
assert consumer.stop_consuming() is None
# Make sure state was updated.
assert consumer._stopped.is_set() is True
assert consumer._consumer_thread is None
# Check mocks.
thread.join.assert_called_once_with()
def test_start_consuming():
creds = mock.Mock(spec=credentials.Credentials)
client = subscriber.Client(credentials=creds)
policy = client.subscribe('sub_name_e')
consumer = _consumer.Consumer()
with mock.patch.object(threading, 'Thread', autospec=True) as Thread:
consumer.start_consuming(policy)
assert consumer._stopped.is_set() is False
Thread.assert_called_once_with(
name=_consumer._BIDIRECTIONAL_CONSUMER_NAME,
target=consumer._blocking_consume,
args=(policy, ),
)
assert consumer._consumer_thread is Thread.return_value
def test_stop_request_generator_queue_non_empty():
# Model scenario tested:
# - The request generator **is** running
# - The request queue **is not** empty
# Expected result:
# - ``_stop_request_generator()`` can't call ``.close()`` (since
# the generator is running) but then returns with ``False`` because
# the queue **is not** empty
consumer = _consumer.Consumer()
# Attach a "fake" queue to the request generator so the generator can
# block on an empty queue while the consumer's queue is not empty.
queue_ = queue.Queue()
received = queue.Queue()
request_generator = basic_queue_generator(queue_, received)
# Make sure the consumer's queue is not empty.
item1 = 'not-empty'
consumer._request_queue.put(item1)
thread = threading.Thread(target=next, args=(request_generator, ))
thread.start()
# Make sure the generator is stuck at the blocked ``.get()``
# in ``thread``.
while not request_generator.gi_running:
pass
assert received.empty()
assert request_generator.gi_frame is not None
response_generator = mock.Mock(spec=('done', ))
response_generator.done.return_value = True
stopped = consumer._stop_request_generator(request_generator,
response_generator)
assert stopped is False
# Make sure the generator is **still** not finished.
assert request_generator.gi_running
assert request_generator.gi_frame is not None
assert consumer._request_queue.get() == item1
# Allow the generator to exit.
item2 = 'just-exit'
queue_.put(item2)
# Wait until it's actually done.
while request_generator.gi_running:
pass
assert received.get() == item2
# Check mocks.
response_generator.done.assert_called_once_with()
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
histogram_data=None):
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer()
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
def test_stop_request_generator_not_running():
# Model scenario tested:
# - The request generator **is not** running
# - The request queue **is not** empty
# Expected result:
# - ``_stop_request_generator()`` successfully calls ``.close()``
consumer = _consumer.Consumer()
queue_ = consumer._request_queue
received = queue.Queue()
request_generator = basic_queue_generator(queue_, received)
item1 = 'unblock-please'
item2 = 'still-here'
queue_.put(item1)
queue_.put(item2)
assert not queue_.empty()
assert received.empty()
thread = threading.Thread(target=next, args=(request_generator, ))
thread.start()
# Make sure the generator is not stuck at the blocked ``.get()``
# in the thread.
while request_generator.gi_running:
pass
assert received.get() == item1
# Make sure it **isn't** done.
assert request_generator.gi_frame is not None
response_generator = mock.Mock(spec=('done', ))
response_generator.done.return_value = True
stopped = consumer._stop_request_generator(request_generator,
response_generator)
assert stopped is True
# Make sure it **is** done.
assert not request_generator.gi_running
assert request_generator.gi_frame is None
assert not queue_.empty()
assert queue_.get() == item2
assert queue_.empty()
# Check mocks.
response_generator.done.assert_called_once_with()
def test_blocking_consume_on_exception():
policy = mock.create_autospec(base.BasePolicy, instance=True)
policy.call_rpc.return_value = iter((mock.sentinel.A, mock.sentinel.B))
exc = TypeError('Bad things!')
policy.on_response.side_effect = exc
consumer = _consumer.Consumer()
consumer.resume()
consumer._consumer_thread = mock.Mock(spec=threading.Thread)
policy.on_exception.side_effect = OnException()
# Establish that we get responses until we are sent the exiting event.
consumer._blocking_consume(policy)
assert consumer._consumer_thread is None
# Check mocks.
policy.call_rpc.assert_called_once()
policy.on_response.assert_called_once_with(mock.sentinel.A)
policy.on_exception.assert_called_once_with(exc)
def __init__(self, client, subscription,
flow_control=types.FlowControl(), histogram_data=None):
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer()
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
""".Histogram: the histogram tracking ack latency."""
self.leased_messages = {}
"""dict[str, float]: A mapping of ack IDs to the local time when the
ack ID was initially leased in seconds since the epoch."""
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
def test_stop_request_generator_running():
# Model scenario tested:
# - The request generator **is** running
# - The request queue **is** empty
# Expected result:
# - ``_stop_request_generator()`` can't call ``.close()`` (since
# the generator is running) but then verifies that the queue is
# empty and sends ``STOP`` into the queue to successfully stop
# the generator
consumer = _consumer.Consumer()
queue_ = consumer._request_queue
received = queue.Queue()
request_generator = basic_queue_generator(queue_, received)
thread = threading.Thread(target=next, args=(request_generator, ))
thread.start()
# Make sure the generator is stuck at the blocked ``.get()``
# in the thread.
while not request_generator.gi_running:
pass
assert received.empty()
assert request_generator.gi_frame is not None
response_generator = mock.Mock(spec=('done', ))
response_generator.done.return_value = True
stopped = consumer._stop_request_generator(request_generator,
response_generator)
assert stopped is True
# Make sure it **is** done, though we may have to wait until
# the generator finishes (it has a few instructions between the
# ``get()`` and the ``break``).
while request_generator.gi_running:
pass
request_generator.close()
assert request_generator.gi_frame is None
assert received.get() == _helper_threads.STOP
assert queue_.empty()
# Check mocks.
response_generator.done.assert_called_once_with()
def test_stop_request_generator_close_failure():
# Model scenario tested:
# - The input isn't actually a generator
# Expected result:
# - ``_stop_request_generator()`` falls through to the ``LOGGER.error``
# case and returns ``False``
consumer = _consumer.Consumer()
request_generator = mock.Mock(spec=('close', ))
request_generator.close.side_effect = TypeError('Really, not a generator')
response_generator = mock.Mock(spec=('done', ))
response_generator.done.return_value = True
stopped = consumer._stop_request_generator(request_generator,
response_generator)
assert stopped is False
# Make sure close() was only called once.
request_generator.close.assert_called_once_with()
response_generator.done.assert_called_once_with()
def test_blocking_consume_iter_exception_while_paused():
policy = mock.create_autospec(base.BasePolicy, instance=True)
exc = TypeError('Bad things!')
policy.call_rpc.return_value = RaisingResponseGenerator(exc, active=False)
consumer = _consumer.Consumer()
# Ensure the consume is paused.
consumer.pause()
consumer._consumer_thread = mock.Mock(spec=threading.Thread)
policy.on_exception.side_effect = OnException()
# Start the thread. It should not block forever but should notice the rpc
# is inactive and raise the exception from the stream and then exit
# because on_exception returns false.
consumer._blocking_consume(policy)
assert consumer._consumer_thread is None
# Check mocks.
policy.call_rpc.assert_called_once()
policy.on_exception.assert_called_once_with(exc)
def __init__(self,
client,
subscription,
flow_control=types.FlowControl(),
histogram_data=None):
"""Instantiate the policy.
Args:
client (~.pubsub_v1.subscriber.client): The subscriber client used
to create this instance.
subscription (str): The name of the subscription. The canonical
format for this is
``projects/{project}/subscriptions/{subscription}``.
flow_control (~.pubsub_v1.types.FlowControl): The flow control
settings.
histogram_data (dict): Optional: A structure to store the histogram
data for predicting appropriate ack times. If set, this should
be a dictionary-like object.
.. note::
Additionally, the histogram relies on the assumption
that the dictionary will properly sort keys provided
that all keys are positive integers. If you are sending
your own dictionary class, ensure this assumption holds
or you will get strange behavior.
"""
self._client = client
self._subscription = subscription
self._consumer = _consumer.Consumer(self)
self._ack_deadline = 10
self._last_histogram_size = 0
self._future = None
self.flow_control = flow_control
self.histogram = _histogram.Histogram(data=histogram_data)
# These are for internal flow control tracking.
# They should not need to be used by subclasses.
self._bytes = 0
self._ack_on_resume = set()
self._paused = False
def test_request_generator_thread():
consumer = _consumer.Consumer()
creds = mock.Mock(spec=credentials.Credentials)
client = subscriber.Client(credentials=creds)
policy = client.subscribe('sub_name_e')
generator = consumer._request_generator_thread(policy)
# The first request that comes from the request generator thread
# should always be the initial request.
initial_request = next(generator)
assert initial_request.subscription == 'sub_name_e'
assert initial_request.stream_ack_deadline_seconds == 10
# Subsequent requests correspond to items placed in the request queue.
consumer.send_request(types.StreamingPullRequest(ack_ids=['i']))
request = next(generator)
assert request.ack_ids == ['i']
# The poison pill should stop the loop.
consumer.send_request(_helper_threads.STOP)
with pytest.raises(StopIteration):
next(generator)
def test_send_request():
consumer = _consumer.Consumer()
request = types.StreamingPullRequest(subscription='foo')
with mock.patch.object(queue.Queue, 'put') as put:
consumer.send_request(request)
put.assert_called_once_with(request)
def create_consumer():
creds = mock.Mock(spec=credentials.Credentials)
client = subscriber.Client(credentials=creds)
subscription = client.subscribe('sub_name_e')
return _consumer.Consumer(policy=subscription)