def test_messages_received(subscriber: SubscriberImpl, async_subscriber,
message_callback, close_callback):
message1 = Message(PubsubMessage(message_id="1")._pb, "", 0, None)
message2 = Message(PubsubMessage(message_id="2")._pb, "", 0, None)
counter = Box[int]()
counter.val = 0
async def on_read() -> Message:
counter.val += 1
if counter.val == 1:
return message1
if counter.val == 2:
return message2
await sleep_forever()
async_subscriber.read.side_effect = on_read
results = Queue()
message_callback.side_effect = lambda m: results.put(m.message_id)
subscriber.add_close_callback(close_callback)
subscriber.__enter__()
assert results.get() == "1"
assert results.get() == "2"
subscriber.close()
async def test_iterator(
default_subscriber,
subscriber_factory,
multiplexed_client: AsyncSubscriberClientInterface,
):
read_queues = wire_queues(default_subscriber.read)
subscription = SubscriptionPath(1, CloudZone.parse("us-central1-a"), "abc")
message = Message(PubsubMessage(message_id="1")._pb, "", 0, None)
async with multiplexed_client:
iterator = await multiplexed_client.subscribe(
subscription, DISABLED_FLOW_CONTROL
)
subscriber_factory.assert_has_calls(
[call(subscription, None, DISABLED_FLOW_CONTROL)]
)
read_fut_1 = asyncio.ensure_future(iterator.__anext__())
assert not read_fut_1.done()
await read_queues.called.get()
default_subscriber.read.assert_has_calls([call()])
await read_queues.results.put(message)
assert await read_fut_1 is message
read_fut_2 = asyncio.ensure_future(iterator.__anext__())
assert not read_fut_2.done()
await read_queues.called.get()
default_subscriber.read.assert_has_calls([call(), call()])
await read_queues.results.put(FailedPrecondition(""))
with pytest.raises(FailedPrecondition):
await read_fut_2
default_subscriber.__aexit__.assert_called_once()
def test_parse_json_message():
attributes = {
'eventType': 'OBJECT_FINALIZE',
'bucketId': 'mybucket',
'objectId': 'myobject',
'objectGeneration': 1234567,
'resource': 'projects/_/buckets/mybucket/objects/myobject#1234567',
'notificationConfig': ('projects/_/buckets/mybucket/'
'notificationConfigs/5'),
'payloadFormat': 'JSON_API_V1'}
data = (b'{'
b' "size": 12345,'
b' "contentType": "text/html",'
b' "metageneration": 1'
b'}')
message = Message(
mock.Mock(data=data, attributes=attributes),
MESSAGE_ID,
mock.Mock())
assert summarize(message) == (
'\tEvent type: OBJECT_FINALIZE\n'
'\tBucket ID: mybucket\n'
'\tObject ID: myobject\n'
'\tGeneration: 1234567\n'
'\tContent type: text/html\n'
'\tSize: 12345\n'
'\tMetageneration: 1\n')
def test_parse_json_message():
attributes = {
"eventType":
"OBJECT_FINALIZE",
"bucketId":
"mybucket",
"objectId":
"myobject",
"objectGeneration":
1234567,
"resource":
"projects/_/buckets/mybucket/objects/myobject#1234567",
"notificationConfig": ("projects/_/buckets/mybucket/"
"notificationConfigs/5"),
"payloadFormat":
"JSON_API_V1",
}
data = (b"{"
b' "size": 12345,'
b' "contentType": "text/html",'
b' "metageneration": 1'
b"}")
message = Message(mock.Mock(data=data, attributes=attributes), MESSAGE_ID,
mock.Mock())
assert summarize(message) == ("\tEvent type: OBJECT_FINALIZE\n"
"\tBucket ID: mybucket\n"
"\tObject ID: myobject\n"
"\tGeneration: 1234567\n"
"\tContent type: text/html\n"
"\tSize: 12345\n"
"\tMetageneration: 1\n")
def __init__(self,
*args: List[Any],
google_cloud_message: Message = None,
**kwargs: Dict[str, Any]) -> None:
if google_cloud_message:
self._message = google_cloud_message
return
self._message = Message(*args, **kwargs)
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
_LOGGER.debug('Using %s to process message with ack_id %s.',
self._callback, msg.ack_id)
message = Message(msg.message, msg.ack_id, self._request_queue)
self._executor.submit(self._callback, message)
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
logger.debug('New message received from Pub/Sub: %r', msg)
logger.debug(self._callback)
message = Message(msg.message, msg.ack_id, self._request_queue)
future = self._executor.submit(self._callback, message)
logger.debug('Result: %s' % future.result())
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, schedule a callback with the executor.
"""
for msg in response.received_messages:
_LOGGER.debug('New message received from Pub/Sub:\n%r', msg)
_LOGGER.debug(self._callback)
message = Message(msg.message, msg.ack_id, self._request_queue)
future = self._executor.submit(self._callback, message)
future.add_done_callback(_callback_completed)
async def test_delivery_from_multiple(subscriber, assigner,
subscriber_factory):
assign_queues = wire_queues(assigner.get_assignment)
async with subscriber:
await assign_queues.called.get()
sub1 = mock_async_context_manager(
MagicMock(spec=AsyncSingleSubscriber))
sub2 = mock_async_context_manager(
MagicMock(spec=AsyncSingleSubscriber))
sub1_queues = wire_queues(sub1.read)
sub2_queues = wire_queues(sub2.read)
subscriber_factory.side_effect = (
lambda partition: sub1 if partition == Partition(1) else sub2)
await assign_queues.results.put({Partition(1), Partition(2)})
await sub1_queues.results.put(
Message(PubsubMessage(message_id="1")._pb, "", 0, None))
await sub2_queues.results.put(
Message(PubsubMessage(message_id="2")._pb, "", 0, None))
message_ids: Set[str] = set()
message_ids.add((await subscriber.read()).message_id)
message_ids.add((await subscriber.read()).message_id)
assert message_ids == {"1", "2"}
def test_construct_subscriber_message_from_google_message():
ack_id = 'some_ack_id'
delivery_attempt = 0
request_queue = Queue()
pubsub_message = PubsubMessage()
pubsub_message.attributes['style'] = 'cool'
google_message = Message(pubsub_message, ack_id, delivery_attempt,
request_queue)
subscriber_message = SubscriberMessage.from_google_cloud(google_message)
assert subscriber_message.ack_id == ack_id
assert subscriber_message.delivery_attempt is None # only an int if >0
assert subscriber_message.attributes['style'] == 'cool'
def _wrap_message(self, message: SequencedMessage.meta.pb) -> Message:
# Rewrap in the proto-plus-python wrapper for passing to the transform
rewrapped = SequencedMessage()
rewrapped._pb = message
cps_message = self._transformer.transform(rewrapped)
offset = message.cursor.offset
ack_id_str = _AckId(self._ack_generation_id, offset).encode()
self._ack_set_tracker.track(offset)
self._messages_by_ack_id[ack_id_str] = _SizedMessage(
cps_message, message.size_bytes)
wrapped_message = Message(
cps_message._pb,
ack_id=ack_id_str,
delivery_attempt=0,
request_queue=self._queue,
)
return wrapped_message
async def read(self) -> Message:
try:
message: SequencedMessage = await self.await_unless_failed(
self._underlying.read())
cps_message = self._transformer.transform(message)
offset = message.cursor.offset
ack_id = _AckId(self._ack_generation_id, offset)
self._ack_set_tracker.track(offset)
self._messages_by_ack_id[ack_id.str()] = _SizedMessage(
cps_message, message.size_bytes)
wrapped_message = Message(
cps_message._pb,
ack_id=ack_id.str(),
delivery_attempt=0,
request_queue=self._queue,
)
return wrapped_message
except GoogleAPICallError as e:
self.fail(e)
raise e
def on_response(self, response):
"""Process all received Pub/Sub messages.
For each message, send a modified acknowledgement request to the
server. This prevents expiration of the message due to buffering by
gRPC or proxy/firewall. This makes the server and client expiration
timer closer to each other thus preventing the message being
redelivered multiple times.
After the messages have all had their ack deadline updated, execute
the callback for each message using the executor.
"""
items = [
base.ModAckRequest(message.ack_id, self.histogram.percentile(99))
for message in response.received_messages
]
self.modify_ack_deadline(items)
for msg in response.received_messages:
_LOGGER.debug('Using %s to process message with ack_id %s.',
self._callback, msg.ack_id)
message = Message(msg.message, msg.ack_id, self._request_queue)
self._executor.submit(self._callback, message)