async def test_iterator(
default_subscriber,
subscriber_factory,
multiplexed_client: AsyncSubscriberClientInterface,
):
read_queues = wire_queues(default_subscriber.read)
subscription = SubscriptionPath(1, CloudZone.parse("us-central1-a"), "abc")
message = Message(PubsubMessage(message_id="1")._pb, "", 0, None)
async with multiplexed_client:
iterator = await multiplexed_client.subscribe(
subscription, DISABLED_FLOW_CONTROL
)
subscriber_factory.assert_has_calls(
[call(subscription, None, DISABLED_FLOW_CONTROL)]
)
read_fut_1 = asyncio.ensure_future(iterator.__anext__())
assert not read_fut_1.done()
await read_queues.called.get()
default_subscriber.read.assert_has_calls([call()])
await read_queues.results.put(message)
assert await read_fut_1 is message
read_fut_2 = asyncio.ensure_future(iterator.__anext__())
assert not read_fut_2.done()
await read_queues.called.get()
default_subscriber.read.assert_has_calls([call(), call()])
await read_queues.results.put(FailedPrecondition(""))
with pytest.raises(FailedPrecondition):
await read_fut_2
default_subscriber.__aexit__.assert_called_once()
def subscription(client: AdminClient,
topic: Topic) -> Generator[Subscription, None, None]:
location = CloudZone(CloudRegion(CLOUD_REGION), ZONE_ID)
subscription_path = SubscriptionPath(PROJECT_NUMBER, location,
SUBSCRIPTION_ID)
subscription = Subscription(
name=str(subscription_path),
topic=topic.name,
delivery_config=Subscription.DeliveryConfig(
delivery_requirement=Subscription.DeliveryConfig.
DeliveryRequirement.DELIVER_IMMEDIATELY, ),
)
try:
response = client.get_subscription(subscription.name)
except NotFound:
# This subscription will start receiving the first message in the topic.
response = client.create_subscription(subscription,
BacklogLocation.BEGINNING)
yield response
try:
client.delete_subscription(response.name)
except NotFound as e:
print(e.message)
def list_topic_subscriptions(
self, topic_path: TopicPath) -> List[SubscriptionPath]:
subscription_strings = [
x for x in self._underlying.list_topic_subscriptions(
name=str(topic_path))
]
return [SubscriptionPath.parse(x) for x in subscription_strings]
def receive_messages(project_number,
cloud_region,
zone_id,
subscription_id,
timeout=90):
# [START pubsublite_quickstart_subscriber]
from concurrent.futures._base import TimeoutError
from google.cloud.pubsublite.cloudpubsub import SubscriberClient
from google.cloud.pubsublite.types import (
CloudRegion,
CloudZone,
FlowControlSettings,
SubscriptionPath,
)
# TODO(developer):
# project_number = 1122334455
# cloud_region = "us-central1"
# zone_id = "a"
# subscription_id = "your-subscription-id"
# timeout = 90
location = CloudZone(CloudRegion(cloud_region), zone_id)
subscription_path = SubscriptionPath(project_number, location,
subscription_id)
# Configure when to pause the message stream for more incoming messages based on the
# maximum size or number of messages that a single-partition subscriber has received,
# whichever condition is met first.
per_partition_flow_control_settings = FlowControlSettings(
# 1,000 outstanding messages. Must be >0.
messages_outstanding=1000,
# 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
bytes_outstanding=10 * 1024 * 1024,
)
def callback(message: PubsubMessage):
message_data = message.data.decode("utf-8")
metadata = MessageMetadata.decode(message.message_id)
print(
f"Received {message_data} of ordering key {message.ordering_key} with id {metadata}."
)
message.ack()
# SubscriberClient() must be used in a `with` block or have __enter__() called before use.
with SubscriberClient() as subscriber_client:
streaming_pull_future = subscriber_client.subscribe(
subscription_path,
callback=callback,
per_partition_flow_control_settings=
per_partition_flow_control_settings,
)
print(f"Listening for messages on {str(subscription_path)}...")
try:
streaming_pull_future.result(timeout=timeout)
except TimeoutError or KeyboardInterrupt:
streaming_pull_future.cancel()
assert streaming_pull_future.done()
def subscribe(
self,
subscription: Union[SubscriptionPath, str],
callback: Callable, # TODO(dpcollins): Change to MessageCallback,
per_partition_flow_control_settings: FlowControlSettings,
fixed_partitions: Optional[Set[Partition]] = None,
) -> StreamingPullFuture:
if isinstance(subscription, str):
subscription = SubscriptionPath.parse(subscription)
callback = cast(MessageCallback, callback)
def create_and_open():
underlying = self._underlying_factory(
subscription, fixed_partitions,
per_partition_flow_control_settings)
subscriber = SubscriberImpl(underlying, callback, self._executor)
future = StreamingPullFuture(subscriber)
subscriber.__enter__()
return future
future = self._multiplexer.create_or_fail(subscription,
create_and_open)
future.add_done_callback(
lambda fut: self._multiplexer.try_erase(subscription, future))
return future
def subscription_path(client):
location = CloudZone(CloudRegion(CLOUD_REGION), ZONE_ID)
subscription_path = str(SubscriptionPath(PROJECT_NUMBER, location, SUBSCRIPTION_ID))
yield subscription_path
try:
client.delete_subscription(subscription_path)
except NotFound:
pass
def test_list_lite_subscriptions_in_project(subscription_path, capsys):
import list_lite_subscriptions_in_project_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
list_lite_subscriptions_in_project_example.list_lite_subscriptions_in_project(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
)
out, _ = capsys.readouterr()
assert "subscription(s) listed in your project and location." in out
def test_get_lite_subscription(subscription_path, capsys):
import get_lite_subscription_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
get_lite_subscription_example.get_lite_subscription(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
subscription_path_object.name,
)
out, _ = capsys.readouterr()
assert f"{subscription_path} exists." in out
def test_update_lite_subscription_example(subscription_path, capsys):
import update_lite_subscription_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
update_lite_subscription_example.update_lite_subscription(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
subscription_path_object.name,
)
out, _ = capsys.readouterr()
assert f"{subscription_path} updated successfully." in out
def create_subscription(
self,
subscription: Subscription,
starting_offset: BacklogLocation = BacklogLocation.END,
) -> Subscription:
path = SubscriptionPath.parse(subscription.name)
return self._underlying.create_subscription(
request={
"parent": str(path.to_location_path()),
"subscription": subscription,
"subscription_id": path.name,
"skip_backlog": (starting_offset == BacklogLocation.END),
})
def seek_lite_subscription(project_number, cloud_region, zone_id,
subscription_id, seek_target, wait_for_operation):
# [START pubsublite_seek_subscription]
from google.api_core.exceptions import NotFound
from google.cloud.pubsublite import AdminClient
from google.cloud.pubsublite.types import CloudRegion, CloudZone, SubscriptionPath
# TODO(developer):
# project_number = 1122334455
# cloud_region = "us-central1"
# zone_id = "a"
# subscription_id = "your-subscription-id"
# seek_target = BacklogLocation.BEGINNING
# wait_for_operation = False
# Possible values for seek_target:
# - BacklogLocation.BEGINNING: replays from the beginning of all retained
# messages.
# - BacklogLocation.END: skips past all current published messages.
# - PublishTime(<datetime>): delivers messages with publish time greater
# than or equal to the specified timestamp.
# - EventTime(<datetime>): seeks to the first message with event time
# greater than or equal to the specified timestamp.
# Waiting for the seek operation to complete is optional. It indicates when
# subscribers for all partitions are receiving messages from the seek
# target. If subscribers are offline, the operation will complete once they
# are online.
cloud_region = CloudRegion(cloud_region)
location = CloudZone(cloud_region, zone_id)
subscription_path = SubscriptionPath(project_number, location,
subscription_id)
client = AdminClient(cloud_region)
try:
# Initiate an out-of-band seek for a subscription to the specified
# target. If an operation is returned, the seek has been successfully
# registered and will eventually propagate to subscribers.
seek_operation = client.seek_subscription(subscription_path,
seek_target)
print(f"Seek operation: {seek_operation.operation.name}")
except NotFound:
print(f"{subscription_path} not found.")
return
if wait_for_operation:
print("Waiting for operation to complete...")
seek_operation.result()
print(f"Operation completed. Metadata:\n{seek_operation.metadata}")
def test_list_lite_subscriptions_in_topic(topic_path, subscription_path, capsys):
import list_lite_subscriptions_in_topic_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
topic_path_object = TopicPath.parse(topic_path)
list_lite_subscriptions_in_topic_example.list_lite_subscriptions_in_topic(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
topic_path_object.name,
)
out, _ = capsys.readouterr()
assert "subscription(s) listed in your topic." in out
def create_subscription(
self,
subscription: Subscription,
starting_offset: BacklogLocation = BacklogLocation.END,
) -> Subscription:
path = SubscriptionPath.parse(subscription.name)
return self._underlying.create_subscription(
request=CreateSubscriptionRequest(
parent=str(path.to_location_path()),
subscription=subscription,
subscription_id=path.name,
skip_backlog=(starting_offset == BacklogLocation.END),
)
)
def test_create_lite_subscription(subscription_path, topic_path, capsys):
import create_lite_subscription_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
topic_path_object = TopicPath.parse(topic_path)
create_lite_subscription_example.create_lite_subscription(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
topic_path_object.name,
subscription_path_object.name,
)
out, _ = capsys.readouterr()
assert f"{subscription_path} created successfully." in out
def test_delete_lite_subscription_example(subscription_path, capsys):
import delete_lite_subscription_example
subscription_path_object = SubscriptionPath.parse(subscription_path)
@backoff.on_exception(backoff.expo, AssertionError, max_time=MAX_TIME)
def eventually_consistent_test():
delete_lite_subscription_example.delete_lite_subscription(
subscription_path_object.project,
subscription_path_object.location.region.name,
subscription_path_object.location.zone_id,
subscription_path_object.name,
)
out, _ = capsys.readouterr()
assert f"{subscription_path} deleted successfully." in out
eventually_consistent_test()
async def subscribe(
self,
subscription: Union[SubscriptionPath, str],
per_partition_flow_control_settings: FlowControlSettings,
fixed_partitions: Optional[Set[Partition]] = None,
) -> AsyncIterator[Message]:
if isinstance(subscription, str):
subscription = SubscriptionPath.parse(subscription)
subscriber = self._underlying_factory(
subscription, fixed_partitions,
per_partition_flow_control_settings)
await subscriber.__aenter__()
self._live_clients.add(subscriber)
return _SubscriberAsyncIterator(
subscriber, lambda: self._try_remove_client(subscriber))
def subscribe(
self,
subscription: Union[SubscriptionPath, str],
callback: MessageCallback,
per_partition_flow_control_settings: FlowControlSettings,
fixed_partitions: Optional[Set[Partition]] = None,
) -> StreamingPullFuture:
if isinstance(subscription, str):
subscription = SubscriptionPath.parse(subscription)
underlying = self._underlying_factory(
subscription, fixed_partitions,
per_partition_flow_control_settings)
subscriber = SubscriberImpl(underlying, callback, self._executor)
future = StreamingPullFuture(subscriber)
subscriber.__enter__()
future.add_done_callback(lambda fut: self._try_remove_client(future))
return future
def create_lite_subscription(project_number, cloud_region, zone_id, topic_id,
subscription_id):
# [START pubsublite_create_subscription]
from google.api_core.exceptions import AlreadyExists
from google.cloud.pubsublite import AdminClient, Subscription
from google.cloud.pubsublite.types import (
CloudRegion,
CloudZone,
SubscriptionPath,
TopicPath,
)
# TODO(developer):
# project_number = 1122334455
# cloud_region = "us-central1"
# zone_id = "a"
# topic_id = "your-topic-id"
# subscription_id = "your-subscription-id"
cloud_region = CloudRegion(cloud_region)
location = CloudZone(cloud_region, zone_id)
topic_path = TopicPath(project_number, location, topic_id)
subscription_path = SubscriptionPath(project_number, location,
subscription_id)
subscription = Subscription(
name=str(subscription_path),
topic=str(topic_path),
delivery_config=Subscription.DeliveryConfig(
# Possible values for delivery_requirement:
# - `DELIVER_IMMEDIATELY`
# - `DELIVER_AFTER_STORED`
# You may choose whether to wait for a published message to be successfully written
# to storage before the server delivers it to subscribers. `DELIVER_IMMEDIATELY` is
# suitable for applications that need higher throughput.
delivery_requirement=Subscription.DeliveryConfig.
DeliveryRequirement.DELIVER_IMMEDIATELY, ),
)
client = AdminClient(cloud_region)
try:
response = client.create_subscription(subscription)
print(f"{response.name} created successfully.")
except AlreadyExists:
print(f"{subscription_path} already exists.")
async def subscribe(
self,
subscription: Union[SubscriptionPath, str],
per_partition_flow_control_settings: FlowControlSettings,
fixed_partitions: Optional[Set[Partition]] = None,
) -> AsyncIterator[Message]:
if isinstance(subscription, str):
subscription = SubscriptionPath.parse(subscription)
async def create_and_open():
client = self._underlying_factory(
subscription, fixed_partitions,
per_partition_flow_control_settings)
await client.__aenter__()
return client
subscriber = await self._multiplexer.get_or_create(
subscription, create_and_open)
return _SubscriberAsyncIterator(
subscriber,
lambda: self._multiplexer.try_erase(subscription, subscriber))
def update_lite_subscription(project_number, cloud_region, zone_id,
subscription_id):
# [START pubsublite_update_subscription]
from google.api_core.exceptions import NotFound
from google.cloud.pubsublite import AdminClient, Subscription
from google.cloud.pubsublite.types import CloudRegion, CloudZone, SubscriptionPath
from google.protobuf.field_mask_pb2 import FieldMask
# TODO(developer):
# project_number = 1122334455
# cloud_region = "us-central1"
# zone_id = "a"
# topic_id = "your-topic-id"
# subscription_id = "your-subscription-id"
cloud_region = CloudRegion(cloud_region)
location = CloudZone(cloud_region, zone_id)
subscription_path = SubscriptionPath(project_number, location,
subscription_id)
field_mask = FieldMask(paths=["delivery_config.delivery_requirement"])
subscription = Subscription(
name=str(subscription_path),
delivery_config=Subscription.DeliveryConfig(
# Possible values for delivery_requirement:
# - `DELIVER_IMMEDIATELY`
# - `DELIVER_AFTER_STORED`
# `DELIVER_AFTER_STORED` requires a published message to be successfully written
# to storage before the server delivers it to subscribers.
delivery_requirement=Subscription.DeliveryConfig.
DeliveryRequirement.DELIVER_AFTER_STORED, ),
)
client = AdminClient(cloud_region)
try:
response = client.update_subscription(subscription, field_mask)
print(f"{response.name} updated successfully.")
except NotFound:
print(f"{subscription_path} not found.")
def get_lite_subscription(project_number, cloud_region, zone_id,
subscription_id):
# [START pubsublite_get_subscription]
from google.api_core.exceptions import NotFound
from google.cloud.pubsublite import AdminClient
from google.cloud.pubsublite.types import CloudRegion, CloudZone, SubscriptionPath
# TODO(developer):
# project_number = 1122334455
# cloud_region = "us-central1"
# zone_id = "a"
# subscription_id = "your-subscription-id"
cloud_region = CloudRegion(cloud_region)
location = CloudZone(cloud_region, zone_id)
subscription_path = SubscriptionPath(project_number, location,
subscription_id)
client = AdminClient(cloud_region)
try:
response = client.get_subscription(subscription_path)
print(f"{response.name} exists.")
except NotFound:
print(f"{subscription_path} not found.")
from concurrent.futures._base import TimeoutError
from google.cloud.pubsublite.cloudpubsub import SubscriberClient
from google.cloud.pubsublite.types import (CloudRegion, CloudZone,
FlowControlSettings,
SubscriptionPath, MessageMetadata)
project_number = 533637743951
cloud_region = "asia-east1"
zone_id = "a"
subscription_id = "test_sub_one"
timeout = 90
location = CloudZone(CloudRegion(cloud_region), zone_id)
subscription_path = SubscriptionPath(project_number, location, subscription_id)
# Configure when to pause the message stream for more incoming messages based on the
# maximum size or number of messages that a single-partition subscriber has received,
# whichever condition is met first.
per_partition_flow_control_settings = FlowControlSettings(
# 1,000 outstanding messages. Must be >0.
messages_outstanding=1000,
# 10 MiB. Must be greater than the allowed size of the largest message (1 MiB).
bytes_outstanding=10 * 1024 * 1024,
)
def callback(message):
message_data = message.data.decode("utf-8")
print(message_data)
# metadata = MessageMetadata.decode(message.message_id)
message.ack()