def build_producers(self):
producers = []
project = "testing"
try:
# Create pubsub topics
publisher = PublisherClient()
publisher.create_topic(publisher.topic_path(project, "test10"))
publisher.create_topic(publisher.topic_path(project, "test20"))
# Create pubsub subscriptions
subscriber = SubscriberClient()
subscriber.create_subscription(
subscriber.subscription_path(project, "test10"),
subscriber.topic_path(project, "test10"))
subscriber.create_subscription(
subscriber.subscription_path(project, "test20"),
subscriber.topic_path(project, "test20"))
except AlreadyExists as _:
pass
producers.append(
StreamBuilderFactory.get_producer_pubsub().set_project_id(
project).set_output_topic("test10").build())
producers.append(
StreamBuilderFactory.get_producer_kafka().set_broker_servers(
"localhost:9092").set_output_topic("test1").build())
producers.append(
StreamBuilderFactory.get_producer_kafka().set_broker_servers(
"localhost:9092").set_output_topic("test3").build())
return producers
def setup_base_subscriber(subscriber: SubscriberClient,
subscriptions: Subscriptions,
**kwargs: Any) -> List[StreamingPullFuture]:
return [
subscriber.subscribe(
subscriber.subscription_path(GOOGLE_PROJECT, subscription_name),
callback=generate_callback(Message, message_handler),
**kwargs) for subscription_name, message_handler in subscriptions
]
def __init__(self, project_id, subscription_name, activity_id=None):
self.subscriber_client = SubscriberClient()
self.subscription_path = self.subscriber_client.subscription_path(
project_id, subscription_name)
self.activity_id = activity_id
self.user_report = user_report_handler.UserReportHandler()
self.db = sqlalchemy.create_engine(
sqlalchemy.engine.url.URL(drivername="mysql+pymysql",
username=_DB_USER,
password=_DB_PASS,
database=_DB_NAME))
def subscription(topic: str, subscriber: SubscriberClient) -> Iterator[str]:
name = topic.replace("/topics/", "/subscriptions/")
try:
subscriber.create_subscription(name, topic)
delete = True
except AlreadyExists:
delete = False
try:
yield name
finally:
if delete:
subscriber.delete_subscription(name)
def subscriber(pubsub: Union[str, PubsubEmulator]) -> SubscriberClient:
if "PUBSUB_EMULATOR_HOST" in os.environ:
host = os.environ["PUBSUB_EMULATOR_HOST"]
try:
# PUBSUB_EMULATOR_HOST will override a channel argument
# so remove it in order to preserve channel options for
# supporting large messages
del os.environ["PUBSUB_EMULATOR_HOST"]
return SubscriberClient(channel=grpc.insecure_channel(
host, options=[("grpc.max_receive_message_length", -1)]))
finally:
os.environ["PUBSUB_EMULATOR_HOST"] = host
else:
return SubscriberClient()
def subscription_path(subscriber_client: pubsub_v1.SubscriberClient,
topic_path: str) -> Generator[str, None, None]:
subscription_path = subscriber_client.subscription_path(
PROJECT_ID, SUBSCRIPTION_ID)
subscription = subscriber_client.create_subscription(request={
"name": subscription_path,
"topic": topic_path
})
yield subscription.name
try:
subscriber_client.delete_subscription(
request={"subscription": subscription_path})
except NotFound:
pass
def create_subscription(client: pubsub_v1.SubscriberClient, project_name: str,
sub_name: str):
"""Creates a new subscription to a given topic."""
print("using pubsub topic: %s \n" % PUBSUB_TOPIC)
name = client.subscription_path(project_name, sub_name)
topic = client.topic_path(project_name, "new_tweets")
print(name)
print(topic)
try:
subscription: sub_client.types.module.Subscription = client.create_subscription(
name, topic)
except:
subscription = "subscription already exists"
#if subscription already exists, should return ALREADY_EXISTS
print('Subscription {} was created.'.format(subscription))
def test_submit_pubsub_topic_not_found(
integration_test: IntegrationTest,
publisher: PublisherClient,
subscriber: SubscriberClient,
subscription: str,
topic: str,
):
publisher.delete_topic(topic)
try:
integration_test.assert_accepted_and_queued()
finally:
subscriber.delete_subscription(subscription)
publisher.create_topic(topic)
subscriber.create_subscription(subscription, topic)
integration_test.assert_flushed_and_delivered()
def get_client_subscription(self) -> SubscriberClient:
"""Obtain the subscriber client for pubsub.
The credentials are obtained from fetch_credentials() function
These credentials are used to authenticate with Google services
If all is correct, it will be created a SubscriberClient for Pubsub
Returns
-------
SubscriberClient object from google cloud library used for Pubsub
Raises
------
RuntimeError: When the Pubsub client cannot be created
"""
streaming_credentials = self.fetch_credentials()
try:
credentials = service_account.Credentials.from_service_account_info(
streaming_credentials)
return SubscriberClient(credentials=credentials)
except Exception:
raise RuntimeError('''
Something unexpected happened while creating Pubsub client
''')
def perform_pull(worker_state: WorkerState, sfm_queue: Queue,
subscriber_client: SubscriberClient, subscription_path: str):
pull_request = PullRequest()
pull_request.max_messages = PROCESSING_WORKER_PULL_REQUEST_MAX_MESSAGES
pull_request.subscription = subscription_path
response: PullResponse = subscriber_client.pull(pull_request)
for received_message in response.received_messages:
# print(f"Received: {received_message.message.data}.")
message_job = _process_message(sfm_queue, received_message)
if not message_job or message_job.bytes_size > REQUEST_BODY_MAX_SIZE - 2:
worker_state.ack_ids.append(received_message.ack_id)
continue
if worker_state.should_flush(message_job):
perform_flush(worker_state, sfm_queue, subscriber_client,
subscription_path)
worker_state.add_job(message_job, received_message.ack_id)
# check if should flush because of time
if worker_state.should_flush():
perform_flush(worker_state, sfm_queue, subscriber_client,
subscription_path)
def pull(
self,
subscription: str,
max_messages: int,
project_id: str,
return_immediately: bool = False,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> List[ReceivedMessage]:
"""
Pulls up to ``max_messages`` messages from Pub/Sub subscription.
:param subscription: the Pub/Sub subscription name to pull from; do not
include the 'projects/{project}/topics/' prefix.
:type subscription: str
:param max_messages: The maximum number of messages to return from
the Pub/Sub API.
:type max_messages: int
:param project_id: Optional, the Google Cloud project ID where the subscription exists.
If set to None or missing, the default project_id from the Google Cloud connection is used.
:type project_id: str
:param return_immediately: If set, the Pub/Sub API will immediately
return if no messages are available. Otherwise, the request will
block for an undisclosed, but bounded period of time
:type return_immediately: bool
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
:return: A list of Pub/Sub ReceivedMessage objects each containing
an ``ackId`` property and a ``message`` property, which includes
the base64-encoded message content. See
https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.subscriptions/pull#ReceivedMessage
"""
subscriber = self.subscriber_client
# noqa E501 # pylint: disable=no-member,line-too-long
subscription_path = SubscriberClient.subscription_path(project_id, subscription)
self.log.info("Pulling max %d messages from subscription (path) %s", max_messages, subscription_path)
try:
# pylint: disable=no-member
response = subscriber.pull(
subscription=subscription_path,
max_messages=max_messages,
return_immediately=return_immediately,
retry=retry,
timeout=timeout,
metadata=metadata,
)
result = getattr(response, 'received_messages', [])
self.log.info("Pulled %d messages from subscription (path) %s", len(result), subscription_path)
return result
except (HttpError, GoogleAPICallError) as e:
raise PubSubException('Error pulling messages from subscription {}'.format(subscription_path), e)
def ensure_topic_and_subscription():
from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
publisher_client = PublisherClient()
try:
publisher_client.create_topic(
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
subscriber_client = SubscriberClient()
try:
subscriber_client.create_subscription(
subscriber_client.subscription_path(PROJECT_ID, SUBSCRIPTION_NAME),
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
def tearDownClass(cls):
super().tearDownClass()
publisher = PublisherClient()
subscriber = SubscriberClient()
for topic in ["test0", "test1"]:
try:
publisher.delete_topic(publisher.topic_path(
cls.project, topic))
except Exception as ex:
raise ex
try:
subscriber.delete_subscription(
subscriber.subscription_path(cls.project, topic))
except Exception as ex:
raise ex
def subscriber_client(self) -> SubscriberClient:
"""
Creates SubscriberClient.
:return: Google Cloud Pub/Sub client object.
:rtype: google.cloud.pubsub_v1.SubscriberClient
"""
return SubscriberClient(credentials=self._get_credentials(), client_info=self.client_info)
def _subscriber(service_account_json: Union[str, None]) -> SubscriberClient:
try:
return SubscriberClient(
credentials=_credentials(service_account_json, 'Subscriber'))
except DefaultCredentialsError:
logging.error(
'`GOOGLE_APPLICATION_CREDENTIALS` environment variable or service account in `address` field are not set!'
)
def subscriber_client(self):
"""
Returns:
google.cloud.pubsub_v1.SubscriberClient
"""
if not hasattr(self, "_subscriber_client"):
setattr(self, "_subscriber_client", SubscriberClient())
return getattr(self, "_subscriber_client")
def subscription_create(project_id: str, topic_name: str,
subscription_name: str):
subscriber = SubscriberClient()
topic_path = subscriber.api.topic_path(project_id, topic_name)
subscription_path = subscriber.api.subscription_path(
project_id, subscription_name)
subscriber.api.create_subscription(subscription_path, topic_path)
def test_set_subscription_policy(
subscriber_client: pubsub_v1.SubscriberClient,
subscription_path: str,
) -> None:
iam.set_subscription_policy(PROJECT_ID, SUBSCRIPTION_ID)
policy = subscriber_client.get_iam_policy(
request={"resource": subscription_path})
assert "roles/pubsub.viewer" in str(policy)
assert "allUsers" in str(policy)
def proto_subscription(subscriber_client: pubsub_v1.SubscriberClient,
proto_topic: str) -> Generator[str, None, None]:
proto_subscription_path = subscriber_client.subscription_path(
PROJECT_ID, PROTO_SUBSCRIPTION_ID)
try:
proto_subscription = subscriber_client.get_subscription(
request={"subscription": proto_subscription_path})
except NotFound:
proto_subscription = subscriber_client.create_subscription(
request={
"name": proto_subscription_path,
"topic": proto_topic
})
yield proto_subscription.name
subscriber_client.delete_subscription(
request={"subscription": proto_subscription.name})
def delete_subscription(
self,
subscription: str,
project_id: Optional[str] = None,
fail_if_not_exists: bool = False,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> None:
"""
Deletes a Pub/Sub subscription, if it exists.
:param subscription: the Pub/Sub subscription name to delete; do not
include the ``projects/{project}/subscriptions/`` prefix.
:param project_id: Optional, the GCP project ID where the subscription exists
If set to None or missing, the default project_id from the GCP connection is used.
:type project_id: str
:type subscription: str
:param fail_if_not_exists: if set, raise an exception if the topic does not exist
:type fail_if_not_exists: bool
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
"""
if not project_id:
raise ValueError("Project ID should be set.")
subscriber = self.subscriber_client
subscription_path = SubscriberClient.subscription_path(
project_id, subscription) # noqa E501 # pylint: disable=no-member,line-too-long
self.log.info("Deleting subscription (path) %s", subscription_path)
try:
# pylint: disable=no-member
subscriber.delete_subscription(subscription=subscription_path,
retry=retry,
timeout=timeout,
metadata=metadata)
except NotFound:
self.log.warning('Subscription does not exist: %s',
subscription_path)
if fail_if_not_exists:
raise PubSubException('Subscription does not exist: {}'.format(
subscription_path))
except GoogleAPICallError as e:
raise PubSubException(
'Error deleting subscription {}'.format(subscription_path), e)
self.log.info("Deleted subscription (path) %s", subscription_path)
def acknowledge(
self,
subscription: str,
ack_ids: List[str],
project_id: Optional[str] = None,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> None:
"""
Acknowledges the messages associated with the ``ack_ids`` from Pub/Sub subscription.
:param subscription: the Pub/Sub subscription name to delete; do not
include the 'projects/{project}/topics/' prefix.
:type subscription: str
:param ack_ids: List of ReceivedMessage ackIds from a previous pull
response
:type ack_ids: list
:param project_id: Optional, the GCP project name or ID in which to create the topic
If set to None or missing, the default project_id from the GCP connection is used.
:type project_id: str
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
"""
if not project_id:
raise ValueError("Project ID should be set.")
subscriber = self.subscriber_client
subscription_path = SubscriberClient.subscription_path(
project_id, subscription) # noqa E501 # pylint: disable=no-member,line-too-long
self.log.info("Acknowledging %d ack_ids from subscription (path) %s",
len(ack_ids), subscription_path)
try:
# pylint: disable=no-member
subscriber.acknowledge(
subscription=subscription_path,
ack_ids=ack_ids,
retry=retry,
timeout=timeout,
metadata=metadata,
)
except (HttpError, GoogleAPICallError) as e:
raise PubSubException(
'Error acknowledging {} messages pulled from subscription {}'.
format(len(ack_ids), subscription_path), e)
self.log.info("Acknowledged ack_ids from subscription (path) %s",
subscription_path)
class PubSubConsumer:
GCLOUD_PUBSUB_PROJECT_ID = 'herbie-app'
def __init__(self):
self._subscriber = SubscriberClient()
def create_subscription(self, topic: str, subscription: str):
topic_path = self._subscriber.topic_path(self.GCLOUD_PUBSUB_PROJECT_ID,
topic)
subscription_path = self._subscriber.subscription_path(
self.GCLOUD_PUBSUB_PROJECT_ID, subscription)
try:
self._subscriber.create_subscription(subscription_path, topic_path)
except AlreadyExists as e:
print(f'Subscription {subscription} already exists.')
pass
def subscribe(self, subscription: str):
project_id = self.GCLOUD_PUBSUB_PROJECT_ID
subscription_path = self._subscriber.subscription_path(
project_id, subscription)
future = self._subscriber.subscribe(subscription_path,
self._subscribe_callback)
print(f'Listening for messages on {subscription_path}')
try:
future.result()
except KeyboardInterrupt:
future.cancel()
def _subscribe_callback(self, message):
print(message.data)
class Subscriber(object):
def __init__(self, project_id, subscription_name, activity_id=None):
self.subscriber_client = SubscriberClient()
self.subscription_path = self.subscriber_client.subscription_path(
project_id, subscription_name)
self.activity_id = activity_id
self.user_report = user_report_handler.UserReportHandler()
self.db = sqlalchemy.create_engine(
sqlalchemy.engine.url.URL(drivername="mysql+pymysql",
username=_DB_USER,
password=_DB_PASS,
database=_DB_NAME))
# To run with local cloud proxy instance add query to the method above:
# query={"unix_socket": "/cloudsql/{}".format(_SQL_CONNECTION)}
def get_streaming_subscription(self):
"""Returns streaming subscription on Google Pub/Sub topic."""
subscription = self.subscriber_client.subscribe(
self.subscription_path, callback=self.get_callback())
return subscription
def _get_user_details_id(self, participant_id):
"""Returns the user_details_id for the given participant_id."""
with self.db.connect() as conn:
query = sqlalchemy.text(
'''SELECT user_details_id FROM participant_study_info
WHERE participant_id=:participant_id''')
result = conn.execute(query,
participant_id=participant_id).fetchone()
if not result:
logging.warning(
'Failed fetching user_id for participant {}'.format(
participant_id))
return None
return result[0]
@abc.abstractmethod
def get_callback(self):
"""Defines callback invoked by the subscription."""
def subscribe_synchronously(project_id: str, subscription_name: str,
callback: Callable):
"""
Creates a pubsub synchronous subscription function for a given project_id and subscription name.
Example usage:
def message_handler(message, ack_callback):
# do something with message
# acknowledge message when done
ack_callback()
subscription_name = f"signals2tsb-{environment}-{signal_list_topic}"
subscribe_synchronously(project_id=project_id, subscription_name=subscription_name, callback=message_handler)
"""
sig_handler = SigHandler()
subscriber = SubscriberClient()
subscription_path = subscriber.subscription_path(
project=project_id, subscription=subscription_name)
def subscribe(message_handler: Callable):
while sig_handler.running:
with LogContext(trace_id=generate_trace_id()):
try:
_pull_and_handle_message(subscriber, subscription_path,
message_handler)
except DeadlineExceeded:
logging.debug(
"Received deadline exceeded event when polling, this is expected if no messages"
)
pass
with LogContext(subscription_path=subscription_path):
logging.info("Set up subscription")
# subscribing to events, one message at a time
subscribe(callback)
def create_push_subscription(
self,
subscription_name: str,
topic_name: str,
endpoint: str,
config: SubscriptionConfig = None) -> Subscription:
subscriber: SubscriberWrapper = SubscriberClient()
subcription_path = subscriber.subscription_path(
self.project_id, subscription_name)
topic_path = self.get_topic_path(topic_name)
if not config:
config = SubscriptionConfig()
logging_extra = {
"subscription_name": subscription_name,
"topic_name": topic_name,
"config": config.dict(),
}
with subscriber:
try:
try:
subscription = subscriber.get_subscription(
request={"subscription": subcription_path})
Logger.info("Push subscription exists", logging_extra)
return subscription
except NotFound:
request = Subscription(
name=subcription_path,
topic=topic_path,
push_config=PushConfig(push_endpoint=endpoint),
ack_deadline_seconds=60,
expiration_policy=ExpirationPolicy(ttl=Duration(
seconds=config.expiration_days *
86400) if config.expiration_days else None),
retry_policy=RetryPolicy(),
message_retention_duration=Duration(
seconds=config.retention_days * 86400))
subscription = subscriber.create_subscription(
request=request)
Logger.info("Push subscription created", logging_extra)
return subscription
except BaseException as ex:
Logger.error("Failed to create push subscription",
exc_info=ex,
extra=logging_extra)
raise ex
def __init__(self,
name,
dist,
emulator_host,
emulator_port,
client_options=None,
credentials=None,
services_service=None,
**config):
services_service(super(Subscriber, self).__init__,
name,
dist,
emulator_host=emulator_host,
emulator_port=emulator_port,
client_options=client_options,
credentials=credentials,
**config)
settings = {}
if client_options is not None:
if isinstance(client_options, (str, type(u''))):
client_options = services_service(
reference.load_object(client_options)[0])
settings['client_options'] = client_options
if emulator_host:
channel = grpc.insecure_channel('{}:{}'.format(
emulator_host, emulator_port))
transport = SubscriberGrpcTransport(channel=channel)
else:
transport = None
if credentials is not None:
settings['credentials'] = services_service(
reference.load_object(credentials)[0])
self.__class__.proxy_target = SubscriberClient(transport=transport,
**settings)
def subscription_path(subscriber_client: pubsub_v1.SubscriberClient,
topic_path: str) -> Generator[str, None, None]:
subscription_path = subscriber_client.subscription_path(
PROJECT_ID, SUBSCRIPTION_ID)
try:
subscription = subscriber_client.create_subscription(
request={
"name": subscription_path,
"topic": topic_path
})
yield subscription.name
except AlreadyExists:
yield subscription_path
subscriber_client.delete_subscription(
request={"subscription": subscription_path})
subscriber_client.close()
def setUpClass(cls):
super().setUpClass()
cls.project = os.environ.get("PUBSUB_PROJECT", "testing")
cls.credentials = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS",
None)
for subscription in ["test0", "test1"]:
try:
publisher = PublisherClient()
publisher.create_topic(
publisher.topic_path(cls.project, subscription))
except AlreadyExists:
pass
try:
subscriber = SubscriberClient()
subscriber.create_subscription(
subscriber.subscription_path(cls.project, subscription),
subscriber.topic_path(cls.project, subscription))
except AlreadyExists:
pass
def create_subscription(
self,
topic: str,
project_id: str,
subscription: Optional[str] = None,
subscription_project_id: Optional[str] = None,
ack_deadline_secs: int = 10,
fail_if_exists: bool = False,
push_config: Optional[Union[dict, PushConfig]] = None,
retain_acked_messages: Optional[bool] = None,
message_retention_duration: Optional[Union[dict, Duration]] = None,
labels: Optional[Dict[str, str]] = None,
enable_message_ordering: bool = False,
expiration_policy: Optional[Union[dict, ExpirationPolicy]] = None,
filter_: Optional[str] = None,
dead_letter_policy: Optional[Union[dict, DeadLetterPolicy]] = None,
retry_policy: Optional[Union[dict, RetryPolicy]] = None,
retry: Optional[Retry] = None,
timeout: Optional[float] = None,
metadata: Optional[Sequence[Tuple[str, str]]] = None,
) -> str:
"""
Creates a Pub/Sub subscription, if it does not already exist.
:param topic: the Pub/Sub topic name that the subscription will be bound
to create; do not include the ``projects/{project}/subscriptions/`` prefix.
:type topic: str
:param project_id: Optional, the Google Cloud project ID of the topic that the subscription will be
bound to. If set to None or missing, the default project_id from the Google Cloud connection
is used.
:type project_id: str
:param subscription: the Pub/Sub subscription name. If empty, a random
name will be generated using the uuid module
:type subscription: str
:param subscription_project_id: the Google Cloud project ID where the subscription
will be created. If unspecified, ``project_id`` will be used.
:type subscription_project_id: str
:param ack_deadline_secs: Number of seconds that a subscriber has to
acknowledge each message pulled from the subscription
:type ack_deadline_secs: int
:param fail_if_exists: if set, raise an exception if the topic
already exists
:type fail_if_exists: bool
:param push_config: If push delivery is used with this subscription,
this field is used to configure it. An empty ``pushConfig`` signifies
that the subscriber will pull and ack messages using API methods.
:type push_config: Union[Dict, google.cloud.pubsub_v1.types.PushConfig]
:param retain_acked_messages: Indicates whether to retain acknowledged
messages. If true, then messages are not expunged from the subscription's
backlog, even if they are acknowledged, until they fall out of the
``message_retention_duration`` window. This must be true if you would
like to Seek to a timestamp.
:type retain_acked_messages: bool
:param message_retention_duration: How long to retain unacknowledged messages
in the subscription's backlog, from the moment a message is published. If
``retain_acked_messages`` is true, then this also configures the
retention of acknowledged messages, and thus configures how far back in
time a ``Seek`` can be done. Defaults to 7 days. Cannot be more than 7
days or less than 10 minutes.
:type message_retention_duration: Union[Dict, google.cloud.pubsub_v1.types.Duration]
:param labels: Client-assigned labels; see
https://cloud.google.com/pubsub/docs/labels
:type labels: Dict[str, str]
:param enable_message_ordering: If true, messages published with the same
ordering_key in PubsubMessage will be delivered to the subscribers in the order
in which they are received by the Pub/Sub system. Otherwise, they may be
delivered in any order.
:type enable_message_ordering: bool
:param expiration_policy: A policy that specifies the conditions for this
subscription’s expiration. A subscription is considered active as long as any
connected subscriber is successfully consuming messages from the subscription or
is issuing operations on the subscription. If expiration_policy is not set,
a default policy with ttl of 31 days will be used. The minimum allowed value for
expiration_policy.ttl is 1 day.
:type expiration_policy: Union[Dict, google.cloud.pubsub_v1.types.ExpirationPolicy`]
:param filter_: An expression written in the Cloud Pub/Sub filter language. If
non-empty, then only PubsubMessages whose attributes field matches the filter are
delivered on this subscription. If empty, then no messages are filtered out.
:type filter_: str
:param dead_letter_policy: A policy that specifies the conditions for dead lettering
messages in this subscription. If dead_letter_policy is not set, dead lettering is
disabled.
:type dead_letter_policy: Union[Dict, google.cloud.pubsub_v1.types.DeadLetterPolicy]
:param retry_policy: A policy that specifies how Pub/Sub retries message delivery
for this subscription. If not set, the default retry policy is applied. This
generally implies that messages will be retried as soon as possible for healthy
subscribers. RetryPolicy will be triggered on NACKs or acknowledgement deadline
exceeded events for a given message.
:type retry_policy: Union[Dict, google.cloud.pubsub_v1.types.RetryPolicy]
:param retry: (Optional) A retry object used to retry requests.
If None is specified, requests will not be retried.
:type retry: google.api_core.retry.Retry
:param timeout: (Optional) The amount of time, in seconds, to wait for the request
to complete. Note that if retry is specified, the timeout applies to each
individual attempt.
:type timeout: float
:param metadata: (Optional) Additional metadata that is provided to the method.
:type metadata: Sequence[Tuple[str, str]]]
:return: subscription name which will be the system-generated value if
the ``subscription`` parameter is not supplied
:rtype: str
"""
subscriber = self.subscriber_client
if not subscription:
subscription = 'sub-{}'.format(uuid4())
if not subscription_project_id:
subscription_project_id = project_id
# Add airflow-version label to the subscription
labels = labels or {}
labels['airflow-version'] = 'v' + version.replace('.', '-').replace('+', '-')
# pylint: disable=no-member
subscription_path = SubscriberClient.subscription_path(subscription_project_id, subscription)
topic_path = SubscriberClient.topic_path(project_id, topic)
self.log.info("Creating subscription (path) %s for topic (path) %a", subscription_path, topic_path)
try:
subscriber.create_subscription(
name=subscription_path,
topic=topic_path,
push_config=push_config,
ack_deadline_seconds=ack_deadline_secs,
retain_acked_messages=retain_acked_messages,
message_retention_duration=message_retention_duration,
labels=labels,
enable_message_ordering=enable_message_ordering,
expiration_policy=expiration_policy,
filter_=filter_,
dead_letter_policy=dead_letter_policy,
retry_policy=retry_policy,
retry=retry,
timeout=timeout,
metadata=metadata,
)
except AlreadyExists:
self.log.warning('Subscription already exists: %s', subscription_path)
if fail_if_exists:
raise PubSubException('Subscription already exists: {}'.format(subscription_path))
except GoogleAPICallError as e:
raise PubSubException('Error creating subscription {}'.format(subscription_path), e)
self.log.info("Created subscription (path) %s for topic (path) %s", subscription_path, topic_path)
return subscription
def setUpClass(cls):
cls.publisher = PublisherClient()
cls.subscriber = SubscriberClient()
