def build_producers(self): producers = [] project = "testing" try: # Create pubsub topics publisher = PublisherClient() publisher.create_topic(publisher.topic_path(project, "test10")) publisher.create_topic(publisher.topic_path(project, "test20")) # Create pubsub subscriptions subscriber = SubscriberClient() subscriber.create_subscription( subscriber.subscription_path(project, "test10"), subscriber.topic_path(project, "test10")) subscriber.create_subscription( subscriber.subscription_path(project, "test20"), subscriber.topic_path(project, "test20")) except AlreadyExists as _: pass producers.append( StreamBuilderFactory.get_producer_pubsub().set_project_id( project).set_output_topic("test10").build()) producers.append( StreamBuilderFactory.get_producer_kafka().set_broker_servers( "localhost:9092").set_output_topic("test1").build()) producers.append( StreamBuilderFactory.get_producer_kafka().set_broker_servers( "localhost:9092").set_output_topic("test3").build()) return producers
class PubSubConsumer: GCLOUD_PUBSUB_PROJECT_ID = 'herbie-app' def __init__(self): self._subscriber = SubscriberClient() def create_subscription(self, topic: str, subscription: str): topic_path = self._subscriber.topic_path(self.GCLOUD_PUBSUB_PROJECT_ID, topic) subscription_path = self._subscriber.subscription_path( self.GCLOUD_PUBSUB_PROJECT_ID, subscription) try: self._subscriber.create_subscription(subscription_path, topic_path) except AlreadyExists as e: print(f'Subscription {subscription} already exists.') pass def subscribe(self, subscription: str): project_id = self.GCLOUD_PUBSUB_PROJECT_ID subscription_path = self._subscriber.subscription_path( project_id, subscription) future = self._subscriber.subscribe(subscription_path, self._subscribe_callback) print(f'Listening for messages on {subscription_path}') try: future.result() except KeyboardInterrupt: future.cancel() def _subscribe_callback(self, message): print(message.data)
def create_subscription(client: pubsub_v1.SubscriberClient, project_name: str, sub_name: str): """Creates a new subscription to a given topic.""" print("using pubsub topic: %s \n" % PUBSUB_TOPIC) name = client.subscription_path(project_name, sub_name) topic = client.topic_path(project_name, "new_tweets") print(name) print(topic) try: subscription: sub_client.types.module.Subscription = client.create_subscription( name, topic) except: subscription = "subscription already exists" #if subscription already exists, should return ALREADY_EXISTS print('Subscription {} was created.'.format(subscription))
def setUpClass(cls): super().setUpClass() cls.project = os.environ.get("PUBSUB_PROJECT", "testing") cls.credentials = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", None) for subscription in ["test0", "test1"]: try: publisher = PublisherClient() publisher.create_topic( publisher.topic_path(cls.project, subscription)) except AlreadyExists: pass try: subscriber = SubscriberClient() subscriber.create_subscription( subscriber.subscription_path(cls.project, subscription), subscriber.topic_path(cls.project, subscription)) except AlreadyExists: pass
def create_subscription( self, topic: str, project_id: str, subscription: Optional[str] = None, subscription_project_id: Optional[str] = None, ack_deadline_secs: int = 10, fail_if_exists: bool = False, push_config: Optional[Union[dict, PushConfig]] = None, retain_acked_messages: Optional[bool] = None, message_retention_duration: Optional[Union[dict, Duration]] = None, labels: Optional[Dict[str, str]] = None, enable_message_ordering: bool = False, expiration_policy: Optional[Union[dict, ExpirationPolicy]] = None, filter_: Optional[str] = None, dead_letter_policy: Optional[Union[dict, DeadLetterPolicy]] = None, retry_policy: Optional[Union[dict, RetryPolicy]] = None, retry: Optional[Retry] = None, timeout: Optional[float] = None, metadata: Optional[Sequence[Tuple[str, str]]] = None, ) -> str: """ Creates a Pub/Sub subscription, if it does not already exist. :param topic: the Pub/Sub topic name that the subscription will be bound to create; do not include the ``projects/{project}/subscriptions/`` prefix. :type topic: str :param project_id: Optional, the Google Cloud project ID of the topic that the subscription will be bound to. If set to None or missing, the default project_id from the Google Cloud connection is used. :type project_id: str :param subscription: the Pub/Sub subscription name. If empty, a random name will be generated using the uuid module :type subscription: str :param subscription_project_id: the Google Cloud project ID where the subscription will be created. If unspecified, ``project_id`` will be used. :type subscription_project_id: str :param ack_deadline_secs: Number of seconds that a subscriber has to acknowledge each message pulled from the subscription :type ack_deadline_secs: int :param fail_if_exists: if set, raise an exception if the topic already exists :type fail_if_exists: bool :param push_config: If push delivery is used with this subscription, this field is used to configure it. An empty ``pushConfig`` signifies that the subscriber will pull and ack messages using API methods. :type push_config: Union[Dict, google.cloud.pubsub_v1.types.PushConfig] :param retain_acked_messages: Indicates whether to retain acknowledged messages. If true, then messages are not expunged from the subscription's backlog, even if they are acknowledged, until they fall out of the ``message_retention_duration`` window. This must be true if you would like to Seek to a timestamp. :type retain_acked_messages: bool :param message_retention_duration: How long to retain unacknowledged messages in the subscription's backlog, from the moment a message is published. If ``retain_acked_messages`` is true, then this also configures the retention of acknowledged messages, and thus configures how far back in time a ``Seek`` can be done. Defaults to 7 days. Cannot be more than 7 days or less than 10 minutes. :type message_retention_duration: Union[Dict, google.cloud.pubsub_v1.types.Duration] :param labels: Client-assigned labels; see https://cloud.google.com/pubsub/docs/labels :type labels: Dict[str, str] :param enable_message_ordering: If true, messages published with the same ordering_key in PubsubMessage will be delivered to the subscribers in the order in which they are received by the Pub/Sub system. Otherwise, they may be delivered in any order. :type enable_message_ordering: bool :param expiration_policy: A policy that specifies the conditions for this subscription’s expiration. A subscription is considered active as long as any connected subscriber is successfully consuming messages from the subscription or is issuing operations on the subscription. If expiration_policy is not set, a default policy with ttl of 31 days will be used. The minimum allowed value for expiration_policy.ttl is 1 day. :type expiration_policy: Union[Dict, google.cloud.pubsub_v1.types.ExpirationPolicy`] :param filter_: An expression written in the Cloud Pub/Sub filter language. If non-empty, then only PubsubMessages whose attributes field matches the filter are delivered on this subscription. If empty, then no messages are filtered out. :type filter_: str :param dead_letter_policy: A policy that specifies the conditions for dead lettering messages in this subscription. If dead_letter_policy is not set, dead lettering is disabled. :type dead_letter_policy: Union[Dict, google.cloud.pubsub_v1.types.DeadLetterPolicy] :param retry_policy: A policy that specifies how Pub/Sub retries message delivery for this subscription. If not set, the default retry policy is applied. This generally implies that messages will be retried as soon as possible for healthy subscribers. RetryPolicy will be triggered on NACKs or acknowledgement deadline exceeded events for a given message. :type retry_policy: Union[Dict, google.cloud.pubsub_v1.types.RetryPolicy] :param retry: (Optional) A retry object used to retry requests. If None is specified, requests will not be retried. :type retry: google.api_core.retry.Retry :param timeout: (Optional) The amount of time, in seconds, to wait for the request to complete. Note that if retry is specified, the timeout applies to each individual attempt. :type timeout: float :param metadata: (Optional) Additional metadata that is provided to the method. :type metadata: Sequence[Tuple[str, str]]] :return: subscription name which will be the system-generated value if the ``subscription`` parameter is not supplied :rtype: str """ subscriber = self.subscriber_client if not subscription: subscription = 'sub-{}'.format(uuid4()) if not subscription_project_id: subscription_project_id = project_id # Add airflow-version label to the subscription labels = labels or {} labels['airflow-version'] = 'v' + version.replace('.', '-').replace('+', '-') # pylint: disable=no-member subscription_path = SubscriberClient.subscription_path(subscription_project_id, subscription) topic_path = SubscriberClient.topic_path(project_id, topic) self.log.info("Creating subscription (path) %s for topic (path) %a", subscription_path, topic_path) try: subscriber.create_subscription( name=subscription_path, topic=topic_path, push_config=push_config, ack_deadline_seconds=ack_deadline_secs, retain_acked_messages=retain_acked_messages, message_retention_duration=message_retention_duration, labels=labels, enable_message_ordering=enable_message_ordering, expiration_policy=expiration_policy, filter_=filter_, dead_letter_policy=dead_letter_policy, retry_policy=retry_policy, retry=retry, timeout=timeout, metadata=metadata, ) except AlreadyExists: self.log.warning('Subscription already exists: %s', subscription_path) if fail_if_exists: raise PubSubException('Subscription already exists: {}'.format(subscription_path)) except GoogleAPICallError as e: raise PubSubException('Error creating subscription {}'.format(subscription_path), e) self.log.info("Created subscription (path) %s for topic (path) %s", subscription_path, topic_path) return subscription
def create_subscription( self, topic: str, project_id: Optional[str] = None, subscription: Optional[str] = None, subscription_project_id: Optional[str] = None, ack_deadline_secs: int = 10, fail_if_exists: bool = False, push_config: Optional[Union[Dict, PushConfig]] = None, retain_acked_messages: Optional[bool] = None, message_retention_duration: Optional[Union[Dict, Duration]] = None, labels: Optional[Dict[str, str]] = None, retry: Optional[Retry] = None, timeout: Optional[float] = None, metadata: Optional[Sequence[Tuple[str, str]]] = None, ) -> str: """ Creates a Pub/Sub subscription, if it does not already exist. :param topic: the Pub/Sub topic name that the subscription will be bound to create; do not include the ``projects/{project}/subscriptions/`` prefix. :type topic: str :param project_id: Optional, the GCP project ID of the topic that the subscription will be bound to. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str :param subscription: the Pub/Sub subscription name. If empty, a random name will be generated using the uuid module :type subscription: str :param subscription_project_id: the GCP project ID where the subscription will be created. If unspecified, ``project_id`` will be used. :type subscription_project_id: str :param ack_deadline_secs: Number of seconds that a subscriber has to acknowledge each message pulled from the subscription :type ack_deadline_secs: int :param fail_if_exists: if set, raise an exception if the topic already exists :type fail_if_exists: bool :param push_config: If push delivery is used with this subscription, this field is used to configure it. An empty ``pushConfig`` signifies that the subscriber will pull and ack messages using API methods. :type push_config: Union[Dict, google.cloud.pubsub_v1.types.PushConfig] :param retain_acked_messages: Indicates whether to retain acknowledged messages. If true, then messages are not expunged from the subscription's backlog, even if they are acknowledged, until they fall out of the ``message_retention_duration`` window. This must be true if you would like to Seek to a timestamp. :type retain_acked_messages: bool :param message_retention_duration: How long to retain unacknowledged messages in the subscription's backlog, from the moment a message is published. If ``retain_acked_messages`` is true, then this also configures the retention of acknowledged messages, and thus configures how far back in time a ``Seek`` can be done. Defaults to 7 days. Cannot be more than 7 days or less than 10 minutes. :type message_retention_duration: Union[Dict, google.cloud.pubsub_v1.types.Duration] :param labels: Client-assigned labels; see https://cloud.google.com/pubsub/docs/labels :type labels: Dict[str, str] :param retry: (Optional) A retry object used to retry requests. If None is specified, requests will not be retried. :type retry: google.api_core.retry.Retry :param timeout: (Optional) The amount of time, in seconds, to wait for the request to complete. Note that if retry is specified, the timeout applies to each individual attempt. :type timeout: float :param metadata: (Optional) Additional metadata that is provided to the method. :type metadata: Sequence[Tuple[str, str]]] :return: subscription name which will be the system-generated value if the ``subscription`` parameter is not supplied :rtype: str """ assert project_id is not None subscriber = self.subscriber_client if not subscription: subscription = 'sub-{}'.format(uuid4()) if not subscription_project_id: subscription_project_id = project_id # Add airflow-version label to the subscription labels = labels or {} labels['airflow-version'] = 'v' + version.replace('.', '-').replace('+', '-') # pylint: disable=no-member subscription_path = SubscriberClient.subscription_path(subscription_project_id, subscription) topic_path = SubscriberClient.topic_path(project_id, topic) self.log.info("Creating subscription (path) %s for topic (path) %a", subscription_path, topic_path) try: subscriber.create_subscription( name=subscription_path, topic=topic_path, push_config=push_config, ack_deadline_seconds=ack_deadline_secs, retain_acked_messages=retain_acked_messages, message_retention_duration=message_retention_duration, labels=labels, retry=retry, timeout=timeout, metadata=metadata, ) except AlreadyExists: self.log.warning('Subscription already exists: %s', subscription_path) if fail_if_exists: raise PubSubException('Subscription already exists: {}'.format(subscription_path)) except GoogleAPICallError as e: raise PubSubException('Error creating subscription {}'.format(subscription_path), e) self.log.info("Created subscription (path) %s for topic (path) %s", subscription_path, topic_path) return subscription