def build_producers(self): producers = [] project = "testing" try: # Create pubsub topics publisher = PublisherClient() publisher.create_topic(publisher.topic_path(project, "test10")) publisher.create_topic(publisher.topic_path(project, "test20")) # Create pubsub subscriptions subscriber = SubscriberClient() subscriber.create_subscription( subscriber.subscription_path(project, "test10"), subscriber.topic_path(project, "test10")) subscriber.create_subscription( subscriber.subscription_path(project, "test20"), subscriber.topic_path(project, "test20")) except AlreadyExists as _: pass producers.append( StreamBuilderFactory.get_producer_pubsub().set_project_id( project).set_output_topic("test10").build()) producers.append( StreamBuilderFactory.get_producer_kafka().set_broker_servers( "localhost:9092").set_output_topic("test1").build()) producers.append( StreamBuilderFactory.get_producer_kafka().set_broker_servers( "localhost:9092").set_output_topic("test3").build()) return producers
class GooglePubSubPublisher: def __init__(self, **kwargs): super().__init__(**kwargs) self._logger = logging.getLogger(__name__) self._publisher = PublisherClient() def send_message(self, message): serializer = message.get_serializer() json_data = JSONRenderer().render(serializer.data) topic_path = self._publisher.topic_path( settings.GCLOUD_PUBSUB_PROJECT_ID, serializer.data['type']) future = self._publisher.publish(topic_path, data=json_data) future.add_done_callback(self._publish_callback) def _publish_callback(self, message_future): if message_future.exception(timeout=30): self._logger.info( f'Publishing message on {message_future.exception()} threw an Exception.' ) else: self._logger.info(message_future.result()) def create_topic(self, topic: str): topic_path = self._publisher.topic_path( settings.GCLOUD_PUBSUB_PROJECT_ID, topic) try: self._publisher.create_topic(topic_path) except AlreadyExists as e: self._logger.info(f'Topic {topic} already exists.')
def check_pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, topic): # check topic in pubsub project_path = pubsub_client.project_path(GOOGLE_CLOUD_PROJECT) topics = [ topic_ref.name for topic_ref in pubsub_client.list_topics(project_path) ] if topic not in topics: log.info(f'create pubsub topic {topic}') pubsub_client.create_topic(topic)
def topic(publisher: PublisherClient, route: Tuple[str, str, str]) -> Iterator[str]: name = route[1] try: publisher.create_topic(name) delete = True except AlreadyExists: delete = False try: yield name finally: if delete: publisher.delete_topic(name)
class PubSubPublisher(Publisher): def __init__(self, project_id, credentials_file=None): logger.info('connecting to pubsub') if credentials_file: service_account_info = json.load(open(credentials_file)) publisher_audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher' credentials_pub = jwt.Credentials.from_service_account_info( service_account_info, audience=publisher_audience, ) self._client = PublisherClient(credentials=credentials_pub) else: self._client = PublisherClient() self._project_id = project_id def _publish(self, topic_id, message, **kwargs): logger.info('publishing message', topic_id=topic_id) topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member response: Future = self._client.publish(topic_path, message.encode('utf-8'), **kwargs) return response def publish(self, topic_id, message: bytes, fulfilment_request_transaction_id: str): response = self._publish(topic_id, message, tx_id=fulfilment_request_transaction_id) try: # Resolve the future message_id = response.result() logger.info( # pragma: no cover 'message published successfully', topic_id=topic_id, message_id=message_id, fulfilment_request_transaction_id=fulfilment_request_transaction_id, ) except Exception as ex: # pylint:disable=broad-except logger.exception( 'message publication failed', topic_id=topic_id, ) raise PublicationFailed(ex) def create_topic(self, topic_id): try: logger.info('creating topic') topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member self._client.create_topic(request={'name': topic_path}) # pylint: disable=no-member except AlreadyExists: logger.info('Topic already exists') except Exception as ex: # pylint:disable=broad-except logger.error( 'failed', exc_info=ex, )
def test_submit_pubsub_topic_not_found( integration_test: IntegrationTest, publisher: PublisherClient, subscriber: SubscriberClient, subscription: str, topic: str, ): publisher.delete_topic(topic) try: integration_test.assert_accepted_and_queued() finally: subscriber.delete_subscription(subscription) publisher.create_topic(topic) subscriber.create_subscription(subscription, topic) integration_test.assert_flushed_and_delivered()
def emulator(node_pools: List[str], options: Dict[str, Any]) -> Iterator[str]: if "emulator" in node_pools: kube_apply("kube/emulator.deploy.yml", **options) kube_apply("kube/emulator.svc.yml") yield "emulator:8000" elif "server" in node_pools: pubsub = PublisherClient() topic = f"projects/{options['project']}/topics/{options['topic']}" try: pubsub.create_topic(topic) except AlreadyExists: pass yield "" pubsub.delete_topic(topic) else: yield ""
def ensure_topic_and_subscription(): from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient publisher_client = PublisherClient() try: publisher_client.create_topic( publisher_client.topic_path(PROJECT_ID, TOPIC_NAME)) except AlreadyExists: pass subscriber_client = SubscriberClient() try: subscriber_client.create_subscription( subscriber_client.subscription_path(PROJECT_ID, SUBSCRIPTION_NAME), publisher_client.topic_path(PROJECT_ID, TOPIC_NAME)) except AlreadyExists: pass
def topic_path( publisher_client: pubsub_v1.PublisherClient ) -> Generator[str, None, None]: topic_path = publisher_client.topic_path(PROJECT_ID, TOPIC_ID) try: topic = publisher_client.create_topic(request={"name": topic_path}) yield topic.name except AlreadyExists: yield topic_path publisher_client.delete_topic(request={"topic": topic_path})
def setUpClass(cls): super().setUpClass() cls.project = os.environ.get("PUBSUB_PROJECT", "testing") cls.credentials = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS", None) for subscription in ["test0", "test1"]: try: publisher = PublisherClient() publisher.create_topic( publisher.topic_path(cls.project, subscription)) except AlreadyExists: pass try: subscriber = SubscriberClient() subscriber.create_subscription( subscriber.subscription_path(cls.project, subscription), subscriber.topic_path(cls.project, subscription)) except AlreadyExists: pass
def topic_path( publisher_client: pubsub_v1.PublisherClient, ) -> Generator[str, None, None]: topic_path = publisher_client.topic_path(PROJECT_ID, TOPIC_ID) try: topic = publisher_client.get_topic(request={"topic": topic_path}) except NotFound: topic = publisher_client.create_topic(request={"name": topic_path}) yield topic.name try: publisher_client.delete_topic(request={"topic": topic.name}) except NotFound: pass
def proto_topic(publisher_client: pubsub_v1.PublisherClient, proto_schema: str) -> Generator[str, None, None]: proto_topic_path = publisher_client.topic_path(PROJECT_ID, PROTO_TOPIC_ID) try: proto_topic = publisher_client.get_topic( request={"topic": proto_topic_path}) except NotFound: proto_topic = publisher_client.create_topic( request={ "name": proto_topic_path, "schema_settings": { "schema": proto_schema, "encoding": Encoding.BINARY, }, }) yield proto_topic.name publisher_client.delete_topic(request={"topic": proto_topic.name})
def avro_topic(publisher_client: pubsub_v1.PublisherClient, avro_schema: str) -> Generator[str, None, None]: from google.pubsub_v1.types import Encoding avro_topic_path = publisher_client.topic_path(PROJECT_ID, AVRO_TOPIC_ID) try: avro_topic = publisher_client.get_topic( request={"topic": avro_topic_path}) except NotFound: avro_topic = publisher_client.create_topic( request={ "name": avro_topic_path, "schema_settings": { "schema": avro_schema, "encoding": Encoding.BINARY, }, }) yield avro_topic.name publisher_client.delete_topic(request={"topic": avro_topic.name})
def pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, project_id): topic_id = resource_prefix() topic_path = pubsub_v1.PublisherClient.topic_path(project_id, topic_id) pubsub_client.create_topic(name=topic_path) yield topic_path pubsub_client.delete_topic(topic=topic_path)