def build_producers(self):
producers = []
project = "testing"
try:
# Create pubsub topics
publisher = PublisherClient()
publisher.create_topic(publisher.topic_path(project, "test10"))
publisher.create_topic(publisher.topic_path(project, "test20"))
# Create pubsub subscriptions
subscriber = SubscriberClient()
subscriber.create_subscription(
subscriber.subscription_path(project, "test10"),
subscriber.topic_path(project, "test10"))
subscriber.create_subscription(
subscriber.subscription_path(project, "test20"),
subscriber.topic_path(project, "test20"))
except AlreadyExists as _:
pass
producers.append(
StreamBuilderFactory.get_producer_pubsub().set_project_id(
project).set_output_topic("test10").build())
producers.append(
StreamBuilderFactory.get_producer_kafka().set_broker_servers(
"localhost:9092").set_output_topic("test1").build())
producers.append(
StreamBuilderFactory.get_producer_kafka().set_broker_servers(
"localhost:9092").set_output_topic("test3").build())
return producers
class GooglePubSubPublisher:
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._logger = logging.getLogger(__name__)
self._publisher = PublisherClient()
def send_message(self, message):
serializer = message.get_serializer()
json_data = JSONRenderer().render(serializer.data)
topic_path = self._publisher.topic_path(
settings.GCLOUD_PUBSUB_PROJECT_ID, serializer.data['type'])
future = self._publisher.publish(topic_path, data=json_data)
future.add_done_callback(self._publish_callback)
def _publish_callback(self, message_future):
if message_future.exception(timeout=30):
self._logger.info(
f'Publishing message on {message_future.exception()} threw an Exception.'
)
else:
self._logger.info(message_future.result())
def create_topic(self, topic: str):
topic_path = self._publisher.topic_path(
settings.GCLOUD_PUBSUB_PROJECT_ID, topic)
try:
self._publisher.create_topic(topic_path)
except AlreadyExists as e:
self._logger.info(f'Topic {topic} already exists.')
def check_pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, topic):
# check topic in pubsub
project_path = pubsub_client.project_path(GOOGLE_CLOUD_PROJECT)
topics = [
topic_ref.name for topic_ref in pubsub_client.list_topics(project_path)
]
if topic not in topics:
log.info(f'create pubsub topic {topic}')
pubsub_client.create_topic(topic)
def topic(publisher: PublisherClient, route: Tuple[str, str, str]) -> Iterator[str]:
name = route[1]
try:
publisher.create_topic(name)
delete = True
except AlreadyExists:
delete = False
try:
yield name
finally:
if delete:
publisher.delete_topic(name)
class PubSubPublisher(Publisher):
def __init__(self, project_id, credentials_file=None):
logger.info('connecting to pubsub')
if credentials_file:
service_account_info = json.load(open(credentials_file))
publisher_audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher'
credentials_pub = jwt.Credentials.from_service_account_info(
service_account_info, audience=publisher_audience,
)
self._client = PublisherClient(credentials=credentials_pub)
else:
self._client = PublisherClient()
self._project_id = project_id
def _publish(self, topic_id, message, **kwargs):
logger.info('publishing message', topic_id=topic_id)
topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member
response: Future = self._client.publish(topic_path, message.encode('utf-8'), **kwargs)
return response
def publish(self, topic_id, message: bytes, fulfilment_request_transaction_id: str):
response = self._publish(topic_id, message, tx_id=fulfilment_request_transaction_id)
try:
# Resolve the future
message_id = response.result()
logger.info( # pragma: no cover
'message published successfully',
topic_id=topic_id,
message_id=message_id,
fulfilment_request_transaction_id=fulfilment_request_transaction_id,
)
except Exception as ex: # pylint:disable=broad-except
logger.exception(
'message publication failed',
topic_id=topic_id,
)
raise PublicationFailed(ex)
def create_topic(self, topic_id):
try:
logger.info('creating topic')
topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member
self._client.create_topic(request={'name': topic_path}) # pylint: disable=no-member
except AlreadyExists:
logger.info('Topic already exists')
except Exception as ex: # pylint:disable=broad-except
logger.error(
'failed', exc_info=ex,
)
def test_submit_pubsub_topic_not_found(
integration_test: IntegrationTest,
publisher: PublisherClient,
subscriber: SubscriberClient,
subscription: str,
topic: str,
):
publisher.delete_topic(topic)
try:
integration_test.assert_accepted_and_queued()
finally:
subscriber.delete_subscription(subscription)
publisher.create_topic(topic)
subscriber.create_subscription(subscription, topic)
integration_test.assert_flushed_and_delivered()
def emulator(node_pools: List[str], options: Dict[str, Any]) -> Iterator[str]:
if "emulator" in node_pools:
kube_apply("kube/emulator.deploy.yml", **options)
kube_apply("kube/emulator.svc.yml")
yield "emulator:8000"
elif "server" in node_pools:
pubsub = PublisherClient()
topic = f"projects/{options['project']}/topics/{options['topic']}"
try:
pubsub.create_topic(topic)
except AlreadyExists:
pass
yield ""
pubsub.delete_topic(topic)
else:
yield ""
def ensure_topic_and_subscription():
from google.cloud.pubsub_v1 import PublisherClient, SubscriberClient
publisher_client = PublisherClient()
try:
publisher_client.create_topic(
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
subscriber_client = SubscriberClient()
try:
subscriber_client.create_subscription(
subscriber_client.subscription_path(PROJECT_ID, SUBSCRIPTION_NAME),
publisher_client.topic_path(PROJECT_ID, TOPIC_NAME))
except AlreadyExists:
pass
def topic_path(
publisher_client: pubsub_v1.PublisherClient
) -> Generator[str, None, None]:
topic_path = publisher_client.topic_path(PROJECT_ID, TOPIC_ID)
try:
topic = publisher_client.create_topic(request={"name": topic_path})
yield topic.name
except AlreadyExists:
yield topic_path
publisher_client.delete_topic(request={"topic": topic_path})
def setUpClass(cls):
super().setUpClass()
cls.project = os.environ.get("PUBSUB_PROJECT", "testing")
cls.credentials = os.environ.get("GOOGLE_APPLICATION_CREDENTIALS",
None)
for subscription in ["test0", "test1"]:
try:
publisher = PublisherClient()
publisher.create_topic(
publisher.topic_path(cls.project, subscription))
except AlreadyExists:
pass
try:
subscriber = SubscriberClient()
subscriber.create_subscription(
subscriber.subscription_path(cls.project, subscription),
subscriber.topic_path(cls.project, subscription))
except AlreadyExists:
pass
def topic_path(
publisher_client: pubsub_v1.PublisherClient,
) -> Generator[str, None, None]:
topic_path = publisher_client.topic_path(PROJECT_ID, TOPIC_ID)
try:
topic = publisher_client.get_topic(request={"topic": topic_path})
except NotFound:
topic = publisher_client.create_topic(request={"name": topic_path})
yield topic.name
try:
publisher_client.delete_topic(request={"topic": topic.name})
except NotFound:
pass
def proto_topic(publisher_client: pubsub_v1.PublisherClient,
proto_schema: str) -> Generator[str, None, None]:
proto_topic_path = publisher_client.topic_path(PROJECT_ID, PROTO_TOPIC_ID)
try:
proto_topic = publisher_client.get_topic(
request={"topic": proto_topic_path})
except NotFound:
proto_topic = publisher_client.create_topic(
request={
"name": proto_topic_path,
"schema_settings": {
"schema": proto_schema,
"encoding": Encoding.BINARY,
},
})
yield proto_topic.name
publisher_client.delete_topic(request={"topic": proto_topic.name})
def avro_topic(publisher_client: pubsub_v1.PublisherClient,
avro_schema: str) -> Generator[str, None, None]:
from google.pubsub_v1.types import Encoding
avro_topic_path = publisher_client.topic_path(PROJECT_ID, AVRO_TOPIC_ID)
try:
avro_topic = publisher_client.get_topic(
request={"topic": avro_topic_path})
except NotFound:
avro_topic = publisher_client.create_topic(
request={
"name": avro_topic_path,
"schema_settings": {
"schema": avro_schema,
"encoding": Encoding.BINARY,
},
})
yield avro_topic.name
publisher_client.delete_topic(request={"topic": avro_topic.name})
def pubsub_topic(pubsub_client: pubsub_v1.PublisherClient, project_id):
topic_id = resource_prefix()
topic_path = pubsub_v1.PublisherClient.topic_path(project_id, topic_id)
pubsub_client.create_topic(name=topic_path)
yield topic_path
pubsub_client.delete_topic(topic=topic_path)
