class PubSubPublisher(Publisher): def __init__(self): self._client = PublisherClient() _, self._project_id = google.auth.default() def _publish(self, topic_id, message): logger.info("publishing message", topic_id=topic_id) topic_path = self._client.topic_path(self._project_id, topic_id) response: Future = self._client.publish(topic_path, message) return response def publish(self, topic_id, message: bytes, fulfilment_request_transaction_id: str): response = self._publish(topic_id, message) try: # Resolve the future message_id = response.result() logger.info( # pragma: no cover "message published successfully", topic_id=topic_id, message_id=message_id, fulfilment_request_transaction_id= fulfilment_request_transaction_id, ) except Exception as ex: # pylint:disable=broad-except logger.exception( "message publication failed", topic_id=topic_id, ) raise PublicationFailed(ex)
def publish_avro_records(project_id, topic_id, avsc_file): """Pulbish a BINARY or JSON encoded message to a topic configured with an Avro schema.""" # [START pubsub_publish_avro_records] from avro.io import BinaryEncoder, DatumWriter import avro import io import json from google.api_core.exceptions import NotFound from google.cloud.pubsub import PublisherClient from google.pubsub_v1.types import Encoding # TODO(developer): Replace these variables before running the sample. # project_id = "your-project-id" # topic_id = "your-topic-id" # avsc_file = "path/to/an/avro/schema/file/(.avsc)/formatted/in/json" publisher_client = PublisherClient() topic_path = publisher_client.topic_path(project_id, topic_id) # Prepare to write Avro records to the binary output stream. avro_schema = avro.schema.parse(open(avsc_file, "rb").read()) writer = DatumWriter(avro_schema) bout = io.BytesIO() # Prepare some data using a Python dictionary that matches the Avro schema record = {"name": "Alaska", "post_abbr": "AK"} try: # Get the topic encoding type. topic = publisher_client.get_topic(request={"topic": topic_path}) encoding = topic.schema_settings.encoding # Encode the data according to the message serialization type. if encoding == Encoding.BINARY: encoder = BinaryEncoder(bout) writer.write(record, encoder) data = bout.getvalue() print(f"Preparing a binary-encoded message:\n{data}") elif encoding == Encoding.JSON: data = json.dumps(record).encode("utf-8") print(f"Preparing a JSON-encoded message:\n{data}") else: print(f"No encoding specified in {topic_path}. Abort.") exit(0) future = publisher_client.publish(topic_path, data) print(f"Published message ID: {future.result()}") except NotFound: print(f"{topic_id} not found.")
def publish_error_report(error: str): publisher = PublisherClient() function_name = os.environ['FUNCTION_TARGET'] error_report_topic = os.environ['ERROR_REPORT_TOPIC'] topic_name = f'projects/{project_id}/topics/{error_report_topic}' try: publisher.publish( topic_name, data=error.encode('utf-8'), projectId=project_id, functionName=function_name, eventTime=str(int(time.time())) ) except Exception as e: # エラー時はリトライしない print(e)
def create_topic_with_schema(project_id, topic_id, schema_id, message_encoding): """Create a topic resource with a schema.""" # [START pubsub_create_topic_with_schema] from google.api_core.exceptions import AlreadyExists, InvalidArgument from google.cloud.pubsub import PublisherClient, SchemaServiceClient from google.pubsub_v1.types import Encoding # TODO(developer): Replace these variables before running the sample. # project_id = "your-project-id" # topic_id = "your-topic-id" # schema_id = "your-schema-id" # Choose either BINARY or JSON as valid message encoding in this topic. # message_encoding = "BINARY" publisher_client = PublisherClient() topic_path = publisher_client.topic_path(project_id, topic_id) schema_client = SchemaServiceClient() schema_path = schema_client.schema_path(project_id, schema_id) if message_encoding == "BINARY": encoding = Encoding.BINARY elif message_encoding == "JSON": encoding = Encoding.JSON else: encoding = Encoding.ENCODING_UNSPECIFIED try: response = publisher_client.create_topic( request={ "name": topic_path, "schema_settings": { "schema": schema_path, "encoding": encoding }, }) print(f"Created a topic:\n{response}") except AlreadyExists: print(f"{topic_id} already exists.") except InvalidArgument: print( "Please choose either BINARY or JSON as a valid message encoding type." )
def publish_proto_messages(project_id, topic_id): """Publish a BINARY or JSON encoded message to a topic configured with a protobuf schema.""" # [START pubsub_publish_proto_messages] from google.api_core.exceptions import NotFound from google.cloud.pubsub import PublisherClient from google.protobuf.json_format import MessageToJson from google.pubsub_v1.types import Encoding from utilities import us_states_pb2 # TODO(developer): Replace these variables before running the sample. # project_id = "your-project-id" # topic_id = "your-topic-id" publisher_client = PublisherClient() topic_path = publisher_client.topic_path(project_id, topic_id) try: # Get the topic encoding type. topic = publisher_client.get_topic(request={"topic": topic_path}) encoding = topic.schema_settings.encoding # Instantiate a protoc-generated class defined in `us-states.proto`. state = us_states_pb2.StateProto() state.name = "Alaska" state.post_abbr = "AK" # Encode the data according to the message serialization type. if encoding == Encoding.BINARY: data = state.SerializeToString() print(f"Preparing a binary-encoded message:\n{data}") elif encoding == Encoding.JSON: json_object = MessageToJson(state) data = str(json_object).encode("utf-8") print(f"Preparing a JSON-encoded message:\n{data}") else: print(f"No encoding specified in {topic_path}. Abort.") exit(0) future = publisher_client.publish(topic_path, data) print(f"Published message ID: {future.result()}") except NotFound: print(f"{topic_id} not found.")
def test_pubsub(self): # create topic if needed client = SubscriberClient() try: topic_id = 'container-analysis-occurrences-v1' topic_name = {"name": f"projects/{PROJECT_ID}/topics/{topic_id}"} publisher = PublisherClient() publisher.create_topic(topic_name) except AlreadyExists: pass subscription_id = 'container-analysis-test-{}'.format(uuid.uuid4()) subscription_name = client.subscription_path(PROJECT_ID, subscription_id) samples.create_occurrence_subscription(subscription_id, PROJECT_ID) # I can not make it pass with multiple messages. My guess is # the server started to dedup? message_count = 1 try: job_done = threading.Event() receiver = MessageReceiver(message_count, job_done) client.subscribe(subscription_name, receiver.pubsub_callback) for i in range(message_count): occ = samples.create_occurrence(self.image_url, self.note_id, PROJECT_ID, PROJECT_ID) time.sleep(SLEEP_TIME) samples.delete_occurrence(basename(occ.name), PROJECT_ID) time.sleep(SLEEP_TIME) # We saw occational failure with 60 seconds timeout, so we bumped it # to 180 seconds. # See also: python-docs-samples/issues/2894 job_done.wait(timeout=180) print('done. msg_count = {}'.format(receiver.msg_count)) assert message_count <= receiver.msg_count finally: # clean up client.delete_subscription({"subscription": subscription_name})
def test_pubsub(self): # create topic if needed client = SubscriberClient() try: topic_id = 'container-analysis-occurrences-v1' topic_name = client.topic_path(PROJECT_ID, topic_id) publisher = PublisherClient() publisher.create_topic(topic_name) except AlreadyExists: pass subscription_id = 'drydockOccurrences' subscription_name = client.subscription_path(PROJECT_ID, subscription_id) samples.create_occurrence_subscription(subscription_id, PROJECT_ID) tries = 0 success = False while not success and tries < TRY_LIMIT: print(tries) tries += 1 receiver = samples.MessageReceiver() client.subscribe(subscription_name, receiver.pubsub_callback) # test adding 3 more occurrences total_created = 3 for _ in range(total_created): occ = samples.create_occurrence(self.image_url, self.note_id, PROJECT_ID, PROJECT_ID) sleep(SLEEP_TIME) samples.delete_occurrence(basename(occ.name), PROJECT_ID) sleep(SLEEP_TIME) print('done. msg_count = {}'.format(receiver.msg_count)) success = receiver.msg_count == total_created if receiver.msg_count != total_created: raise AssertionError # clean up client.delete_subscription(subscription_name)
def test_pubsub(self): # create topic if needed client = SubscriberClient() try: topic_id = 'container-analysis-occurrences-v1' topic_name = client.topic_path(PROJECT_ID, topic_id) publisher = PublisherClient() publisher.create_topic(topic_name) except AlreadyExists: pass subscription_id = 'container-analysis-test-{}'.format(uuid.uuid4()) subscription_name = client.subscription_path(PROJECT_ID, subscription_id) samples.create_occurrence_subscription(subscription_id, PROJECT_ID) # I can not make it pass with multiple messages. My guess is # the server started to dedup? message_count = 1 try: job_done = threading.Event() receiver = MessageReceiver(message_count, job_done) client.subscribe(subscription_name, receiver.pubsub_callback) for i in range(message_count): occ = samples.create_occurrence(self.image_url, self.note_id, PROJECT_ID, PROJECT_ID) time.sleep(SLEEP_TIME) samples.delete_occurrence(basename(occ.name), PROJECT_ID) time.sleep(SLEEP_TIME) job_done.wait(timeout=60) print('done. msg_count = {}'.format(receiver.msg_count)) assert message_count <= receiver.msg_count finally: # clean up client.delete_subscription(subscription_name)
def clients(): cleanup() run_cmd = shlex.split( f"docker run -d -p 8085:8085 --name {CONTAINER} google/cloud-sdk " "gcloud beta emulators pubsub start " f"--project={PROJECT} --host-port=0.0.0.0:8085" ) subprocess.check_call(run_cmd) def predicate(): cmd = shlex.split(f"docker logs {CONTAINER}") logs = subprocess.check_output(cmd, stderr=subprocess.STDOUT) return b"INFO: Server started, listening on 8085" in logs wait_for(predicate, 10, period=0.1) os.environ["PUBSUB_EMULATOR_HOST"] = "localhost:8085" os.environ["PUBSUB_PROJECT_ID"] = PROJECT try: yield (PublisherClient(), SubscriberClient()) finally: cleanup(fail=True)
def __init__(self): self._client = PublisherClient() _, self._project_id = google.auth.default()
def publisher_client(): yield PublisherClient()
from google.api_core.exceptions import AlreadyExists from google.cloud.pubsub import SchemaServiceClient from google.pubsub_v1.types import Schema from google.cloud import pubsub_v1 from google.cloud.pubsub import PublisherClient import json from google.pubsub_v1.types import Encoding publisher = pubsub_v1.PublisherClient() publisher_client = PublisherClient() project_id = "gcplayproject" schema_id = "userdetails" avsc_file = "/home/googlecloud/mynewdata/avro.avsc" topic_id = "schematopic" topic_path = publisher.topic_path(project_id, topic_id) project_path = f"projects/{project_id}" # Read a JSON-formatted Avro schema file as a string. with open(avsc_file, "rb") as f: avsc_source = f.read().decode("utf-8") schema_client = SchemaServiceClient() schema_path = schema_client.schema_path(project_id, schema_id) schema = Schema(name=schema_path, type_=Schema.Type.AVRO, definition=avsc_source) def createschema():
def publisher_client() -> Generator[pubsub_v1.PublisherClient, None, None]: yield PublisherClient()
def publish(self, data): from google.cloud.pubsub import PublisherClient client = PublisherClient() sub_path = client.topic_path(self.project, self._tf_name) client.publish(sub_path, data).result()