class PubSubPublisher(Publisher):
def __init__(self):
self._client = PublisherClient()
_, self._project_id = google.auth.default()
def _publish(self, topic_id, message):
logger.info("publishing message", topic_id=topic_id)
topic_path = self._client.topic_path(self._project_id, topic_id)
response: Future = self._client.publish(topic_path, message)
return response
def publish(self, topic_id, message: bytes,
fulfilment_request_transaction_id: str):
response = self._publish(topic_id, message)
try:
# Resolve the future
message_id = response.result()
logger.info( # pragma: no cover
"message published successfully",
topic_id=topic_id,
message_id=message_id,
fulfilment_request_transaction_id=
fulfilment_request_transaction_id,
)
except Exception as ex: # pylint:disable=broad-except
logger.exception(
"message publication failed",
topic_id=topic_id,
)
raise PublicationFailed(ex)
def publish_avro_records(project_id, topic_id, avsc_file):
"""Pulbish a BINARY or JSON encoded message to a topic configured with an Avro schema."""
# [START pubsub_publish_avro_records]
from avro.io import BinaryEncoder, DatumWriter
import avro
import io
import json
from google.api_core.exceptions import NotFound
from google.cloud.pubsub import PublisherClient
from google.pubsub_v1.types import Encoding
# TODO(developer): Replace these variables before running the sample.
# project_id = "your-project-id"
# topic_id = "your-topic-id"
# avsc_file = "path/to/an/avro/schema/file/(.avsc)/formatted/in/json"
publisher_client = PublisherClient()
topic_path = publisher_client.topic_path(project_id, topic_id)
# Prepare to write Avro records to the binary output stream.
avro_schema = avro.schema.parse(open(avsc_file, "rb").read())
writer = DatumWriter(avro_schema)
bout = io.BytesIO()
# Prepare some data using a Python dictionary that matches the Avro schema
record = {"name": "Alaska", "post_abbr": "AK"}
try:
# Get the topic encoding type.
topic = publisher_client.get_topic(request={"topic": topic_path})
encoding = topic.schema_settings.encoding
# Encode the data according to the message serialization type.
if encoding == Encoding.BINARY:
encoder = BinaryEncoder(bout)
writer.write(record, encoder)
data = bout.getvalue()
print(f"Preparing a binary-encoded message:\n{data}")
elif encoding == Encoding.JSON:
data = json.dumps(record).encode("utf-8")
print(f"Preparing a JSON-encoded message:\n{data}")
else:
print(f"No encoding specified in {topic_path}. Abort.")
exit(0)
future = publisher_client.publish(topic_path, data)
print(f"Published message ID: {future.result()}")
except NotFound:
print(f"{topic_id} not found.")
def publish_error_report(error: str):
publisher = PublisherClient()
function_name = os.environ['FUNCTION_TARGET']
error_report_topic = os.environ['ERROR_REPORT_TOPIC']
topic_name = f'projects/{project_id}/topics/{error_report_topic}'
try:
publisher.publish(
topic_name,
data=error.encode('utf-8'),
projectId=project_id,
functionName=function_name,
eventTime=str(int(time.time()))
)
except Exception as e:
# エラー時はリトライしない
print(e)
def create_topic_with_schema(project_id, topic_id, schema_id,
message_encoding):
"""Create a topic resource with a schema."""
# [START pubsub_create_topic_with_schema]
from google.api_core.exceptions import AlreadyExists, InvalidArgument
from google.cloud.pubsub import PublisherClient, SchemaServiceClient
from google.pubsub_v1.types import Encoding
# TODO(developer): Replace these variables before running the sample.
# project_id = "your-project-id"
# topic_id = "your-topic-id"
# schema_id = "your-schema-id"
# Choose either BINARY or JSON as valid message encoding in this topic.
# message_encoding = "BINARY"
publisher_client = PublisherClient()
topic_path = publisher_client.topic_path(project_id, topic_id)
schema_client = SchemaServiceClient()
schema_path = schema_client.schema_path(project_id, schema_id)
if message_encoding == "BINARY":
encoding = Encoding.BINARY
elif message_encoding == "JSON":
encoding = Encoding.JSON
else:
encoding = Encoding.ENCODING_UNSPECIFIED
try:
response = publisher_client.create_topic(
request={
"name": topic_path,
"schema_settings": {
"schema": schema_path,
"encoding": encoding
},
})
print(f"Created a topic:\n{response}")
except AlreadyExists:
print(f"{topic_id} already exists.")
except InvalidArgument:
print(
"Please choose either BINARY or JSON as a valid message encoding type."
)
def publish_proto_messages(project_id, topic_id):
"""Publish a BINARY or JSON encoded message to a topic configured with a protobuf schema."""
# [START pubsub_publish_proto_messages]
from google.api_core.exceptions import NotFound
from google.cloud.pubsub import PublisherClient
from google.protobuf.json_format import MessageToJson
from google.pubsub_v1.types import Encoding
from utilities import us_states_pb2
# TODO(developer): Replace these variables before running the sample.
# project_id = "your-project-id"
# topic_id = "your-topic-id"
publisher_client = PublisherClient()
topic_path = publisher_client.topic_path(project_id, topic_id)
try:
# Get the topic encoding type.
topic = publisher_client.get_topic(request={"topic": topic_path})
encoding = topic.schema_settings.encoding
# Instantiate a protoc-generated class defined in `us-states.proto`.
state = us_states_pb2.StateProto()
state.name = "Alaska"
state.post_abbr = "AK"
# Encode the data according to the message serialization type.
if encoding == Encoding.BINARY:
data = state.SerializeToString()
print(f"Preparing a binary-encoded message:\n{data}")
elif encoding == Encoding.JSON:
json_object = MessageToJson(state)
data = str(json_object).encode("utf-8")
print(f"Preparing a JSON-encoded message:\n{data}")
else:
print(f"No encoding specified in {topic_path}. Abort.")
exit(0)
future = publisher_client.publish(topic_path, data)
print(f"Published message ID: {future.result()}")
except NotFound:
print(f"{topic_id} not found.")
def test_pubsub(self):
# create topic if needed
client = SubscriberClient()
try:
topic_id = 'container-analysis-occurrences-v1'
topic_name = {"name": f"projects/{PROJECT_ID}/topics/{topic_id}"}
publisher = PublisherClient()
publisher.create_topic(topic_name)
except AlreadyExists:
pass
subscription_id = 'container-analysis-test-{}'.format(uuid.uuid4())
subscription_name = client.subscription_path(PROJECT_ID,
subscription_id)
samples.create_occurrence_subscription(subscription_id, PROJECT_ID)
# I can not make it pass with multiple messages. My guess is
# the server started to dedup?
message_count = 1
try:
job_done = threading.Event()
receiver = MessageReceiver(message_count, job_done)
client.subscribe(subscription_name, receiver.pubsub_callback)
for i in range(message_count):
occ = samples.create_occurrence(self.image_url, self.note_id,
PROJECT_ID, PROJECT_ID)
time.sleep(SLEEP_TIME)
samples.delete_occurrence(basename(occ.name), PROJECT_ID)
time.sleep(SLEEP_TIME)
# We saw occational failure with 60 seconds timeout, so we bumped it
# to 180 seconds.
# See also: python-docs-samples/issues/2894
job_done.wait(timeout=180)
print('done. msg_count = {}'.format(receiver.msg_count))
assert message_count <= receiver.msg_count
finally:
# clean up
client.delete_subscription({"subscription": subscription_name})
def test_pubsub(self):
# create topic if needed
client = SubscriberClient()
try:
topic_id = 'container-analysis-occurrences-v1'
topic_name = client.topic_path(PROJECT_ID, topic_id)
publisher = PublisherClient()
publisher.create_topic(topic_name)
except AlreadyExists:
pass
subscription_id = 'drydockOccurrences'
subscription_name = client.subscription_path(PROJECT_ID,
subscription_id)
samples.create_occurrence_subscription(subscription_id, PROJECT_ID)
tries = 0
success = False
while not success and tries < TRY_LIMIT:
print(tries)
tries += 1
receiver = samples.MessageReceiver()
client.subscribe(subscription_name, receiver.pubsub_callback)
# test adding 3 more occurrences
total_created = 3
for _ in range(total_created):
occ = samples.create_occurrence(self.image_url, self.note_id,
PROJECT_ID, PROJECT_ID)
sleep(SLEEP_TIME)
samples.delete_occurrence(basename(occ.name), PROJECT_ID)
sleep(SLEEP_TIME)
print('done. msg_count = {}'.format(receiver.msg_count))
success = receiver.msg_count == total_created
if receiver.msg_count != total_created:
raise AssertionError
# clean up
client.delete_subscription(subscription_name)
def test_pubsub(self):
# create topic if needed
client = SubscriberClient()
try:
topic_id = 'container-analysis-occurrences-v1'
topic_name = client.topic_path(PROJECT_ID, topic_id)
publisher = PublisherClient()
publisher.create_topic(topic_name)
except AlreadyExists:
pass
subscription_id = 'container-analysis-test-{}'.format(uuid.uuid4())
subscription_name = client.subscription_path(PROJECT_ID,
subscription_id)
samples.create_occurrence_subscription(subscription_id, PROJECT_ID)
# I can not make it pass with multiple messages. My guess is
# the server started to dedup?
message_count = 1
try:
job_done = threading.Event()
receiver = MessageReceiver(message_count, job_done)
client.subscribe(subscription_name, receiver.pubsub_callback)
for i in range(message_count):
occ = samples.create_occurrence(self.image_url, self.note_id,
PROJECT_ID, PROJECT_ID)
time.sleep(SLEEP_TIME)
samples.delete_occurrence(basename(occ.name), PROJECT_ID)
time.sleep(SLEEP_TIME)
job_done.wait(timeout=60)
print('done. msg_count = {}'.format(receiver.msg_count))
assert message_count <= receiver.msg_count
finally:
# clean up
client.delete_subscription(subscription_name)
def clients():
cleanup()
run_cmd = shlex.split(
f"docker run -d -p 8085:8085 --name {CONTAINER} google/cloud-sdk "
"gcloud beta emulators pubsub start "
f"--project={PROJECT} --host-port=0.0.0.0:8085"
)
subprocess.check_call(run_cmd)
def predicate():
cmd = shlex.split(f"docker logs {CONTAINER}")
logs = subprocess.check_output(cmd, stderr=subprocess.STDOUT)
return b"INFO: Server started, listening on 8085" in logs
wait_for(predicate, 10, period=0.1)
os.environ["PUBSUB_EMULATOR_HOST"] = "localhost:8085"
os.environ["PUBSUB_PROJECT_ID"] = PROJECT
try:
yield (PublisherClient(), SubscriberClient())
finally:
cleanup(fail=True)
def __init__(self):
self._client = PublisherClient()
_, self._project_id = google.auth.default()
def publisher_client():
yield PublisherClient()
from google.api_core.exceptions import AlreadyExists
from google.cloud.pubsub import SchemaServiceClient
from google.pubsub_v1.types import Schema
from google.cloud import pubsub_v1
from google.cloud.pubsub import PublisherClient
import json
from google.pubsub_v1.types import Encoding
publisher = pubsub_v1.PublisherClient()
publisher_client = PublisherClient()
project_id = "gcplayproject"
schema_id = "userdetails"
avsc_file = "/home/googlecloud/mynewdata/avro.avsc"
topic_id = "schematopic"
topic_path = publisher.topic_path(project_id, topic_id)
project_path = f"projects/{project_id}"
# Read a JSON-formatted Avro schema file as a string.
with open(avsc_file, "rb") as f:
avsc_source = f.read().decode("utf-8")
schema_client = SchemaServiceClient()
schema_path = schema_client.schema_path(project_id, schema_id)
schema = Schema(name=schema_path,
type_=Schema.Type.AVRO,
definition=avsc_source)
def createschema():
def publisher_client() -> Generator[pubsub_v1.PublisherClient, None, None]:
yield PublisherClient()
def publish(self, data):
from google.cloud.pubsub import PublisherClient
client = PublisherClient()
sub_path = client.topic_path(self.project, self._tf_name)
client.publish(sub_path, data).result()