class Publish(object):
_increment = 0
def _createKey(self):
'''
:return: returns a unique key for each entry.
'''
key = datetime.datetime.now().strftime(
'%Y-%m-%d_%H-%M-%S') + '_' + str(self._increment)
self._increment += 1
return key
def __init__(self, projectId, topic, separateLines=False):
self._topicPath = 'projects/{project}/topics/{topic}'.format(
project=projectId, topic=topic)
self._publisher = PublisherClient()
self._separateLines = separateLines
def process(self, data):
'''
Will write data as a series of JSON objects, one per line. NOTE that this is not a JSON list of JSON objects. Big Query will ingest the series of JSON objects on separate lines.
:param data: a list of dicts.
'''
rows = map(lambda row: json.dumps(row), data)
if self._separateLines:
for row in rows:
key = self._createKey()
try:
self._publisher.publish(
self._topicPath,
data=json.dumps(row).encode('utf-8'),
query=key)
except Exception as ex:
print(
json.dumps({
'log':
'ERROR Error publishing {key} to {topic}: {error}'.
format(key=key,
topic=self._topicPath,
error=str(ex))
}))
else:
key = self._createKey()
try:
self._publisher.publish(self._topicPath,
data=json.dumps(
'\n'.join(rows)).encode('utf-8'),
query=self._createKey())
except Exception as ex:
print(
json.dumps({
'log':
'ERROR Error publishing {key} to {topic}: {error}'.
format(key=key, topic=self._topicPath, error=str(ex))
}))
def test_sub(publisher_client: pubsub_v1.PublisherClient, topic_path: str,
subscription_path: str, capsys: CaptureFixture) -> None:
publisher_client.publish(topic_path, b"Hello World!")
import sub
sub.sub(PROJECT_ID, SUBSCRIPTION_ID, 10)
out, _ = capsys.readouterr()
assert f"Listening for messages on {subscription_path}" in out
assert "Received" in out
assert "Acknowledged" in out
class PubSubSubmitter(): # pylint: disable=no-self-use
def __init__(self, project_id, topic):
self.publisher = PublisherClient()
self.topic_path = 'projects/{project_id}/topics/{topic}'.format(
project_id=project_id,
topic=topic,
)
def send_message(self, message, queue, tx_id): # pylint: disable=unused-argument
logger.info('sending message')
self.publisher.publish(self.topic_path, str(message).encode('utf8'))
return True
class GooglePubSubPublisher:
def __init__(self, **kwargs):
super().__init__(**kwargs)
self._logger = logging.getLogger(__name__)
self._publisher = PublisherClient()
def send_message(self, message):
serializer = message.get_serializer()
json_data = JSONRenderer().render(serializer.data)
topic_path = self._publisher.topic_path(
settings.GCLOUD_PUBSUB_PROJECT_ID, serializer.data['type'])
future = self._publisher.publish(topic_path, data=json_data)
future.add_done_callback(self._publish_callback)
def _publish_callback(self, message_future):
if message_future.exception(timeout=30):
self._logger.info(
f'Publishing message on {message_future.exception()} threw an Exception.'
)
else:
self._logger.info(message_future.result())
def create_topic(self, topic: str):
topic_path = self._publisher.topic_path(
settings.GCLOUD_PUBSUB_PROJECT_ID, topic)
try:
self._publisher.create_topic(topic_path)
except AlreadyExists as e:
self._logger.info(f'Topic {topic} already exists.')
class PubSubPublisher(Publisher):
def __init__(self):
self._client = PublisherClient()
_, self._project_id = google.auth.default()
def _publish(self, topic_id, message):
logger.info("publishing message", topic_id=topic_id)
topic_path = self._client.topic_path(self._project_id, topic_id)
response: Future = self._client.publish(topic_path, message)
return response
def publish(self, topic_id, message: bytes):
response = self._publish(topic_id, message)
try:
# Resolve the future
message_id = response.result()
logger.info( # pragma: no cover
"message published successfully",
topic_id=topic_id,
message_id=message_id,
)
except Exception as ex: # pylint:disable=broad-except
logger.exception(
"message publication failed",
topic_id=topic_id,
)
raise PublicationFailed(ex)
class PublishService:
_publisher: PublisherClient
_topic_name: str
_request: Request
def __init__(self, topic_name: str, request: Request):
self._publisher = PublisherClient()
self._topic_name = topic_name
self._request = request
def _publish_message(self, message_name, message):
metadata = Gobits.from_request(request=self._request)
try:
my_gobits = [metadata.to_json()]
except: # noqa: E722
my_gobits = []
message_to_publish = {"gobits": my_gobits, message_name: message}
self._publisher.publish(
self._topic_name,
bytes(json.dumps(message_to_publish).encode("utf-8")))
class PubSubPublisher:
_client_config = {
"interfaces": {
"google.pubsub.v1.Publisher": {
"retry_params": {
"messaging": {
"total_timeout_millis": consts.PUBSUB_TIMEOUT_MS,
}
}
}
}
}
def __init__(self, project_id):
self.project_id = project_id
self.client = PublisherClient(client_config=self._client_config)
self.futures = dict()
def _get_callback(self, f, data):
def callback(f):
if f.exception():
logging.error(f"Please handle {f.exeption()} for {data}.")
if data in self.futures:
self.futures.pop(data)
return callback
def publish_data(self, topic_name, data, timeout=60):
""" Publish Pub/Sub Data
:param topic_name: String name of topic
:param data: String data being processed
:param wait_for_ack: Bool if user wants to wait for ack
:param timeout: Int seconds to wait
"""
if isinstance(data, dict):
data = json.dumps(data)
self.futures.update({data: None})
# When you publish a message, the client returns a future.
topic_path = self.client.topic_path(self.project_id, topic_name)
future = self.client.publish(
topic_path, data=data.encode("utf-8")
)
self.futures[data] = future
# Publish failures shall be handled in the callback function.
future.add_done_callback(self._get_callback(future, data))
return data
async def submit(
request: Request,
client: PublisherClient,
q: SQLiteAckQueue,
topic: str,
metadata_headers: Dict[str, str],
timeout: Optional[float],
**kwargs
) -> response.HTTPResponse:
"""Deliver request to the pubsub topic.
Deliver to the local queue to be retried on transient errors.
"""
data = request.body
attrs = {
key: value
for key, value in dict(
submission_timestamp=datetime.utcnow().isoformat() + "Z",
uri=request.path,
protocol="HTTP/" + request.version,
method=request.method,
args=request.query_string,
remote_addr=request.ip,
host=request.host,
**{
attr: request.headers.get(header)
for header, attr in metadata_headers.items()
}
).items()
if value is not None
}
# assert valid pubsub message
for value in attrs.values():
if len(value.encode("utf8")) > 1024:
# attribute exceeds value size limit of 1024 bytes
# https://cloud.google.com/pubsub/quotas#resource_limits
return response.text(
"header too large\n", HTTP_STATUS.REQUEST_HEADER_FIELDS_TOO_LARGE
)
try:
future = client.publish(topic, data, **attrs)
await asyncio.wait_for(async_wrap(future), timeout)
except Exception:
# api call failure, write to queue
try:
q.put((topic, data, attrs))
except DatabaseError:
# sqlite queue is probably out of space
return response.text("", HTTP_STATUS.INSUFFICIENT_STORAGE)
return response.text("")
class PubSubPublisher(Publisher):
def __init__(self, project_id, credentials_file=None):
logger.info('connecting to pubsub')
if credentials_file:
service_account_info = json.load(open(credentials_file))
publisher_audience = 'https://pubsub.googleapis.com/google.pubsub.v1.Publisher'
credentials_pub = jwt.Credentials.from_service_account_info(
service_account_info, audience=publisher_audience,
)
self._client = PublisherClient(credentials=credentials_pub)
else:
self._client = PublisherClient()
self._project_id = project_id
def _publish(self, topic_id, message, **kwargs):
logger.info('publishing message', topic_id=topic_id)
topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member
response: Future = self._client.publish(topic_path, message.encode('utf-8'), **kwargs)
return response
def publish(self, topic_id, message: bytes, fulfilment_request_transaction_id: str):
response = self._publish(topic_id, message, tx_id=fulfilment_request_transaction_id)
try:
# Resolve the future
message_id = response.result()
logger.info( # pragma: no cover
'message published successfully',
topic_id=topic_id,
message_id=message_id,
fulfilment_request_transaction_id=fulfilment_request_transaction_id,
)
except Exception as ex: # pylint:disable=broad-except
logger.exception(
'message publication failed',
topic_id=topic_id,
)
raise PublicationFailed(ex)
def create_topic(self, topic_id):
try:
logger.info('creating topic')
topic_path = self._client.topic_path(self._project_id, topic_id) # pylint: disable=no-member
self._client.create_topic(request={'name': topic_path}) # pylint: disable=no-member
except AlreadyExists:
logger.info('Topic already exists')
except Exception as ex: # pylint:disable=broad-except
logger.error(
'failed', exc_info=ex,
)
def publish_message(response_text, patient_id, event_attributes):
publisher = PublisherClient()
# pylint: disable=no-member
topic_path = publisher.topic_path(
os.environ.get("PROJECT_NAME"), os.environ.get("TOPIC_NAME")
)
data = response_text.encode("utf-8")
future = publisher.publish(
topic_path,
data=data,
**event_attributes,
patient_id=patient_id,
)
message_id = future.result()
logger.info(f"Successfully published message: {message_id}")
print(f"Successfully published message: {message_id}")
class EmsPublisherClient:
def __init__(self):
self.__client = PublisherClient()
def publish(self, topic: str, data: bytes, **attrs) -> Future:
return self.__client.publish(topic=topic, data=data, **attrs)
def topic_create_if_not_exists(self, project_id: str, topic_name: str):
topic_path = self.__client.api.topic_path(project_id, topic_name)
try:
self.__client.api.create_topic(topic_path)
LOGGER.info("Topic %s created in project %s", topic_name,
project_id)
except AlreadyExists:
LOGGER.info("Topic %s already exists in project %s", topic_name,
project_id)
def delete_topic_if_exists(self, project_id: str, topic_name: str):
topic_path = self.__client.api.topic_path(project_id, topic_name)
try:
self.__client.api.delete_topic(topic_path)
LOGGER.info("Topic %s deleted in project %s", topic_name,
project_id)
except NotFound:
LOGGER.info("Topic %s not found in project %s", topic_name,
project_id)
@staticmethod
def subscription_create(project_id: str, topic_name: str,
subscription_name: str):
subscriber = SubscriberClient()
topic_path = subscriber.api.topic_path(project_id, topic_name)
subscription_path = subscriber.api.subscription_path(
project_id, subscription_name)
subscriber.api.create_subscription(subscription_path, topic_path)
async def test_batch_reject_message(client: PublisherClient):
client.batch_settings = types.BatchSettings(1, 0, 1)
with pytest.raises(ValueError):
client.publish("", b"..")
for batch in client._batches.values():
await batch.result
def _send(p: pubsub_v1.PublisherClient, m) -> None:
p.publish(TOPIC_PATH, str.encode(json.dumps(m)))
import json
import random
from kaggle_environments import make
from google.cloud.pubsub_v1 import PublisherClient
SEED = 42
AGENTS = ["random", "random", "random", "random"]
TAGS = [{"key": "type", "value": "DEBUG"}]
random.seed(SEED)
random.seed(SEED)
env = make("halite")
_ = env.reset(num_agents=len(AGENTS))
_ = env.run(copy(AGENTS))
payload = {
"env": env.toJSON(),
"agents": AGENTS,
"seed": SEED,
"tags": TAGS,
}
publisher = PublisherClient()
topic_name = 'projects/kaggle-halite/topics/match'
result = publisher.publish(topic_name, json.dumps(payload).encode("ascii"))
result.result()
