async def get_client(self):
if not self.client:
session = await self.get_session()
self.client = PublisherClient(
service_file=self.service_file, session=session
)
return self.client
def get_topic(self):
publisher = pubsub_v1.PublisherClient()
project_id = os.getenv('GCP_PROJECT')
topic_path = PublisherClient.topic_path(project_id, self.topic)
try:
publisher.create_topic(request={"name": topic_path})
except google.api_core.exceptions.AlreadyExists:
pass
finally:
return topic_path
def start_subscriber(loop, topic):
client = SubscriberClient()
project_id = os.getenv('GCP_PROJECT')
topic_path = PublisherClient.topic_path(project_id, topic)
subscription_path = PublisherClient.subscription_path(project_id, topic)
# Create subscription if it doesn't already exist
try:
client.create_subscription(subscription_path, topic_path)
except google.api_core.exceptions.PermissionDenied as e:
raise TypeError('Please set the GCP_PROJECT environment variable') from e
# Subscribe to the subscription, receiving a Future that acts as a keepalive
keep_alive = client.subscribe(subscription_path, message_callback)
# Have the client run forever, pulling messages from this subscription,
# passing them to the specified callback function, and wrapping it in an
# asyncio task.
client.run_forever(keep_alive)
async def start_subscriber(topic):
client = SubscriberClient()
project_id = os.getenv("GCP_PROJECT")
topic_path = PublisherClient.topic_path(project_id, topic)
subscription_path = PublisherClient.subscription_path(project_id, topic)
# Create subscription if it doesn't already exist
try:
await client.create_subscription(subscription_path, topic_path)
except aiohttp.client_exceptions.ClientResponseError as e:
if e.status == 409: # Subscription exists
pass
else:
raise TypeError("Please set the GCP_PROJECT environment variable") from e
# For demo with Pub/Sub emulator, maybe ack_deadline_cache_timeout 300
# On GCP, default seems fine.
# For more options, check gcloud-aio docs:
# https://github.com/talkiq/gcloud-aio/tree/master/pubsub
await subscribe(subscription_path, message_callback, client, ack_deadline_cache_timeout=300)
def __init__(self,
topic: Optional[str] = None,
key: Optional[str] = None,
service_file: Optional[Union[str, IO[AnyStr]]] = None,
client: Optional[PublisherClient] = None,
session: Optional[aiohttp.ClientSession] = None,
ordering_key: Optional[Union[str, io.IOBase]] = None,
numeric_type=float):
'''
Backend using Google Cloud Platform Pub/Sub. Use requires an account with Google Cloud Platform.
Free tier allows 10GB messages per month.
Both the environment variables GCP_PROJECT='<project_id>' and GOOGLE_APPLICATION_CREDENTIALS='/path/key.json'
may be required.
topic: str
Topic name. Defaults to 'cryptofeed-{key}', for example 'cryptofeed-trades'
key: str
Setting key lets you override the symbol name.
The defaults are related to the data
being stored, i.e. trade, funding, etc
service_file: str or file obj
Loads credentials from a service account file.
If not provided, credentials will be loaded from the environment variable
'GOOGLE_APPLICATION_CREDENTIALS'. If inside a Google Cloud environment
that has a default service account, such as Compute Engine, Google Kubernetes Engine,
or App Engine the environment variable will already be set.
https://cloud.google.com/bigquery/docs/authentication/service-account-file
https://cloud.google.com/docs/authentication/production
client: PublisherClient
Allows gcloud.aio.pubsub.PublisherClient reuse
session: ClientSession
Allows aiohttp.ClientSession resuse
ordering_key: str
if messages have the same ordering key and you publish the messages
to the same region, subscribers can receive the messages in order
https://cloud.google.com/pubsub/docs/publisher#using_ordering_keys
'''
self.key = key or self.default_key
self.ordering_key = ordering_key
self.numeric_type = numeric_type
self.topic = topic or f'cryptofeed-{self.key}'
self.topic_path = self.get_topic()
self.session = session or aiohttp.ClientSession()
self.client = client or PublisherClient(service_file=service_file,
session=self.session)