async def connect(self):
"""Connect the service worker to th3e NATS/STAN Queue.
Also handles reconnecting when the network has dropped the connection.
Both the NATS and the STAN clients need to be reinstantiated to work correctly.
"""
logger.info('Connecting...')
if self.nc:
try:
logger.debug('close old NATS client')
await self.nc.close()
except asyncio.CancelledError as err:
logger.debug('closing stale connection err:%s', err)
finally:
self.nc = None
self.nc = NATS()
self.sc = STAN()
nats_connection_options = {
**self.config.NATS_CONNECTION_OPTIONS,
**{
'loop': self._loop,
'error_cb': error_cb
},
**self.nats_connection_options
}
stan_connection_options = {
**self.config.STAN_CONNECTION_OPTIONS,
**{
'nats': self.nc,
'conn_lost_cb': self._stan_conn_lost_cb,
'loop': self._loop
},
**self.stan_connection_options
}
subscription_options = {
**self.config.SUBSCRIPTION_OPTIONS,
**{
'cb': self.cb_handler
},
**self.subscription_options
}
await self.nc.connect(**nats_connection_options)
await self.sc.connect(**stan_connection_options)
await self.sc.subscribe(**subscription_options)
logger.info(
'Subscribe the callback: %s to the queue: %s.',
subscription_options.get('cb').__name__
if subscription_options.get('cb') else 'no_call_back',
subscription_options.get('queue'))
async def conn_lost_cb(error):
logger.info('Connection lost:%s', error)
for i in range(0, 100):
try:
logger.info('Reconnecting, attempt=%i...', i)
await self.connect()
except Exception as e: # pylint: disable=broad-except; catch all errors from client framework
logger.error('Error %s',
e.with_traceback(),
stack_info=True)
continue
break
async def cb_subscription_handler(msg: nats.aio.client.Msg):
"""Use Callback to process Queue Msg objects."""
try:
logger.info('Received raw message seq:%s, data= %s', msg.sequence, msg.data.decode())
payment_token = extract_payment_token(msg)
logger.debug('Extracted payment token: %s', payment_token)
process_filing(payment_token, FLASK_APP)
except OperationalError as err:
logger.error('Queue Blocked - Database Issue: %s', json.dumps(payment_token), exc_info=True)
raise err # We don't want to handle the error, as a DB down would drain the queue
except (QueueException, Exception): # pylint: disable=broad-except
# Catch Exception so that any error is still caught and the message is removed from the queue
capture_message('Queue Error:' + json.dumps(payment_token), level='error')
logger.error('Queue Error: %s', json.dumps(payment_token), exc_info=True)
async def reconnected_cb():
"""Connect to the NATS services.
This gets called when the client successfully connects, or reconnects.
"""
logger.info('Connected to NATS at %s...', nc.connected_url.netloc)
async def run(loop): # pylint: disable=too-many-locals
"""Run the main application loop for the service.
This runs the main top level service functions for working with the Queue.
"""
# NATS client connections
nc = NATS()
sc = STAN()
async def reconnected_cb():
"""Connect to the NATS services.
This gets called when the client successfully connects, or reconnects.
"""
logger.info('Connected to NATS at %s...', nc.connected_url.netloc)
async def close():
"""Close the stream and nats connections."""
await sc.close()
await nc.close()
# Connection and Queue configuration.
def nats_connection_options():
return {
'servers':
os.getenv('NATS_SERVERS', 'nats://127.0.0.1:4222').split(','),
'io_loop':
loop,
'error_cb':
error_cb,
'closed_cb':
closed_cb,
'reconnected_cb':
reconnected_cb,
'name':
os.getenv('NATS_CLIENT_NAME', 'entity.filing.worker')
}
def stan_connection_options():
return {
'cluster_id': os.getenv('NATS_CLUSTER_ID', 'test-cluster'),
'client_id': str(random.SystemRandom().getrandbits(0x58)),
'nats': nc
}
def subscription_options():
return {
'subject': os.getenv('NATS_SUBJECT', 'entity.filings'),
'queue': os.getenv('NATS_QUEUE', 'filing-worker'),
'durable_name':
os.getenv('NATS_QUEUE', 'filing-worker') + '_durable',
'cb': cb_subscription_handler
}
try:
# Connect to the NATS server, and then use that for the streaming connection.
await nc.connect(**nats_connection_options())
await sc.connect(**stan_connection_options())
# Attach the callback queue
await sc.subscribe(**subscription_options())
logger.info('Subscribe the callback: %s to the queue: %s.',
subscription_options().get('cb').__name__,
subscription_options().get('queue'))
# register the signal handler
for sig in ('SIGINT', 'SIGTERM'):
loop.add_signal_handler(
getattr(signal, sig),
functools.partial(signal_handler,
sig_loop=loop,
sig_nc=nc,
task=close))
except Exception as e: # pylint: disable=broad-except
# TODO tighten this error and decide when to bail on the infinite reconnect
logger.error(e)