async def run(loop): # pylint: disable=too-many-locals """Run the main application loop for the service. This runs the main top level service functions for working with the Queue. """ service = ServiceWorker(loop=loop, cb_handler=cb_subscription_handler) probe = Probes(components=[service], loop=loop) async def close(): await service.close() my_loop = asyncio.get_running_loop() await asyncio.sleep(0.1, loop=my_loop) my_loop.stop() try: await probe.start() await service.connect() # register the signal handler for sig in ('SIGINT', 'SIGTERM'): loop.add_signal_handler( getattr(signal, sig), functools.partial(signal_handler, sig_loop=loop, task=close)) except Exception as e: # pylint: disable=broad-except # TODO tighten this error and decide when to bail on the infinite reconnect logger.error(e)
def process_filing(payment_token, flask_app): """Render the filings contained in the submission.""" if not flask_app: raise QueueException('Flask App not available.') with flask_app.app_context(): # try to find the filing 5 times before putting back on the queue - in case payment token ends up on the queue # before it is assigned to filing. counter = 1 filing_submission = None while not filing_submission and counter <= 5: filing_submission = get_filing_by_payment_id(payment_token['paymentToken'].get('id')) counter += 1 sleep(0.2) if not filing_submission: raise FilingException if filing_submission.status == Filing.Status.COMPLETED.value: logger.warning('Queue: Attempting to reprocess business.id=%s, filing.id=%s payment=%s', filing_submission.business_id, filing_submission.id, payment_token) return if payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED': # TODO - need to surface TRANSACTION_FAILED, but Filings manages its own status # filing_submission.status = Filing.Status.ERROR filing_submission.payment_completion_date = datetime.datetime.utcnow() db.session.add(filing_submission) db.session.commit() return legal_filings = filing_submission.legal_filings() # TODO: handle case where there are no legal_filings uow = versioning_manager.unit_of_work(db.session) transaction = uow.create_transaction(db.session) if not payment_token['paymentToken'].get('statusCode') == 'TRANSACTION_FAILED': if not payment_token['paymentToken'].get('statusCode') == Filing.Status.COMPLETED.value: logger.error('Unknown payment status given: %s', payment_token['paymentToken'].get('statusCode')) raise QueueException business = Business.find_by_internal_id(filing_submission.business_id) for filing in legal_filings: if filing.get('annualReport'): annual_report.process(business, filing) if filing.get('changeOfAddress'): change_of_address.process(business, filing) if filing.get('changeOfDirectors'): change_of_directors.process(business, filing) filing_submission.transaction_id = transaction.id db.session.add(business) filing_submission.payment_completion_date = datetime.datetime.utcnow() db.session.add(filing_submission) db.session.commit() return
async def run(loop, token): # pylint: disable=too-many-locals """Run the main application loop for the service. This runs the main top level service functions for working with the Queue. """ # NATS client connections nc = NATS() sc = STAN() async def close(): """Close the stream and nats connections.""" await sc.close() await nc.close() # Connection and Queue configuration. def nats_connection_options(): return { 'servers': os.getenv('NATS_SERVERS', 'nats://127.0.0.1:4222').split(','), 'io_loop': loop, 'error_cb': error_cb, 'name': os.getenv('NATS_CLIENT_NAME', 'entity.filing.tester') } def stan_connection_options(): return { 'cluster_id': os.getenv('NATS_CLUSTER_ID', 'test-cluster'), 'client_id': str(random.SystemRandom().getrandbits(0x58)), 'nats': nc } def subscription_options(): return { 'subject': os.getenv('NATS_SUBJECT', 'entity.filings'), 'queue': os.getenv('NATS_QUEUE', 'filing-worker'), 'durable_name': os.getenv('NATS_QUEUE', 'filing-worker') + '_durable' } try: # Connect to the NATS server, and then use that for the streaming connection. await nc.connect(**nats_connection_options()) await sc.connect(**stan_connection_options()) # register the signal handler for sig in ('SIGINT', 'SIGTERM'): loop.add_signal_handler(getattr(signal, sig), functools.partial(signal_handler, sig_loop=loop, sig_nc=nc, task=close) ) payload = {'paymentToken': {'id': token, 'statusCode': 'COMPLETED'}} await sc.publish(subject=subscription_options().get('subject'), payload=json.dumps(payload).encode('utf-8')) except Exception as e: # pylint: disable=broad-except # TODO tighten this error and decide when to bail on the infinite reconnect logger.error(e)
async def conn_lost_cb(error): logger.info('Connection lost:%s', error) for i in range(0, 100): try: logger.info('Reconnecting, attempt=%i...', i) await self.connect() except Exception as e: # pylint: disable=broad-except; catch all errors from client framework logger.error('Error %s', e.with_traceback(), stack_info=True) continue break
async def cb_subscription_handler(msg: nats.aio.client.Msg): """Use Callback to process Queue Msg objects.""" try: logger.info('Received raw message seq:%s, data= %s', msg.sequence, msg.data.decode()) payment_token = extract_payment_token(msg) logger.debug('Extracted payment token: %s', payment_token) process_filing(payment_token, FLASK_APP) except OperationalError as err: logger.error('Queue Blocked - Database Issue: %s', json.dumps(payment_token), exc_info=True) raise err # We don't want to handle the error, as a DB down would drain the queue except (QueueException, Exception): # pylint: disable=broad-except # Catch Exception so that any error is still caught and the message is removed from the queue capture_message('Queue Error:' + json.dumps(payment_token), level='error') logger.error('Queue Error: %s', json.dumps(payment_token), exc_info=True)
def process(business: Business, filing: Filing): """Render the annual_report onto the business model objects.""" agm_date = filing['annualReport'].get('annualGeneralMeetingDate') ar_date = filing['annualReport'].get('annualReportDate') if agm_date and validations.annual_report.requires_agm(business): agm_date = datetime.date.fromisoformat(agm_date) if ar_date: ar_date = datetime.date.fromisoformat(ar_date) else: # should never get here (schema validation should prevent this from making it to the filer) logger.error( 'No annualReportDate given for in annual report. Filing id: %s', filing.id) business.last_agm_date = agm_date business.last_ar_date = ar_date
def process(business: Business, filing: Filing): """Render the change_of_directors onto the business model objects.""" new_directors = filing['changeOfDirectors'].get('directors') for new_director in new_directors: if 'appointed' in new_director['actions']: # create address address = create_address(new_director['deliveryAddress'], Address.DELIVERY) # add new director to the list business.directors.append( Director(first_name=new_director['officer'].get( 'firstName', '').upper(), middle_initial=new_director['officer'].get( 'middleInitial', '').upper(), last_name=new_director['officer'].get('lastName', '').upper(), title=new_director.get('title', '').upper(), appointment_date=new_director.get('appointmentDate'), cessation_date=new_director.get('cessationDate'), delivery_address=address)) if any([action != 'appointed' for action in new_director['actions']]): # get name of director in json for comparison * new_director_name = \ new_director['officer'].get('firstName') + new_director['officer'].get('middleInitial') + \ new_director['officer'].get('lastName') \ if 'nameChanged' not in new_director['actions'] \ else new_director['officer'].get('prevFirstName') + \ new_director['officer'].get('prevMiddleInitial') + new_director['officer'].get('prevLastName') if not new_director_name: logger.error('Could not resolve director name from json %s.', new_director) raise QueueException for director in business.directors: # get name of director in database for comparison * director_name = director.first_name + director.middle_initial + director.last_name if director_name.upper() == new_director_name.upper(): update_director(director, new_director) break
await asyncio.sleep(0.1, loop=my_loop) my_loop.stop() try: await probe.start() await service.connect() # register the signal handler for sig in ('SIGINT', 'SIGTERM'): loop.add_signal_handler( getattr(signal, sig), functools.partial(signal_handler, sig_loop=loop, task=close)) except Exception as e: # pylint: disable=broad-except # TODO tighten this error and decide when to bail on the infinite reconnect logger.error(e) if __name__ == '__main__': try: event_loop = asyncio.get_event_loop() event_loop.run_until_complete(run(event_loop)) event_loop.run_forever() except Exception as err: # pylint: disable=broad-except; Catching all errors from the frameworks logger.error('problem in running the service: %s', err, stack_info=True, exc_info=True) finally: event_loop.close()
async def run(loop): # pylint: disable=too-many-locals """Run the main application loop for the service. This runs the main top level service functions for working with the Queue. """ # NATS client connections nc = NATS() sc = STAN() async def reconnected_cb(): """Connect to the NATS services. This gets called when the client successfully connects, or reconnects. """ logger.info('Connected to NATS at %s...', nc.connected_url.netloc) async def close(): """Close the stream and nats connections.""" await sc.close() await nc.close() # Connection and Queue configuration. def nats_connection_options(): return { 'servers': os.getenv('NATS_SERVERS', 'nats://127.0.0.1:4222').split(','), 'io_loop': loop, 'error_cb': error_cb, 'closed_cb': closed_cb, 'reconnected_cb': reconnected_cb, 'name': os.getenv('NATS_CLIENT_NAME', 'entity.filing.worker') } def stan_connection_options(): return { 'cluster_id': os.getenv('NATS_CLUSTER_ID', 'test-cluster'), 'client_id': str(random.SystemRandom().getrandbits(0x58)), 'nats': nc } def subscription_options(): return { 'subject': os.getenv('NATS_SUBJECT', 'entity.filings'), 'queue': os.getenv('NATS_QUEUE', 'filing-worker'), 'durable_name': os.getenv('NATS_QUEUE', 'filing-worker') + '_durable', 'cb': cb_subscription_handler } try: # Connect to the NATS server, and then use that for the streaming connection. await nc.connect(**nats_connection_options()) await sc.connect(**stan_connection_options()) # Attach the callback queue await sc.subscribe(**subscription_options()) logger.info('Subscribe the callback: %s to the queue: %s.', subscription_options().get('cb').__name__, subscription_options().get('queue')) # register the signal handler for sig in ('SIGINT', 'SIGTERM'): loop.add_signal_handler( getattr(signal, sig), functools.partial(signal_handler, sig_loop=loop, sig_nc=nc, task=close)) except Exception as e: # pylint: disable=broad-except # TODO tighten this error and decide when to bail on the infinite reconnect logger.error(e)