def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(TURNSTILE_SUMMARY) is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists(STATIONS_TABLE) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application( [(r"/", MainHandler, {"weather": weather_model, "lines": lines})] ) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( WEATHER, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( STATIONS_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "^org.chicago.cta.station.arrivals.", lines.process_message, offset_earliest=True, ), KafkaConsumer( TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if not topic_check.topic_exists(TURNSTILE_ENTRIES_TABLE): logger.error( "Ensure that the KSQL Command has run successfully before running the web server!" ) raise Exception("Ensure that the KSQL Command has run") if not topic_check.topic_exists(STATION_MASTER_DATA): logger.error( "Ensure that Faust Streaming is running successfully before running the web server!" ) raise Exception("Ensure that Faust Streaming is running successfully") weather_model = Weather() lines = Lines() application = tornado.web.Application( [(r"/", MainHandler, {"weather": weather_model, "lines": lines})] ) application.listen(8888) consumers = [ KafkaConsumer(WEATHER_STATUS, weather_model.process_message, offset_earliest=True), KafkaConsumer(STATION_MASTER_DATA, lines.process_message, offset_earliest=True, is_avro=False), KafkaConsumer(TRAIN_ARRIVAL, lines.process_message, offset_earliest=True), KafkaConsumer( TURNSTILE_ENTRIES_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), ] logger.info("Open a web browser to http://localhost:8888 to see the Transit Status Page") try: for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists("TURNSTILE_SUMMARY") is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_pattern_match("faust.stations.transformed") is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(WEB_SERVER_PORT) print("Building consumers....") # Kafka consumers for the application consumers = [ KafkaConsumer( "weather", weather_model.process_message, offset_earliest=True, ), KafkaConsumer( "faust.stations.transformed", lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "^station.arrivals.*", lines.process_message, offset_earliest=True, ), KafkaConsumer( "TURNSTILE_SUMMARY", lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( f"Open a web browser to http://localhost:{WEB_SERVER_PORT} to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(TURNSTILE_SUMMARY_TOPIC) is False: logger.fatal( "ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists(STATIONS_TOPIC_V1) is False: logger.fatal( "ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( id="weather.consumer", topic_name_pattern=WEATHER_TOPIC_V1, message_handler=weather_model.process_message, offset_earliest=True, ), KafkaConsumer( id="stations.table.consumer", topic_name_pattern=STATIONS_TOPIC_V1, message_handler=lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer(id="arrival.consumer", topic_name_pattern=f"^{ARRIVALS_TOPIC_PREFIX}", message_handler=lines.process_message, offset_earliest=True), KafkaConsumer( id="turnstile.summary.consumer", topic_name_pattern=TURNSTILE_SUMMARY_TOPIC, message_handler=lines.process_message, offset_earliest=True, is_avro=False, ) ] try: logger.info( "open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists("TURNSTILE_SUMMARY") is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists( "com.udacity.project.chicago_transportation.station.transformed" ) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines.get_lines() })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( "com.udacity.project.chicago_transportation.weather.update", weather_model.process_message, offset_earliest=True, ), KafkaConsumer( "com.udacity.project.chicago_transportation.station.transformed", lines.process_station_update_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "com.udacity.project.chicago_transportation.arrival", lines.process_new_arrival_message, offset_earliest=True, ), KafkaConsumer( "TURNSTILE_SUMMARY", lines.process_turnstile_update_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): ''' Runs the Tornado Server and begins Kafka consumption ''' if not utils.topic_exists(constants.TOPIC_TURNSTILE_SUMMARY): logger.fatal( 'Ensure that the KSQL Command has run successfully before running the web server!' ) exit(1) if not utils.topic_exists(constants.TOPIC_STATIONS_TABLE_V1): logger.fatal( 'Ensure that Faust Streaming is running successfully before running the web server!' ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r'/', MainHandler, { 'weather': weather_model, 'lines': lines })]) application.listen(constants.SERVER_PORT) # Build kafka consumers consumers = [ KafkaConsumer( constants.TOPIC_WEATHER_V1, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( constants.TOPIC_STATIONS_TABLE_V1, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( constants.TOPIC_ARRIVALS_REGEX, lines.process_message, offset_earliest=True, ), KafkaConsumer( constants.TOPIC_TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ) ] try: logger.info( f'Transit Status Page running at http://localhost:{constants.SERVER_PORT}' ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: logger.info('shutting down server') tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(config.TOPIC_TURNSTILE_SUMMARY) is False: logger.fatal("Ensure that the KSQL Command has run successfully!") exit(1) if topic_check.topic_pattern_match("stations.table") is False: logger.fatal("Ensure that Faust Streaming is running successfully!") exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(config.WEB_SERVER_PORT) # Build kafka consumers consumers = [ KafkaConsumer( config.TOPIC_WEATHER, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( config.TOPIC_FAUST_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( config.TOPIC_STATIONS, lines.process_message, offset_earliest=True, ), KafkaConsumer( config.TOPIC_TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( f"Open a web browser to http://localhost:{config.WEB_SERVER_PORT} to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close() except Exception as ex: logger.fatal(f"server.py error: {ex}") exit(1)
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists( config['topics.consumers']['turnstile.summary']) is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists( config['topics.consumers']['faust.station.transformed']) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( config['topics.producers']['weather'], weather_model.process_message, offset_earliest=True, ), KafkaConsumer( config['topics.consumers']['faust.station.transformed'], lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( f"^{config['topics.producers']['station.arrival.prefix']}.*", lines.process_message, offset_earliest=True, ), KafkaConsumer( config['topics.consumers']['turnstile.summary'], lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()