def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(TURNSTILE_SUMMARY) is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists(STATIONS_TABLE) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application( [(r"/", MainHandler, {"weather": weather_model, "lines": lines})] ) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( WEATHER, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( STATIONS_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "^org.chicago.cta.station.arrivals.", lines.process_message, offset_earliest=True, ), KafkaConsumer( TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def worker_proc(gee): logging.info("worker({}) start ...".format(os.getpid())) kafka_consumer = None try: gen_table_pri_dict() kafka_db_config = { 'bootstrap_servers': Config.kafka_bootstrap_servers, 'group_id': Config.kafka_group, 'ip': Config.db_ip, 'port': Config.db_port, 'username': Config.db_username, 'password': Config.db_password, 'dbname': Config.db_name } kafka_consumer = KafkaConsumer(kafka_db_config) kafka_consumer.subscribe([Config.kafka_topic]) while not gee.is_stop(): kafka_consumer.process() except GracefulExitException: logging.info("worker({}) got GracefulExitException".format( os.getpid())) except Exception, ex: logging.error("Exception:{}".format(ex)) logging.error(traceback.format_exc())
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if not topic_check.topic_exists(TURNSTILE_ENTRIES_TABLE): logger.error( "Ensure that the KSQL Command has run successfully before running the web server!" ) raise Exception("Ensure that the KSQL Command has run") if not topic_check.topic_exists(STATION_MASTER_DATA): logger.error( "Ensure that Faust Streaming is running successfully before running the web server!" ) raise Exception("Ensure that Faust Streaming is running successfully") weather_model = Weather() lines = Lines() application = tornado.web.Application( [(r"/", MainHandler, {"weather": weather_model, "lines": lines})] ) application.listen(8888) consumers = [ KafkaConsumer(WEATHER_STATUS, weather_model.process_message, offset_earliest=True), KafkaConsumer(STATION_MASTER_DATA, lines.process_message, offset_earliest=True, is_avro=False), KafkaConsumer(TRAIN_ARRIVAL, lines.process_message, offset_earliest=True), KafkaConsumer( TURNSTILE_ENTRIES_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), ] logger.info("Open a web browser to http://localhost:8888 to see the Transit Status Page") try: for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists("TURNSTILE_SUMMARY") is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_pattern_match("faust.stations.transformed") is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(WEB_SERVER_PORT) print("Building consumers....") # Kafka consumers for the application consumers = [ KafkaConsumer( "weather", weather_model.process_message, offset_earliest=True, ), KafkaConsumer( "faust.stations.transformed", lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "^station.arrivals.*", lines.process_message, offset_earliest=True, ), KafkaConsumer( "TURNSTILE_SUMMARY", lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( f"Open a web browser to http://localhost:{WEB_SERVER_PORT} to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(TURNSTILE_SUMMARY_TOPIC) is False: logger.fatal( "ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists(STATIONS_TOPIC_V1) is False: logger.fatal( "ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( id="weather.consumer", topic_name_pattern=WEATHER_TOPIC_V1, message_handler=weather_model.process_message, offset_earliest=True, ), KafkaConsumer( id="stations.table.consumer", topic_name_pattern=STATIONS_TOPIC_V1, message_handler=lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer(id="arrival.consumer", topic_name_pattern=f"^{ARRIVALS_TOPIC_PREFIX}", message_handler=lines.process_message, offset_earliest=True), KafkaConsumer( id="turnstile.summary.consumer", topic_name_pattern=TURNSTILE_SUMMARY_TOPIC, message_handler=lines.process_message, offset_earliest=True, is_avro=False, ) ] try: logger.info( "open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def runner(offsets): fltr = Filter() queues = {} bmp_map = {} offset_update_freq = config.offset_update_freq for host in config.bagheera_nodes: for topic in config.topics: for partition in config.partitions: queue = Queue.Queue(256) queues[(host, topic, partition)] = queue bmp = BagheeraMessageProcessor(queue) bmp_map[id(bmp)] = (host, topic, partition) offset = offsets[(host, topic, partition)] kc = KafkaConsumer(host, {}, topic, partition, bmp.processor, offset, offset_update_freq) t = threading.Thread(target=kc.process_messages_forever) t.start() strtime = datetime.datetime.now().strftime('%Y%m%d%H%M%S') fos = FileOutputStream("redacted/redacted_%s.gz" % (strtime)) gos = GZIPOutputStream(fos) writer = PrintWriter(gos) uf_fos = FileOutputStream("raw/unfiltered_%s.gz" % (strtime)) uf_gos = GZIPOutputStream(uf_fos) uf_writer = PrintWriter(uf_gos) err_fos = FileOutputStream("errors/errors_%s.gz" % (strtime)) err_gos = GZIPOutputStream(err_fos) err_writer = PrintWriter(err_gos) count = 0 while True: for htp, q in queues.iteritems(): try: v = q.get(False) except Queue.Empty: continue if v[1] == 'PUT': count = count + 1 pid, op, ts, ipaddr, doc_id, payload = v json_payload = JSON.toJSONString(payload) uf_writer.println(json_payload) #TODO: figure out less braindead way of working with # java.util.HashMaps in jython try: filtered = json.loads(json_payload) # System.out.println('%s %s %d %s %s %s' % (htp[1], op, ts, ipaddr, doc_id, json_payload)) fltr.filter_document(filtered) filtered['doc_id'] = doc_id writer.println(json.dumps(filtered)) #print doc_id except: err_writer.println(doc_id + " " + json_payload) if count % 10000 == 0: print ts
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists("TURNSTILE_SUMMARY") is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists( "com.udacity.project.chicago_transportation.station.transformed" ) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines.get_lines() })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( "com.udacity.project.chicago_transportation.weather.update", weather_model.process_message, offset_earliest=True, ), KafkaConsumer( "com.udacity.project.chicago_transportation.station.transformed", lines.process_station_update_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( "com.udacity.project.chicago_transportation.arrival", lines.process_new_arrival_message, offset_earliest=True, ), KafkaConsumer( "TURNSTILE_SUMMARY", lines.process_turnstile_update_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): ''' Runs the Tornado Server and begins Kafka consumption ''' if not utils.topic_exists(constants.TOPIC_TURNSTILE_SUMMARY): logger.fatal( 'Ensure that the KSQL Command has run successfully before running the web server!' ) exit(1) if not utils.topic_exists(constants.TOPIC_STATIONS_TABLE_V1): logger.fatal( 'Ensure that Faust Streaming is running successfully before running the web server!' ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r'/', MainHandler, { 'weather': weather_model, 'lines': lines })]) application.listen(constants.SERVER_PORT) # Build kafka consumers consumers = [ KafkaConsumer( constants.TOPIC_WEATHER_V1, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( constants.TOPIC_STATIONS_TABLE_V1, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( constants.TOPIC_ARRIVALS_REGEX, lines.process_message, offset_earliest=True, ), KafkaConsumer( constants.TOPIC_TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ) ] try: logger.info( f'Transit Status Page running at http://localhost:{constants.SERVER_PORT}' ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt: logger.info('shutting down server') tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists(config.TOPIC_TURNSTILE_SUMMARY) is False: logger.fatal("Ensure that the KSQL Command has run successfully!") exit(1) if topic_check.topic_pattern_match("stations.table") is False: logger.fatal("Ensure that Faust Streaming is running successfully!") exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(config.WEB_SERVER_PORT) # Build kafka consumers consumers = [ KafkaConsumer( config.TOPIC_WEATHER, weather_model.process_message, offset_earliest=True, ), KafkaConsumer( config.TOPIC_FAUST_TABLE, lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( config.TOPIC_STATIONS, lines.process_message, offset_earliest=True, ), KafkaConsumer( config.TOPIC_TURNSTILE_SUMMARY, lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( f"Open a web browser to http://localhost:{config.WEB_SERVER_PORT} to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close() except Exception as ex: logger.fatal(f"server.py error: {ex}") exit(1)
def run_server(): """Runs the Tornado Server and begins Kafka consumption""" if topic_check.topic_exists( config['topics.consumers']['turnstile.summary']) is False: logger.fatal( "Ensure that the KSQL Command has run successfully before running the web server!" ) exit(1) if topic_check.topic_exists( config['topics.consumers']['faust.station.transformed']) is False: logger.fatal( "Ensure that Faust Streaming is running successfully before running the web server!" ) exit(1) weather_model = Weather() lines = Lines() application = tornado.web.Application([(r"/", MainHandler, { "weather": weather_model, "lines": lines })]) application.listen(8888) # Build kafka consumers consumers = [ KafkaConsumer( config['topics.producers']['weather'], weather_model.process_message, offset_earliest=True, ), KafkaConsumer( config['topics.consumers']['faust.station.transformed'], lines.process_message, offset_earliest=True, is_avro=False, ), KafkaConsumer( f"^{config['topics.producers']['station.arrival.prefix']}.*", lines.process_message, offset_earliest=True, ), KafkaConsumer( config['topics.consumers']['turnstile.summary'], lines.process_message, offset_earliest=True, is_avro=False, ), ] try: logger.info( "Open a web browser to http://localhost:8888 to see the Transit Status Page" ) for consumer in consumers: tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume) tornado.ioloop.IOLoop.current().start() except KeyboardInterrupt as e: logger.info("shutting down server") tornado.ioloop.IOLoop.current().stop() for consumer in consumers: consumer.close()