def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(TURNSTILE_SUMMARY) is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(STATIONS_TABLE) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application(
[(r"/", MainHandler, {"weather": weather_model, "lines": lines})]
)
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
WEATHER,
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
STATIONS_TABLE,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"^org.chicago.cta.station.arrivals.",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
TURNSTILE_SUMMARY,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def worker_proc(gee):
logging.info("worker({}) start ...".format(os.getpid()))
kafka_consumer = None
try:
gen_table_pri_dict()
kafka_db_config = {
'bootstrap_servers': Config.kafka_bootstrap_servers,
'group_id': Config.kafka_group,
'ip': Config.db_ip,
'port': Config.db_port,
'username': Config.db_username,
'password': Config.db_password,
'dbname': Config.db_name
}
kafka_consumer = KafkaConsumer(kafka_db_config)
kafka_consumer.subscribe([Config.kafka_topic])
while not gee.is_stop():
kafka_consumer.process()
except GracefulExitException:
logging.info("worker({}) got GracefulExitException".format(
os.getpid()))
except Exception, ex:
logging.error("Exception:{}".format(ex))
logging.error(traceback.format_exc())
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if not topic_check.topic_exists(TURNSTILE_ENTRIES_TABLE):
logger.error(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
raise Exception("Ensure that the KSQL Command has run")
if not topic_check.topic_exists(STATION_MASTER_DATA):
logger.error(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
raise Exception("Ensure that Faust Streaming is running successfully")
weather_model = Weather()
lines = Lines()
application = tornado.web.Application(
[(r"/", MainHandler, {"weather": weather_model, "lines": lines})]
)
application.listen(8888)
consumers = [
KafkaConsumer(WEATHER_STATUS, weather_model.process_message, offset_earliest=True),
KafkaConsumer(STATION_MASTER_DATA, lines.process_message,
offset_earliest=True, is_avro=False),
KafkaConsumer(TRAIN_ARRIVAL, lines.process_message, offset_earliest=True),
KafkaConsumer(
TURNSTILE_ENTRIES_TABLE, lines.process_message, offset_earliest=True, is_avro=False,
),
]
logger.info("Open a web browser to http://localhost:8888 to see the Transit Status Page")
try:
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_pattern_match("faust.stations.transformed") is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(WEB_SERVER_PORT)
print("Building consumers....")
# Kafka consumers for the application
consumers = [
KafkaConsumer(
"weather",
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
"faust.stations.transformed",
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"^station.arrivals.*",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
"TURNSTILE_SUMMARY",
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
f"Open a web browser to http://localhost:{WEB_SERVER_PORT} to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(TURNSTILE_SUMMARY_TOPIC) is False:
logger.fatal(
"ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(STATIONS_TOPIC_V1) is False:
logger.fatal(
"ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
id="weather.consumer",
topic_name_pattern=WEATHER_TOPIC_V1,
message_handler=weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
id="stations.table.consumer",
topic_name_pattern=STATIONS_TOPIC_V1,
message_handler=lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(id="arrival.consumer",
topic_name_pattern=f"^{ARRIVALS_TOPIC_PREFIX}",
message_handler=lines.process_message,
offset_earliest=True),
KafkaConsumer(
id="turnstile.summary.consumer",
topic_name_pattern=TURNSTILE_SUMMARY_TOPIC,
message_handler=lines.process_message,
offset_earliest=True,
is_avro=False,
)
]
try:
logger.info(
"open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def runner(offsets):
fltr = Filter()
queues = {}
bmp_map = {}
offset_update_freq = config.offset_update_freq
for host in config.bagheera_nodes:
for topic in config.topics:
for partition in config.partitions:
queue = Queue.Queue(256)
queues[(host, topic, partition)] = queue
bmp = BagheeraMessageProcessor(queue)
bmp_map[id(bmp)] = (host, topic, partition)
offset = offsets[(host, topic, partition)]
kc = KafkaConsumer(host, {}, topic, partition, bmp.processor,
offset, offset_update_freq)
t = threading.Thread(target=kc.process_messages_forever)
t.start()
strtime = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
fos = FileOutputStream("redacted/redacted_%s.gz" % (strtime))
gos = GZIPOutputStream(fos)
writer = PrintWriter(gos)
uf_fos = FileOutputStream("raw/unfiltered_%s.gz" % (strtime))
uf_gos = GZIPOutputStream(uf_fos)
uf_writer = PrintWriter(uf_gos)
err_fos = FileOutputStream("errors/errors_%s.gz" % (strtime))
err_gos = GZIPOutputStream(err_fos)
err_writer = PrintWriter(err_gos)
count = 0
while True:
for htp, q in queues.iteritems():
try:
v = q.get(False)
except Queue.Empty:
continue
if v[1] == 'PUT':
count = count + 1
pid, op, ts, ipaddr, doc_id, payload = v
json_payload = JSON.toJSONString(payload)
uf_writer.println(json_payload)
#TODO: figure out less braindead way of working with
# java.util.HashMaps in jython
try:
filtered = json.loads(json_payload)
# System.out.println('%s %s %d %s %s %s' % (htp[1], op, ts, ipaddr, doc_id, json_payload))
fltr.filter_document(filtered)
filtered['doc_id'] = doc_id
writer.println(json.dumps(filtered))
#print doc_id
except:
err_writer.println(doc_id + " " + json_payload)
if count % 10000 == 0:
print ts
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(
"com.udacity.project.chicago_transportation.station.transformed"
) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines.get_lines()
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
"com.udacity.project.chicago_transportation.weather.update",
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
"com.udacity.project.chicago_transportation.station.transformed",
lines.process_station_update_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"com.udacity.project.chicago_transportation.arrival",
lines.process_new_arrival_message,
offset_earliest=True,
),
KafkaConsumer(
"TURNSTILE_SUMMARY",
lines.process_turnstile_update_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
'''
Runs the Tornado Server and begins Kafka consumption
'''
if not utils.topic_exists(constants.TOPIC_TURNSTILE_SUMMARY):
logger.fatal(
'Ensure that the KSQL Command has run successfully before running the web server!'
)
exit(1)
if not utils.topic_exists(constants.TOPIC_STATIONS_TABLE_V1):
logger.fatal(
'Ensure that Faust Streaming is running successfully before running the web server!'
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r'/', MainHandler, {
'weather': weather_model,
'lines': lines
})])
application.listen(constants.SERVER_PORT)
# Build kafka consumers
consumers = [
KafkaConsumer(
constants.TOPIC_WEATHER_V1,
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
constants.TOPIC_STATIONS_TABLE_V1,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
constants.TOPIC_ARRIVALS_REGEX,
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
constants.TOPIC_TURNSTILE_SUMMARY,
lines.process_message,
offset_earliest=True,
is_avro=False,
)
]
try:
logger.info(
f'Transit Status Page running at http://localhost:{constants.SERVER_PORT}'
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
logger.info('shutting down server')
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(config.TOPIC_TURNSTILE_SUMMARY) is False:
logger.fatal("Ensure that the KSQL Command has run successfully!")
exit(1)
if topic_check.topic_pattern_match("stations.table") is False:
logger.fatal("Ensure that Faust Streaming is running successfully!")
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(config.WEB_SERVER_PORT)
# Build kafka consumers
consumers = [
KafkaConsumer(
config.TOPIC_WEATHER,
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
config.TOPIC_FAUST_TABLE,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
config.TOPIC_STATIONS,
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
config.TOPIC_TURNSTILE_SUMMARY,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
f"Open a web browser to http://localhost:{config.WEB_SERVER_PORT} to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
except Exception as ex:
logger.fatal(f"server.py error: {ex}")
exit(1)
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(
config['topics.consumers']['turnstile.summary']) is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(
config['topics.consumers']['faust.station.transformed']) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
config['topics.producers']['weather'],
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
config['topics.consumers']['faust.station.transformed'],
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
f"^{config['topics.producers']['station.arrival.prefix']}.*",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
config['topics.consumers']['turnstile.summary'],
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()