def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
print("Checking of TURNSTILE_SUMMARY topic...")
print(topic_check.topic_exists("TURNSTILE_SUMMARY"))
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
print("Topic found")
return
logging.debug("executing ksql statement...")
print("Posting data")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={
"Content-Type": "application/vnd.ksql.v1+json",
"Accept": "application/vnd.ksql.v1+json"
},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
# Ensure that a 2XX status code was returned
print("Done!")
resp.raise_for_status()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(TURNSTILE_SUMMARY) is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(STATIONS_TABLE) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application(
[(r"/", MainHandler, {"weather": weather_model, "lines": lines})]
)
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
WEATHER,
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
STATIONS_TABLE,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"^org.chicago.cta.station.arrivals.",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
TURNSTILE_SUMMARY,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
data = json.dumps(
{
"ksql": json.dumps(KSQL_STATEMENT),
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
)
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json; charset=utf-8"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
),
)
# Ensure that a 2XX status code was returned
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as e:
print(e)
logger.info("Error with KSQL POST request.")
def __init__(
self,
topic_name_pattern,
message_handler,
is_avro=True,
offset_earliest=False,
sleep_secs=1.0,
consume_timeout=0.1,
):
"""Creates a consumer object for asynchronous use"""
self.topic_name_pattern = topic_name_pattern
self.message_handler = message_handler
self.sleep_secs = sleep_secs
self.consume_timeout = consume_timeout
self.offset_earliest = offset_earliest
self.broker_properties = {
"bootstrap.servers": "PLAINTEXT://localhost:9092",
"auto.offset.reset": "earliest" if offset_earliest else "latest",
"group.id": 0
}
if is_avro is True:
self.broker_properties["schema.registry.url"] = "http://localhost:8081"
self.consumer = AvroConsumer(config=self.broker_properties)
else:
self.consumer = Consumer(config=self.broker_properties)
if topic_exists(topic_name_pattern):
self.consumer.subscribe(topics=[self.topic_name_pattern], on_assign=self.on_assign)
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists(AGGREGATED_TABLE_NAME) is True:
logger.info(f"Table {RAW_TABLE_NAME} already exists. Exiting...")
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
try:
resp.raise_for_status()
logger.info("Successfully created tables")
except Exception as err:
logger.error(json.dumps(resp.json()))
raise err
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
),
)
# Ensure that a 2XX status code was returned
if resp.status_code == 200:
logger.info(f"Created turnstile summary table using ksql.")
try:
resp.raise_for_status()
except:
print(f"Some error occured while running ksql commands")
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
print('Topic Existed...............................')
time.sleep(0.5)
return
else :
print('Not Existed................................')
logging.debug("executing ksql statement...")
# TODO: Complete the following KSQL Statement
# Directions: Use KSQL to combine the Stations Topic and the Turnstile Topic
time.sleep(0.5)
print('Line 52' + str(KSQL_URL))
resp = requests.post(
#f"{KSQL_URL}/ksql",
str(KSQL_URL)+'/ksql',
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
),
)
print('Line 64')
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("cta.trains.monitor.turnstile.summarized"):
logger.info("Turnstile summary topic already exists")
return
logger.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
),
)
# Check whether response status reflect successfull run
resp.raise_for_status()
if resp.status_code == 200:
logger.info("Turnstile ksql tables created successfully")
else:
logger.warning("Error creating Turnstile ksql tables")
logger.info(f"Response status code: {resp.status_code}")
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
#print('f**k')
#logger.info('Hasime')
if topic_check.topic_exists("TURNSTILE_SUMMARY") == True:
print('f**k')
return
#logger.info('OWARI')
#print('f**k')
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists(
config['topics.consumers']['turnstile.summary']) is True:
logger.debug('Table creation has been completed')
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{config['ksql']['url']}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
try:
# Ensure that a 2XX status code was returned
resp.raise_for_status()
except requests.exceptions.HTTPError as e:
logger.error(f"KSQL consumption error: {e.response.text}")
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
try:
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
except Exception as ex:
logger.warning(ex)
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
print(resp.content)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def create_topic(self):
"""Creates the producer topic if it does not already exist"""
# TODO: Write code that creates the topic for this producer if it does not already exist on
logger.info(f"check topic {self.topic_name}")
if topic_check.topic_exists(self.topic_name.strip()):
logger.info(f"topic {self.topic_name} exist")
return
# if not Producer.TOPIC_LIST_CACHED or len(Producer.TOPIC_LIST_CACHED) == 0:
# Producer.TOPIC_LIST_CACHED = topic_check.list_topics()
# logger.info(f"get list of topics cached,,,{Producer.TOPIC_LIST_CACHED}")
#
# if self.topic_name in Producer.TOPIC_LIST_CACHED:
# logger.info(f"topic {self.topic_name} exist")
# return
client = AdminClient({"bootstrap.servers": self.BROKER_URL})
futures = client.create_topics([
NewTopic(topic=self.topic_name,
num_partitions=self.num_partitions,
replication_factor=self.num_replicas)
])
for _, future in futures.items():
try:
future.result()
except Exception as e:
logger.error("error when creating topics", e)
raise e
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
),
)
# Ensure that a 2XX status code was returned
try:
resp.raise_for_status()
if resp.status_code >= 200 and resp.status_code < 300:
logger.debug(f"{resp.status_code}")
except:
logger.error(f"{resp.status_code}")
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
# Ensure that a 2XX status code was returned
try:
resp.raise_for_status()
except:
print(
f"failed creating ksql statement: {json.dumps(resp.json(), indent=2)}"
)
exit(1)
print("ksql statement created successfully.")
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
print("Creating KSQL table")
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
try:
resp.raise_for_status()
except:
print(
f"Failed to send data to Connector {json.dumps(resp.json(), indent=2)}"
)
return
print("KSQL Statement sent with success")
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if not topic_check.topic_exists(TURNSTILE_ENTRIES_TABLE):
logger.error(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
raise Exception("Ensure that the KSQL Command has run")
if not topic_check.topic_exists(STATION_MASTER_DATA):
logger.error(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
raise Exception("Ensure that Faust Streaming is running successfully")
weather_model = Weather()
lines = Lines()
application = tornado.web.Application(
[(r"/", MainHandler, {"weather": weather_model, "lines": lines})]
)
application.listen(8888)
consumers = [
KafkaConsumer(WEATHER_STATUS, weather_model.process_message, offset_earliest=True),
KafkaConsumer(STATION_MASTER_DATA, lines.process_message,
offset_earliest=True, is_avro=False),
KafkaConsumer(TRAIN_ARRIVAL, lines.process_message, offset_earliest=True),
KafkaConsumer(
TURNSTILE_ENTRIES_TABLE, lines.process_message, offset_earliest=True, is_avro=False,
),
]
logger.info("Open a web browser to http://localhost:8888 to see the Transit Status Page")
try:
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
# print("gogo")
ksql_data = {
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"},
}
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps(ksql_data),
)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("turnstile_summary") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
url_str = "http://localhost:8088/ksql"
# url_str = "http://ksql:8088/ksql"
resp = requests.post(
url_str,
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
return
logging.debug("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={"Content-Type": "application/vnd.ksql.v1+json"},
data=json.dumps({
"ksql": KSQL_STATEMENT,
"streamsProperties": {
"ksql.streams.auto.offset.reset": "earliest"
},
}),
)
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as err:
print(err)
print('ksql working')
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is True:
logging.info('The topic TURNSTILE_SUMMARY already created')
return
logging.info("executing ksql statement...")
resp = requests.post(
f"{KSQL_URL}/ksql",
headers={
"Content-Type" : "application/vnd.ksql.v1+json",
"Accept" : "application/vnd.ksql.v1+json"},
data=json.dumps(
{
"ksql": KSQL_STATEMENT,
"streamsProperties": {"ksql.streams.auto.offset.reset": "earliest"}
}
)
)
# Ensure that a 2XX status code was returned
resp.raise_for_status()
def execute_statement():
"""Executes the KSQL statement against the KSQL API"""
if topic_check.topic_exists("turnstile_summary") is True:
return
logging.debug("executing ksql statement...")
headers = {
"Content-Type": "application/vnd.ksql.v1+json; charset=utf-8",
"Accept": "application/vnd.ksql.v1+json"
}
stream_properties = {"ksql.streams.auto.offset.reset": "earliest"}
resp = requests.post(f"{KSQL_URL}/ksql",
headers=headers,
data=json.dumps({
"ksql":
KSQL_STATEMENT,
"streamsProperties":
stream_properties,
}))
# Ensure that a 2XX status code was returned
try:
resp.raise_for_status()
except requests.exceptions.HTTPError as err:
print(f"error message: {err}")
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(
"com.udacity.project.chicago_transportation.station.transformed"
) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines.get_lines()
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
"com.udacity.project.chicago_transportation.weather.update",
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
"com.udacity.project.chicago_transportation.station.transformed",
lines.process_station_update_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"com.udacity.project.chicago_transportation.arrival",
lines.process_new_arrival_message,
offset_earliest=True,
),
KafkaConsumer(
"TURNSTILE_SUMMARY",
lines.process_turnstile_update_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(
config['topics.consumers']['turnstile.summary']) is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(
config['topics.consumers']['faust.station.transformed']) is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
config['topics.producers']['weather'],
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
config['topics.consumers']['faust.station.transformed'],
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
f"^{config['topics.producers']['station.arrival.prefix']}.*",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
config['topics.consumers']['turnstile.summary'],
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
"Open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists("TURNSTILE_SUMMARY") is False:
logger.fatal(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_pattern_match("faust.stations.transformed") is False:
logger.fatal(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(WEB_SERVER_PORT)
print("Building consumers....")
# Kafka consumers for the application
consumers = [
KafkaConsumer(
"weather",
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
"faust.stations.transformed",
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
"^station.arrivals.*",
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
"TURNSTILE_SUMMARY",
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
f"Open a web browser to http://localhost:{WEB_SERVER_PORT} to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(TURNSTILE_SUMMARY_TOPIC) is False:
logger.fatal(
"ensure that the KSQL Command has run successfully before running the web server!"
)
exit(1)
if topic_check.topic_exists(STATIONS_TOPIC_V1) is False:
logger.fatal(
"ensure that Faust Streaming is running successfully before running the web server!"
)
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(8888)
# Build kafka consumers
consumers = [
KafkaConsumer(
id="weather.consumer",
topic_name_pattern=WEATHER_TOPIC_V1,
message_handler=weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
id="stations.table.consumer",
topic_name_pattern=STATIONS_TOPIC_V1,
message_handler=lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(id="arrival.consumer",
topic_name_pattern=f"^{ARRIVALS_TOPIC_PREFIX}",
message_handler=lines.process_message,
offset_earliest=True),
KafkaConsumer(
id="turnstile.summary.consumer",
topic_name_pattern=TURNSTILE_SUMMARY_TOPIC,
message_handler=lines.process_message,
offset_earliest=True,
is_avro=False,
)
]
try:
logger.info(
"open a web browser to http://localhost:8888 to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if topic_check.topic_exists(config.TOPIC_TURNSTILE_SUMMARY) is False:
logger.fatal("Ensure that the KSQL Command has run successfully!")
exit(1)
if topic_check.topic_pattern_match("stations.table") is False:
logger.fatal("Ensure that Faust Streaming is running successfully!")
exit(1)
weather_model = Weather()
lines = Lines()
application = tornado.web.Application([(r"/", MainHandler, {
"weather": weather_model,
"lines": lines
})])
application.listen(config.WEB_SERVER_PORT)
# Build kafka consumers
consumers = [
KafkaConsumer(
config.TOPIC_WEATHER,
weather_model.process_message,
offset_earliest=True,
),
KafkaConsumer(
config.TOPIC_FAUST_TABLE,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
KafkaConsumer(
config.TOPIC_STATIONS,
lines.process_message,
offset_earliest=True,
),
KafkaConsumer(
config.TOPIC_TURNSTILE_SUMMARY,
lines.process_message,
offset_earliest=True,
is_avro=False,
),
]
try:
logger.info(
f"Open a web browser to http://localhost:{config.WEB_SERVER_PORT} to see the Transit Status Page"
)
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt as e:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
except Exception as ex:
logger.fatal(f"server.py error: {ex}")
exit(1)
