def wrapper(*args, **kwargs):
logger.info(f"Waiting for {decorated_function.__name__}")
result = None
stored_exception = None
end_time = time.time() + timeout
while time.time() < end_time:
try:
result = decorated_function(*args, **kwargs)
stored_exception = None
except Exception as e:
if caught_exception is None or not isinstance(
e, caught_exception):
raise
stored_exception = e
time.sleep(0.3)
continue
if result:
return
else:
time.sleep(0.3)
continue
if not result:
if stored_exception:
raise stored_exception
else:
raise Exception("Dependency not ready")
def configure_connector():
"""Starts and configures the Kafka Connect connector"""
logger.debug("creating or updating kafka connect connector...")
resp = requests.get(f"{KAFKA_CONNECT_URL}/connectors/{CONNECTOR_NAME}")
if resp.status_code == 200:
logger.info("Kafka Connect JDBC connector already created skipping recreation")
return
resp = requests.post(
f"{KAFKA_CONNECT_URL}/connectors",
headers={"Content-Type": "application/json"},
data=json.dumps({
"name": CONNECTOR_NAME,
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "false",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schemas.enable": "false",
"batch.max.rows": "500",
"connection.url": STATION_DB_JDBC_URL,
"table.whitelist": CONNECT_STATION_TABLE,
"mode": "incrementing",
"incrementing.column.name": "stop_id",
"topic.prefix": CONNECT_PREFIX,
"poll.interval.ms": "86400000", # 1 day
}
}),
)
resp.raise_for_status()
logger.info("Kafka Connect JDBC connector created successfully")
def on_assign(self, consumer, partitions):
"""Callback for when topic assignment takes place"""
if self.offset_earliest:
for partition in partitions:
partition.offset = OFFSET_BEGINNING
logger.info(f"partitions assigned for {self.topic_name_pattern}")
consumer.assign(partitions)
def run(self):
curr_time = datetime.datetime.utcnow().replace(hour=0,
minute=0,
second=0,
microsecond=0)
logger.info("Beginning simulation, press Ctrl+C to exit at any time")
logger.info("loading Kafka Connect JDBC source connector")
configure_connector()
logger.info("beginning cta train simulation")
weather = Weather(curr_time.month)
try:
while True:
logger.debug(f"simulation running: {curr_time.isoformat()}")
# Send weather on the top of the hour
if curr_time.minute == 0:
weather.run(curr_time.month)
for line in self.train_lines:
line.run(curr_time, self.time_step)
curr_time = curr_time + self.time_step
time.sleep(self.sleep_seconds)
except KeyboardInterrupt:
logger.info("Shutting down")
for line in self.train_lines:
line.close()
def run_server():
"""Runs the Tornado Server and begins Kafka consumption"""
if not topic_check.topic_exists(TURNSTILE_ENTRIES_TABLE):
logger.error(
"Ensure that the KSQL Command has run successfully before running the web server!"
)
raise Exception("Ensure that the KSQL Command has run")
if not topic_check.topic_exists(STATION_MASTER_DATA):
logger.error(
"Ensure that Faust Streaming is running successfully before running the web server!"
)
raise Exception("Ensure that Faust Streaming is running successfully")
weather_model = Weather()
lines = Lines()
application = tornado.web.Application(
[(r"/", MainHandler, {"weather": weather_model, "lines": lines})]
)
application.listen(8888)
consumers = [
KafkaConsumer(WEATHER_STATUS, weather_model.process_message, offset_earliest=True),
KafkaConsumer(STATION_MASTER_DATA, lines.process_message,
offset_earliest=True, is_avro=False),
KafkaConsumer(TRAIN_ARRIVAL, lines.process_message, offset_earliest=True),
KafkaConsumer(
TURNSTILE_ENTRIES_TABLE, lines.process_message, offset_earliest=True, is_avro=False,
),
]
logger.info("Open a web browser to http://localhost:8888 to see the Transit Status Page")
try:
for consumer in consumers:
tornado.ioloop.IOLoop.current().spawn_callback(consumer.consume)
tornado.ioloop.IOLoop.current().start()
except KeyboardInterrupt:
logger.info("shutting down server")
tornado.ioloop.IOLoop.current().stop()
for consumer in consumers:
consumer.close()
async def transform_stations(stations):
logger.info("Station transform stream started")
async for station in stations:
line_color = None
if station.red:
line_color = "red"
elif station.blue:
line_color = "blue"
elif station.green:
line_color = "green"
else:
logger.warning(f"Received unknown line: {station}")
continue
transformed_station = TransformedStation(
station_id=station.station_id,
station_name=station.station_name,
order=station.order,
line=line_color,
)
table[station.stop_id] = transformed_station
"""Used in local development (and docker) to wait until all services started"""
from shared_helpers.logging import logger
from shared_helpers.wait_until import (
check_kafka,
check_schema_registry,
check_rest_proxy,
check_kafka_connect,
)
if __name__ == "__main__":
check_kafka()
check_schema_registry()
check_rest_proxy()
check_kafka_connect()
logger.info(f"All dependencies are there!")