def run(self, month):
self._set_weather(month)
resp = requests.post(
f"{KAKFA_REST_PROXY_URL}/topics/{Weather.topic_name}",
headers={"Content-Type": "application/vnd.kafka.avro.v2+json"},
data=json.dumps({
"value_schema":
Weather.value_schema,
"key_schema":
Weather.key_schema,
"records": [
{
"key": {
"timestamp": int(time.time())
},
"value": {
"temperature": self.temp,
"status": self.status.name
},
},
],
}),
)
resp.raise_for_status()
logger.debug(
f"sent weather data to kafka, temp: {self.temp}, status: {self.status.name}"
)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
key_schema=self.key_schema,
value_schema=self.value_schema,
)
except Exception:
logger.exception(f"Error producing for station {self.name}")
raise
logger.debug(
f"produced arrival of train {train} at station {self.name}")
def configure_connector():
"""Starts and configures the Kafka Connect connector"""
logger.debug("creating or updating kafka connect connector...")
resp = requests.get(f"{KAFKA_CONNECT_URL}/connectors/{CONNECTOR_NAME}")
if resp.status_code == 200:
logger.info("Kafka Connect JDBC connector already created skipping recreation")
return
resp = requests.post(
f"{KAFKA_CONNECT_URL}/connectors",
headers={"Content-Type": "application/json"},
data=json.dumps({
"name": CONNECTOR_NAME,
"config": {
"connector.class": "io.confluent.connect.jdbc.JdbcSourceConnector",
"key.converter": "org.apache.kafka.connect.json.JsonConverter",
"key.converter.schemas.enable": "false",
"value.converter": "org.apache.kafka.connect.json.JsonConverter",
"value.converter.schemas.enable": "false",
"batch.max.rows": "500",
"connection.url": STATION_DB_JDBC_URL,
"table.whitelist": CONNECT_STATION_TABLE,
"mode": "incrementing",
"incrementing.column.name": "stop_id",
"topic.prefix": CONNECT_PREFIX,
"poll.interval.ms": "86400000", # 1 day
}
}),
)
resp.raise_for_status()
logger.info("Kafka Connect JDBC connector created successfully")
def run(self):
curr_time = datetime.datetime.utcnow().replace(hour=0,
minute=0,
second=0,
microsecond=0)
logger.info("Beginning simulation, press Ctrl+C to exit at any time")
logger.info("loading Kafka Connect JDBC source connector")
configure_connector()
logger.info("beginning cta train simulation")
weather = Weather(curr_time.month)
try:
while True:
logger.debug(f"simulation running: {curr_time.isoformat()}")
# Send weather on the top of the hour
if curr_time.minute == 0:
weather.run(curr_time.month)
for line in self.train_lines:
line.run(curr_time, self.time_step)
curr_time = curr_time + self.time_step
time.sleep(self.sleep_seconds)
except KeyboardInterrupt:
logger.info("Shutting down")
for line in self.train_lines:
line.close()
def __init__(self, group_id=None):
"""Initializes a Producer object with basic settings"""
schema_registry = CachedSchemaRegistryClient({"url": KAFKA_SCHEMA_REGISTRY_URL})
producer_config = {"bootstrap.servers": KAFKA_BOOTSTRAP_SERVERS}
if group_id:
producer_config["client.id"] = group_id
self.producer = AvroProducer(producer_config, schema_registry=schema_registry)
logger.debug(f"Created producer with id {group_id}")
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"num_entries": num_entries,
},
key_schema=self.key_schema,
value_schema=self.value_schema,
)
except Exception:
logger.exception(
f"Error producing for turnstile at {self.station.name}")
raise
logger.debug(
f"produced turnstile event at station {self.station.name}")
def get(self):
"""Responds to get requests"""
logger.debug("rendering and writing handler template")
self.write(
MainHandler.template.generate(weather=self.weather, lines=self.lines)
)