class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below, done
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
topic_name = f"station_topic_{station_name}_{color.name}" # TODO: Come up with a better topic name, done
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined, done
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# TODO: Complete this function by producing an arrival message to Kafka, done
logger.info(
f"{self.topic_name}: train {train.train_id} arrived from direction \
{direction} and prev_direction {prev_direction} \
and prev_station_id {prev_station_id} ")
#print(f"station_id: {self.station_id}")
#print(f"trainid: {train.train_id}")
#print(f"dir: {direction}")
#print(f"colorname: {self.color.name}")
#print(f"statusname: {train.status.name}")
#print(f"prevstid: {prev_station_id}")
#print(f"prevdir: {prev_direction}")
if not prev_station_id:
prev_station_id = 0
if not prev_direction:
prev_direction = 'None'
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
})
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#""
topic_name = f"org.udacity.project.{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
logger.info()
self.producer.produce(
self.topic_name,
key={"timestamp": self.time_millis()},
value={
self.train: train,
self.direction: direction,
self.prev_station_id: prev_station_id,
self.prev_direction: prev_direction,
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = "chicago.transport.arrivals" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# logger.info("arrival kafka integration incomplete - skipping")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#topic_name = f"com.udacity.station.arrivals.v1" #topic name convention: <namespace or domain>.<schema or model>.<event>
super().__init__(
topic_name=f"com.udacity.station.arrivals.v1",
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# Producing an arrival message to Kafka
#
#
logger.info("arrival kafka integration")
value = {
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
}
#double check json value
logger.info("Values for train arrivals are: %s ", json.dumps(value))
try:
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=value)
except Exception as e:
logger.fatal(e)
raise e
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema: RecordSchema = load_schema("arrival_key.json")
value_schema: RecordSchema = load_schema("arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.station_id = int(station_id)
self.name = name
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
station_name = normalize_station_name(self.name)
topic_name = join_topic_name(CtaTopics.ARRIVALS_PREFIX, station_name)
super().__init__(
topic_name=topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
self.producer.poll(0)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super().close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
#print('Inside station, station name: ',name)
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#print('Inside station, station name 2: ',station_name)
topic_name = f"cta.kafka.stations.{station_name}"
num_partitions = 1
num_replicas = 1
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=num_partitions,
num_replicas=num_replicas,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
#print('Inside station, station id: ',station_id)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
logger.info("arrival kafka integration started")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"line": self.color.name,
"direction": direction,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
)
logger.info("arrival kafka integration is complete")
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
#print('Inside arrive_a :',train,' ',prev_station_id,' ',prev_direction)
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
#print('Inside arrive_b :',train,' ',prev_station_id,' ',prev_direction)
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "org.chicago.cta.station.arrivals.v1" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
# TODO: value_schema=Station.value_schema, # TODO: Uncomment once schema is defined
# TODO: num_partitions=???,
# TODO: num_replicas=???,
value_schema=Station.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
logger.info(f"arrival message to kafka for station id: {self.station_id} train:{train.train_id} direction: {direction} line:{self.color._name_}")
#self.producer.produce(
# topic=self.topic_name,
# key={"timestamp": self.time_millis()},
# value={
# #
# #
# # TODO: Configure this
# #
# #
# },
#)
try:
self.producer.produce(
topic=self.topic_name,
value={
"station_id":self.station_id,
"train_id":train.train_id, ##station does not have train id as attributes
"direction":direction,
"line":self.color._name_,#not coming in parameters. not present in station attributes
"train_status":train.status._name_, #KMN - Train ID BL000 is in service - status is a property - not callable - in service =1 <enum 'status'>
"prev_station_id":prev_station_id, ## what does previous mean? coming blank
"prev_direction":prev_direction # coming blank
},
value_schema=self.value_schema, #how to open this https://github.com/apache/avro/blob/master/lang/py3/avro/schema.py self.value_schema.namespace
key={"timestamp": self.time_millis()},
key_schema=self.key_schema
)
except Exception as e:
logger.warning(f"Encountered Error - {e} for station_id : {self.station_id} train_id : {train.train_id} direction : {direction} line : {self.color._name_}")
pass
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station():
"""Defines a single station"""
_producer: Producer = None
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
Station._init_producer_singleton()
@classmethod
def _init_producer_singleton(cls):
if cls._producer is None:
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
cls._producer = Producer(
'com.udacity.project.chicago_transportation.arrival',
key_schema=key_schema,
value_schema=value_schema
)
def run(self, train: Train, direction: str, prev_station_id: int, prev_direction: str):
"""Simulates train arrivals at this station"""
Station._producer.produce({
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
})
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
Station._producer.close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
topic_name = f"com.company.station.arrivals" # TODO: Come up with a better topic name
super().__init__(
topic_name=topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# TODO: Complete this function by producing an arrival message to Kafka
try:
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color._name_,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
value_schema=self.value_schema)
except Exception as e:
logger.warning(
f"Encountered Error - {e} for station_id : {self.station_id} train_id : {train.train_id} direction : {direction} line : {self.color._name_}"
)
#raise e
#except Exception as e:
# logger.fatal(e)
# raise e
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
# logger.info("Loaded: "+str(type(value_schema)))
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# logger.info(f"station_id:{station_id}\n station_name: {name}\n color: {color}\n")
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"com.udacity.project1.station.arrivals.{station_name}.v1"
logger.info(f"topic_name:{topic_name}")
super().__init__(topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
# logger.info(f"{self.station_id}\n{self.color}\n{self.dir_a}\n{self.dir_b}\n{self.a_train}\n{self.b_train}")
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
# TODO: Complete this function by producing an arrival message to Kafka
#
logger.info("arrival kafka integration complete - running")
logger.info(
f"{self.station_id},{train.train_id},{direction},{self.color.name},{train.status}"
)
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"train_id": train.train_id,
"station_id": self.station_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
})
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
# :: Use value schema from `schemas/station_value.json.
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# :: Init (Topic created in base class)
topic_name = ARRIVAL_MSG_TOPIC_NAME
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# :: Producing an arrival message to Kafka
logger.info(
"---------------- 1] Producing an arrival message to Kafka -----------------"
)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"train_status": train.status,
"line": self.color,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "org.chicago.cta.station.arrivals" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=2,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
# logger.info("arrival kafka integration incomplete - skipping")
# requests.delete('http://localhost:8081/subjects/{}-key'.format(self.topic_name))
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
'station_id': self.station_id,
'train_id': train.train_id,
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id': prev_station_id,
'prev_direction': prev_direction,
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
# TODO: Come up with a better topic name
topic_name = f"org.chicago.cta.station.arrivals.{station_name}"
# TODO: Include/fill the following in the call to super.__init__():
# value_schema=Station.value_schema,
# num_partitions=???,
# num_replicas=???,
# call the super to instantiate super's vars also incl. self.producer
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
# schemas have already been set in instance creation hence commented out
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
# key_schema=Station.key_schema,
# value_schema=Station.value_schema,
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | " \
"departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
'''
Defines a single station
'''
def __init__(
self,
station_id,
name,
color,
direction_a=None,
direction_b=None
):
key_schema = utils.load_avro_schema('arrival_key.json')
value_schema = utils.load_avro_schema('arrival_value.json')
super().__init__(
constants.TOPIC_ARRIVALS_V1,
key_schema,
value_schema=value_schema,
num_partitions=5
)
self.station_id = int(station_id)
self.name = name
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
'''
Simulates train arrivals at this station
'''
logger.info(f'train {train.train_id} arrived to station {self.station_id}')
self.producer.produce(
topic=self.topic_name,
key={'timestamp': self.time_millis()},
value={
'station_id': self.station_id,
'train_id': train.train_id,
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id': prev_station_id,
'prev_direction': prev_direction
}
)
def __str__(self):
return 'Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | '.format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else '---',
self.dir_a.name if self.dir_a is not None else '---',
self.b_train.train_id if self.b_train is not None else '---',
self.dir_b.name if self.dir_b is not None else '---'
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
'''
Denotes a train arrival at this station in the 'a' direction
'''
self.a_train = train
self.run(train, 'a', prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
'''
Denotes a train arrival at this station in the 'b' direction
'''
self.b_train = train
self.run(train, 'b', prev_station_id, prev_direction)
def close(self):
'''
Prepares the producer for exit by cleaning up the producer
'''
self.turnstile.close()
super().close()
class Station(Producer):
"""Defines a single station"""
logger.info(
f"loading {Path(__file__).parents[0]}/schemas/arrival_key.json")
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
logger.info(
f"loading {Path(__file__).parents[0]}/schemas/arrival_value.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
logger.info(f"station name {station_name}")
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "com.transitchicago.station.arrivals" # TODO: Come up with a better topic name
logger.info(f"topic name {topic_name}")
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
logger.info(
f"producing {self.topic_name} for station {self.station_id} train {train.train_id} with status {train.status.name} and color {self.color.name}"
)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
#
#
# TODO: Configure this
#
#
'station_id': self.station_id,
'train_id': train.train_id, # see train.py
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id': prev_station_id,
'prev_direction': prev_direction
},
)
logger.info(
f"produced {self.topic_name} for station {self.station_id} train {train.train_id}"
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_kafka_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"org.chicago.cta.station.arrivals.{station_kafka_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
# Produces an arrival message to Kafka
#
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": "arrival",
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"org.chicago.cta.station.arrivals.{station_name}"
super().__init__(topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
key = {"timestamp": self.time_millis()}
station = self._build_station(direction, prev_direction,
prev_station_id, train)
self._produce(key, station)
logger.debug(
f"Arriving at station {self.name}: {train}. Direction is {direction}"
)
def _build_station(self, direction, prev_direction, prev_station_id,
train):
"""Convenience method for generating a new Station object"""
station = {
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
}
return station
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
# TODO: (done) Define this value schema in `schemas/arrival_value.json, then uncomment the below
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: (done) Complete the below by deciding on a topic name, number of partitions, and number of replicas
topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.{station_name}.v1"
# topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.v1" # single topic for all station arrivals
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1, # TODO: find out optimal partitions value
num_replicas=1, # TODO: find out optimal replicas value
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# TODO: Complete this function by producing an arrival message to Kafka
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
# TODO:
value_schema=Station.value_schema,
num_partitions=2,
# TODO:
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
#logger.info("arrival kafka integration incomplete - skipping")
# make sure the arrival events to kafka are paired with Avro key and value schemas
# look at train.py and line.py to get the properties of those instances (train and line)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
# TODO: Configure this
"station_id" : self.station_id ,
"train_id" : train.train_id, # to get train_id, look at `self.train_id` in train.py
"direction" : direction,
"line" : self.color.name, # to get the line , look at `self.color.name` in line.py
"train_status" : train.status.name, # to get train status, look at `self.status.name` in train.py
"prev_station_id" : prev_station_id,
"prev_direction" : prev_direction
},
)
logger.info(f"producing arrival event to kafka is complete")
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
# Load Avro Key Schema
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
# Load Avro Value Schema
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
topic_name = f"org.chicago.cta.station.arrivals.v1"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1, # Given that the key is a timestamp, num_partitions should be 1 to guarantee order
num_replicas=1 # There is only 1 kafka broker in this simulation
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
logger.info("Train arriving - sending message to kafka")
self.producer.produce(
topic=self.topic_name,
key_schema=self.key_schema,
value_schema=self.value_schema,
key={"timestamp": self.time_millis()},
value={
'station_id': self.station_id,
'train_id': train.train_id,
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id': prev_station_id,
'prev_direction': prev_direction
}
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "com.udacity.station.arrival" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
missing = ":station/missing"
value = {
"station_id": self.station_id or -1,
"train_status": train.status.name,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"prev_station_id": prev_station_id or -1,
"prev_direction": prev_direction or missing
}
try:
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=value)
except Exception as e:
print(e)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of replicas
topic_name = 'org.chicago.cta.station.arrivals.v1' # TODO: Come up with a better topic name
super().__init__(
topic_name,
# TODO: value_schema=Station.value_schema, # TODO: Uncomment once schema is defined
# TODO: num_partitions=???,
# TODO: num_replicas=???,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
# TODO: Complete this function by producing an arrival message to Kafka
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
# TODO: Configure this
'station_id': self.station_id,
'train_id': train.train_id,
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id': prev_station_id,
'prev_direction': prev_direction
},
)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
# station_name = (
# self.name.lower()
# .replace("/", "_and_")
# .replace(" ", "_")
# .replace("-", "_")
# .replace("'", "")
# )
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#topic_name = f"com.cta.station.arrivals.{station_name}"
super().__init__(
topic_name="com.cta.station.arrivals.v1",
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
#logger.info("arrival kafka integration incomplete - skipping")
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
)
except Exception as e:
logger.fatal(e)
raise (e)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
super().__init__(
topic_name="org.chicago.cta.station.arrivals.v1",
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction,
},
)
except Exception as e:
logger.error(e)
raise e
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
# class variables
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
# instamce variables
# self.topic_name
# self.name
# self.station_id
# self.color
# self.dir_a
# self.dir_b
# self.a_train
# self.b_train
# self.turnstile
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
# No longer used.
# station_name = (
# self.name.lower()
# .replace("/", "_and_")
# .replace(" ", "_")
# .replace("-", "_")
# .replace("'", "")
# )
# Unique stations 91.
# yellow (not r, b, and g) = 42
# red = 66
# blue = 66
# green = 56
# ---------------------------------
# # of Station-lines 230
#
# Since our reporting is train-line based, and key is timestamp, we will have to aggregate on metro line.
# Topic per station vs station-line: Topic per station as per station-line is will have a mesaages per few minutes instead of seconds.
# Assuming busy station with three lines crossing, with topic per station, we can expect appx 4-6 msgs/min.
#
# Ex: com.cta.stations.addison.red, com.cta.stations.addison.blue, com.cta.stations.ohare.blue.
#topic_name = f"{CTAConstants.TRAIN_ARRIVAL_TOPIC_PREFIX}.{color}"
#topic_name = f"{CTAConstants.TRAIN_ARRIVAL_TOPIC_PREFIX}{station_name}.{color}"
topic_name = f"{CTAConstants.TRAIN_ARRIVAL_TOPIC_PREFIX}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=2,
num_replicas=1,
# num_replicas=2,
)
self.topic_name = topic_name
self.station_id = int(station_id)
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
colors = CTAConstants.colors
if (colors.red == color):
self.color = "red"
elif (colors.blue == color):
self.color = "blue"
elif (colors.green == color):
self.color = "green"
else:
self.color = "yellow" #Unknown.
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
logger.debug(
f"arrival kafka integration. sin: {self.station_id}, tin: {train.train_id}, dir: {direction}, col: {self.color}, p.sin: {prev_station_id}, pdir:{prev_direction}"
)
# if (prev_station_id is None):
# val = {
# "station_id": self.station_id,
# "train_id": train.train_id,
# "direction": direction,
# "line": self.color,
# "train_status":train.status.name},
# else:
val = {
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
}
self.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=val)
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
logger.debug("station - init")
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
#topic_name = f"org.chicago.cta.station.arrivals.{station_name}" # TODO: Come up with a better topic name
topic_name = f"org.chicago.cta.station.arrivals" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=1,
num_replicas=1)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
logger.debug("station - initiarized")
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
#logger.info("arrival kafka integration incomplete - skipping")
logger.debug(f"line <- self.color: {self.color}")
logger.debug(f"prev_station_id: {prev_station_id}")
logger.debug(f"prev_direction: {prev_direction}")
logger.debug(f"self.value_schema: {self.value_schema}")
#prev_direction = "empty" if prev_direction == None else prev_direction
#logger.info(f"prev_direction: {prev_direction}")
value = {
#
#
# TODO: Configure this
#
#
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
}
logger.debug(f"value: {value}")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
#
#
# TODO: Configure this
#
#
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
} #,
#value_schema=Station.value_schema,
#key_schema=Station.key_schema
)
logger.info(f"run - {self.topic_name}")
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"org.cta.chicago.station.arrivals.{station_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
'station_id': self.station_id,
'train_id': train.train_id,
'direction': direction,
'line': self.color.name,
'train_status': train.status.name,
'prev_station_id':
str(prev_station_id) if prev_station_id else "---",
'prev_direction':
str(prev_direction) if prev_direction else "---",
})
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# I'm deciding for 2 replicas to have a failure reliance design. If one replica is unavailable we won't miss data
# I'm deciding for 3 as the number of partioon as this application is not very thoughput intensive. Once I finish the implementation if I noticed the
# messaging traffic is too high, I can increase this number
topic_name = f"org.cta.station.arrivals.{station_name}.v1"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#logger.info(f"train: {train}, prev_station_id:{prev_station_id}, prev_direction:{prev_direction}")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=asdict(
ArrivalEvent(
station_id=str(self.station_id),
train_id=train.train_id,
direction=direction,
line=self.color.name, #IntEnum from line.py
train_status=train.status.name, #IntEnum from train.py
prev_station_id=prev_station_id,
prev_direction=prev_direction)))
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"{super().NAMESPACE}.stations.arrivals.{station_name}"
super().__init__(
topic_name=topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=3,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
try:
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station_id,
"train_id": train.train_id,
"direction": direction,
"line": self.color.name,
"train_status": train.status.name,
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
value_schema=Station.value_schema,
key_schema=Station.key_schema)
except KeyboardInterrupt as e:
logger.info("Shutting down")
super().close()
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
class Station(Producer):
"""Defines a single station"""
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_key.json")
#
# TODO: Define this value schema in `schemas/station_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/arrival_value.json")
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
# TODO:
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
# https://www.confluent.io/blog/how-choose-number-topics-partitions-kafka-cluster/
# p : single partition for production
# c : single partition for consumption
# t : target throughput
# choose at least max(t/p, t/c)
# partions = max(throughput/#producers, throughput/#consumers)
# Partitions = Max(Overall Throughput/Producer Throughput, Overall Throughput/Consumer Throughput)
# Example from video, with 3 Producers and 5 Consumers, each operating at 10MB/s per single producer/consumer
# partition: Max(100MBs/(3 * 10MB/s), 100MBs/(5 * 10MB/s)) = Max(2) ~= *4 partitions needed*
# TODO:
num_partitions=
2, # higher partition leads to higher throughput but high latency
# TODO:
num_replicas=1, # replicas shared between brokers
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def run(self, train, direction, prev_station_id, prev_direction):
"""Simulates train arrivals at this station"""
#
#
# TODO: Complete this function by producing an arrival message to Kafka
#
#
#logger.info("arrival kafka integration incomplete - skipping")
# make sure the arrival events to kafka are paired with Avro key and value schemas
# look at train.py and line.py to get the properties of those instances (train and line)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
# TODO: Configure this
"station_id": self.station_id,
"train_id": train.
train_id, # to get train_id, look at `self.train_id` in train.py
"direction": direction,
"line": self.color.
name, # to get the line , look at `self.color.name` in line.py
"train_status": train.status.
name, # to get train status, look at `self.status.name` in train.py
"prev_station_id": prev_station_id,
"prev_direction": prev_direction
},
)
logger.info(f"producing arrival event to kafka is complete")
def __str__(self):
return "Station | {:^5} | {:<30} | Direction A: | {:^5} | departing to {:<30} | Direction B: | {:^5} | departing to {:<30} | ".format(
self.station_id,
self.name,
self.a_train.train_id if self.a_train is not None else "---",
self.dir_a.name if self.dir_a is not None else "---",
self.b_train.train_id if self.b_train is not None else "---",
self.dir_b.name if self.dir_b is not None else "---",
)
def __repr__(self):
return str(self)
def arrive_a(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'a' direction"""
self.a_train = train
self.run(train, "a", prev_station_id, prev_direction)
def arrive_b(self, train, prev_station_id, prev_direction):
"""Denotes a train arrival at this station in the 'b' direction"""
self.b_train = train
self.run(train, "b", prev_station_id, prev_direction)
def close(self):
"""Prepares the producer for exit by cleaning up the producer"""
self.turnstile.close()
super(Station, self).close()
