def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
topic_name = "from_{}_station".format(
station_name) # TODO: Come up with a better topic name
super().__init__(
topic_name, # each station has its own topic to produce to
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=3, # Choose number of partitions & replicas
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
super().__init__(
topic_name="com.udacity.station.arrivals",
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
topic_name = f"com.company.station.arrivals" # TODO: Come up with a better topic name
super().__init__(
topic_name=topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
#print('Inside station, station name: ',name)
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#print('Inside station, station name 2: ',station_name)
topic_name = f"cta.kafka.stations.{station_name}"
num_partitions = 1
num_replicas = 1
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=num_partitions,
num_replicas=num_replicas,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "org.chicago.cta.station.arrivals.data" #f"com.name.station.{station_name}"#"org_chicago_transit_stations_data" # topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
# Fix a topic name, number of partitions, and number of replicas
topic_name = "org.chicago.transit.stations"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"nanodegre.kafka.chicago_cta.station_{station_name}.v1"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.station_name = station_name
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# logger.info(f"station_id:{station_id}\n station_name: {name}\n color: {color}\n")
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"com.udacity.project1.station.arrivals.{station_name}.v1"
logger.info(f"topic_name:{topic_name}")
super().__init__(topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"{constants.STATION_TOPIC_PREFIX}.{station_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color.name
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"org.chicago.cta.station.arrivals.{station_name}" # <domain>.<model>.<event type>
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=3,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"{Producer.TOPIC_PREFIX}.station.{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=20,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# :: Init (Topic created in base class)
topic_name = ARRIVAL_MSG_TOPIC_NAME
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(
self,
station_id,
name,
color,
direction_a=None,
direction_b=None
):
key_schema = utils.load_avro_schema('arrival_key.json')
value_schema = utils.load_avro_schema('arrival_value.json')
super().__init__(
constants.TOPIC_ARRIVALS_V1,
key_schema,
value_schema=value_schema,
num_partitions=5
)
self.station_id = int(station_id)
self.name = name
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
super().__init__(
"org.chicago.cta.station.arrivals.v1",
key_schema = Station.key_schema,
value_schema = Station.value_schema,
num_partitions = 1,
num_replicas = 1
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
num_partitions = 1
num_replicas = 1
#
#
topic_name = "kafka_station"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=num_partitions,
num_replicas=num_replicas,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
logger.info(f"station name {station_name}")
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "com.transitchicago.station.arrivals" # TODO: Come up with a better topic name
logger.info(f"topic name {topic_name}")
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
#topic_name = "chicago.cta.station.arrivals.v1" # TODO: Come up with a better topic name
super().__init__(
topic_name="chicago.cta.station.arrivals",
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"org.cta.station.arrivals.v1.{station_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
num_partitions=10,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# I'm deciding for 2 replicas to have a failure reliance design. If one replica is unavailable we won't miss data
# I'm deciding for 3 as the number of partioon as this application is not very thoughput intensive. Once I finish the implementation if I noticed the
# messaging traffic is too high, I can increase this number
topic_name = f"org.cta.station.arrivals.{station_name}.v1"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=2,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
topic_name = f"org.chicago.cta.station.arrivals.v1"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1, # Given that the key is a timestamp, num_partitions should be 1 to guarantee order
num_replicas=1 # There is only 1 kafka broker in this simulation
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
topic_name = f"{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
self.station_descriptive_name = (self.name.lower().replace(
"/", "_and_").replace(" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
TOPIC_NAME,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=2,
num_replicas=2,
)
self.station_id = int(station_id)
self.dir_a = direction_a
self.dir_b = direction_b
self.color = color
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
# TODO: Complete topic name, number of partitions, and number of replicas
#
topic_name = (
f"org.chicago.cta.station.{station_name}" # TODO: Come up with a better topic name
)
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema, # TODO: Uncomment once schema is defined
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
station_name = (
self.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"{Producer.TOPIC_PREFIX}.station.{station_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=20,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
station_name = (name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
self.name = name
topic_name = f"com.udacity.station.arrivals.{station_name}"
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: (done) Complete the below by deciding on a topic name, number of partitions, and number of replicas
topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.{station_name}.v1"
# topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.v1" # single topic for all station arrivals
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=1, # TODO: find out optimal partitions value
num_replicas=1, # TODO: find out optimal replicas value
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
topic_name = (
"streaming.project1.station" # TODO: Come up with a better topic name
)
super().__init__(
topic_name,
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=self.num_partitions,
num_replicas=self.num_replicas,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
Station._init_producer_singleton()
def __init__(self,
station_id,
name,
color,
direction_a=None,
direction_b=None):
self.name = name
station_name = (self.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = f"{station_name}" # TODO: Come up with a better topic name
super().__init__(
topic_name,
key_schema=Station.key_schema,
# TODO:
value_schema=Station.
value_schema, # TODO: Uncomment once schema is defined
# https://www.confluent.io/blog/how-choose-number-topics-partitions-kafka-cluster/
# p : single partition for production
# c : single partition for consumption
# t : target throughput
# choose at least max(t/p, t/c)
# partions = max(throughput/#producers, throughput/#consumers)
# Partitions = Max(Overall Throughput/Producer Throughput, Overall Throughput/Consumer Throughput)
# Example from video, with 3 Producers and 5 Consumers, each operating at 10MB/s per single producer/consumer
# partition: Max(100MBs/(3 * 10MB/s), 100MBs/(5 * 10MB/s)) = Max(2) ~= *4 partitions needed*
# TODO:
num_partitions=
2, # higher partition leads to higher throughput but high latency
# TODO:
num_replicas=1, # replicas shared between brokers
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None):
self.name = name
super().__init__(
topic_name="org.chicago.cta.station.arrivals.v1",
key_schema=Station.key_schema,
value_schema=Station.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station_id = int(station_id)
self.color = color
self.dir_a = direction_a
self.dir_b = direction_b
self.a_train = None
self.b_train = None
self.turnstile = Turnstile(self)
