def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas topic_name = "from_{}_station".format( station_name) # TODO: Come up with a better topic name super().__init__( topic_name, # each station has its own topic to produce to key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=3, # Choose number of partitions & replicas num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas super().__init__( topic_name="com.udacity.station.arrivals", key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=5, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas topic_name = f"com.company.station.arrivals" # TODO: Come up with a better topic name super().__init__( topic_name=topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=3, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name #print('Inside station, station name: ',name) station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) #print('Inside station, station name 2: ',station_name) topic_name = f"cta.kafka.stations.{station_name}" num_partitions = 1 num_replicas = 1 super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=num_partitions, num_replicas=num_replicas, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = "org.chicago.cta.station.arrivals.data" #f"com.name.station.{station_name}"#"org_chicago_transit_stations_data" # topic name super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=1, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # Fix a topic name, number of partitions, and number of replicas topic_name = "org.chicago.transit.stations" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=5, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) topic_name = f"nanodegre.kafka.chicago_cta.station_{station_name}.v1" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=3, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.station_name = station_name self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # logger.info(f"station_id:{station_id}\n station_name: {name}\n color: {color}\n") # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = f"com.udacity.project1.station.arrivals.{station_name}.v1" logger.info(f"topic_name:{topic_name}") super().__init__(topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, num_replicas=1) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) topic_name = f"{constants.STATION_TOPIC_PREFIX}.{station_name}" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, num_replicas=1, ) self.station_id = int(station_id) self.color = color.name self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = f"org.chicago.cta.station.arrivals.{station_name}" # <domain>.<model>.<event type> super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, num_replicas=3, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = f"{Producer.TOPIC_PREFIX}.station.{station_name}" # TODO: Come up with a better topic name super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=20, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # :: Init (Topic created in base class) topic_name = ARRIVAL_MSG_TOPIC_NAME super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=3, num_replicas=2, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__( self, station_id, name, color, direction_a=None, direction_b=None ): key_schema = utils.load_avro_schema('arrival_key.json') value_schema = utils.load_avro_schema('arrival_value.json') super().__init__( constants.TOPIC_ARRIVALS_V1, key_schema, value_schema=value_schema, num_partitions=5 ) self.station_id = int(station_id) self.name = name self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) super().__init__( "org.chicago.cta.station.arrivals.v1", key_schema = Station.key_schema, value_schema = Station.value_schema, num_partitions = 1, num_replicas = 1 ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # num_partitions = 1 num_replicas = 1 # # topic_name = "kafka_station" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=num_partitions, num_replicas=num_replicas, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) logger.info(f"station name {station_name}") # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = "com.transitchicago.station.arrivals" # TODO: Come up with a better topic name logger.info(f"topic name {topic_name}") super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=1, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # #topic_name = "chicago.cta.station.arrivals.v1" # TODO: Come up with a better topic name super().__init__( topic_name="chicago.cta.station.arrivals", key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=5, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) topic_name = f"org.cta.station.arrivals.v1.{station_name}" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station. value_schema, # TODO: Uncomment once schema is defined num_partitions=10, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # I'm deciding for 2 replicas to have a failure reliance design. If one replica is unavailable we won't miss data # I'm deciding for 3 as the number of partioon as this application is not very thoughput intensive. Once I finish the implementation if I noticed the # messaging traffic is too high, I can increase this number topic_name = f"org.cta.station.arrivals.{station_name}.v1" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=3, num_replicas=2, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) topic_name = f"org.chicago.cta.station.arrivals.v1" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, # Given that the key is a timestamp, num_partitions should be 1 to guarantee order num_replicas=1 # There is only 1 kafka broker in this simulation ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) topic_name = f"{station_name}" # TODO: Come up with a better topic name super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=3, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name self.station_descriptive_name = (self.name.lower().replace( "/", "_and_").replace(" ", "_").replace("-", "_").replace("'", "")) super().__init__( TOPIC_NAME, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=2, num_replicas=2, ) self.station_id = int(station_id) self.dir_a = direction_a self.dir_b = direction_b self.color = color self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # TODO: Complete topic name, number of partitions, and number of replicas # topic_name = ( f"org.chicago.cta.station.{station_name}" # TODO: Come up with a better topic name ) super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, # TODO: Uncomment once schema is defined num_partitions=1, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = ( self.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # topic name, number of partitions, and number of # replicas # # topic_name = f"{Producer.TOPIC_PREFIX}.station.{station_name}" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=20, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): station_name = (name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) self.name = name topic_name = f"com.udacity.station.arrivals.{station_name}" super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # TODO: (done) Complete the below by deciding on a topic name, number of partitions, and number of replicas topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.{station_name}.v1" # topic_name = f"{self.TOPIC_BASE_NAME}.station.arrivals.v1" # single topic for all station arrivals super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=1, # TODO: find out optimal partitions value num_replicas=1, # TODO: find out optimal replicas value ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas topic_name = ( "streaming.project1.station" # TODO: Come up with a better topic name ) super().__init__( topic_name, key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=self.num_partitions, num_replicas=self.num_replicas, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self) Station._init_producer_singleton()
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name station_name = (self.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name = f"{station_name}" # TODO: Come up with a better topic name super().__init__( topic_name, key_schema=Station.key_schema, # TODO: value_schema=Station. value_schema, # TODO: Uncomment once schema is defined # https://www.confluent.io/blog/how-choose-number-topics-partitions-kafka-cluster/ # p : single partition for production # c : single partition for consumption # t : target throughput # choose at least max(t/p, t/c) # partions = max(throughput/#producers, throughput/#consumers) # Partitions = Max(Overall Throughput/Producer Throughput, Overall Throughput/Consumer Throughput) # Example from video, with 3 Producers and 5 Consumers, each operating at 10MB/s per single producer/consumer # partition: Max(100MBs/(3 * 10MB/s), 100MBs/(5 * 10MB/s)) = Max(2) ~= *4 partitions needed* # TODO: num_partitions= 2, # higher partition leads to higher throughput but high latency # TODO: num_replicas=1, # replicas shared between brokers ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)
def __init__(self, station_id, name, color, direction_a=None, direction_b=None): self.name = name super().__init__( topic_name="org.chicago.cta.station.arrivals.v1", key_schema=Station.key_schema, value_schema=Station.value_schema, num_partitions=5, num_replicas=1, ) self.station_id = int(station_id) self.color = color self.dir_a = direction_a self.dir_b = direction_b self.a_train = None self.b_train = None self.turnstile = Turnstile(self)