def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # topic_name="com.udacity.topic.turnstile" super().__init__( topic_name, # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, # TODO: value_schema=Turnstile.value_schema, #TODO: Uncomment once schema is defined # TODO: num_partitions=1, # TODO: num_replicas=1 ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # super().__init__( # TODO: Come up with a better topic name topic_name="org.cta.station.turnstile", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=Producer.num_partitions, num_replicas=Producer.num_replicas, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" self.station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # self.topic_name = f"org.chicago.cta.station.turnstile_entry" # <domain>.<model>.<event type> super().__init__( self.topic_name, # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=3, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # super().__init__( "com.udacity.cta.turnstile.v1", # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, #TODO: Uncomment once schema is defined # https://www.confluent.io/blog/how-choose-number-topics-partitions-kafka-cluster/ # p : single partition for production # c : single partition for consumption # t : target throughput # choose at least max(t/p, t/c) # TODO: num_partitions=2, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" print('init turnstile') station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas #self.station.color.name" # topic_name = f"{Producer.TOPIC_PREFIX}.turnstiles" super().__init__( topic_name, # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, #TODO: Uncomment once schema is defined num_partitions=20, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # Complete: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # super().__init__( f"com.cta.station.arrival_dtl.turnstile", # Complete: Come up with a better topic name key_schema=Turnstile.key_schema, # Complete: value_schema=Turnstile.value_schema, TODO: Uncomment once schema is defined # Complete: num_partitions=???, # Complete: num_replicas=???, value_schema=Turnstile.value_schema, num_partitions=5, num_replicas=1 # complete: num_replicas=???, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # super().__init__( # f"{station_name}", # TODO: Come up with a better topic name f"org.chicago.cta.turnstiles", key_schema=Turnstile.key_schema, # TODO: value_schema=Turnstile.value_schema, TODO: Uncomment once schema is defined # TODO: num_partitions=???, # TODO: num_replicas=???, value_schema = Turnstile.value_schema ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # DONE: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # # super().__init__( f"il.cta.{station_name}.turnstile", # DONE: Come up with a better topic name key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, # DONE: Uncomment once schema is defined num_partitions=3, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = ( station.name.lower() .replace("/", "_and_") .replace(" ", "_") .replace("-", "_") .replace("'", "") ) # # # # # self.topic_name = "turnstile.v1" super().__init__( "turnstile.v1", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" super().__init__(topic_name="org.chicago.cta.station.turnstile.v1", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=5, num_replicas=1) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" super().__init__( topic_name=f"com.udacity.turnstiles", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" super().__init__( topic_name=CtaTopics.TURNSTILES, key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__(topic_name="org.chicago.cta.turnstiles", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" super().__init__( TOPIC_NAME, key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) topic_name = f"udacity.com.km.turnstiles" super().__init__(topic_name, key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__(f"com.udacity.cta.turnstile", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, num_replicas=1) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__( f"turnstile.entries.v1", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station, create_topic=True): """Create the Turnstile""" super().__init__( topic_name="com-cta-station-turnstile", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=5, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
class Turnstile(Producer): key_schema = avro.load( f"{Path(__file__).parents[0]}/schemas/turnstile_key.json") # # TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below # value_schema = avro.load( f"{Path(__file__).parents[0]}/schemas/turnstile_value.json") def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # # # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas # topic_name = "chicago.turnstiles" super().__init__( topic_name, # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, # TODO: value_schema=Turnstile.value_schema, TODO: Uncomment once schema is defined # TODO: num_partitions=???, # TODO: num_replicas=???, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1) self.station = station self.turnstile_hardware = TurnstileHardware(station) def run(self, timestamp, time_step): """Simulates riders entering through the turnstile.""" num_entries = self.turnstile_hardware.get_entries(timestamp, time_step) logger.info( f"Producing turnstile data \n Topic Name: {self.topic_name} \n Station ID: {self.station.station_id}" ) # # # TODO: Complete this function by emitting a message to the turnstile topic for the number # of entries that were calculated entries = self.turnstile_hardware.get_entries(timestamp, time_step) for entry in range(entries): self.producer.produce(topic=self.topic_name, key={"timestamp": self.time_millis()}, value={ "station_id": self.station.station_id, "station_name": self.station.name, "line": self.station.color.name })
def __init__(self, station): """Create the Turnstile""" topic_name = f"chicago_transit.turnstile" super().__init__( topic_name, key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=10, num_replicas=2, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) self.station = station self.turnstile_hardware = TurnstileHardware(station) super().__init__( "udacity.cta.turnstiles", # TODO: Come up with a better topic name key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, num_replicas=1)
def __init__(self, station, create_topic=True): """Create the Turnstile""" # TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas super().__init__( topic_name="org.chicago.cta.station.turnstile.v1", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=5, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__( f"{self.TURNSTILE_TOPIC_NAME}", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__( config["topics.producers"]['turnstile'], key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__( f"{Turnstile.topic_namespace}.{Turnstile.__name__.lower()}.{Turnstile.event_type}", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" self.station_name = (station.name.lower().replace( "/", "_and_").replace(" ", "_").replace("-", "_").replace("'", "")) super().__init__( f"com.cta.v1.turnstile_events", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, # Order of turnstile events is not important num_replicas=1 # Only 1 kafka broker ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) # (DONE)TODO: Complete the below by deciding on a topic name, number of partitions, and number of # replicas super().__init__(f"cta.status_dashboard.turnstiles", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=1, num_replicas=1) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" self.station_name = (station.name.lower().replace( "/", "_and_").replace(" ", "_").replace("-", "_").replace("'", "")) super().__init__( 'org.chicago.cta.station.turnstile.v1', key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=2, num_replicas=2, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) topic_name = f"org.chicago.transit.looptrain.{station_name}.turnstile" super().__init__( topic_name, key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=3, num_replicas=1, ) self.station = station self.turnstile_hardware = TurnstileHardware(station)
def __init__(self, station): """Create the Turnstile""" station_name = (station.name.lower().replace("/", "_and_").replace( " ", "_").replace("-", "_").replace("'", "")) super().__init__( f"org.chicago.cta.turnstile", key_schema=Turnstile.key_schema, value_schema=Turnstile.value_schema, num_partitions=2, num_replicas=1, # increase replicas when used in production ) self.station = station self.turnstile_hardware = TurnstileHardware(station)