class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
topic_name = "chicago.turnstiles"
super().__init__(
topic_name, # TODO: Come up with a better topic name
key_schema=Turnstile.key_schema,
# TODO: value_schema=Turnstile.value_schema, TODO: Uncomment once schema is defined
# TODO: num_partitions=???,
# TODO: num_replicas=???,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(
f"Producing turnstile data \n Topic Name: {self.topic_name} \n Station ID: {self.station.station_id}"
)
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for entry in range(entries):
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
})
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"udacity.com.km.turnstiles"
super().__init__(topic_name,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#logger.info("turnstile kafka integration incomplete - skipping")
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.station.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
})
class Turnstile(Producer):
key_schema: RecordSchema = load_schema("turnstile_key.json")
value_schema: RecordSchema = load_schema("turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
super().__init__(
topic_name=CtaTopics.TURNSTILES,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.debug(
f"[{timestamp.isoformat()}] Riders count: {num_entries} @ {self.station.name}"
)
for _ in range(num_entries):
self.producer.poll(0)
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
},
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
### Not sure how to proceed with KSQL assignment when having topic for every station turnstile
# f"org.chicago.cta.station.turnstile.{station_name}",
"org.chicago.cta.station.turnstile", # Using single topic for all turnstiles
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=10,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for i in repeat(None, num_entries):
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
})
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
f"org.chicago.cta.turnstiles",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info("turnstile kafka integration incomplete - skipping")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={"num_entries": num_entries},
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
topic_name = f"org.chicago.transit.looptrain.{station_name}.turnstile"
super().__init__(
topic_name,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(
f"Sending turnstile events to kafka topic {self.topic_name}")
turnstile_message = {
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"num_entries": num_entries
}
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=turnstile_message)
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json"
)
def __init__(self, station):
"""Create the Turnstile"""
station_name = (
station.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
f"com.udacity.turnstile.v3", # TODO: Come up with a better topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema, #TODO: Uncomment once schema is defined
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#logger.info("turnstile kafka integration incomplete - skipping")
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
for _ in range(num_entries):
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
}
)
except Exception as e:
logger.fatal(e)
raise e
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
super().__init__(
topic_name=f"com.udacity.turnstiles",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
key_schema=self.key_schema,
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"num_entries": num_entries,
},
value_schema=self.value_schema)
logger.info("turnstile data emmited")
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
f"org.chicago.cta.trunstile",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for _ in itertools.repeat(None, num_entries):
self.kafka_avro_producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
'station_id': self.station.station_id,
'station_name': self.station.name,
'line': self.station.color.name
})
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
super().__init__(
"org.chicago.cta.turnstile",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for _ in range(num_entries):
self.producer.produce(topic=self.topic_name,
key_schema=self.key_schema,
value_schema=self.value_schema,
key={"timestamp": self.time_millis()},
value={
"station_name": self.station.name,
"station_id": self.station.station_id,
"line": self.station.color.name
})
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json"
)
def __init__(self, station, create_topic=True):
"""Create the Turnstile"""
super().__init__(
topic_name="com-cta-station-turnstile",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=5,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info("turnstile kafka integration incomplete - skipping")
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
},
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
self.station_name = (station.name.lower().replace(
"/", "_and_").replace(" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
'from_turnstile',
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=2,
num_replicas=2,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for entry in range(num_entries):
self.producer.produce(topic=f'from_turnstile',
key={"timestamp": int(self.time_millis())},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
})
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
super().__init__(topic_name="org.chicago.cta.turnstiles",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
logger.debug("turnstile run function working")
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"entries": num_entries
})
class Turnstile():
_producer: Producer = None
def __init__(self, station):
"""Create the Turnstile"""
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
Turnstile._init_producer_singleton()
@classmethod
def _init_producer_singleton(cls):
if cls._producer is None:
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
cls._producer = Producer(
'com.udacity.project.chicago_transportation.station.turstile_entries',
key_schema=key_schema,
value_schema=value_schema)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
for _ in range(num_entries):
Turnstile._producer.produce({
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
})
def close(self):
Turnstile._producer.close()
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
#value_schema = avro.load(
# f"{Path(__file__).parents[0]}/schemas/turnstile_value.json"
#)
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
topic_name = "oturnstile",
super().__init__(
topic_name,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.debug(
"%s riders have entered station %s at %s",
num_entries,
self.station.name,
timestamp.isoformat(),
)
for _ in range(num_entries):
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
},
)
except Exception as e:
logger.error(e)
raise e
logger.info("turnstile kafka integration Succesfull - skipping")
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
# done
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
# Done
topic_name = "chicago.cta.train.stations.turnstile"
super().__init__(
topic_name, # TODO: Come up with a better topic name -done
key_schema=Turnstile.key_schema,
value_schema=Turnstile.
value_schema, #TODO: Uncomment once schema is defined
num_partitions=2,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(
f"turnstile entries {num_entries} for station {self.station.name}")
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
# done
for _ in range(num_entries):
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
'station_id': self.station.station_id,
'station_name': self.station.name,
'line': self.station.color.name,
},
)
except Exception as e:
logger.error("Failed")
raise e
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_value.json" )
value_schema = avro.loads("""
{
"type": "record",
"name": "turnstile.value",
"namespace": "com.udacity.turnstile",
"fields":[
{"name": "station_id", "type": "int"},
{"name": "station_name", "type": ["null", "string"]},
{"name": "line","type": ["null", "string"] }
]
}""")
def __init__(self, station):
"""Create the Turnstile"""
station_name = ( station.name.lower().replace("/", "_and_").replace(" ", "_").replace("-", "_").replace("'", "") )
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
super().__init__(
f"com.udacity.cta.turnstile", # TODO: Come up with a better topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema, #TODO: Uncomment once schema is defined
num_partitions=1, # TODO
num_replicas=1 # TODO,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#logger.info("turnstile kafka integration incomplete - skipping")
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#logger.info(json.dumps(self.f_asdict(num_entries)))
for _ in range (num_entries):
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
#value_schema=Turnstile.value_schema,
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
}
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
topic_name=
"org.chicago.cta.turnstile.data", #f"com.name.turnstile.{station_name}",#"org_chicago_transit_turnstile_data", # topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
# Emitting a message to the Kafka turnstile topic for the number
# of entries that were calculated
for i in range(num_entries):
try:
self.producer.produce(
topic=self.topic_name,
#key_schema=self.key_schema,
#value_schema=self.value_schema,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
},
)
logger.info(
f"succeeded:{self.station.station_id},{self.station.name},{self.station.color.name}"
)
except:
logger.info(
f"failed:{self.station.station_id},{self.station.name},{self.station.color.name}"
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
# Define the value schema in `schemas/turnstile_value.json`
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
super().__init__(
topic_name="org.chicago.cta.turnstile", # create topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=4,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#logger.info("turnstile kafka integration incomplete - skipping")
# Produce a message to the turnstile topic for the number of entries that were calculated
# input timestamp is in datetime.datetime format. Transform the data to timestamp format (long int)
timestamp = int(
datetime.timestamp(timestamp)
) # Transform time to timestamp format (long int), python3 only support int
self.producer.produce(topic=self.topic_name,
key={"timestamp": timestamp},
key_schema=Turnstile.key_schema,
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"num_entries": num_entries
},
value_schema=avro.loads("""{
"namespace": "org.chicago.cta",
"type": "record",
"name": "turnstile_entry",
"fields": [
{"name": "station_id", "type": "int"},
{"name": "station_name", "type": "string"},
{"name": "line", "type": "string"},
{"name": "num_entries", "type": "int"}
]
}"""))
self.producer.flush()
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (
station.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# DONE: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
f"il.cta.{station_name}.turnstile", # DONE: Come up with a better topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema, # DONE: Uncomment once schema is defined
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(f"turnstile of stations {self.station.station_id}-{self.station.name} get {num_entries} numbers of entries")
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
Key = {"timestamp": datetime.timestamp(timestamp)}
Values = {
"station.id": self.station.station_id,
"station_name": self.station.name,
"entries": num_entries
}
self.producer.produce(
topic = self.topic_name,
key = Key,
key_schema = self.key_schema,
value = Values,
value_schema = self.value_schema
)
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json"
)
def __init__(self, station):
"""Create the Turnstile"""
station_name = (
station.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
# TODO: Come up with a better topic name
topic_name="org.cta.station.turnstile",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=Producer.num_partitions,
num_replicas=Producer.num_replicas,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
message = f"Total number of riders who entered {self.station.name} at {timestamp.isoformat()} is {num_entries}"
logger.info(message)
for _ in range(num_entries):
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line":self.station.color.name
},
)
class Turnstile():
#key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#filelocation=os.path.abspath(__file__)
filelocation = os.path.dirname(__file__)
key_schema = avro.load(filelocation + "/schemas/turnstile_key.json")
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
value_schema = avro.load(
filelocation + "/schemas/turnstile_value.json"
#f"{Path(__file__).parents[0]}/schemas/turnstile_value.json"
)
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: Complete the below by deciding on a
# topic name, number of partitions, and number of replicas
super().__init__(
#f"com.udacity.station.turnstile.v1", # TODO: Come up with a better topic name
"org.chicago.cta.station.turnstile.v1",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=4,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
# TODO: Complete this function by emitting a message
# to the turnstile topic for the number of entries that were calculated
#logger.debug(f"Passenger count :{num_entries} on {self.station.name} | {timestamp.isoformat()}")
#Kenny Added
logger.debug("Passenger count :" + str(num_entries) + "on" +
str(self.station.name) + '|' + str(timestamp.isoformat()))
# produce a message from 0 to the number of entries
for _ in range(num_entries):
try:
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id":
self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
})
except Exception as e:
logger.fatal(e)
raise e
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
# We create one single topic here.
# See: https://knowledge.udacity.com/questions/69131
f"{TURNSTILE_TOPIC_V1}",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=3,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
# TODO: Confer https://knowledge.udacity.com/questions/417521
# regarding further instructions.
for _ in range(0, num_entries):
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
},
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below, done
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
super().__init__(
"turnstiles_topic", # TODO: Come up with a better topic name, done
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(
f"{self.topic_name} : turnstile at station {self.station.station_id} turned for {num_entries} times at {timestamp}"
)
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated, done
try:
for _ in range(num_entries):
#print(f"Produced entry with {self.station.station_id}, \
# {self.station.name} and {self.station.color.name}")
self.producer.produce(
topic="turnstiles_topic",
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
}, #stationid and name from HW class, but line?, done
value_schema=Turnstile.value_schema)
except Exception as e:
logger.info(f"turnstile kafka integration incomplete - skipping \
with Error: {e}")
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
NUM_PARTITIONS = 3
NUM_REPLICAS = 1
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
print(f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
print(value_schema)
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__("turnstile_event",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=self.NUM_PARTITIONS,
num_replicas=self.NUM_REPLICAS)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info("turnstile kafka integration incomplete - skipping")
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
logger.debug(self.value_schema)
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"line": self.station.color.name,
"station_name": self.station.name
},
value_schema=self.value_schema,
key_schema=self.key_schema)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
TURNSTILE_TOPIC_NAME,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=NUM_PARTITIONS_PER_TOPIC,
num_replicas=NUM_REPLICAS_PER_TOPIC,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
timestamp = self.time_millis()
try:
self.producer.produce(
topic=self.topic_name,
key={"timestamp": timestamp},
value_schema=self.value_schema,
key_schema=self.key_schema,
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
"num_entries": num_entries
},
)
logger.debug("Turnstile produce success")
except Exception as e:
logger.error(
f"error produce turnstile data {self.station.station_id}" +
str(e))
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
self.station_name = (
station.name.lower()
.replace("/", "_and_")
.replace(" ", "_")
.replace("-", "_")
.replace("'", "")
)
#
#
# TODO: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
self.topic_name = f"org.chicago.cta.station.turnstile_entry" # <domain>.<model>.<event type>
super().__init__(
self.topic_name, # TODO: Come up with a better topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=1,
num_replicas=3,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
for i in range(num_entries):
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_name": self.station_name,
"station_id": self.station.station_id,
"line": self.station.color.name
},
)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
print('init turnstile')
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# topic name, number of partitions, and number of
# replicas
#self.station.color.name"
#
topic_name = f"{Producer.TOPIC_PREFIX}.turnstiles"
super().__init__(
topic_name,
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=20,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
# logger.info("turnstile kafka integration incomplete - skipping")
#
#
# emitting a message to the turnstile topic for the number
# of entries that were calculated
#
#
payload = {
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
}
self.producer.produce(key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value=payload)
class Turnstile(Producer):
key_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# DONE: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(
f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
station_name = (station.name.lower().replace("/", "_and_").replace(
" ", "_").replace("-", "_").replace("'", ""))
#
#
# DONE: Complete the below by deciding on a topic name, number of partitions, and number of
# replicas
#
#
super().__init__(
f"nd.project.opt.turnstile", # DONE: Come up with a better topic name
key_schema=Turnstile.key_schema,
value_schema=Turnstile.
value_schema, # DONE: Uncomment once schema is defined
num_partitions=10, # DONE
num_replicas=3, # DONE
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
logger.info(
f"turnstile event: num_entries: {num_entries}; station_id: {self.station.station_id}; station_name: {self.station.name}"
)
#
#
# DONE: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
for entry in range(num_entries):
self.producer.produce(topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name,
},
key_schema=self.key_schema,
value_schema=self.value_schema)
class Turnstile(Producer):
key_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_key.json")
#
# TODO: Define this value schema in `schemas/turnstile_value.json, then uncomment the below
#
value_schema = avro.load(f"{Path(__file__).parents[0]}/schemas/turnstile_value.json")
def __init__(self, station):
"""Create the Turnstile"""
#
# TODO: Complete the below by deciding on a topic name, number of
# partitions, and number of replicas
#
# TODO: Include/fill the following in the call to super.__init__():
# value_schema=Station.value_schema,
# num_partitions=???,
# num_replicas=???,
super().__init__(
# TODO: Come up with a better topic name
topic_name="org.chicago.cta.station.turnstile.v1",
key_schema=Turnstile.key_schema,
value_schema=Turnstile.value_schema,
num_partitions=3,
num_replicas=1,
)
self.station = station
self.turnstile_hardware = TurnstileHardware(station)
def run(self, timestamp, time_step):
"""Simulates riders entering through the turnstile."""
num_entries = self.turnstile_hardware.get_entries(timestamp, time_step)
#
#
# TODO: Complete this function by emitting a message to the turnstile topic for the number
# of entries that were calculated
#
msg = f"{num_entries} riders have entered station {self.station.name} at {timestamp.isoformat()}"
logger.info(msg)
for _ in range(num_entries):
self.producer.produce(
topic=self.topic_name,
key={"timestamp": self.time_millis()},
value={
"station_id": self.station.station_id,
"station_name": self.station.name,
"line": self.station.color.name
}
)
