def from_my_json(j: MyPassageTimeJson) -> PassageTime:
return PassageTime(
arrival=Maybe.of(
j.get("arrival")).map(lambda j: ArrivalTime.from_my_json(j)),
departure=Maybe.of(j.get("departure")).map(
lambda j: DepartureTime.from_my_json(j)),
)
def from_my_json(cls: Type[T], j: MyArrivalDepartureJson) -> T:
return cls(
scheduled=Maybe.of(j.get("scheduled")).map(
lambda s: datetime.fromisoformat(cast(str, s))),
actual_or_prediction=Maybe.of(j.get("actual_or_prediction")).map(
lambda s: datetime.fromisoformat(cast(str, s))),
service_mode=Maybe.of(cast(Optional[int], j.get("service_mode"))),
type=Maybe.of(cast(Optional[int], j.get("type"))),
direction_text=Maybe.of(
cast(Optional[str], j.get("direction_text"))),
)
def stop_by_id(c: connection, s: m.StopId) -> Maybe[Stop]:
with c.cursor() as cu:
cu.execute(
"""
select * from stops
where id = %s
""",
[s.raw],
)
return Maybe.of(cu.fetchone()).map(Stop.from_db_row)
def from_web_timetable(wt: WebTimetable, stops_by_name: Dict[str, Stop],
route_name: str) -> Timetable:
tvs = set()
for t in wt.variants():
if t[0] == route_name:
stops = tuple(
Maybe.justs(
unique_justseen(stops_from_names(t[1],
stops_by_name))))
tvs.add(TimetableVariant(t[0], stops))
return Timetable(wt.table.caption.string, tvs)
def from_json(cls: Type[T], json: Dict[str, Any]) -> T:
return cls(
scheduled=Maybe.of(json.get("scheduled_passage_time_utc")).map(
lambda x: datetime.fromtimestamp(x, timezone(timedelta(
hours=1))).replace(tzinfo=None)),
actual_or_prediction=Maybe.of(
json.get("actual_passage_time_utc")).map(
lambda x: datetime.fromtimestamp(
x, timezone(timedelta(hours=1))).replace(tzinfo=None)),
service_mode=Maybe.of(json.get("service_mode")),
type=Maybe.of(json.get("type")),
direction_text=Maybe.of(json.get(
"multilingual_direction_text")).bind(lambda j: Maybe.of(
cast(Dict[str, str], j).get("defaultValue"))),
)
def live_test(stop_id: StopId) -> None:
pd.set_option("display.max_rows", 1_000_000_000)
pd.set_option("display.max_columns", 1_000_000_000)
pd.set_option("display.width", 1_000_000_000)
warnings.simplefilter("ignore")
stop = stop_by_id(default_connection(), stop_id).value
routes_by_id = db.routes_by_id()
routes_by_name = db.routes_by_name()
timetables = [
tt.value for tt in db.timetables(routes_by_name["220"].id)
if isinstance(tt, Right)
]
variants = {v for timetable in timetables for v in timetable.variants}
filtered_variants = {v for v in variants if stop in v.stops}
timetable = sorted(filtered_variants, key=lambda v: len(v.stops))[-1]
route_sections = list(prediction.route_sections(timetable.stops))
snapshots: Dict[VehicleId, List[BusSnapshot]] = {}
print("Training predictors…")
preprocessed_by_timetable = dict(read_preprocessed_data("220"))
preprocessed = preprocessed_by_timetable[timetable]
bins = [time(i) for i in range(23)] + [time(23, 59, 59, 999_999)]
average_predictors = train_average_predictors(preprocessed,
stop.name + " [arrival]")
binned_predictors = train_binned_average_predictors(
preprocessed, stop.name + " [arrival]", bins)
day_binned_predictors = train_day_binned_predictors(
preprocessed, stop.name + " [arrival]", bins)
print("Finished training.")
responses: Deque[DataFrame] = deque(maxlen=720)
arrived_passages: Dict[PassageId, datetime] = {}
stats_output_file = Path(
f"/Users/Noel/Developer/Projects/Busboy/data/live/{stop.name}-{datetime.now().date().isoformat()}.csv"
)
while True:
try:
print("-" * 80)
loop_start = datetime.now()
response = (stop_passage(
stop.id,
timeout=5).dataframe().sort_values("scheduled_arrival"))
response["poll_time"] = loop_start
response = response[response["predicted_arrival"] > loop_start -
timedelta(minutes=10)]
response["route"] = response["route"].apply(
lambda r: Maybe.of(routes_by_id.get(RouteId(r))).map(
lambda r: r.name).or_else(None))
response = response[response["route"] == "220"]
response["sections"] = [
containing_sections(route_sections, r.longitude, r.latitude)
for r in response.itertuples()
]
arrived = [
arrived_passages.get(p.id.value, pd.NaT)
for p in response["passage"]
]
my_predictions = [pd.NaT] * len(response["passage"])
binned_predictions = [pd.NaT] * len(response["passage"])
response["day_binned_prediction"] = pd.NaT
last_stops = [None] * len(response["passage"])
for row, passage in enumerate(response["passage"]):
if isinstance(passage.vehicle, Just):
snapshot = BusSnapshot.from_passage(passage, loop_start)
snapshots.setdefault(passage.vehicle.value,
[]).append(snapshot)
timetable_journeys = separate_journeys(
snapshots[passage.vehicle.value],
{timetable},
{timetable: route_sections},
)
journeys = timetable_journeys.get(timetable)
if journeys is not None:
journeys = journeys.fillna(value=pd.NaT)
nonempty = journeys.loc[:, journeys.notna().any()]
if len(nonempty.columns) > 0:
last = nonempty.columns[-1]
last_stops[row] = last
if last in average_predictors:
predicted_travel_time = average_predictors[
last].predict(journeys)
predicted_arrival = journeys[last].iloc[
-1] + timedelta(
seconds=predicted_travel_time[-1])
my_predictions[row] = predicted_arrival
binned_predictor = binned_predictors(
last, loop_start.time()) # should be last_time
if binned_predictor is not None:
predicted_travel_time = binned_predictor.predict(
journeys)
predicted_arrival = journeys[last].iloc[
-1] + timedelta(
seconds=predicted_travel_time[-1])
binned_predictions[row] = predicted_arrival
day_binned_predictor = day_binned_predictors(
last, loop_start) # should be last_day
if day_binned_predictor is not None:
predicted_travel_time = day_binned_predictor.predict(
journeys)
predicted_arrival = journeys[last].iloc[
-1] + timedelta(
seconds=predicted_travel_time[-1])
if not pd.isnull(predicted_arrival):
response["day_binned_prediction"][
row] = predicted_arrival
if ((stop.name + " [arrival]") in nonempty.columns
and passage.id.value
not in arrived_passages):
# discard journey data
arrival_time = nonempty[stop.name +
" [arrival]"].iloc[-1]
arrived[row] = arrival_time
arrived_passages[
passage.id.value] = arrival_time
# display prediction stats
prediction_results = evaluate_predictions(
responses, passage, arrival_time)
prediction_results[
"route"] = passage.route.map(
lambda id: id.raw).optional()
prediction_results["target_stop"] = stop.id
if stats_output_file.exists():
prediction_results.to_csv(
str(stats_output_file),
mode="a",
header=False,
index=False,
)
else:
prediction_results.to_csv(
str(stats_output_file), index=False)
response["average_time_prediction"] = my_predictions
response["binned_average_time_prediction"] = binned_predictions
response["arrived"] = arrived
response["last_stop"] = last_stops
display(response)
responses.appendleft(response)
# delete old arrival data
for passage, arrival_time in list(arrived_passages.items()):
if arrival_time < loop_start - timedelta(minutes=10):
del arrived_passages[passage]
sleep(max(0, 10 - (datetime.now() - loop_start).total_seconds()))
except KeyboardInterrupt:
print("Exiting")
return
except Exception as e:
# print(f"Got an exception: {e}")
traceback.print_exc()
print()
print("Continuing…")
sleep(max(0, 10 - (datetime.now() - loop_start).total_seconds()))
def from_json(json: Dict[str, Any]) -> PassageTime:
a = Maybe.of(json.get("arrival_data")).map(ArrivalTime.from_json)
d = Maybe.of(json.get("departure_data")).map(DepartureTime.from_json)
return PassageTime(arrival=a, departure=d)
def from_json(json: Dict[str, Any]) -> Passage:
time = PassageTime.from_json(json)
try:
return Passage(
id=Maybe.of(json.get("duid")).map(PassageId),
last_modified=Maybe.of(
json.get("last_modification_timestamp")).map(
lambda j: datetime.fromtimestamp(
j / 1000, timezone(timedelta(hours=1))).replace(
tzinfo=None)),
is_deleted=Maybe.of(json.get("is_deleted")),
route=Maybe.of(json.get("route_duid")).bind(lambda j: Maybe.of(
cast(Dict[str, str], j).get("duid"))).map(RouteId),
direction=Maybe.of(json.get("direction")),
trip=Maybe.of(json.get("trip_duid")).map(
lambda j: cast(Dict[str, str], j)).bind(
lambda j: Maybe.of(j.get("duid"))).map(TripId),
stop=Maybe.of(json.get("stop_point_duid")).map(
lambda j: cast(Dict[str, str], j)).bind(
lambda j: Maybe.of(j.get("duid"))).map(StopId),
vehicle=Maybe.of(json.get("vehicle_duid")).map(
lambda j: cast(Dict[str, str], j)).bind(
lambda j: Maybe.of(j.get("duid"))).map(VehicleId),
time=time,
congestion=Maybe.of(json.get("congestion_level")),
accuracy=Maybe.of(json.get("accuracy_level")),
status=Maybe.of(json.get("status")),
is_accessible=Maybe.of(json.get("is_accessible")).map(bool),
latitude=Maybe.of(json.get("latitude")),
longitude=Maybe.of(json.get("longitude")),
bearing=Maybe.of(json.get("bearing")),
pattern=Maybe.of(
json.get("pattern_duid")).bind(lambda j: Maybe.of(
cast(Dict[str, str], j).get("duid"))).map(PatternId),
has_bike_rack=Maybe.of(json.get("has_bike_rack")).map(bool),
category=Maybe.of(json.get("category")),
)
except KeyError as e:
raise Exception(json, e)