def get_arrival_history(as_of_date: datetime, time_zone: pytz.timezone,
increment: timedelta, agency: str, route_ids: list,
start_hour: int, continue_index):
start_dt = time_zone.localize(
datetime(as_of_date.year,
as_of_date.month,
as_of_date.day,
hour=start_hour))
end_dt = start_dt + increment
start_time = int(start_dt.timestamp())
end_time = int(end_dt.timestamp())
print(f"time = [{start_dt}, {end_dt})")
t1 = time.time()
state = trynapi.get_state(agency, as_of_date, start_time, end_time,
route_ids)
print(f'retrieved state in {round(time.time()-t1,1)} sec')
for i, route_id in enumerate(route_ids):
if continue_index is not None and i < continue_index:
continue
route_state = state.get_for_route(route_id)
if route_state is None:
print(f'no state for route {route_id}')
continue
route_config = nextbus.get_route_config(agency, route_id)
t1 = time.time()
arrivals_df = eclipses.find_arrivals(route_state, route_config,
as_of_date, time_zone)
history = arrival_history.from_data_frame(agency, route_id,
arrivals_df, start_time,
end_time)
print(f'{route_id}: {round(time.time()-t1,1)} saving arrival history')
arrival_history.save_for_date(history, d, args.s3)
print(f'{route_id}: {round(time.time()-t1,2)} done')
def compute_arrivals_for_date_and_start_hour(d: date,
start_hour: int,
agency: config.Agency,
route_ids: list,
save_to_s3=True):
tz = agency.tz
start_dt = tz.localize(datetime(d.year, d.month, d.day, hour=start_hour))
end_dt = start_dt + timedelta(days=1)
start_time = int(start_dt.timestamp())
end_time = int(end_dt.timestamp())
print(f"time = [{start_dt}, {end_dt})")
t1 = time.time()
state = trynapi.get_state(agency.id, d, start_time, end_time, route_ids)
print(f'retrieved state in {round(time.time()-t1,1)} sec')
for i, route_id in enumerate(route_ids):
route_state = state.get_for_route(route_id)
if route_state is None:
print(f'no state for route {route_id}')
continue
route_config = agency.get_route_config(route_id)
t1 = time.time()
arrivals_df = eclipses.find_arrivals(agency, route_state, route_config,
d)
history = arrival_history.from_data_frame(agency.id, route_id,
arrivals_df, start_time,
end_time)
print(f'{route_id}: {round(time.time()-t1,1)} saving arrival history')
arrival_history.save_for_date(history, d, save_to_s3)
print(f'{route_id}: {round(time.time()-t1,2)} done')
print(f"time = [{start_dt}, {end_dt}) (chunk_minutes={chunk_minutes})")
route_state_map = {}
chunk_start_time = start_time
while chunk_start_time < end_time:
chunk_end_time = min(chunk_start_time + 60 * chunk_minutes,
end_time)
# trynapi returns all route states in the UTC minute containing the end timestamp, *inclusive*.
# This would normally cause trynapi to return duplicate route states at the end of one chunk and
# the beginning of the next chunk. Since chunk_end_time is always the first second in a UTC minute,
# subtracting 1 from the corresponding millisecond will be the last millisecond in the previous minute,
# so it should avoid fetching duplicate vehicle states at chunk boundaries
chunk_state = trynapi.get_state(agency, chunk_start_time * 1000,
chunk_end_time * 1000 - 1,
route_ids)
if 'message' in chunk_state: # trynapi returns an internal server error if you ask for too much data at once
raise Exception(
f"trynapi error for time range {chunk_start_time}-{chunk_end_time}: {chunk_state['message']}"
)
if not ('data' in chunk_state):
print(chunk_state)
raise Exception(f'trynapi returned no data')
for chunk_route_state in chunk_state['data']['trynState'][
'routes']:
route_id = chunk_route_state['rid']
if route_id not in route_state_map:
tz = pytz.timezone('US/Pacific')
local_start = datetime.fromtimestamp(start_time, tz)
local_end = datetime.fromtimestamp(end_time, tz)
agency = 'sf-muni'
if re.match('^[\w\-]+$', agency) is None:
raise Exception(f"Invalid agency: {agency}")
for route_id in route_ids:
if re.match('^[\w\-]+$', route_id) is None:
raise Exception(f"Invalid route id: {route_id}")
source_dir = os.path.dirname(os.path.realpath(__file__))
local_path = os.path.join(
source_dir, 'data',
f"state_{agency}_{'+'.join(route_ids)}_{start_time}_{end_time}.json")
print(f"route = {route_ids}")
print(f"start = {local_start} ({start_time})")
print(f"end = {local_end} ({end_time})")
print(f"local_path = {local_path}")
agency = 'sf-muni'
data = trynapi.get_state(agency, start_time * 1000, end_time * 1000,
route_ids)
f = open(local_path, "w")
f.write(json.dumps(data))
f.close()
route_ids = args.route
agency_id = 'sf-muni'
if route_ids is None:
route_ids = [route.id for route in nextbus.get_route_list(agency_id)]
date_str = args.date
d = util.parse_date(date_str)
start_time_str = args.start_time
if start_time_str is None:
start_time_str = '03:00'
end_time_str = args.end_time
if end_time_str is None:
end_time_str = '03:00+1'
tz = pytz.timezone('US/Pacific')
local_start = util.get_localized_datetime(d, start_time_str, tz)
local_end = util.get_localized_datetime(d, end_time_str, tz)
print(f"route_ids = {route_ids}")
print(f"start = {local_start}")
print(f"end = {local_end}")
state = trynapi.get_state(agency_id, d, local_start.timestamp(),
local_end.timestamp(), route_ids)
if args.date:
dates = util.get_dates_in_range(args.date, args.date)
elif args.start_date is not None and args.end_date is not None:
dates = util.get_dates_in_range(args.start_date, args.end_date)
else:
raise Exception('missing date, start-date, or end-date')
tz = pytz.timezone('America/Los_Angeles')
incr = timedelta(days=1)
for d in dates:
start_dt = tz.localize(datetime(d.year,d.month, d.day, hour=3)) # start each "day" at 3 AM local time so midnight-3am buses are associated with previous day
end_dt = start_dt + incr
start_time = int(start_dt.timestamp())
end_time = int(end_dt.timestamp())
print(f"time = [{start_dt}, {end_dt})")
route_state_map = trynapi.get_state(agency, start_time, end_time, route_ids, args.cache_state)
for route_id in route_ids:
if route_id not in route_state_map:
print(f'no state for route {route_id}')
continue
route_state = route_state_map[route_id]
history = arrival_history.compute_from_state(agency, route_id, start_time, end_time, route_state)
arrival_history.save_for_date(history, d, args.s3)