def routes(): route_list = nextbus.get_route_list('sf-muni') data = [{ 'id': route.id, 'title': route.title, 'directions': [{ 'id': dir.id, 'title': dir.title, 'name': dir.name, 'stops': dir.get_stop_ids() } for dir in route.get_direction_infos()], 'stops': { stop.id: { 'title': stop.title, 'lat': stop.lat, 'lon': stop.lon } for stop in route.get_stop_infos() } } for route in route_list] res = Response( json.dumps(data), mimetype='application/json') # no prettyprinting to save bandwidth if not DEBUG: res.headers['Cache-Control'] = 'max-age=3600' return res
parser.add_argument('--start-date', help='Start date (yyyy-mm-dd)') parser.add_argument('--end-date', help='End date (yyyy-mm-dd), inclusive') parser.add_argument('--s3', dest='s3', action='store_true', help='store in s3') parser.add_argument('--stat', nargs='*') parser.set_defaults(s3=False) args = parser.parse_args() agency_id = 'sf-muni' tz = pytz.timezone('US/Pacific') routes = nextbus.get_route_list(agency_id) if args.date: dates = util.get_dates_in_range(args.date, args.date) elif args.start_date is not None and args.end_date is not None: dates = util.get_dates_in_range(args.start_date, args.end_date) else: raise Exception('missing date, start-date, or end-date') stat_ids = args.stat for d in dates: compute_trip_times(d, tz, agency_id, routes,
def save_routes_for_agency(agency: config.Agency, save_to_s3=True): agency_id = agency.id gtfs_cache_dir = f'{util.get_data_dir()}/gtfs-{agency_id}' download_gtfs_data(agency, gtfs_cache_dir) feed = ptg.load_geo_feed(gtfs_cache_dir, {}) print(f"Loading {agency_id} routes...") routes_df = feed.routes if agency.gtfs_agency_id is not None: routes_df = routes_df[routes_df.agency_id == agency.gtfs_agency_id] routes_data = [] print(f"Loading {agency_id} trips...") trips_df = feed.trips trips_df['direction_id'] = trips_df['direction_id'].astype(str) print(f"Loading {agency_id} stop times...") stop_times_df = feed.stop_times print(f"Loading {agency_id} shapes...") shapes_df = feed.shapes print(f"Loading {agency_id} stops...") stops_df = feed.stops # gtfs_stop_ids_map allows looking up row from stops.txt via GTFS stop_id gtfs_stop_ids_map = {stop.stop_id: stop for stop in stops_df.itertuples()} stop_id_gtfs_field = agency.stop_id_gtfs_field # get OpenTransit stop ID for GTFS stop_id (may be the same) def normalize_gtfs_stop_id(gtfs_stop_id): if stop_id_gtfs_field != 'stop_id': return getattr(gtfs_stop_ids_map[gtfs_stop_id], stop_id_gtfs_field) else: return gtfs_stop_id # stops_map allows looking up row from stops.txt via OpenTransit stop ID if stop_id_gtfs_field != 'stop_id': stops_map = {getattr(stop, stop_id_gtfs_field): stop for stop in stops_df.itertuples()} else: stops_map = gtfs_stop_ids_map if agency.provider == 'nextbus': nextbus_route_order = [route.id for route in nextbus.get_route_list(agency.nextbus_id)] for route in routes_df.itertuples(): gtfs_route_id = route.route_id short_name = route.route_short_name long_name = route.route_long_name if isinstance(short_name, str) and isinstance(long_name, str): title = f'{short_name} - {long_name}' elif isinstance(short_name, str): title = short_name else: title = long_name type = int(route.route_type) if hasattr(route, 'route_type') else None url = route.route_url if hasattr(route, 'route_url') and isinstance(route.route_url, str) else None #color = route.route_color #text_color = route.route_text_color route_id = getattr(route, agency.route_id_gtfs_field) if agency.provider == 'nextbus': route_id = route_id.replace('-', '_') # hack to handle muni route IDs where e.g. GTFS has "T-OWL" but nextbus has "T_OWL" try: nextbus_route_config = nextbus.get_route_config(agency.nextbus_id, route_id) title = nextbus_route_config.title except Exception as ex: print(ex) continue try: sort_order = nextbus_route_order.index(route_id) except ValueError as ex: print(ex) sort_order = None else: sort_order = int(route.route_sort_order) if hasattr(route, 'route_sort_order') else None print(f'route {route_id} {title}') route_data = { 'id': route_id, 'title': title, 'url': url, 'type': type, #'color': color, #'text_color': text_color, 'gtfs_route_id': gtfs_route_id, 'sort_order': sort_order, 'stops': {}, 'directions': [], } directions = [] route_directions_df = feed.get('route_directions.txt') # unofficial trimet gtfs extension if not route_directions_df.empty: route_directions_df = route_directions_df[route_directions_df['route_id'] == gtfs_route_id] else: route_directions_df = None routes_data.append(route_data) route_trips_df = trips_df[trips_df['route_id'] == gtfs_route_id] route_direction_id_values = route_trips_df['direction_id'].values def add_custom_direction(custom_direction_info): direction_id = custom_direction_info['id'] print(f' custom direction = {direction_id}') gtfs_direction_id = custom_direction_info['gtfs_direction_id'] direction_trips_df = route_trips_df[route_direction_id_values == gtfs_direction_id] included_stop_ids = custom_direction_info.get('included_stop_ids', []) excluded_stop_ids = custom_direction_info.get('excluded_stop_ids', []) shapes = get_unique_shapes( direction_trips_df=direction_trips_df, stop_times_df=stop_times_df, stops_map=stops_map, normalize_gtfs_stop_id=normalize_gtfs_stop_id ) def contains_included_stops(shape_stop_ids): min_index = 0 for stop_id in included_stop_ids: try: index = shape_stop_ids.index(stop_id, min_index) except ValueError: return False min_index = index + 1 # stops must appear in same order as in included_stop_ids return True def contains_excluded_stop(shape_stop_ids): for stop_id in excluded_stop_ids: try: index = shape_stop_ids.index(stop_id) return True except ValueError: pass return False matching_shapes = [] for shape in shapes: shape_stop_ids = shape['stop_ids'] if contains_included_stops(shape_stop_ids) and not contains_excluded_stop(shape_stop_ids): matching_shapes.append(shape) if len(matching_shapes) != 1: matching_shape_ids = [shape['shape_id'] for shape in matching_shapes] error_message = f'{len(matching_shapes)} shapes found for route {route_id} with GTFS direction ID {gtfs_direction_id}' if len(included_stop_ids) > 0: error_message += f" including {','.join(included_stop_ids)}" if len(excluded_stop_ids) > 0: error_message += f" excluding {','.join(excluded_stop_ids)}" if len(matching_shape_ids) > 0: error_message += f": {','.join(matching_shape_ids)}" raise Exception(error_message) matching_shape = matching_shapes[0] matching_shape_id = matching_shape['shape_id'] matching_shape_count = matching_shape['count'] print(f' matching shape = {matching_shape_id} ({matching_shape_count} times)') add_direction( id=direction_id, gtfs_shape_id=matching_shape_id, gtfs_direction_id=gtfs_direction_id, stop_ids=matching_shape['stop_ids'], title=custom_direction_info.get('title', None) ) def add_default_direction(direction_id): print(f' default direction = {direction_id}') direction_trips_df = route_trips_df[route_direction_id_values == direction_id] shapes = get_unique_shapes( direction_trips_df=direction_trips_df, stop_times_df=stop_times_df, stops_map=stops_map, normalize_gtfs_stop_id=normalize_gtfs_stop_id) best_shape = shapes[0] best_shape_id = best_shape['shape_id'] best_shape_count = best_shape['count'] print(f' most common shape = {best_shape_id} ({best_shape_count} times)') add_direction( id=direction_id, gtfs_shape_id=best_shape_id, gtfs_direction_id=direction_id, stop_ids=best_shape['stop_ids'] ) def add_direction(id, gtfs_shape_id, gtfs_direction_id, stop_ids, title = None): if title is None: default_direction_info = agency.default_directions.get(gtfs_direction_id, {}) title_prefix = default_direction_info.get('title_prefix', None) last_stop_id = stop_ids[-1] last_stop = stops_map[last_stop_id] if title_prefix is not None: title = f"{title_prefix} to {last_stop.stop_name}" else: title = f"To {last_stop.stop_name}" print(f' title = {title}') dir_data = { 'id': id, 'title': title, 'gtfs_shape_id': gtfs_shape_id, 'gtfs_direction_id': gtfs_direction_id, 'stops': stop_ids, 'stop_geometry': {}, } route_data['directions'].append(dir_data) for stop_id in stop_ids: stop = stops_map[stop_id] stop_data = { 'id': stop_id, 'lat': round(stop.geometry.y, 5), # stop_lat in gtfs 'lon': round(stop.geometry.x, 5), # stop_lon in gtfs 'title': stop.stop_name, 'url': stop.stop_url if hasattr(stop, 'stop_url') and isinstance(stop.stop_url, str) else None, } route_data['stops'][stop_id] = stop_data geometry = shapes_df[shapes_df['shape_id'] == gtfs_shape_id]['geometry'].values[0] # partridge returns GTFS geometries for each shape_id as a shapely LineString # (https://shapely.readthedocs.io/en/stable/manual.html#linestrings). # Each coordinate is an array in [lon,lat] format (note: longitude first, latitude second) dir_data['coords'] = [ { 'lat': round(coord[1], 5), 'lon': round(coord[0], 5) } for coord in geometry.coords ] if agency.provider == 'nextbus': # match nextbus direction IDs with GTFS direction IDs best_nextbus_dir_info, best_terminal_dist = match_nextbus_direction(nextbus_route_config, geometry) print(f' {direction_id} = {best_nextbus_dir_info.id} (terminal_dist={int(best_terminal_dist)}) {" (questionable match)" if best_terminal_dist > 300 else ""}') # dir_data['title'] = best_nextbus_dir_info.title dir_data['nextbus_direction_id'] = best_nextbus_dir_info.id start_lat = geometry.coords[0][1] start_lon = geometry.coords[0][0] #print(f" start_lat = {start_lat} start_lon = {start_lon}") deg_lat_dist = util.haver_distance(start_lat, start_lon, start_lat-0.1, start_lon)*10 deg_lon_dist = util.haver_distance(start_lat, start_lon, start_lat, start_lon-0.1)*10 # projection function from lon/lat coordinates in degrees (z ignored) to x/y coordinates in meters. # satisfying the interface of shapely.ops.transform (https://shapely.readthedocs.io/en/stable/manual.html#shapely.ops.transform). # This makes it possible to use shapely methods to calculate the distance in meters between geometries def project_xy(lon, lat, z=None): return (round((lon - start_lon) * deg_lon_dist, 1), round((lat - start_lat) * deg_lat_dist, 1)) xy_geometry = shapely.ops.transform(project_xy, geometry) shape_lon_lat = np.array(geometry).T shape_lon = shape_lon_lat[0] shape_lat = shape_lon_lat[1] shape_prev_lon = np.r_[shape_lon[0], shape_lon[:-1]] shape_prev_lat = np.r_[shape_lat[0], shape_lat[:-1]] # shape_cumulative_dist[i] is the cumulative distance in meters along the shape geometry from 0th to ith coordinate shape_cumulative_dist = np.cumsum(util.haver_distance(shape_lon, shape_lat, shape_prev_lon, shape_prev_lat)) shape_lines_xy = [shapely.geometry.LineString(xy_geometry.coords[i:i+2]) for i in range(0, len(xy_geometry.coords) - 1)] # this is the total distance of the GTFS shape, which may not be exactly the same as the # distance along the route between the first and last Nextbus stop dir_data['distance'] = int(shape_cumulative_dist[-1]) print(f" distance = {dir_data['distance']}") # Find each stop along the route shape, so that the frontend can draw line segments between stops along the shape start_index = 0 for stop_id in stop_ids: stop_info = route_data['stops'][stop_id] # Need to project lon/lat coords to x/y in order for shapely to determine the distance between # a point and a line (shapely doesn't support distance for lon/lat coords) stop_xy = shapely.geometry.Point(project_xy(stop_info['lon'], stop_info['lat'])) stop_geometry = get_stop_geometry(stop_xy, shape_lines_xy, shape_cumulative_dist, start_index) if stop_geometry['offset'] > 100: print(f" !! bad geometry for stop {stop_id}: {stop_geometry['offset']} m from route line segment") continue dir_data['stop_geometry'][stop_id] = stop_geometry start_index = stop_geometry['after_index'] if route_id in agency.custom_directions: for custom_direction_info in agency.custom_directions[route_id]: add_custom_direction(custom_direction_info) else: for direction_id in np.unique(route_direction_id_values): add_default_direction(direction_id) if routes_data[0]['sort_order'] is not None: sort_key = lambda route_data: route_data['sort_order'] else: sort_key = lambda route_data: route_data['id'] routes_data = sorted(routes_data, key=sort_key) data_str = json.dumps({ 'version': routeconfig.DefaultVersion, 'routes': routes_data }, separators=(',', ':')) cache_path = routeconfig.get_cache_path(agency_id) with open(cache_path, "w") as f: f.write(data_str) if save_to_s3: s3 = boto3.resource('s3') s3_path = routeconfig.get_s3_path(agency_id) s3_bucket = config.s3_bucket print(f'saving to s3://{s3_bucket}/{s3_path}') object = s3.Object(s3_bucket, s3_path) object.put( Body=gzip.compress(bytes(data_str, 'utf-8')), CacheControl='max-age=86400', ContentType='application/json', ContentEncoding='gzip', ACL='public-read' )
parser.add_argument('--route', nargs='*') parser.add_argument('--date', help='Date (yyyy-mm-dd)') parser.add_argument('--start-date', help='Start date (yyyy-mm-dd)') parser.add_argument('--end-date', help='End date (yyyy-mm-dd), inclusive') parser.add_argument('--s3', dest='s3', action='store_true', help='store in s3') parser.set_defaults(s3=False) args = parser.parse_args() route_ids = args.route agency = 'sf-muni' if route_ids is None: route_ids = [route.id for route in nextbus.get_route_list(agency)] date_str = args.date if args.date: dates = util.get_dates_in_range(args.date, args.date) elif args.start_date is not None and args.end_date is not None: dates = util.get_dates_in_range(args.start_date, args.end_date) else: raise Exception('missing date, start-date, or end-date') tz = pytz.timezone('America/Los_Angeles') incr = timedelta(days=1) for d in dates:
def routes(): route_list = nextbus.get_route_list('sf-muni') data = [{'id': route.id, 'title': route.title} for route in route_list] return Response(json.dumps(data, indent=2), mimetype='application/json')