def handle(self, dirname, outdir, **kw):
def store(route):
filename = "%s%s.%s.json" % (route['borough'], route['route_no'], route['day_of_week'])
path = os.path.join(outdir, filename)
if os.path.exists(path):
import pdb;pdb.set_trace()
out = open(path, "w")
print >>out, dumps(route)
out.close()
for route in parse_schedule_dir(dirname):
store(route)
def handle_buses(dirname):
feed = transitfeed.Loader("mta_data/bus-gtfs.zip", memory_db=False).Load() #base data
current_borough = None
#capture multiple stops with different box ids
stop_name_to_stop = {}
last_route = None
for route_rec in parse_schedule_dir(os.path.join(dirname, 'surface-bus'), 'stif'):
#gtfs files are organized by borough (not bus prefix)
if current_borough != route_rec['borough']:
if current_borough:
feed.Validate()
feed.WriteGoogleTransitFeed('mta_data/bus-%s.zip' % current_borough)
feed = transitfeed.Loader("mta_data/gtfs.zip", memory_db=False).Load()
stop_name_to_stop = {}
current_borough = route_rec['borough']
if route_rec['route_name_flag'] == 'X':
#express buses
borough = 'X'
else:
borough = route_rec['borough'].title()
name = "%s%s" % (borough,
route_rec['route_no'])
print "processing route:", name
if last_route != name:
_shape_by_stops_cache.clear()
last_route = name
long_name = route_rec["street_name"]
if long_name.startswith(name):
long_name = long_name[len(name):].strip()
period = feed.GetServicePeriod(route_rec['day_of_week'].upper())
stop_hexid_to_stop = {}
#add all stops
for stop_rec in route_rec['stops']:
location = "%s at %s" % (stop_rec['street1'], stop_rec['street2'])
location = rename_location.get(location, location)
#check for duplicate box ids
box_no = str(stop_rec['box_no'])
stop_id = stop_rec['stop_id']
if location in stop_name_to_stop:
stop_hexid_to_stop[stop_id] = stop_name_to_stop[location]
continue
lat = stop_rec['latitude'] / 1000000.0
lng = stop_rec['longitude'] / 1000000.0
#special case for QVDEP:
if location == 'Queens Village Depot':
lat, lng = 40.726711, -73.734779
#not in NYC area
if not (-72 > lng > -75) or not (41 > lat > 39):
print "bad lat, lng", lat, lng
import pdb;pdb.set_trace()
#now, try to find a nearby stop
nearest = feed.GetNearestStops(lat, lng, 1)
if len(nearest) and not ' LANE ' in nearest[0].stop_name:
#sometimes bus stops really are like 1 m away because
#they are two lanes in a multilane terminal area
nearest = nearest[0]
else:
nearest = None
if nearest and abs(nearest.stop_lat - lat) + abs(nearest.stop_lon - lng) < 0.000001:
stop_hexid_to_stop[stop_id] = nearest
stop_name_to_stop[location] = nearest
else:
stop = transitfeed.Stop(
lng=lng,
lat=lat,
stop_id=box_no,
name=location
)
stop_hexid_to_stop[stop_id] = stop
stop_name_to_stop[location] = stop
#figure out headsigns
headsigns = dict((sign['headsign_id'], sign['headsign']) for sign in route_rec['headsigns'])
#get possible shapes
names = set()
if name in extra_names:
extra = extra_names[name]
if isinstance(extra, str):
names.add(extra)
else:
names.update(extra)
for trip_rec in route_rec['trips']:
route_name = trip_rec['route_name']
names.add(route_name)
nameq = models.Q(route = name)
for rname in names:
rname = fix_leading_zeros.get(rname, rname)
nameq |= models.Q(route__iexact = rname)
if rname in extra_names:
extra = extra_names[rname]
if isinstance(extra, str):
if extra not in names:
nameq |= models.Q(route__iexact = extra_names[rname])
else:
for rname in extra:
if rname not in names:
nameq |= models.Q(route__iexact = extra_names[rname])
shapes = list(MTABusRoute.objects.filter(nameq))
shapes_by_direction = {}
for shape in shapes:
if not shape.rt_dir in shapes_by_direction:
shapes_by_direction[shape.rt_dir] = []
shapes_by_direction[shape.rt_dir].append(shape)
del shapes
#now trips
for trip_rec in route_rec['trips']:
if trip_rec['trip_type'] > 1:
#these are trips to/from the depot
continue
if trip_rec['UNKNOWN_1'].startswith('-'):
#these trips are bogus -- their stops are out-of-order.
continue
hid = trip_rec['headsign_id']
headsign = headsigns.get(hid, long_name)
route = route_for_trip(feed, trip_rec, headsign)
trip = route.AddTrip(feed, headsign, service_period=period)
stops = []
for tripstop_rec in trip_rec['stops']:
stop_id = tripstop_rec['stop_id']
stop_time = google_time_from_centiminutes(tripstop_rec['minutes'])
stop = stop_hexid_to_stop[stop_id]
if not stop.stop_id in feed.stops:
feed.AddStopObject(stop)
trip.AddStopTime(stop,
stop_time=stop_time)
stops.append(stop)
#find the appropriate shape from the shapefiles
trip_route_name = rename_routes.get(trip_rec['route_name'], trip_rec['route_name'])
direction = trip_rec['direction']
if trip_route_name in fix_direction:
direction = fix_direction[trip_route_name].get(direction, direction)
shapes = shapes_by_direction.get(direction)
shape = find_shape_by_stops(feed, shapes, stops, 'bus_route_table')
if shape:
trip.shape_id = shape.shape_id
feed.Validate()
feed.WriteGoogleTransitFeed('mta_data/bus-%s.zip' % current_borough)
def handle_subway(dirname):
feed = transitfeed.Loader("mta_data/subway-gtfs.zip", memory_db=False).Load() #base data
dow_num_to_service_period = {'1' : 'WD',
'2' : 'SAT',
'3' : 'SUN'}
for route_rec in parse_schedule_dir(os.path.join(dirname, 'rail-subway'), 'rtif'):
route_rec['day_of_week'] = dow_num_to_service_period[route_rec['day_of_week']]
period = feed.GetServicePeriod(route_rec['day_of_week'])
route_id = route_rec['line']
try:
route = feed.GetRoute(route_id)
except KeyError:
route = transitfeed.Route(route_id=route_id,
short_name=route_id,
route_type="Subway")
feed.AddRouteObject(route)
feed.AddFareRuleObject(transitfeed.FareRule("regular", route_id))
stops_by_id = {}
for stop in route_rec['stops']:
stop_id = stop['location_id']
stops_by_id[stop_id] = stop
for trip in route_rec['subway_trips']:
line_name = trip['line_name'][-1:] #GS -> S
if not line_name:
continue # a bogus trip
if line_name == '2':
if 'NOSTRAND AVE.' in [stops_by_id[stop['stop_id']]['full_name'] for stop in trip['stops']]:
#the 2 train, as far as I can tell, does not
#actually run on the 3/4 track in Brooklyn.
continue
direction = trip['direction']
headsign = subway_headsign[route_id][direction] #fixme: depends on actual destination of train
gtfs_trip = route.AddTrip(feed, headsign, service_period=period)
for tripstop in trip['stops']:
if tripstop['is_real_stop'] != 'S':
continue
if tripstop['stop_id'] == '138':
#Cortland St
continue
stop_id = tripstop['stop_id']
if stop_id in feed.stops:
gtfs_stop = feed.GetStop(stop_id)
else:
stop = stops_by_id[stop_id]
#first, the table
translated_full_name = stop['full_name']
#remove anything following a dot or a dash or an open paren
for endchar in ['.','-','(',' ']:
if endchar in translated_full_name:
translated_full_name = translated_full_name[:translated_full_name.index(endchar)]
#abbreviations
for long, abbreviated in abbreviations.items():
translated_full_name = translated_full_name.replace(long, abbreviated)
translated_full_name = translated_full_name.strip()
translated_full_name = translated_full_name.replace(" ", " ")
number_st_re = re.compile('^(\d+)')
number_st = number_st_re.match(translated_full_name)
is_num = False
if number_st:
translated_full_name = number_st.group(1)
is_num = True
translated_full_name = rename_subway_stops.get(stop['full_name'], translated_full_name
)
possible_stops = MTASubwayStop.objects.filter(routes__contains=line_name, facility__contains=translated_full_name)
#the A makes C stops late at night
if line_name in addl_lines:
possible_stops = list(possible_stops)
for addl_line in addl_lines[line_name]:
addl_possible_stops = MTASubwayStop.objects.filter(routes__contains=addl_line, facility__contains=translated_full_name)
for addl_stop in addl_possible_stops:
if not addl_stop in possible_stops:
possible_stops.append(addl_stop)
#hard coded stop id
if stop_id in stop_id_to_gid:
possible_stops = MTASubwayStop.objects.filter(gid = stop_id_to_gid[stop_id])
if is_num:
#we need to make sure our search for 23rd st doesn't
#bring up 232nd st.
new_possible_stops = []
num_re = re.compile("(^|\D)" + translated_full_name + "($|\D)")
for possible_stop in possible_stops:
if num_re.search(possible_stop.facility):
new_possible_stops.append(possible_stop)
possible_stops = new_possible_stops
if len(possible_stops) == 0:
print "no stops for %s on the %s (translated to %s)" % (stop['full_name'], line_name, translated_full_name)
print "possible stops are %s" % sorted([s.facility for s in MTASubwayStop.objects.filter(routes__contains=line_name)])
import pdb;pdb.set_trace()
continue
elif len(possible_stops) > 1:
print "too many stops for %s on the %s (translated to %s): %s" % (stop['full_name'], line_name, translated_full_name, [(s.facility, s.gid, s.routes) for s in possible_stops])
import pdb;pdb.set_trace()
continue
else:
dbstop = possible_stops[0]
gtfs_stop = transitfeed.Stop(
lng=dbstop.the_geom.x,
lat=dbstop.the_geom.y,
stop_id=stop_id,
name=dbstop.facility
)
feed.AddStopObject(gtfs_stop)
stop_time = google_time_from_centiminutes(tripstop['stop_time'])
gtfs_trip.AddStopTime(gtfs_stop, stop_time=stop_time)
feed.WriteGoogleTransitFeed('mta_data/subway.zip')
def handle_subway(dirname):
feed = transitfeed.Loader("mta_data/subway-gtfs.zip", memory_db=False).Load() #base data
dow_num_to_service_period = {'1' : 'WD',
'2' : 'SAT',
'3' : 'SUN'}
for route_rec in parse_schedule_dir(os.path.join(dirname, 'rail-subway'), 'rtif'):
route_rec['day_of_week'] = dow_num_to_service_period[route_rec['day_of_week']]
period = feed.GetServicePeriod(route_rec['day_of_week'])
route_id = route_rec['line']
try:
route = feed.GetRoute(route_id)
except KeyError:
route = transitfeed.Route(route_id=route_id,
short_name=route_id,
route_type="Subway")
feed.AddRouteObject(route)
feed.AddFareRuleObject(transitfeed.FareRule("regular", route_id))
stops_by_id = {}
for stop in route_rec['stops']:
stop_id = stop['location_id']
stops_by_id[stop_id] = stop
for trip in route_rec['subway_trips']:
line_name = trip['line_name'][-1:] #GS -> S
if not line_name:
continue # a bogus trip
if line_name == '2':
if 'NOSTRAND AVE.' in [stops_by_id[stop['stop_id']]['full_name'] for stop in trip['stops']]:
#the 2 train, as far as I can tell, does not
#actually run on the 3/4 track in Brooklyn.
continue
direction = trip['direction']
headsign = subway_headsign[route_id][direction] #fixme: depends on actual destination of train
gtfs_trip = route.AddTrip(feed, headsign, service_period=period)
for tripstop in trip['stops']:
if tripstop['is_real_stop'] != 'S':
continue
if tripstop['stop_id'] == '138':
#Cortland St
continue
direction = trip['direction']
stop_id = tripstop['stop_id']
dir_stop_id = stop_id + "_" + direction
if dir_stop_id in feed.stops:
gtfs_stop = feed.GetStop(dir_stop_id)
elif stop_id in feed.stops:
gtfs_stop = feed.GetStop(stop_id)
else:
dbstop = MTASubwayStop.objects.filter(gid=stop_id_to_gid[stop_id])[0]
if stop_id in end_of_line:
gtfs_stop = transitfeed.Stop(
lng=dbstop.the_geom.x,
lat=dbstop.the_geom.y,
stop_id=stop_id,
name=dbstop.facility
)
feed.AddStopObject(gtfs_stop)
else:
for new_direction in 'NS':
new_gtfs_stop = transitfeed.Stop(
lng=dbstop.the_geom.x,
lat=dbstop.the_geom.y,
stop_id=stop_id + "_" + new_direction,
name=dbstop.facility
)
feed.AddStopObject(new_gtfs_stop)
if new_direction == direction:
gtfs_stop = new_gtfs_stop
stop_time = google_time_from_centiminutes(tripstop['stop_time'])
gtfs_trip.AddStopTime(gtfs_stop, stop_time=stop_time)
feed.WriteGoogleTransitFeed('mta_data/subway.zip')
#add back extra files
z = zipfile.ZipFile('mta_data/subway.zip', 'a', zipfile.ZIP_DEFLATED)
z.write('mta_data/subway-gtfs/entrances.txt', 'entrances.txt')
z.write('mta_data/subway-gtfs/stop_entrances.txt', 'stop_entrances.txt')
z.close()