def main():
usage = """usage: python dedupe.py <graphdb_filename>"""
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
exit(-1)
graphdb_filename = args[0]
gtfsdb = GTFSDatabase( graphdb_filename )
query = """
SELECT count(*), monday, tuesday, wednesday, thursday, friday, saturday, sunday, start_date, end_date
FROM calendar
GROUP BY monday, tuesday, wednesday, thursday, friday, saturday, sunday, start_date, end_date"""
duped_periods = gtfsdb.execute( query )
equivilants = []
for count, m,t,w,th,f,s,su,start_date,end_date in duped_periods:
# no need to check for dupes if there's only one
if count==1:
continue
#print count, m, t, w, th, f, s, su, start_date, end_date
# get service_ids for this dow/start_date/end_date combination
service_ids = [x[0] for x in list( gtfsdb.execute( "SELECT service_id FROM calendar where monday=? and tuesday=? and wednesday=? and thursday=? and friday=? and saturday=? and sunday=? and start_date=? and end_date=?", (m,t,w,th,f,s,su,start_date,end_date) ) ) ]
# group by service periods with the same set of exceptions
exception_set_grouper = {}
for service_id in service_ids:
exception_set = list(gtfsdb.execute( "SELECT date, exception_type FROM calendar_dates WHERE service_id=?", (service_id,) ) )
exception_set.sort()
exception_set = tuple(exception_set)
exception_set_grouper[exception_set] = exception_set_grouper.get(exception_set,[])
exception_set_grouper[exception_set].append( service_id )
# extend list of equivilants
for i, exception_set_group in enumerate( exception_set_grouper.values() ):
equivilants.append( ("%d%d%d%d%d%d%d-%s-%s-%d"%(m,t,w,th,f,s,su,start_date,end_date,i), exception_set_group) )
for new_name, old_names in equivilants:
for old_name in old_names:
print old_name, new_name
c = gtfsdb.conn.cursor()
c.execute( "UPDATE calendar SET service_id=? WHERE service_id=?", (new_name, old_name) )
c.execute( "UPDATE calendar_dates SET service_id=? WHERE service_id=?", (new_name, old_name) )
c.execute( "UPDATE trips SET service_id=? WHERE service_id=?", (new_name, old_name) )
gtfsdb.conn.commit()
c.close()
class HeadwayBoardEvent:
def __init__(self, gtfsdb_filename, timezone_name="America/Los_Angeles"):
self.gtfsdb = GTFSDatabase(gtfsdb_filename)
self.timezone_name = timezone_name
@staticmethod
def applies_to(vertex1, edge, vertex2):
return edge is not None and isinstance(edge.payload,
graphserver.core.HeadwayBoard)
def __call__(self, vertex1, edge, vertex2, context):
event_time = vertex2.state.time
trip_id = vertex2.state.trip_id
stop_id = vertex1.label.split("-")[-1]
route_desc = "-".join(
list(
self.gtfsdb.execute(
"SELECT routes.route_short_name, routes.route_long_name FROM routes, trips WHERE routes.route_id=trips.route_id AND trip_id=?",
(trip_id, )))[0])
stop_desc = list(
self.gtfsdb.execute(
"SELECT stop_name FROM stops WHERE stop_id = ?",
(stop_id, )))[0][0]
lat, lon = list(
self.gtfsdb.execute(
"SELECT stop_lat, stop_lon FROM stops WHERE stop_id = ?",
(stop_id, )))[0]
what = "Board the %s" % route_desc
where = stop_desc
when = "about %s" % str(
TimeHelpers.unix_to_localtime(event_time, self.timezone_name))
geom = (lon, lat)
return NarrativeEvent(what, where, when, geom)
class AlightEvent:
def __init__(self, gtfsdb_filename, timezone_name="America/Los_Angeles"):
self.gtfsdb = GTFSDatabase(gtfsdb_filename)
self.timezone_name = timezone_name
@staticmethod
def applies_to(vertex1, edge, vertex2):
return edge is not None and isinstance(edge.payload,
graphserver.core.TripAlight)
def __call__(self, vertex1, edge, vertex2, context):
event_time = vertex1.state.time
stop_id = vertex2.label.split("-")[-1]
stop_desc = list(
self.gtfsdb.execute(
"SELECT stop_name FROM stops WHERE stop_id = ?",
(stop_id, )))[0][0]
lat, lon = list(
self.gtfsdb.execute(
"SELECT stop_lat, stop_lon FROM stops WHERE stop_id = ?",
(stop_id, )))[0]
what = "Alight"
where = stop_desc
when = str(
TimeHelpers.unix_to_localtime(event_time, self.timezone_name))
geom = (lon, lat)
return NarrativeEvent(what, where, when, geom)
def main():
usage = """usage: python dedupe.py <graphdb_filename>"""
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 1:
parser.print_help()
exit(-1)
graphdb_filename = args[0]
gtfsdb = GTFSDatabase( graphdb_filename )
query = """
SELECT count(*), monday, tuesday, wednesday, thursday, friday, saturday, sunday, start_date, end_date
FROM calendar
GROUP BY monday, tuesday, wednesday, thursday, friday, saturday, sunday, start_date, end_date"""
duped_periods = gtfsdb.execute( query )
equivilants = []
for count, m,t,w,th,f,s,su,start_date,end_date in duped_periods:
# no need to check for dupes if there's only one
if count==1:
continue
#print count, m, t, w, th, f, s, su, start_date, end_date
# get service_ids for this dow/start_date/end_date combination
service_ids = [x[0] for x in list( gtfsdb.execute( "SELECT service_id FROM calendar where monday=? and tuesday=? and wednesday=? and thursday=? and friday=? and saturday=? and sunday=? and start_date=? and end_date=?", (m,t,w,th,f,s,su,start_date,end_date) ) ) ]
# group by service periods with the same set of exceptions
exception_set_grouper = {}
for service_id in service_ids:
exception_set = list(gtfsdb.execute( "SELECT date, exception_type FROM calendar_dates WHERE service_id=?", (service_id,) ) )
exception_set.sort()
exception_set = tuple(exception_set)
exception_set_grouper[exception_set] = exception_set_grouper.get(exception_set,[])
exception_set_grouper[exception_set].append( service_id )
# extend list of equivilants
for i, exception_set_group in enumerate( exception_set_grouper.values() ):
equivilants.append( ("%d%d%d%d%d%d%d-%s-%s-%d"%(m,t,w,th,f,s,su,start_date,end_date,i), exception_set_group) )
for new_name, old_names in equivilants:
for old_name in old_names:
print old_name, new_name
c = gtfsdb.conn.cursor()
c.execute( "UPDATE calendar SET service_id=? WHERE service_id=?", (new_name, old_name) )
c.execute( "UPDATE calendar_dates SET service_id=? WHERE service_id=?", (new_name, old_name) )
c.execute( "UPDATE trips SET service_id=? WHERE service_id=?", (new_name, old_name) )
gtfsdb.conn.commit()
c.close()
class AlightEvent:
def __init__(self, gtfsdb_filename, timezone_name="America/Los_Angeles"):
self.gtfsdb = GTFSDatabase( gtfsdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(vertex1, edge, vertex2):
return edge is not None and isinstance(edge.payload, graphserver.core.TripAlight)
def __call__(self, vertex1, edge, vertex2, context):
event_time = vertex1.state.time
stop_id = vertex2.label.split("-")[-1]
stop_desc = list( self.gtfsdb.execute( "SELECT stop_name FROM stops WHERE stop_id = ?", (stop_id,) ) )[0][0]
lat, lon = list( self.gtfsdb.execute( "SELECT stop_lat, stop_lon FROM stops WHERE stop_id = ?", (stop_id,) ) )[0]
what = "Alight"
where = stop_desc
when = str(TimeHelpers.unix_to_localtime( event_time, self.timezone_name ))
geom = (lon,lat)
return NarrativeEvent(what, where, when, geom)
class HeadwayBoardEvent:
def __init__(self, gtfsdb_filename, timezone_name="America/Los_Angeles"):
self.gtfsdb = GTFSDatabase( gtfsdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(vertex1, edge, vertex2):
return edge is not None and isinstance(edge.payload, graphserver.core.HeadwayBoard)
def __call__(self, vertex1, edge, vertex2, context):
event_time = vertex2.state.time
trip_id = vertex2.state.trip_id
stop_id = vertex1.label.split("-")[-1]
route_desc = "-".join(list( self.gtfsdb.execute( "SELECT routes.route_short_name, routes.route_long_name FROM routes, trips WHERE routes.route_id=trips.route_id AND trip_id=?", (trip_id,) ) )[0])
stop_desc = list( self.gtfsdb.execute( "SELECT stop_name FROM stops WHERE stop_id = ?", (stop_id,) ) )[0][0]
lat, lon = list( self.gtfsdb.execute( "SELECT stop_lat, stop_lon FROM stops WHERE stop_id = ?", (stop_id,) ) )[0]
what = "Board the %s"%route_desc
where = stop_desc
when = "about %s"%str(TimeHelpers.unix_to_localtime( event_time, self.timezone_name ))
geom = (lon,lat)
return NarrativeEvent(what, where, when, geom)
def main():
gtfsdb = GTFSDatabase( "data/washingtondc.gtfsdb" )
osmdb = OSMDB( "data/washingtondc.osmdb" )
ll,bb,rr,tt = list(gtfsdb.execute( "SELECT min(stop_lon), min(stop_lat), max(stop_lon), max(stop_lat) FROM stops" ))[0]
from prender import processing
mr = processing.MapRenderer()
mr.start(ll,bb,rr,tt,4000) #left,bottom,right,top,width
mr.smooth()
mr.strokeWeight(0.000001)
mr.background(255,255,255)
mr.stroke(128,128,128)
render_osmdb(osmdb, mr)
mr.stroke(0,0,0)
render_gtfsdb(gtfsdb, mr)
mr.saveLocal("map.png")
mr.stop()
# requires graphserver to be installed
from graphserver.ext.gtfs.gtfsdb import GTFSDatabase
if len(sys.argv) != 2:
print 'usage: datecheck.py inputfile.gtfsdb'
exit(1)
else:
gtfsdb_file = sys.argv[1]
try:
with open(gtfsdb_file) as f:
db = GTFSDatabase(gtfsdb_file)
except IOError as e:
print 'gtfsdb file %s cannot be opened' % gtfsdb_file
exit(1)
# check that all routes gs reports running are actually running on each day
for line in sys.stdin.readlines():
trip_id, fromid, fromtime, toid, totime = line.split()
if trip_id == 'walk':
continue
service_id = list(
db.execute('select service_id from trips where trip_id = ?',
(trip_id, )))[0][0]
print trip_id, '___', service_id
# and date > 20130415 and date < 20130420
for line in db.execute(
'select date from calendar_dates where service_id = ? and date = 20130417 order by date',
(service_id, )):
print line[0]
wo = WalkOptions()
wo.max_walk = 2000
wo.walking_overage = 0.0
wo.walking_speed = 1.0 # trimet uses 0.03 miles / 1 minute - but it uses straight line distance as well
wo.transfer_penalty = 60 * 10
wo.walking_reluctance = 1.5
wo.max_transfers = 5
wo.transfer_slack = 60 * 4
wo_transit = wo
print "Fetching grid from OSMDB..."
grid = list(osmdb.execute("SELECT x, y, vertex FROM grid"))
max_x, max_y = osmdb.execute("SELECT max(x), max(y) FROM grid").next()
print "Finding unique GTFS station linking points..."
stop_vertices = [e[0] for e in gtfsdb.execute("SELECT DISTINCT osm_vertex FROM osm_links")]
#test
#stop_vertices = stop_vertices[:10]
def save_image(spt, fname) :
print "saving grid image..."
im = Image.new("L", (max_x, max_y))
for (x, y, vertex) in grid :
v = spt.get_vertex('osm-%s'%vertex)
if v != None :
c = ((v.best_state.time - t0) / (120. * 60)) * 255
im.putpixel((x, max_y - y - 1), 255 - c)
# be careful that img dir exists
im.save('img/%s.png' % (fname))
# equirectangular / sinusoidal projection
def distance (lat1, lon1, lat2, lon2) :
avg_lat = (lat1 + lat2) / 2
xscale = math.cos(math.radians(avg_lat))
return distance (lat1, lon1, lat2, lon2, xscale)
def distance (lat1, lon1, lat2, lon2, xscale) :
dlon = lon2 - lon1
dlat = lat2 - lat1
dlon *= xscale
d2 = dlon * dlon + dlat * dlat
return math.sqrt(d2) * 111111.111
stops = {}
for sid, sname, lat, lon, routes in db.execute(all_query) :
stops[sid] = {'name': sname, 'lat': lat, 'lon': lon, 'routes': set(routes.split(','))}
# can also compare squared distances in scaled meters
transfers = []
n_processed = 0
for sid, param in stops.items():
if verbose :
print sid, sname
xscale = math.cos(math.radians(param['lat']))
range_lon = range_lat * xscale
routes = copy(param['routes'])
# print xscale, range_lat, range_lon
for sid2, sname2, lat2, lon2 in db.execute(near_query, {'range_lat': range_lat, 'range_lon': range_lon, 'lat': param['lat'], 'lon': param['lon']}):
if sid == sid2:
continue
avg_lat = (lat1 + lat2) / 2
xscale = math.cos(math.radians(avg_lat))
return distance(lat1, lon1, lat2, lon2, xscale)
def distance(lat1, lon1, lat2, lon2, xscale):
dlon = lon2 - lon1
dlat = lat2 - lat1
dlon *= xscale
d2 = dlon * dlon + dlat * dlat
return math.sqrt(d2) * 111111.111
stops = {}
for sid, sname, lat, lon, routes in db.execute(all_query):
stops[sid] = {
'name': sname,
'lat': lat,
'lon': lon,
'routes': set(routes.split(','))
}
# can also compare squared distances in scaled meters
transfers = []
n_processed = 0
for sid, param in stops.items():
if verbose:
print sid, sname
xscale = math.cos(math.radians(param['lat']))
range_lon = range_lat * xscale
import sys, struct
# requires graphserver to be installed
from graphserver.ext.gtfs.gtfsdb import GTFSDatabase
if len(sys.argv) != 2:
print "usage: datecheck.py inputfile.gtfsdb"
exit(1)
else:
gtfsdb_file = sys.argv[1]
try:
with open(gtfsdb_file) as f:
db = GTFSDatabase(gtfsdb_file)
except IOError as e:
print "gtfsdb file %s cannot be opened" % gtfsdb_file
exit(1)
# check that all routes gs reports running are actually running on each day
for line in sys.stdin.readlines():
trip_id, fromid, fromtime, toid, totime = line.split()
if trip_id == "walk":
continue
service_id = list(db.execute("select service_id from trips where trip_id = ?", (trip_id,)))[0][0]
print trip_id, "___", service_id
# and date > 20130415 and date < 20130420
for line in db.execute(
"select date from calendar_dates where service_id = ? and date = 20130417 order by date", (service_id,)
):
print line[0]
