def populate_profile_db(osmdb_name, profiledb_name, dem_basenames, resolution):
ddb = OSMDB(osmdb_name)
elevs = ElevationPile()
for dem_basename in dem_basenames:
elevs.add(dem_basename)
pdb = ProfileDB(profiledb_name, overwrite=True)
n = ddb.count_edges()
print "Profiling %d way segments" % n
for i, (id, parent_id, node1, node2, dist, geom,
tags) in enumerate(ddb.edges()):
if i % 1000 == 0: print "%d/%d" % (i, n)
raw_profile = elevs.profile(geom, resolution)
profile = []
tunnel = tags.get('tunnel')
bridge = tags.get('bridge')
if tunnel == 'yes' or tunnel == 'true' or bridge == 'yes' or bridge == 'true':
if len(raw_profile) > 0:
ss, ee = raw_profile[0]
if ee is not None: profile.append((ss, ee))
if len(raw_profile) > 1:
ss, ee = raw_profile[-1]
if ee is not None: profile.append((ss, ee))
else:
for ss, ee in raw_profile:
if ee is not None: profile.append((ss, ee))
pdb.store(id, profile)
pdb.conn.commit()
def populate_profile_db( osmdb_name, profiledb_name, dem_basenames, resolution ):
ddb = OSMDB( osmdb_name )
elevs = ElevationPile()
for dem_basename in dem_basenames:
elevs.add( dem_basename )
pdb = ProfileDB( profiledb_name, overwrite=True )
n = ddb.count_edges()
print "Profiling %d way segments"%n
for i, (id, parent_id, node1, node2, dist, geom, tags) in enumerate( ddb.edges() ):
if i%1000==0: print "%d/%d"%(i,n)
raw_profile = elevs.profile( geom, resolution )
profile = []
tunnel = tags.get('tunnel')
bridge = tags.get('bridge')
if tunnel == 'yes' or tunnel == 'true' or bridge == 'yes' or bridge == 'true':
if len(raw_profile) > 0:
ss, ee = raw_profile[0]
if ee is not None: profile.append( (ss,ee) )
if len(raw_profile) > 1:
ss, ee = raw_profile[-1]
if ee is not None: profile.append( (ss,ee) )
else:
for ss, ee in raw_profile:
if ee is not None: profile.append( (ss,ee) )
pdb.store( id, profile )
pdb.conn.commit()
class StreetEndEvent:
def __init__(self, osmdb_filename, timezone_name="America/Los_Angeles"):
self.osmdb = OSMDB(osmdb_filename)
self.timezone_name = timezone_name
@staticmethod
def applies_to(edge1, vertex, edge2):
# if edge1 is not a street and edge2 is
return (edge2 is None or not isinstance(edge2.payload, graphserver.core.Street)) and \
(edge1 and isinstance(edge1.payload, graphserver.core.Street))
def __call__(self, edge1, vertex, edge2, context):
osm_way1 = edge1.payload.name.split("-")[0]
street_name1 = self.osmdb.way(osm_way1).tags.get('name', "unnamed")
osm_id = vertex.label.split("-")[1]
osm_node_id, osm_node_tags, osm_node_lat, osm_node_lon, osm_node_refcount = self.osmdb.node(
osm_id)
average_speed = context['sumlength'] / (
vertex.state.time - context['lastturntime']
) if vertex.state.time - context['lastturntime'] > 0 else 100000000
what = "arrive walking after %dm, %0.1f rise, %0.1f fall (%0.1fm/s)" % (
context['sumlength'], context['sumrise'], context['sumfall'],
average_speed)
where = "on %s" % (street_name1)
when = "about %s" % str(
TimeHelpers.unix_to_localtime(vertex.state.time,
self.timezone_name))
geom = [osm_node_lon, osm_node_lat]
return NarrativeEvent(what, where, when, geom)
def main():
usage = """usage: python gdb_link_osm_gtfs.py <graphdb_filename> <osmdb_filename> <gtfsdb_filename>"""
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 3:
parser.print_help()
exit(-1)
graphdb_filename = args[0]
osmdb_filename = args[1]
gtfsdb_filename = args[2]
gtfsdb = GTFSDatabase( gtfsdb_filename )
osmdb = OSMDB( osmdb_filename )
gdb = GraphDatabase( graphdb_filename )
n_stops = gtfsdb.count_stops()
for i, (stop_id, stop_name, stop_lat, stop_lon) in enumerate( gtfsdb.stops() ):
print "%d/%d"%(i,n_stops)
nd_id, nd_lat, nd_lon, nd_dist = osmdb.nearest_node( stop_lat, stop_lon )
station_vertex_id = "sta-%s"%stop_id
osm_vertex_id = "osm-%s"%nd_id
print station_vertex_id, osm_vertex_id
gdb.add_edge( station_vertex_id, osm_vertex_id, Link() )
gdb.add_edge( osm_vertex_id, station_vertex_id, Link() )
class ModifiedOSMReverseGeocoder:
def __init__(self, osmdb_filename, range=0.005):
self.osmdb = OSMDB(osmdb_filename)
self.range = range
def __call__(self, lat, lon):
#vertex = self.nearest_node(lat, lon, use_index=False, range=self.range)
vertex = self.nearest_node(lat, lon, use_index=True, range=self.range)
if vertex:
return NearbyVertex(*vertex)
else:
return None
def bounds(self):
"""return tuple representing bounding box of reverse geocoder with form
# (left, bottom, right, top)"""
return self.osmdb.bounds()
def nearest_node(self, lat, lon, use_index=True, range=0.005):
c = self.osmdb.get_cursor()
if use_index and self.osmdb.index:
id = list(self.osmdb.index.nearest((lon, lat), 1))[0]
c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id, ))
else:
c.execute(
"SELECT id, lat, lon FROM nodes WHERE " +
"endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?",
(lat - range, lat + range, lon - range, lon + range))
dists = [(nid, nlat, nlon, self.haversine_miles(nlon, nlat, lon, lat))
for nid, nlat, nlon in c]
if not dists:
return (None, None, None, None)
return min(dists, key=lambda x: x[3])
def haversine_miles(self, lon1, lat1, lon2, lat2):
"""Calculate the great circle distance between two points
on the earth (specified in decimal degrees)"""
# convert decimal degrees to radians
lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
# haversine formula
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
miles = 3963.0 * c
return miles
def delete_orphan_nodes(db_conn_string):
db = OSMDB(db_conn_string, rtree_index=False)
filter = DeleteOrphanNodesFilter()
filter.run(db, *[])
# reindex the database
db.index = Rtree()
db.index_endnodes()
class ModifiedOSMReverseGeocoder:
def __init__(self, osmdb_filename, range=0.005):
self.osmdb = OSMDB(osmdb_filename)
self.range = range
def __call__(self, lat, lon):
#vertex = self.nearest_node(lat, lon, use_index=False, range=self.range)
vertex = self.nearest_node(lat, lon, use_index=True, range=self.range)
if vertex:
return NearbyVertex(*vertex)
else:
return None
def bounds(self):
"""return tuple representing bounding box of reverse geocoder with form
# (left, bottom, right, top)"""
return self.osmdb.bounds()
def nearest_node(self, lat, lon, use_index=True, range=0.005):
c = self.osmdb.get_cursor()
if use_index and self.osmdb.index:
id = list(self.osmdb.index.nearest((lon, lat), 1))[0]
c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id,))
else:
c.execute("SELECT id, lat, lon FROM nodes WHERE " +
"endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?",
(lat-range, lat+range, lon-range, lon+range))
dists = [(nid, nlat, nlon, self.haversine_miles(nlon, nlat, lon, lat)) for nid, nlat, nlon in c]
if not dists:
return (None, None, None, None)
return min(dists, key=lambda x:x[3])
def haversine_miles(self, lon1, lat1, lon2, lat2):
"""Calculate the great circle distance between two points
on the earth (specified in decimal degrees)"""
# convert decimal degrees to radians
lon1, lat1, lon2, lat2 = map(radians, [lon1, lat1, lon2, lat2])
# haversine formula
dlon = lon2 - lon1
dlat = lat2 - lat1
a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2
c = 2 * atan2(sqrt(a), sqrt(1 - a))
miles = 3963.0 * c
return miles
class StreetTurnEvent:
def __init__(self, osmdb_filename, timezone_name = "America/Los_Angeles"):
self.osmdb = OSMDB( osmdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(edge1, vertex, edge2):
return edge1 and edge2 and isinstance(edge1.payload, graphserver.core.Street) and isinstance(edge2.payload, graphserver.core.Street) \
and edge1.payload.way != edge2.payload.way
def __call__(self, edge1, vertex, edge2, context):
osm_id = vertex.label.split("-")[1]
# figure out which direction to turn
osm_way_id1 = edge1.payload.name.split("-")[0]
osm_way_id2 = edge2.payload.name.split("-")[0]
osm_edge1 = self.osmdb.edge( edge1.payload.name )
osm_edge2 = self.osmdb.edge( edge2.payload.name )
osm_edge1_endnode = osm_edge1[3]
osm_edge2_startnode = osm_edge2[2]
osm_edge1_geom = osm_edge1[5]
osm_edge2_geom = osm_edge2[5]
if osm_id != osm_edge1_endnode:
osm_edge1_geom.reverse()
if osm_id != osm_edge2_startnode:
osm_edge2_geom.reverse()
endseg = osm_edge1_geom[-2:]
startseg = osm_edge2_geom[:2]
direction = turn_narrative( endseg[0], endseg[1], startseg[0], startseg[1] )
street_name1 = self.osmdb.way( osm_way_id1 ).tags.get("name", "unnamed")
street_name2 = self.osmdb.way( osm_way_id2 ).tags.get("name", "unnamed")
osm_node_id, osm_node_tags, osm_node_lat, osm_node_lon, osm_node_refcount = self.osmdb.node( osm_id )
average_speed = context['sumlength']/(vertex.state.time-context['lastturntime']) if vertex.state.time-context['lastturntime']>0 else 100000000
what = "%s onto %s after %dm, %0.1fm rise, %0.1fm fall (%0.1fm/s)"%(direction, street_name2, context['sumlength'], context['sumrise'], context['sumfall'], average_speed)
where = "%s & %s"%(street_name1, street_name2)
when = "about %s"%str(TimeHelpers.unix_to_localtime( vertex.state.time, self.timezone_name ))
geom = (osm_node_lon, osm_node_lat)
context['sumlength'] = 0
context['sumrise'] = 0
context['sumfall'] = 0
context['lastturntime'] = vertex.state.time
return NarrativeEvent(what,where,when,geom)
class OSMReverseGeocoder:
def __init__(self, osmdb_filename):
self.osmdb = OSMDB( osmdb_filename )
def __call__(self, lat, lon):
nearby_vertex = list(self.osmdb.nearest_node(lat, lon))
return "osm-%s"%(nearby_vertex[0])
def bounds(self):
"""return tuple representing bounding box of reverse geocoder with form (left, bottom, right, top)"""
return self.osmdb.bounds()
class OSMReverseGeocoder:
def __init__(self, osmdb_filename):
self.osmdb = OSMDB(osmdb_filename)
def __call__(self, lat, lon):
nearby_vertex = list(self.osmdb.nearest_node(lat, lon))
return "osm-%s" % (nearby_vertex[0])
def bounds(self):
"""return tuple representing bounding box of reverse geocoder with form (left, bottom, right, top)"""
return self.osmdb.bounds()
def main(osm_xml, osm_db, gtfs_db):
print "CREATING AND POPULATING OSM DATABASE"
osmdb = OSMDB( osm_db, overwrite=True )
osmdb.populate( osm_xml, reporter=sys.stdout )
print "CLEANING UP OSM DATABASE"
StitchDisjunctGraphs().run( osmdb )
osmdb.create_and_populate_edges_table( tolerant=True )
PurgeDisjunctGraphsFilter().run( osmdb ) #, threshold = 1000 )
print "SPLITTING OSM EDGES FOR BETTER STATION LINKING"
gtfsdb = GTFSDatabase (gtfs_db)
station_split (osmdb, gtfsdb)
print "DONE."
sys.exit(0)
class StreetStartEvent:
def __init__(self, osmdb_filename, timezone_name="America/Los_Angeles"):
self.osmdb = OSMDB(osmdb_filename)
self.timezone_name = timezone_name
@staticmethod
def applies_to(edge1, vertex, edge2):
# if edge1 is not a street and edge2 is
return (edge1 is None or not isinstance(edge1.payload, graphserver.core.Street)) and \
(edge2 and isinstance(edge2.payload, graphserver.core.Street))
def __call__(self, edge1, vertex, edge2, context):
context['streetgeom'] = []
context['sumlength'] = 0
context['sumrise'] = 0
context['sumfall'] = 0
context['lastturntime'] = vertex.state.time
osm_way2 = edge2.payload.name.split("-")[0]
street_name2 = self.osmdb.way(osm_way2).tags.get('name', "unnamed")
osm_id = vertex.label.split("-")[1]
osm_node_id, osm_node_tags, osm_node_lat, osm_node_lon, osm_node_refcount = self.osmdb.node(
osm_id)
osm_edge2 = self.osmdb.edge(edge2.payload.name)
osm_edge2_startnode = osm_edge2[2]
osm_edge2_geom = osm_edge2[5]
if osm_id != osm_edge2_startnode:
osm_edge2_geom.reverse()
startseg = osm_edge2_geom[:2]
direction = description_from_north(startseg[0], startseg[1])
what = "start walking"
where = "on %s facing %s" % (street_name2, direction)
when = "about %s" % str(
TimeHelpers.unix_to_localtime(vertex.state.time,
self.timezone_name))
geom = [osm_node_lon, osm_node_lat]
return NarrativeEvent(what, where, when, geom)
def main():
usage = """usage: python gdb_import_osm.py <graphdb_filename> <osmdb_filename>"""
parser = OptionParser(usage=usage)
parser.add_option(
"-n",
"--namespace",
dest="namespace",
default="osm",
help="prefix all imported vertices with namespace string")
parser.add_option(
"-s",
"--slog",
action="append",
dest="slog_strings",
default=[],
help=
"specify slog for highway type, in highway_type:slog form. For example, 'motorway:10.5'"
)
parser.add_option(
"-p",
"--profiledb",
dest="profiledb_filename",
default=None,
help="specify profiledb to annotate streets with rise/fall data")
(options, args) = parser.parse_args()
if len(args) != 2:
parser.print_help()
exit(-1)
slogs = {}
for slog_string in options.slog_strings:
highway_type, slog_penalty = slog_string.split(":")
slogs[highway_type] = float(slog_penalty)
print "slog values: %s" % slogs
graphdb_filename = args[0]
osmdb_filename = args[1]
print "importing osmdb '%s' into graphdb '%s'" % (osmdb_filename,
graphdb_filename)
profiledb = ProfileDB(
options.profiledb_filename) if options.profiledb_filename else None
osmdb = OSMDB(osmdb_filename)
gdb = GraphDatabase(graphdb_filename, overwrite=False)
gdb_import_osm(gdb, osmdb, options.namespace, slogs, profiledb)
print "done"
class StreetStartEvent:
def __init__(self, osmdb_filename, timezone_name = "America/Los_Angeles"):
self.osmdb = OSMDB( osmdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(edge1, vertex, edge2):
# if edge1 is not a street and edge2 is
return (edge1 is None or not isinstance(edge1.payload, graphserver.core.Street)) and \
(edge2 and isinstance(edge2.payload, graphserver.core.Street))
def __call__(self, edge1, vertex, edge2, context):
context['streetgeom'] = []
context['sumlength'] = 0
context['sumrise'] = 0
context['sumfall'] = 0
context['lastturntime'] = vertex.state.time
osm_way2 = edge2.payload.name.split("-")[0]
street_name2 = self.osmdb.way( osm_way2 ).tags.get('name', "unnamed")
osm_id = vertex.label.split("-")[1]
osm_node_id, osm_node_tags, osm_node_lat, osm_node_lon, osm_node_refcount = self.osmdb.node( osm_id )
osm_edge2 = self.osmdb.edge( edge2.payload.name )
osm_edge2_startnode = osm_edge2[2]
osm_edge2_geom = osm_edge2[5]
if osm_id != osm_edge2_startnode:
osm_edge2_geom.reverse()
startseg = osm_edge2_geom[:2]
direction = description_from_north( startseg[0], startseg[1] )
what = "start walking"
where = "on %s facing %s"%(street_name2, direction)
when = "about %s"%str(TimeHelpers.unix_to_localtime( vertex.state.time, self.timezone_name ))
geom = [osm_node_lon, osm_node_lat]
return NarrativeEvent(what,where,when,geom)
class StreetEndEvent:
def __init__(self, osmdb_filename, timezone_name = "America/Los_Angeles"):
self.osmdb = OSMDB( osmdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(edge1, vertex, edge2):
# if edge1 is not a street and edge2 is
return (edge2 is None or not isinstance(edge2.payload, graphserver.core.Street)) and \
(edge1 and isinstance(edge1.payload, graphserver.core.Street))
def __call__(self, edge1, vertex, edge2, context):
osm_way1 = edge1.payload.name.split("-")[0]
street_name1 = self.osmdb.way( osm_way1 ).tags.get('name', "unnamed")
osm_id = vertex.label.split("-")[1]
osm_node_id, osm_node_tags, osm_node_lat, osm_node_lon, osm_node_refcount = self.osmdb.node( osm_id )
average_speed = context['sumlength']/(vertex.state.time-context['lastturntime']) if vertex.state.time-context['lastturntime']>0 else 100000000
what = "arrive walking after %dm, %0.1f rise, %0.1f fall (%0.1fm/s)"%(context['sumlength'], context['sumrise'], context['sumfall'], average_speed)
where = "on %s"%(street_name1)
when = "about %s"%str(TimeHelpers.unix_to_localtime( vertex.state.time, self.timezone_name ))
geom = [osm_node_lon, osm_node_lat]
return NarrativeEvent(what,where,when,geom)
def __init__(self, settings_filename):
settings = yaml.load( open( settings_filename ) )
self.home_point = settings['center']
# create cache of osm-node positions
self.osmdb = OSMDB( settings['osmdb_filename'] )
self.gtfsdb = GTFSDatabase( settings['gtfsdb_filename'] )
self.port = settings['port']
self.node_positions = {}
self.index = Rtree()
for node_id, tags, lat, lon in self.osmdb.nodes():
self.node_positions[node_id] = (lon,lat)
self.index.add( int(node_id), (lon,lat,lon,lat) )
# incarnate graph from graphdb
graphdb = GraphDatabase( settings['graphdb_filename'] )
self.graph = graphdb.incarnate()
class StreetEvent:
def __init__(self, osmdb_filename, timezone_name="America/Los_Angeles"):
self.osmdb = OSMDB( osmdb_filename )
self.timezone_name = timezone_name
@staticmethod
def applies_to(vertex1, edge, vertex2):
return edge is not None and isinstance(edge.payload, graphserver.core.Street)
def __call__(self, vertex1, edge, vertex2, context):
# adds to the variable set up by the StreetStartEvent
geometry_chunk = self.osmdb.edge( edge.payload.name )[5]
if len(context['streetgeom'])==0 or len(geometry_chunk)==0 or context['streetgeom'][-1] == geometry_chunk[0]:
context['streetgeom'].extend( geometry_chunk )
else:
context['streetgeom'].extend( reversed( geometry_chunk ) )
context['sumlength'] += edge.payload.length
context['sumrise'] += edge.payload.rise
context['sumfall'] += edge.payload.fall
return None
#!/usr/bin/env python
from graphserver.ext.osm.osmdb import OSMDB
from PIL import Image
from math import log, sqrt
osmdb = OSMDB ('/home/andrew/data/pdx/testgrid.osmdb' )
print "Fetching grid from OSMDB..."
grid = list(osmdb.execute("SELECT g.x, g.y, gp.surf, gp.pop FROM grid AS g, grid_pop AS gp WHERE g.rowid = gp.rowid"))
max_x, max_y = osmdb.execute("SELECT max(x), max(y) FROM grid").next()
max_pop, = osmdb.execute("SELECT max(pop) FROM grid_pop").next()
max_surf, = osmdb.execute("SELECT max(surf) FROM grid_pop").next()
max_x += 1
max_y += 1
print max_surf, max_pop
print "saving image..."
im_surf = Image.new("L", (max_x, max_y))
im_pop = Image.new("L", (max_x, max_y))
for (x, y, surf, pop) in grid :
# s = int((sqrt(surf)/sqrt(max_surf)) * 255)
s = int((surf/max_surf) * 255)
im_surf.putpixel((x, max_y - y - 1), s)
# p = int((sqrt(pop)/sqrt(max_pop)) * 255)
p = int((pop / max_pop) * 255)
im_pop.putpixel((x, max_y - y - 1), p)
im_pop.save('population.png')
def __init__(self, osmdb_filename, range=0.005):
self.osmdb = OSMDB(osmdb_filename)
self.range = range
def FindDisjunctGraphs (dbname):
db = OSMDB(dbname)
#should really be done before simplifying and splitting
#fuse_nodes(db)
c = db.cursor()
c.execute("DROP table if exists graph_nodes")
c.execute("DROP table if exists graph_edges")
c.execute("CREATE table graph_nodes (graph_num INTEGER, node_id TEXT, WKT_GEOMETRY TEXT)")
c.execute("CREATE table graph_edges (graph_num INTEGER, edge_id TEXT, WKT_GEOMETRY TEXT)")
c.execute("CREATE index graph_nodes_id_indx ON graph_nodes(node_id)")
c.execute("CREATE index graph_edges_id_indx ON graph_edges(edge_id)")
c.close()
g = Graph()
t0 = time.time()
vertices = {}
print "load vertices into memory"
for row in db.execute("SELECT DISTINCT start_nd from edges"):
g.add_vertex(str(row[0]))
vertices[str(row[0])] = 0
#print str(row[0])
for row in db.execute("SELECT DISTINCT end_nd from edges"):
g.add_vertex(str(row[0]))
vertices[str(row[0])] = 0
#k = vertices.keys()
#k.sort()
#print k, len(k)
print "load edges into memory"
for start_nd, end_nd in db.execute("SELECT start_nd, end_nd from edges"):
g.add_edge(start_nd, end_nd, Link())
g.add_edge(end_nd, start_nd, Link())
#print start_nd, end_nd
db.conn.commit()
t1 = time.time()
print "populating graph took: %f"%(t1-t0)
t0 = t1
print len(vertices)
iteration = 1
c = db.cursor()
while True:
#c.execute("SELECT id from nodes where id not in (SELECT node_id from graph_nodes) LIMIT 1")
try:
vertex, dummy = vertices.popitem()
#print vertex
except:
break
spt = g.shortest_path_tree(vertex, None, State(1,0))
print spt.size
for v in spt.vertices:
lat, lon = c.execute("SELECT lat, lon from nodes where id=?", (v.label, )).next()
c.execute("INSERT into graph_nodes VALUES (?, ?, ?)", (iteration, v.label, "POINT(%f %f)" % (lon, lat)))
for e in v.outgoing: # this gives a wierd maze graph, should do for all edges outside loop.
lat1, lon1 = c.execute("SELECT lat, lon from nodes where id=?", (e.from_v.label, )).next()
lat2, lon2 = c.execute("SELECT lat, lon from nodes where id=?", (e.to_v.label, )).next()
c.execute("INSERT into graph_edges VALUES (?, ?, ?)",
(iteration, e.from_v.label + '->' + e.to_v.label, "LINESTRING(%f %f, %f %f)" % (lon1, lat1, lon2, lat2)))
#print v.label
vertices.pop(v.label, None)
g.remove_vertex(v.label, True, True)
#print v.label
spt.destroy()
t1 = time.time()
print "pass %s took: %f nvertices %d"%(iteration, t1-t0, len(vertices))
t0 = t1
iteration += 1
c.close()
db.conn.commit()
g.destroy()
# audit
for gnum, count in db.execute("SELECT graph_num, count(*) FROM graph_nodes GROUP BY graph_num"):
print "FOUND: %s=%s" % (gnum, count)
def __init__(self, osmdb_filename, timezone_name="America/Los_Angeles"):
self.osmdb = OSMDB(osmdb_filename)
self.timezone_name = timezone_name
def __init__(self, osmdb_filename):
self.osmdb = OSMDB(osmdb_filename)
def __init__(self, osmdb_filename, timezone_name = "America/Los_Angeles"):
self.osmdb = OSMDB( osmdb_filename )
self.timezone_name = timezone_name
def __init__(self, osmdb_filename):
self.osmdb = OSMDB( osmdb_filename )
#!/usr/bin/env python
from graphserver.ext.osm.osmdb import OSMDB
o = OSMDB("test.osmdb")
o.make_grid()
class RouteServer(Servable):
def __init__(self, ch_basename, osmdb_filename, profiledb_filename):
graphdb = GraphDatabase( graphdb_filename )
self.osmdb = OSMDB( osmdb_filename )
self.profiledb = ProfileDB( profiledb_filename )
self.ch = reincarnate_ch( ch_basename )
self.shortcut_cache = ShortcutCache( ch_basename+".scc" )
def vertices(self):
return "\n".join( [vv.label for vv in self.graph.vertices] )
vertices.mime = "text/plain"
def path(self, lat1, lng1, lat2, lng2, transfer_penalty=0, walking_speed=1.0, hill_reluctance=20, narrative=True, jsoncallback=None):
t0 = time.time()
origin = "osm-%s"%self.osmdb.nearest_node( lat1, lng1 )[0]
dest = "osm-%s"%self.osmdb.nearest_node( lat2, lng2 )[0]
endpoint_find_time = time.time()-t0
print origin, dest
t0 = time.time()
wo = WalkOptions()
#wo.transfer_penalty=transfer_penalty
#wo.walking_speed=walking_speed
wo.walking_speed=4
wo.walking_overage = 0
wo.hill_reluctance = 20
wo.turn_penalty = 15
edgepayloads = self.ch.shortest_path( origin, dest, State(1,0), wo )
wo.destroy()
route_find_time = time.time()-t0
t0 = time.time()
names = []
geoms = []
profile = Profile()
total_dist = 0
total_elev = 0
if narrative:
names, total_dist = get_full_route_narrative( self.osmdb, edgepayloads )
for edgepayload in edgepayloads:
geom, profile_seg = self.shortcut_cache.get( edgepayload.external_id )
#geom = get_ep_geom( self.osmdb, edgepayload )
#profile_seg = get_ep_profile( self.profiledb, edgepayload )
geoms.extend( geom )
profile.add( profile_seg )
route_desc_time = time.time()-t0
ret = json.dumps( (names,
encode_pairs( [(lat, lon) for lon, lat in geoms] ),
profile.concat(300),
{ 'route_find_time':route_find_time,
'route_desc_time':route_desc_time,
'endpoint_find_time':endpoint_find_time,},
{ 'total_dist':total_dist,
'total_elev':total_elev}) )
if jsoncallback:
return "%s(%s)"%(jsoncallback,ret)
else:
return ret
"""
def path_raw(self, origin, dest, currtime):
wo = WalkOptions()
spt = self.graph.shortest_path_tree( origin, dest, State(1,currtime), wo )
wo.destroy()
vertices, edges = spt.path( dest )
ret = "\n".join([str(x) for x in vertices]) + "\n\n" + "\n".join([str(x) for x in edges])
spt.destroy()
return ret
"""
def bounds(self, jsoncallback=None):
ret = json.dumps( self.osmdb.bounds() )
if jsoncallback:
return "%s(%s)"%(jsoncallback,ret)
else:
return ret
def __init__(self, ch_basename, osmdb_filename, profiledb_filename):
graphdb = GraphDatabase( graphdb_filename )
self.osmdb = OSMDB( osmdb_filename )
self.profiledb = ProfileDB( profiledb_filename )
self.ch = reincarnate_ch( ch_basename )
self.shortcut_cache = ShortcutCache( ch_basename+".scc" )
class RouteServer(Servable):
def __init__(self, ch_basename, osmdb_filename, profiledb_filename):
graphdb = GraphDatabase(graphdb_filename)
self.osmdb = OSMDB(osmdb_filename)
self.profiledb = ProfileDB(profiledb_filename)
self.ch = reincarnate_ch(ch_basename)
self.shortcut_cache = ShortcutCache(ch_basename + ".scc")
def vertices(self):
return "\n".join([vv.label for vv in self.graph.vertices])
vertices.mime = "text/plain"
def path(self,
lat1,
lng1,
lat2,
lng2,
transfer_penalty=0,
walking_speed=1.0,
hill_reluctance=20,
narrative=True,
jsoncallback=None):
t0 = time.time()
origin = "osm-%s" % self.osmdb.nearest_node(lat1, lng1)[0]
dest = "osm-%s" % self.osmdb.nearest_node(lat2, lng2)[0]
endpoint_find_time = time.time() - t0
print origin, dest
t0 = time.time()
wo = WalkOptions()
#wo.transfer_penalty=transfer_penalty
#wo.walking_speed=walking_speed
wo.walking_speed = 4
wo.walking_overage = 0
wo.hill_reluctance = 20
wo.turn_penalty = 15
edgepayloads = self.ch.shortest_path(origin, dest, State(1, 0), wo)
wo.destroy()
route_find_time = time.time() - t0
t0 = time.time()
names = []
geoms = []
profile = Profile()
total_dist = 0
total_elev = 0
if narrative:
names, total_dist = get_full_route_narrative(
self.osmdb, edgepayloads)
for edgepayload in edgepayloads:
geom, profile_seg = self.shortcut_cache.get(
edgepayload.external_id)
#geom = get_ep_geom( self.osmdb, edgepayload )
#profile_seg = get_ep_profile( self.profiledb, edgepayload )
geoms.extend(geom)
profile.add(profile_seg)
route_desc_time = time.time() - t0
ret = json.dumps(
(names, encode_pairs([(lat, lon) for lon, lat in geoms]),
profile.concat(300), {
'route_find_time': route_find_time,
'route_desc_time': route_desc_time,
'endpoint_find_time': endpoint_find_time,
}, {
'total_dist': total_dist,
'total_elev': total_elev
}))
if jsoncallback:
return "%s(%s)" % (jsoncallback, ret)
else:
return ret
"""
def path_raw(self, origin, dest, currtime):
wo = WalkOptions()
spt = self.graph.shortest_path_tree( origin, dest, State(1,currtime), wo )
wo.destroy()
vertices, edges = spt.path( dest )
ret = "\n".join([str(x) for x in vertices]) + "\n\n" + "\n".join([str(x) for x in edges])
spt.destroy()
return ret
"""
def bounds(self, jsoncallback=None):
ret = json.dumps(self.osmdb.bounds())
if jsoncallback:
return "%s(%s)" % (jsoncallback, ret)
else:
return ret
i = 0
while i < len(l):
while isinstance(l[i], ltypes):
if not l[i]:
l.pop(i)
i -= 1
break
else:
l[i:i + 1] = l[i]
i += 1
return ltype(l)
if __name__ == '__main__':
COLORS = {
'motorway': ((1,0,0,1), 2),
'motorway_link': ((1,0,0,1), 1.5),
}
out, filename, w, h = sys.argv[1:]
db = OSMDB(filename)
b = db.bounds()
r = GeographicCanvas(int(w), int(h), b, mode='svg', fobj=open(out,'wb'))
r.background()
for way_id, parent_id, from_nd, to_nd, dist, geom, tags in db.edges():
color, stroke = COLORS.get(tags.get('highway'),((0,0,0,.8),0.8))
r.line(coords=flatten(geom), cstroke=color, stroke=stroke)
#r.write_to_png(out)
def __init__(self, ch_basename, osmdb_filename, profiledb_filename):
graphdb = GraphDatabase(graphdb_filename)
self.osmdb = OSMDB(osmdb_filename)
self.profiledb = ProfileDB(profiledb_filename)
self.ch = reincarnate_ch(ch_basename)
self.shortcut_cache = ShortcutCache(ch_basename + ".scc")
from PIL import Image
from multiprocessing import Pool
os.environ['TZ'] = 'US/Pacific'
time.tzset()
t0s = "Mon May 17 08:50:00 2010"
t0t = time.strptime(t0s)
d0s = time.strftime('%a %b %d %Y', t0t)
t0 = time.mktime(t0t)
print 'search date: ', d0s
print 'search time: ', time.ctime(t0), t0
gtfsdb = GTFSDatabase ('./trimet.gtfsdb')
gdb = GraphDatabase ('./test.gdb' )
osmdb = OSMDB ('./testgrid.osmdb' )
g = gdb.incarnate ()
# FOOT - would be better if i could specify 0 boardings not 0 transfers
wo = WalkOptions()
wo.max_walk = 2000
wo.walking_overage = 0.0
wo.walking_speed = 1.0 # trimet uses 0.03 miles / 1 minute - but it uses straight line distance as well
wo.transfer_penalty = 99999
wo.walking_reluctance = 1
wo.max_transfers = 0
# make much higher?
wo.transfer_slack = 60 * 5
wo_foot = wo
# TRANSIT
def main():
usage = """usage: python gdb_import_osm.py <graphdb_filename> <osmdb_filename>"""
parser = OptionParser(usage=usage)
parser.add_option(
"-n",
"--namespace",
dest="namespace",
default="osm",
help="prefix all imported vertices with namespace string")
parser.add_option(
"-s",
"--slog",
action="append",
dest="slog_strings",
default=[],
help=
"specify slog for highway type, in highway_type:slog form. For example, 'motorway:10.5'"
)
parser.add_option(
"-p",
"--profiledb",
dest="profiledb_filename",
default=None,
help="specify profiledb to annotate streets with rise/fall data")
parser.add_option(
"-c",
"--slog_config",
dest="slog_config",
default=None,
metavar="CONFIG.yaml",
help="file containing slog parameters for highways, cycleways, etc")
(options, args) = parser.parse_args()
if len(args) != 2:
parser.print_help()
exit(-1)
slogs = {}
slog_config = {}
if options.slog_config:
slog_config = yaml.load(open(options.slog_config).read())
for highway_type, slog_penalty in slog_config.get('slogs', {}).items():
slogs[highway_type] = float(slog_penalty)
for slog_string in options.slog_strings:
highway_type, slog_penalty = slog_string.split(":")
slogs[highway_type] = float(slog_penalty)
print "slog values: %s" % slogs
slog_config['slogs'] = slogs
if slog_config.get('slog_function'):
slog_config['slog_function'] = import_object(
slog_config['slog_function'])
graphdb_filename = args[0]
osmdb_filename = args[1]
print "importing osmdb '%s' into graphdb '%s'" % (osmdb_filename,
graphdb_filename)
profiledb = ProfileDB(
options.profiledb_filename) if options.profiledb_filename else None
osmdb = OSMDB(osmdb_filename)
gdb = GraphDatabase(graphdb_filename, overwrite=False)
gdb_import_osm(gdb, osmdb, options.namespace, slog_config, profiledb)
print "done"
class ContourServer(Servable):
def __init__(self, settings_filename):
settings = yaml.load( open( settings_filename ) )
self.home_point = settings['center']
# create cache of osm-node positions
self.osmdb = OSMDB( settings['osmdb_filename'] )
self.gtfsdb = GTFSDatabase( settings['gtfsdb_filename'] )
self.port = settings['port']
self.node_positions = {}
self.index = Rtree()
for node_id, tags, lat, lon in self.osmdb.nodes():
self.node_positions[node_id] = (lon,lat)
self.index.add( int(node_id), (lon,lat,lon,lat) )
# incarnate graph from graphdb
graphdb = GraphDatabase( settings['graphdb_filename'] )
self.graph = graphdb.incarnate()
def strgraph(self):
return str(self.graph)
def vertices(self):
return "\n".join( [vv.label for vv in self.graph.vertices] )
def _get_important_routes(self, spt):
# set edge thicknesses
t0 = time.time()
print "setting thicknesses"
spt.set_thicknesses( vertex_label )
t1 = time.time()
print "took %s"%(t1-t0)
t0 = time.time()
print "finding gateway boardings"
# use thicknesses to determine important boardings
origin = spt.get_vertex( vertex_label )
sum_thickness = sum( [edge.thickness for edge in origin.outgoing] )
important_boardings = sorted( filter(lambda x: x.payload.__class__==core.TripBoard and \
x.thickness/float(sum_thickness) > 0.01, spt.edges),
key = lambda x:x.thickness )
for edge in important_boardings:
print "gateway to %f%% vertices"%(100*edge.thickness/float(sum_thickness))
print "hop onto trip '%s' at stop '%s', time '%s'"%(edge.to_v.payload.trip_id, edge.from_v.label, edge.to_v.payload.time)
print "took %ss"%(time.time()-t0)
def _points(self, vertex_label, starttime, cutoff, speed):
starttime = starttime or time.time()
#=== find shortest path tree ===
print "Finding shortest path tree"
t0 = time.time()
wo = WalkOptions()
wo.walking_speed = speed
spt = self.graph.shortest_path_tree( vertex_label, None, State(1,starttime), wo, maxtime=starttime+int(cutoff*1.25) )
wo.destroy()
t1 = time.time()
print "took %s s"%(t1-t0)
#=== cobble together ETA surface ===
print "Creating ETA surface from OSM points..."
points = []
t0 = time.time()
for vertex in spt.vertices:
if "osm" in vertex.label:
x, y = self.node_positions[vertex.label[3:]]
points.append( (x, y, vertex.payload.time-starttime) )
t1 = time.time()
print "Took %s s"%(t1-t0)
spt.destroy()
return points
def _contour(self, vertex_label, starttime, cutoff, step=None, speed=0.85):
points = self._points( vertex_label, starttime, cutoff, speed )
#=== create contour ===
print "creating contour...",
t0 = time.time()
contours = travel_time_contour( points, cutoff=cutoff, cellsize=0.004, fudge=1.7, step=step )
print "%s sec"%(time.time()-t0)
print "done. here you go..."
return contours
def label_contour(self, vertex_label, starttime=None, cutoff=1800):
starttime = starttime or time.time()
return json.dumps( self._contour( vertex_label, starttime, cutoff ) )
def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85):
if step is not None and step < 600:
raise Exception( "Step cannot be less than 600 seconds" )
starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" )
#=== get osm vertex ==
print( "getting nearest vertex" )
#find osmid of origin intersection
t0 = time.time()
range = 0.001
bbox = (lon-range, lat-range, lon+range, lat+range)
candidates = self.index.intersection( bbox )
vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates )
t1 = time.time()
print( "done, took %s seconds"%(t1-t0) )
#vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon )
if vlabel is None:
return json.dumps( "NO NEARBY INTERSECTION" )
print( "found - %s"%vlabel )
contours = self._contour( "osm"+vlabel, starttime, cutoff, step, speed )
if encoded:
encoded_contours = []
for contour in contours:
encoded_contour = []
for ring in contour:
encoded_contour.append( encode_pairs( [(lat,lon) for lon,lat in ring] ) )
encoded_contours.append( encoded_contour )
contours = encoded_contours
return json.dumps( contours )
def _surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed, cellsize=0.004):
starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" )
#=== get osm vertex ==
print( "getting nearest vertex" )
#find osmid of origin intersection
t0 = time.time()
range = 0.001
bbox = (lon-range, lat-range, lon+range, lat+range)
candidates = self.index.intersection( bbox )
vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates )
t1 = time.time()
print( "done, took %s seconds"%(t1-t0) )
#vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon )
if vlabel is None:
return json.dumps( "NO NEARBY INTERSECTION" )
print( "found - %s"%vlabel )
#=== get points which comprise ETA surface ===
points = self._points( "osm"+vlabel, starttime, cutoff, speed )
#=== create regular grid from ETA surface ===
print "creating surface...",
t0 = time.time()
ret = points_to_surface_grid( points, cutoff=cutoff, fudge=1.1, margin=2, closure_tolerance=0.05, cellsize=cellsize )
#ret = travel_time_surface( points, cutoff=cutoff, cellsize=0.004, fudge=1.7 )
print "%s sec"%(time.time()-t0)
print "done. here you go..."
return ret
def surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85):
return json.dumps( self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed).to_matrix() )
def transitability(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85):
grid = self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed)
if type(grid) == str:
return None
ret = 0
for i in range(10):
contourslice=sum( [len(filter(lambda x:x[2]<=(cutoff/10.0)*(i+1),col)) for col in grid] )
print contourslice
ret += contourslice
return ret
def transitability_surface(self, left, bottom, right, top, res, year, month, day, hour, minute, second, cutoff, speed=0.85):
step = (right-left)/res
return json.dumps([[(lon,lat,self.transitability(lat,lon,year,month,day,hour,minute,second,cutoff,speed)) for lat in frange(bottom,top,step)] for lon in frange(left,right,step)])
def nodes(self):
return "\n".join( ["%s-%s"%(k,v) for k,v in self.node_positions.items()] )
def nearest_node(self, lat, lon):
range = 0.005
bbox = (lon-range, lat-range, lon+range, lat+range)
print bbox
print self.index.intersection( bbox )
return json.dumps(self.osmdb.nearest_node( lat, lon ))
def bounds(self):
return json.dumps(self.osmdb.bounds())
def run_test_server(self):
Servable.run_test_server(self, self.port)
def __init__(self, osmdb_filename, range=0.005):
self.osmdb = OSMDB(osmdb_filename)
self.range = range
def get( self, external_id ):
packed_geom, packed_profile = list(self.execute( "SELECT geom, profile FROM ep_geoms WHERE id=?", (external_id,) ))[0]
return unpack_coords( packed_geom ), unpack_coords( packed_profile )
import sys
def selftest():
assert description_from_north( (0,0), (0,1) ) == "north"
assert description_from_north( (0,0), (1,1) ) == "northeast"
assert description_from_north( (0,0), (1,0) ) == "east"
assert description_from_north( (0,0), (1,-1) ) == "southeast"
assert description_from_north( (0,0), (0,-1) ) == "south"
assert description_from_north( (0,0), (-1,-1) ) == "southwest"
assert description_from_north( (0,0), (-1,0) ) == "west"
assert description_from_north( (0,0), (-1,1) ) == "northwest"
if __name__=='__main__':
#selftest()
print "usage: python shortcut_cache.py basename"
basename = sys.argv[1]
ch = reincarnate_ch( basename )
osmdb = OSMDB( basename+".osmdb" )
profiledb = ProfileDB( basename+".profiledb" )
scc = ShortcutCache( basename+".scc", overwrite=True )
scc.ingest( osmdb, profiledb, ch.upgraph )
scc.ingest( osmdb, profiledb, ch.downgraph )
if __name__=='__main__':
usage = """usage: python zzzz.py <graph_database> <assist_graph_database> <osm_database> <gtfs_database>"""
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 4:
parser.print_help()
exit(-1)
graph_db = args[0]
assist_graph_db = args[1]
osm_db = args[2]
gtfs_db = args[3]
graphdb = GraphDatabase( graph_db )
assistgraphdb = GraphDatabase( assist_graph_db )
osmdb = OSMDB( osm_db )
gtfsdb = GTFSDatabase( gtfs_db )
g = graphdb.incarnate()
ag = assistgraphdb.incarnate()
nodes = {}
for id, tags, lat, lon, endnode_refs in osmdb.nodes():
nodes['osm-' + id] = (lat, lon)
for id, name, lat, lon in gtfsdb.stops():
nodes['sta-' + id] = (lat, lon)
os.environ['TZ'] = 'US/Pacific'
time.tzset()
t0s = "Tue Nov 16 07:50:30 2010"
t0t = time.strptime(t0s)
d0s = time.strftime('%a %b %d %Y', t0t)
