def main():
usage = """usage: python gdb_link_osm_gtfs.py <graphdb_filename> <osmdb_filename> <gtfsdb_filename>"""
parser = OptionParser(usage=usage)
(options, args) = parser.parse_args()
if len(args) != 3:
parser.print_help()
exit(-1)
graphdb_filename = args[0]
osmdb_filename = args[1]
gtfsdb_filename = args[2]
gtfsdb = GTFSDatabase( gtfsdb_filename )
osmdb = OSMDB( osmdb_filename )
gdb = GraphDatabase( graphdb_filename )
n_stops = gtfsdb.count_stops()
for i, (stop_id, stop_name, stop_lat, stop_lon) in enumerate( gtfsdb.stops() ):
print "%d/%d"%(i,n_stops)
nd_id, nd_lat, nd_lon, nd_dist = osmdb.nearest_node( stop_lat, stop_lon )
station_vertex_id = "sta-%s"%stop_id
osm_vertex_id = "osm-%s"%nd_id
print station_vertex_id, osm_vertex_id
gdb.add_edge( station_vertex_id, osm_vertex_id, Link() )
gdb.add_edge( osm_vertex_id, station_vertex_id, Link() )
def links_process_cache(self, qb):
from_and_where = self.links_cache_from_and_where(
'gtfsdb_cache_links', qb)
count_links_sql = "SELECT COUNT(*) %s" % (from_and_where, )
nlinks = qb.db.sql(count_links_sql)[0]['count']
cache_links_sql = ("""
SELECT
node_stack_id
, transit_stop_id
%s
""" % (from_and_where, ))
cache_links = qb.db.sql(cache_links_sql)
for i, (cache_link) in enumerate(cache_links):
node_id = cache_link['node_stack_id']
stop_id = cache_link['transit_stop_id']
# Every once in a while, print a debug message
if (i % 1000) == 0:
log.debug(
'links_process_cache: progress: on stop # %d of %d...' % (
i,
nlinks,
))
#log.debug(' >> node_id: %s / stop_id: %s' % (node_id, stop_id,))
try:
self.gserver.add_edge(stop_id, node_id, Link())
self.gserver.add_edge(node_id, stop_id, Link())
except VertexNotFoundError:
# 2014.09.19/BUG nnnn: Planner p2 very much broken (only returns
# bike-only routes; never includes transit), so make this an info
# so logcheck stops complaining when cron runs gtfsdb_build_cache.
log.info(
'link_graphs: no vertex?!.1: node_id: %s / stop_id: %s' % (
node_id,
stop_id,
))
def test_walk_back(self):
l = Link()
before = l.walk_back(State(1,0), WalkOptions())
assert before.time == 0
assert before.weight == 0
assert before.dist_walked == 0.0
assert before.prev_edge.type == 3
assert before.prev_edge.name == "LINK"
assert before.num_agencies == 1
def test_walk(self):
l = Link()
after = l.walk(State(1, 0), WalkOptions())
assert after.time == 0
assert after.weight == 0
assert after.dist_walked == 0
assert after.prev_edge.type == 3
assert after.prev_edge.name == "LINK"
assert after.num_agencies == 1
def test_walk_back(self):
l = Link()
before = l.walk_back(State(1, 0), WalkOptions())
assert before.time == 0
assert before.weight == 0
assert before.dist_walked == 0.0
assert before.prev_edge.type == 3
assert before.prev_edge.name == "LINK"
assert before.num_agencies == 1
def test_walk(self):
l = Link()
after = l.walk(State(1,0), WalkOptions())
assert after.time==0
assert after.weight==0
assert after.dist_walked==0
assert after.prev_edge.type == 3
assert after.prev_edge.name == "LINK"
assert after.num_agencies == 1
def filter(self, osmdb, *args):
g = Graph()
t0 = time.time()
vertices = {}
print "load vertices into memory"
for row in osmdb.execute("SELECT id from nodes"):
g.add_vertex(str(row[0]))
vertices[str(row[0])] = 0
print "load ways into memory"
for way in osmdb.ways():
g.add_edge(way.nds[0], way.nds[-1], Link())
g.add_edge(way.nds[-1], way.nds[0], Link())
t1 = time.time()
print "populating graph took: %f" % (t1 - t0)
t0 = t1
iteration = 1
c = osmdb.cursor()
while True:
#c.execute("SELECT id from nodes where id not in (SELECT node_id from graph_nodes) LIMIT 1")
try:
vertex, dummy = vertices.popitem()
except:
break
spt = g.shortest_path_tree(vertex, None, State(1, 0))
for v in spt.vertices:
vertices.pop(v.label, None)
c.execute("INSERT into graph_nodes VALUES (?, ?)",
(iteration, v.label))
spt.destroy()
t1 = time.time()
print "pass %s took: %f" % (iteration, t1 - t0)
t0 = t1
iteration += 1
c.close()
osmdb.conn.commit()
g.destroy()
# audit
for gnum, count in osmdb.execute(
"SELECT graph_num, count(*) FROM graph_nodes GROUP BY graph_num"
):
print "FOUND: %s=%s" % (gnum, count)
def gtfsdb_to_transfer_edges(self):
# load transfers
if self.reporter:
self.reporter.write("Loading transfers to graph...\n")
# keep track to avoid redundancies
# this assumes that transfer relationships are bi-directional.
# TODO this implementation is also incomplete - it's theoretically possible that
# a transfers.txt table could contain "A,A,3,", which would mean you can't transfer
# at A.
seen = set([])
for stop_id1, stop_id2, conn_type, min_transfer_time in self.gtfsdb.execute(
"SELECT * FROM transfers"):
s1 = "sta-%s" % stop_id1
s2 = "sta-%s" % stop_id2
# TODO - what is the semantics of this? see note above
if s1 == s2:
continue
key = ".".join(sorted([s1, s2]))
if key not in seen:
seen.add(key)
else:
continue
assert conn_type == None or type(conn_type) == int
if conn_type in (
0, None
): # This is a recommended transfer point between two routes
if min_transfer_time in ("", None):
yield (s1, s2, Link())
yield (s2, s1, Link())
else:
yield (s1, s2, ElapseTime(int(min_transfer_time)))
yield (s2, s1, ElapseTime(int(min_transfer_time)))
elif conn_type == 1: # This is a timed transfer point between two routes
yield (s1, s2, Link())
yield (s2, s1, Link())
elif conn_type == 2: # This transfer requires a minimum amount of time
yield (s1, s2, ElapseTime(int(min_transfer_time)))
yield (s2, s1, ElapseTime(int(min_transfer_time)))
elif conn_type == 3: # Transfers are not possible between routes at this location.
print "WARNING: Support for no-transfer (transfers.txt transfer_type=3) not implemented."
def test_one(self):
"""getSize returns one after one entry"""
bb = Vertex("BB")
ee = Edge(self.aa, bb, Link())
self.path.addSegment(bb, ee)
self.assertEqual(self.path.num_elements, 1)
def test_ten(self):
"""getSize returns ten after ten entries"""
for i in range(10):
aa = Vertex("AA")
bb = Vertex("BB")
payload = Link()
self.path.addSegment(bb, Edge(aa, bb, payload))
self.assertEquals(self.path.num_elements, 10)
def test_expand(self):
"""vertices gettable after resizing"""
# the path length right before a vector expansion
pathlen = 50
# make a bunch of fake segments
segments = []
for i in range(pathlen):
vv = Vertex(str(i))
ee = Edge(vv, vv, Link())
segments.append((vv, ee))
# add those segments to the path
for vv, ee in segments:
self.path.addSegment(vv, ee)
# check that they're alright
# check the odd-duck vertex
self.assertEqual(self.path.getVertex(0).label, "A")
# check the bunch of fake segments added
for i in range(1, pathlen + 1):
self.assertEqual(i - 1, int(self.path.getVertex(i).label))
#
# getting towards the real test - add a segment after the vectors have
# been expanded
#
# add it
vv = Vertex("B")
ee = Edge(vv, vv, Link())
self.path.addSegment(vv, ee)
# get it
self.assertEqual(self.path.getVertex(51).label, "B")
def test_two(self):
"""get a vertex, edge after adding two segments"""
ee1 = Edge(self.aa, self.bb, Link())
ee2 = Edge(self.bb, self.aa, Link())
self.path.addSegment(self.bb, ee1)
self.path.addSegment(self.aa, ee2)
# out of bounds
self.assertRaises(IndexError, self.path.getVertex, -1)
self.assertRaises(IndexError, self.path.getEdge, -1)
# vertices in bounds
self.assertEqual(self.path.getVertex(0).soul, self.aa.soul)
self.assertEqual(self.path.getVertex(1).soul, self.bb.soul)
self.assertEqual(self.path.getVertex(2).soul, self.aa.soul)
# edges in bounds
self.assertEqual(self.path.getEdge(0).soul, ee1.soul)
self.assertEqual(self.path.getEdge(1).soul, ee2.soul)
# out of bounds again
self.assertRaises(IndexError, self.path.getVertex, 3)
self.assertRaises(IndexError, self.path.getEdge, 2)
def test_basic(self):
g = Graph()
g.add_vertex("A")
g.add_vertex("B")
g.add_edge("A", "B", Link())
g.add_edge("A", "B", Street("foo", 20.0))
gdb_file = os.path.dirname(__file__) + "unit_test.db"
if os.path.exists(gdb_file):
os.remove(gdb_file)
gdb = GraphDatabase(gdb_file)
gdb.populate(g)
list(gdb.execute("select * from resources"))
assert "A" in list(gdb.all_vertex_labels())
assert "B" in list(gdb.all_vertex_labels())
assert glen(gdb.all_edges()) == 2
assert glen(gdb.all_outgoing("A")) == 2
assert glen(gdb.all_outgoing("B")) == 0
assert glen(gdb.all_incoming("A")) == 0
assert glen(gdb.all_incoming("B")) == 2
assert glen(gdb.resources()) == 0
assert gdb.num_vertices() == 2
assert gdb.num_edges() == 2
g.destroy()
g = gdb.incarnate()
list(gdb.execute("select * from resources"))
assert "A" in list(gdb.all_vertex_labels())
assert "B" in list(gdb.all_vertex_labels())
assert glen(gdb.all_edges()) == 2
assert glen(gdb.all_outgoing("A")) == 2
assert glen(gdb.all_outgoing("B")) == 0
assert glen(gdb.all_incoming("A")) == 0
assert glen(gdb.all_incoming("B")) == 2
assert glen(gdb.resources()) == 0
assert gdb.num_vertices() == 2
assert gdb.num_edges() == 2
os.remove(gdb_file)
def link_test(self):
l = Link()
assert l
assert str(l) == "<Link name='LINK'/>"
def test_getstate(self):
l = Link()
assert l.__getstate__() == tuple([])
def test_name(self):
l = Link()
assert l.name == "LINK"
def test_destroy(self):
l = Link()
l.destroy()
assert l.soul == None
def test_destroy(self):
l = Link()
l.destroy()
assert l.soul==None
def test_getstate(self):
l = Link()
assert l.__getstate__() == tuple([])
def test_hello_world(self):
g = Graph()
g.add_vertex("Seattle")
g.add_vertex("Portland")
g.add_edge("Seattle", "Portland", Street("I-5 south", 5000))
g.add_edge("Portland", "Seattle", Street("I-5 north", 5500))
spt = g.shortest_path_tree("Seattle", "Portland",
State(g.numagencies, 0), WalkOptions())
assert spt.get_vertex(
"Seattle").outgoing[0].payload.name == "I-5 south"
g.add_vertex("Portland-busstop")
g.add_vertex("Seattle-busstop")
g.add_edge("Seattle", "Seattle-busstop", Link())
g.add_edge("Seattle-busstop", "Seattle", Link())
g.add_edge("Portland", "Portland-busstop", Link())
g.add_edge("Portland-busstop", "Portland", Link())
spt = g.shortest_path_tree("Seattle", "Seattle-busstop",
State(g.numagencies, 0), WalkOptions())
assert spt.get_vertex(
"Seattle-busstop").incoming[0].payload.__class__ == Link
spt.destroy()
spt = g.shortest_path_tree("Seattle-busstop", "Portland",
State(g.numagencies, 0), WalkOptions())
assert spt.get_vertex(
"Portland").incoming[0].payload.__class__ == Street
spt.destroy()
sc = ServiceCalendar()
sc.add_period(0, 86400, ["WKDY", "SAT"])
tz = Timezone()
tz.add_period(TimezonePeriod(0, 86400, 0))
g.add_vertex("Portland-busstop-onbus")
g.add_vertex("Seattle-busstop-onbus")
tb = TripBoard("WKDY", sc, tz, 0)
tb.add_boarding("A", 10, 0)
tb.add_boarding("B", 15, 0)
tb.add_boarding("C", 400, 0)
cr = Crossing()
al = TripAlight("WKDY", sc, tz, 0)
al.add_alighting("A", 10 + 20, 0)
al.add_alighting("B", 15 + 20, 0)
al.add_alighting("C", 400 + 20, 0)
g.add_edge("Seattle-busstop", "Seattle-busstop-onbus", tb)
g.add_edge("Seattle-busstop-onbus", "Portland-busstop-onbus", cr)
g.add_edge("Portland-busstop-onbus", "Portland-busstop", al)
spt = g.shortest_path_tree("Seattle", "Portland",
State(g.numagencies, 0), WalkOptions())
assert spt.get_vertex("Portland").incoming[0].from_v.incoming[
0].from_v.incoming[0].from_v.incoming[0].from_v.incoming[
0].from_v.label == "Seattle"
spt = g.shortest_path_tree("Seattle", "Portland",
State(g.numagencies, 0), WalkOptions())
vertices, edges = spt.path("Portland")
assert [v.label for v in vertices] == [
'Seattle', 'Seattle-busstop', "Seattle-busstop-onbus",
"Portland-busstop-onbus", 'Portland-busstop', 'Portland'
]
assert [e.payload.__class__ for e in edges
] == [Link, TripBoard, Crossing, TripAlight, Link]
spt.destroy()
g.destroy()
def setUp(self):
self.aa = Vertex("A")
self.bb = Vertex("B")
self.ep = Link()
self.path = Path(self.aa)
def link_graphs_slow(self, qb, db_transit):
log.debug('link_graphs: linking Cyclopath and Transit networks')
# NOTE: We load all byways into the graph, including those tagged
# 'prohibited' and 'closed', but we only ever link those not tagged as
# such with transit stops.
qb = qb.clone(skip_clauses=True, skip_filtport=True, db_clone=True)
g.assurt(qb.finalized)
# MAGIC HACK ALERT
tagprefs = {}
tagprefs['prohibited'] = ratings.t_avoid
tagprefs['closed'] = ratings.t_avoid
#
rating_func = self.ratings.rating_func(qb.username, tagprefs, self)
# MAGIC NUMBER: Min rating.
rating_min = 0.5
# The transit data is lat,lon, as opposed to SRID-encoded x,y.
is_latlon = True
n_stops = db_transit.count_stops()
# NOTE: 2011.06.26: This loops takes a while. For me [lb], 55 secs.
# NOTE: 2011.08.08: Find nearest node using GRAC SQL is time consuming!
# On the order of minutes and minutes...
for i, (stop_id, stop_name, stop_lat, stop_lon,) \
in enumerate(db_transit.stops()):
# Every once in a while, print a debug message.
if (i % 150) == 0:
log.debug('link_graphs: progress: on stop # %d of %d...' % (
i,
n_stops,
))
log.debug(' >> id: %s / name: %s / lat: %s / lon: %s' % (
stop_id,
stop_name,
stop_lat,
stop_lon,
))
# NOTE: The (x,y) point is lon first, then lat.
stop_xy = (
stop_lon,
stop_lat,
)
nearest_byway = route.One.byway_closest_xy(qb, stop_name, stop_xy,
rating_func, rating_min,
is_latlon)
nearest_node = nearest_byway.nearest_node_id()
# FIXME: What if the node is on a one-way? What if the node is tagged with
# something that the user marks 'avoid'? In both cases, transit stop might be
# Unreachable.
if nearest_node is not None:
node_id = str(nearest_node)
# NOTE: If we don't cast to string, it's unicode, and db.insert
# doesn't quote it.
stop_id = 'sta-%s' % (str(stop_id), )
if node_id != '':
try:
self.gserver.add_edge(stop_id, node_id, Link())
self.gserver.add_edge(node_id, stop_id, Link())
except VertexNotFoundError:
# 2014.09.19/BUG nnnn: Planner p2 very much broken, so make
# this info instead of warning and add to list of p2 bugs.
log.info(
'link_graphs: no vertex?!.2: node_id: %s / stop_id: %s'
% (
node_id,
stop_id,
))
else:
log.warning(
'link_graphs: no node name?!: node_id: %s / stop_id: %s'
% (
node_id,
stop_id,
))
else:
log.warning(
'link_graphs: no nearest node?!: node_id: %s / stop_id: %s'
% (
node_id,
stop_id,
))
log.warning(' >> lat, lon: (%s, %s)' % (stop_lat, stop_lon))
qb.db.close()