Пример #1
0
 def links_process_cache(self, qb):
     from_and_where = self.links_cache_from_and_where(
         'gtfsdb_cache_links', qb)
     count_links_sql = "SELECT COUNT(*) %s" % (from_and_where, )
     nlinks = qb.db.sql(count_links_sql)[0]['count']
     cache_links_sql = ("""
      SELECT
         node_stack_id
         , transit_stop_id
      %s
      """ % (from_and_where, ))
     cache_links = qb.db.sql(cache_links_sql)
     for i, (cache_link) in enumerate(cache_links):
         node_id = cache_link['node_stack_id']
         stop_id = cache_link['transit_stop_id']
         # Every once in a while, print a debug message
         if (i % 1000) == 0:
             log.debug(
                 'links_process_cache: progress: on stop # %d of %d...' % (
                     i,
                     nlinks,
                 ))
             #log.debug(' >> node_id: %s / stop_id: %s' % (node_id, stop_id,))
         try:
             self.gserver.add_edge(stop_id, node_id, Link())
             self.gserver.add_edge(node_id, stop_id, Link())
         except VertexNotFoundError:
             # 2014.09.19/BUG nnnn: Planner p2 very much broken (only returns
             # bike-only routes; never includes transit), so make this an info
             # so logcheck stops complaining when cron runs gtfsdb_build_cache.
             log.info(
                 'link_graphs: no vertex?!.1: node_id: %s / stop_id: %s' % (
                     node_id,
                     stop_id,
                 ))
Пример #2
0
def main():
    usage = """usage: python gdb_link_osm_gtfs.py <graphdb_filename> <osmdb_filename> <gtfsdb_filename>"""
    parser = OptionParser(usage=usage)
    
    (options, args) = parser.parse_args()
    
    if len(args) != 3:
        parser.print_help()
        exit(-1)
        
    graphdb_filename = args[0]
    osmdb_filename   = args[1]
    gtfsdb_filename  = args[2]
    
    gtfsdb = GTFSDatabase( gtfsdb_filename )
    osmdb = OSMDB( osmdb_filename )
    gdb = GraphDatabase( graphdb_filename )

    n_stops = gtfsdb.count_stops()

    for i, (stop_id, stop_name, stop_lat, stop_lon) in enumerate( gtfsdb.stops() ):
        print "%d/%d"%(i,n_stops)
        
        nd_id, nd_lat, nd_lon, nd_dist = osmdb.nearest_node( stop_lat, stop_lon )
        station_vertex_id = "sta-%s"%stop_id
        osm_vertex_id = "osm-%s"%nd_id
        
        print station_vertex_id, osm_vertex_id
        
        gdb.add_edge( station_vertex_id, osm_vertex_id, Link() )
        gdb.add_edge( osm_vertex_id, station_vertex_id, Link() )
Пример #3
0
    def filter(self, osmdb, *args):
        g = Graph()
        t0 = time.time()

        vertices = {}
        print "load vertices into memory"
        for row in osmdb.execute("SELECT id from nodes"):
            g.add_vertex(str(row[0]))
            vertices[str(row[0])] = 0

        print "load ways into memory"
        for way in osmdb.ways():
            g.add_edge(way.nds[0], way.nds[-1], Link())
            g.add_edge(way.nds[-1], way.nds[0], Link())

        t1 = time.time()
        print "populating graph took: %f" % (t1 - t0)
        t0 = t1

        iteration = 1
        c = osmdb.cursor()
        while True:
            #c.execute("SELECT id from nodes where id not in (SELECT node_id from graph_nodes) LIMIT 1")
            try:
                vertex, dummy = vertices.popitem()
            except:
                break
            spt = g.shortest_path_tree(vertex, None, State(1, 0))
            for v in spt.vertices:
                vertices.pop(v.label, None)
                c.execute("INSERT into graph_nodes VALUES (?, ?)",
                          (iteration, v.label))
            spt.destroy()

            t1 = time.time()
            print "pass %s took: %f" % (iteration, t1 - t0)
            t0 = t1
            iteration += 1
        c.close()

        osmdb.conn.commit()
        g.destroy()
        # audit
        for gnum, count in osmdb.execute(
                "SELECT graph_num, count(*) FROM graph_nodes GROUP BY graph_num"
        ):
            print "FOUND: %s=%s" % (gnum, count)
Пример #4
0
    def gtfsdb_to_transfer_edges(self):

        # load transfers
        if self.reporter:
            self.reporter.write("Loading transfers to graph...\n")

        # keep track to avoid redundancies
        # this assumes that transfer relationships are bi-directional.
        # TODO this implementation is also incomplete - it's theoretically possible that
        # a transfers.txt table could contain "A,A,3,", which would mean you can't transfer
        # at A.
        seen = set([])
        for stop_id1, stop_id2, conn_type, min_transfer_time in self.gtfsdb.execute(
                "SELECT * FROM transfers"):
            s1 = "sta-%s" % stop_id1
            s2 = "sta-%s" % stop_id2

            # TODO - what is the semantics of this? see note above
            if s1 == s2:
                continue

            key = ".".join(sorted([s1, s2]))
            if key not in seen:
                seen.add(key)
            else:
                continue

            assert conn_type == None or type(conn_type) == int
            if conn_type in (
                    0, None
            ):  # This is a recommended transfer point between two routes
                if min_transfer_time in ("", None):
                    yield (s1, s2, Link())
                    yield (s2, s1, Link())
                else:
                    yield (s1, s2, ElapseTime(int(min_transfer_time)))
                    yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 1:  # This is a timed transfer point between two routes
                yield (s1, s2, Link())
                yield (s2, s1, Link())
            elif conn_type == 2:  # This transfer requires a minimum amount of time
                yield (s1, s2, ElapseTime(int(min_transfer_time)))
                yield (s2, s1, ElapseTime(int(min_transfer_time)))
            elif conn_type == 3:  # Transfers are not possible between routes at this location.
                print "WARNING: Support for no-transfer (transfers.txt transfer_type=3) not implemented."
Пример #5
0
    def test_one(self):
        """getSize returns one after one entry"""

        bb = Vertex("BB")
        ee = Edge(self.aa, bb, Link())

        self.path.addSegment(bb, ee)

        self.assertEqual(self.path.num_elements, 1)
Пример #6
0
    def test_ten(self):
        """getSize returns ten after ten entries"""

        for i in range(10):
            aa = Vertex("AA")
            bb = Vertex("BB")
            payload = Link()
            self.path.addSegment(bb, Edge(aa, bb, payload))

        self.assertEquals(self.path.num_elements, 10)
Пример #7
0
    def test_walk_back(self):
        l = Link()

        before = l.walk_back(State(1, 0), WalkOptions())

        assert before.time == 0
        assert before.weight == 0
        assert before.dist_walked == 0.0
        assert before.prev_edge.type == 3
        assert before.prev_edge.name == "LINK"
        assert before.num_agencies == 1
Пример #8
0
    def test_walk(self):
        l = Link()

        after = l.walk(State(1, 0), WalkOptions())

        assert after.time == 0
        assert after.weight == 0
        assert after.dist_walked == 0
        assert after.prev_edge.type == 3
        assert after.prev_edge.name == "LINK"
        assert after.num_agencies == 1
Пример #9
0
    def test_expand(self):
        """vertices gettable after resizing"""

        # the path length right before a vector expansion
        pathlen = 50

        # make a bunch of fake segments
        segments = []
        for i in range(pathlen):
            vv = Vertex(str(i))
            ee = Edge(vv, vv, Link())
            segments.append((vv, ee))

        # add those segments to the path
        for vv, ee in segments:
            self.path.addSegment(vv, ee)

        # check that they're alright
        # check the odd-duck vertex
        self.assertEqual(self.path.getVertex(0).label, "A")

        # check the bunch of fake segments added
        for i in range(1, pathlen + 1):
            self.assertEqual(i - 1, int(self.path.getVertex(i).label))

        #
        # getting towards the real test - add a segment after the vectors have
        # been expanded
        #

        # add it
        vv = Vertex("B")
        ee = Edge(vv, vv, Link())
        self.path.addSegment(vv, ee)

        # get it
        self.assertEqual(self.path.getVertex(51).label, "B")
Пример #10
0
    def test_two(self):
        """get a vertex, edge after adding two segments"""

        ee1 = Edge(self.aa, self.bb, Link())
        ee2 = Edge(self.bb, self.aa, Link())
        self.path.addSegment(self.bb, ee1)
        self.path.addSegment(self.aa, ee2)

        # out of bounds
        self.assertRaises(IndexError, self.path.getVertex, -1)
        self.assertRaises(IndexError, self.path.getEdge, -1)

        # vertices in bounds
        self.assertEqual(self.path.getVertex(0).soul, self.aa.soul)
        self.assertEqual(self.path.getVertex(1).soul, self.bb.soul)
        self.assertEqual(self.path.getVertex(2).soul, self.aa.soul)

        # edges in bounds
        self.assertEqual(self.path.getEdge(0).soul, ee1.soul)
        self.assertEqual(self.path.getEdge(1).soul, ee2.soul)

        # out of bounds again
        self.assertRaises(IndexError, self.path.getVertex, 3)
        self.assertRaises(IndexError, self.path.getEdge, 2)
Пример #11
0
    def test_basic(self):
        g = Graph()
        g.add_vertex("A")
        g.add_vertex("B")
        g.add_edge("A", "B", Link())
        g.add_edge("A", "B", Street("foo", 20.0))
        gdb_file = os.path.dirname(__file__) + "unit_test.db"
        if os.path.exists(gdb_file):
            os.remove(gdb_file)
        gdb = GraphDatabase(gdb_file)
        gdb.populate(g)

        list(gdb.execute("select * from resources"))
        assert "A" in list(gdb.all_vertex_labels())
        assert "B" in list(gdb.all_vertex_labels())
        assert glen(gdb.all_edges()) == 2
        assert glen(gdb.all_outgoing("A")) == 2
        assert glen(gdb.all_outgoing("B")) == 0
        assert glen(gdb.all_incoming("A")) == 0
        assert glen(gdb.all_incoming("B")) == 2
        assert glen(gdb.resources()) == 0
        assert gdb.num_vertices() == 2
        assert gdb.num_edges() == 2

        g.destroy()
        g = gdb.incarnate()

        list(gdb.execute("select * from resources"))
        assert "A" in list(gdb.all_vertex_labels())
        assert "B" in list(gdb.all_vertex_labels())
        assert glen(gdb.all_edges()) == 2
        assert glen(gdb.all_outgoing("A")) == 2
        assert glen(gdb.all_outgoing("B")) == 0
        assert glen(gdb.all_incoming("A")) == 0
        assert glen(gdb.all_incoming("B")) == 2
        assert glen(gdb.resources()) == 0
        assert gdb.num_vertices() == 2
        assert gdb.num_edges() == 2

        os.remove(gdb_file)
Пример #12
0
    def link_graphs_slow(self, qb, db_transit):

        log.debug('link_graphs: linking Cyclopath and Transit networks')

        # NOTE: We load all byways into the graph, including those tagged
        # 'prohibited' and 'closed', but we only ever link those not tagged as
        # such with transit stops.

        qb = qb.clone(skip_clauses=True, skip_filtport=True, db_clone=True)
        g.assurt(qb.finalized)

        # MAGIC HACK ALERT
        tagprefs = {}
        tagprefs['prohibited'] = ratings.t_avoid
        tagprefs['closed'] = ratings.t_avoid
        #
        rating_func = self.ratings.rating_func(qb.username, tagprefs, self)
        # MAGIC NUMBER: Min rating.
        rating_min = 0.5
        # The transit data is lat,lon, as opposed to SRID-encoded x,y.
        is_latlon = True

        n_stops = db_transit.count_stops()
        # NOTE: 2011.06.26: This loops takes a while. For me [lb], 55 secs.
        # NOTE: 2011.08.08: Find nearest node using GRAC SQL is time consuming!
        #                   On the order of minutes and minutes...

        for i, (stop_id, stop_name, stop_lat, stop_lon,) \
            in enumerate(db_transit.stops()):
            # Every once in a while, print a debug message.
            if (i % 150) == 0:
                log.debug('link_graphs: progress: on stop # %d of %d...' % (
                    i,
                    n_stops,
                ))
                log.debug(' >> id: %s / name: %s / lat: %s / lon: %s' % (
                    stop_id,
                    stop_name,
                    stop_lat,
                    stop_lon,
                ))
            # NOTE: The (x,y) point is lon first, then lat.
            stop_xy = (
                stop_lon,
                stop_lat,
            )
            nearest_byway = route.One.byway_closest_xy(qb, stop_name, stop_xy,
                                                       rating_func, rating_min,
                                                       is_latlon)
            nearest_node = nearest_byway.nearest_node_id()
            # FIXME: What if the node is on a one-way? What if the node is tagged with
            # something that the user marks 'avoid'? In both cases, transit stop might be
            # Unreachable.
            if nearest_node is not None:
                node_id = str(nearest_node)
                # NOTE: If we don't cast to string, it's unicode, and db.insert
                # doesn't quote it.
                stop_id = 'sta-%s' % (str(stop_id), )
                if node_id != '':
                    try:
                        self.gserver.add_edge(stop_id, node_id, Link())
                        self.gserver.add_edge(node_id, stop_id, Link())
                    except VertexNotFoundError:
                        # 2014.09.19/BUG nnnn: Planner p2 very much broken, so make
                        # this info instead of warning and add to list of p2 bugs.
                        log.info(
                            'link_graphs: no vertex?!.2: node_id: %s / stop_id: %s'
                            % (
                                node_id,
                                stop_id,
                            ))
                else:
                    log.warning(
                        'link_graphs: no node name?!: node_id: %s / stop_id: %s'
                        % (
                            node_id,
                            stop_id,
                        ))
            else:
                log.warning(
                    'link_graphs: no nearest node?!: node_id: %s / stop_id: %s'
                    % (
                        node_id,
                        stop_id,
                    ))
                log.warning(' >> lat, lon: (%s, %s)' % (stop_lat, stop_lon))

        qb.db.close()
Пример #13
0
 def link_test(self):
     l = Link()
     assert l
     assert str(l) == "<Link name='LINK'/>"
Пример #14
0
 def test_getstate(self):
     l = Link()
     assert l.__getstate__() == tuple([])
Пример #15
0
 def test_name(self):
     l = Link()
     assert l.name == "LINK"
Пример #16
0
    def test_destroy(self):
        l = Link()
        l.destroy()

        assert l.soul == None
Пример #17
0
    def test_hello_world(self):
        g = Graph()

        g.add_vertex("Seattle")
        g.add_vertex("Portland")

        g.add_edge("Seattle", "Portland", Street("I-5 south", 5000))
        g.add_edge("Portland", "Seattle", Street("I-5 north", 5500))

        spt = g.shortest_path_tree("Seattle", "Portland",
                                   State(g.numagencies, 0), WalkOptions())

        assert spt.get_vertex(
            "Seattle").outgoing[0].payload.name == "I-5 south"

        g.add_vertex("Portland-busstop")
        g.add_vertex("Seattle-busstop")

        g.add_edge("Seattle", "Seattle-busstop", Link())
        g.add_edge("Seattle-busstop", "Seattle", Link())
        g.add_edge("Portland", "Portland-busstop", Link())
        g.add_edge("Portland-busstop", "Portland", Link())

        spt = g.shortest_path_tree("Seattle", "Seattle-busstop",
                                   State(g.numagencies, 0), WalkOptions())
        assert spt.get_vertex(
            "Seattle-busstop").incoming[0].payload.__class__ == Link
        spt.destroy()

        spt = g.shortest_path_tree("Seattle-busstop", "Portland",
                                   State(g.numagencies, 0), WalkOptions())
        assert spt.get_vertex(
            "Portland").incoming[0].payload.__class__ == Street
        spt.destroy()

        sc = ServiceCalendar()
        sc.add_period(0, 86400, ["WKDY", "SAT"])
        tz = Timezone()
        tz.add_period(TimezonePeriod(0, 86400, 0))

        g.add_vertex("Portland-busstop-onbus")
        g.add_vertex("Seattle-busstop-onbus")

        tb = TripBoard("WKDY", sc, tz, 0)
        tb.add_boarding("A", 10, 0)
        tb.add_boarding("B", 15, 0)
        tb.add_boarding("C", 400, 0)

        cr = Crossing()

        al = TripAlight("WKDY", sc, tz, 0)
        al.add_alighting("A", 10 + 20, 0)
        al.add_alighting("B", 15 + 20, 0)
        al.add_alighting("C", 400 + 20, 0)

        g.add_edge("Seattle-busstop", "Seattle-busstop-onbus", tb)
        g.add_edge("Seattle-busstop-onbus", "Portland-busstop-onbus", cr)
        g.add_edge("Portland-busstop-onbus", "Portland-busstop", al)

        spt = g.shortest_path_tree("Seattle", "Portland",
                                   State(g.numagencies, 0), WalkOptions())

        assert spt.get_vertex("Portland").incoming[0].from_v.incoming[
            0].from_v.incoming[0].from_v.incoming[0].from_v.incoming[
                0].from_v.label == "Seattle"

        spt = g.shortest_path_tree("Seattle", "Portland",
                                   State(g.numagencies, 0), WalkOptions())
        vertices, edges = spt.path("Portland")

        assert [v.label for v in vertices] == [
            'Seattle', 'Seattle-busstop', "Seattle-busstop-onbus",
            "Portland-busstop-onbus", 'Portland-busstop', 'Portland'
        ]
        assert [e.payload.__class__ for e in edges
                ] == [Link, TripBoard, Crossing, TripAlight, Link]

        spt.destroy()
        g.destroy()
Пример #18
0
 def setUp(self):
     self.aa = Vertex("A")
     self.bb = Vertex("B")
     self.ep = Link()
     self.path = Path(self.aa)