class SampleSpace(object): def __init__(self): self._index = Rtree() self._locations = [] self._values = [] def __setitem__(self, location, value): i = len(self._locations) self._locations.append(location) self._values.append(value) self._index.add(i, self._locations[i]) def __getitem__(self, location): js = list(self._index.nearest(location, 3)) if len(js) == 0: return 0 if len(js) == 1: return self._values[js[0]] ds = [sqrt(sum([(self._locations[j][i]-location[i])**2 for i in range(2)])) for j in js] for i in range(len(js)): if ds[i] == 0: return self._values[js[i]] R = max(ds) nums = [((R - d)/(R * d))**2 for d in ds] den = sum(nums) ws = [num/den for num in nums] return sum([self._values[js[i]] * ws[i] for i in range(len(js))]) def __iter__(self): for i in range(len(self._values)): yield self._locations[i], self._values[i]
What points are within this 2-dimensional box. RTREE helps us speed up other calculations by only considering legitimate objects for more complex comparisons. ''' from rtree import Rtree # instantiate the Rtree class idx = Rtree() # create a selection boundary (2d bounding box) minx, miny, maxx, maxy = (0.0, 0.0, 1.0, 1.0) # add an item to the index idx.add(0, (minx, miny, maxx, maxy)) # Intersect another bounding box with the original bounding box. list(idx.intersection((1.0, 1.0, 2.0, 2.0))) #[0L] # Point out the accuracy of the spatial calculation list(idx.intersection((1.0000001, 1.0000001, 2.0, 2.0))) #[] #add another item to the index. - show EXACT dimensiion matching index.add(id=id, (left, bottom, right, top)) object = [n for n in index.intersection((left, bottom, right, top))] #[id] # Find nearest object / bounding box idx.add(1, (minx, miny, maxx, maxy)) list(idx.nearest((1.0000001, 1.0000001, 2.0, 2.0), 1)) #[0L, 1L]
class MapMatcher(): """A very nice MapMatching class """ def __init__(self): self.GPS = GPS() self.idx = Rtree() self.nodeidx = Rtree() self.G = None self.edgeindex_edge = {} self.edgecounter = 0 self.nodecounter = 0 self.gps_points = []; self.edge_id__count = {} self.node_counter__node = {} self.distance_matrix = {} def saveGraph(self, filename): """Saves the graph as a YAML file """ nx.write_yaml(self.G,filename) def readGraphFromYAMLFile(self, filename): """Loads a graph from an YAML file. """ self.G = nx.read_yaml(filename) # TODO: buiild up the indexes !!! def addNodeToIndex(self, node): """Adds a node to the node index (RTree) """ # self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node) self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1])) self.node_counter__node[self.nodecounter] = node def addEdgeToIndex(self, edge): """Add an edge to the edhe index. """ self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge) # print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()) self.edgeindex_edge[self.edgecounter] = edge self.edgecounter = self.edgecounter + 1 def openShape(self, inFile, index=0): self.shapeFile = ogr.Open(inFile) if self.shapeFile is None: print "Failed to open " + inFile + ".\n" sys.exit( 1 ) else: print "SHP file successfully read" def getfieldinfo(self, lyr, feature, flds): f = feature return [f.GetField(f.GetFieldIndex(x)) for x in flds] def addlyr(self, G,lyr, fields): point_coords__nodes = {} for findex in xrange(lyr.GetFeatureCount()): f = lyr.GetFeature(findex) flddata = self.getfieldinfo(lyr, f, fields) g = f.geometry() attributes = dict(zip(fields, flddata)) attributes["ShpName"] = lyr.GetName() if g.GetGeometryType() == 2: #linestring last = g.GetPointCount() - 1 p_from = g.GetPoint_2D(0) p_to = g.GetPoint_2D(last) # check whether we have a node in the index intersection_mask = (p_from[0]-INTERSECTION_MASK/2, p_from[1]-INTERSECTION_MASK/2, p_from[0]+INTERSECTION_MASK/2, p_from[1]+INTERSECTION_MASK/2) results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: #print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pfrom self.nodeidx.add(self.nodecounter, (p_from[0], p_from[1], p_from[0], p_from[1])) # print p_from self.nodecounter = self.nodecounter + 1 else: #print len(results) #print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pfrom = self.node_counter__node[results[0]] intersection_mask = (p_to[0]-INTERSECTION_MASK/2, p_to[1]-INTERSECTION_MASK/2, p_to[0]+INTERSECTION_MASK/2, p_to[1]+INTERSECTION_MASK/2) # print intersection_mask results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: #print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pto self.nodeidx.add(self.nodecounter, (p_to[0], p_to[1], p_to[0], p_to[1])) self.nodecounter = self.nodecounter + 1 else: #print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pto = self.node_counter__node[results[0]] shly_geom = shapely.wkt.loads(g.ExportToWkt()) e = Edge(pfrom, pto, attributes, geometry = shly_geom) # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter) self.addEdgeToIndex(e) return G def shapeToGraph(self, inFile, uniqueId="FID"): """Loads a shapefile and builds the graph. uniqueId is the name of a unique field in the shape file. """ # self.G = nx.readwrite.nx_shp.read_shp(inFile) self.G = nx.MultiGraph() self.shapeFileUniqueId = uniqueId lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory for lyrindex in xrange(lyrcount): lyr = self.shapeFile.GetLayerByIndex(lyrindex) flds = [x.GetName() for x in lyr.schema] self.G=self.addlyr(self.G, lyr, flds) self.routefinder = RouteFinder(self.G, euclideanOD=self.distance_matrix) def readGPS(self, inFile): """Parses a shapefile and build the GPS object """ self.GPS.readFromShapeFile(inFile) self.gps_points = self.GPS.getGPSPoints() def maxGPSDistance(self): """Calculate the maximum distance of two consecutive GPS Points """ # TODO check whether GPS points are already there # TODO: move into sl.gps.GPS() maxDistance = 0 gps_point = self.gps_points[0] for gpspoint in self.gps_points: distance = gpspoint.getGeometry().distance(gps_point.getGeometry()) gps_point = gps_point if distance > maxDistance: maxDistance = distance return maxDistance def nearPoints(self): """Sums up the gps point per edge segment. Stores in self.edge_id__count """ # initialize the edge counter for edge in self.G.edges(): self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0 for point in self.gps_points: nearest_edge = self.getNearestEdge(point) # print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id')) self.addPointCountToEdge(nearest_edge) def addPointCountToEdge(self, edge): """Increments the point counter for the given edge by one. """ attributes = edge.getAttributes() if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)): self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1 else: self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1 edge.setAttributes(attributes) def getNearestEdge(self, point): """Returns the edge closes to a Shapely entity given (point) """ edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True) edges = [e.object for e in edge] if len(edges) == 1: result = edges[0] else: dist = 99999999999999999999999999999999999999999 for edge in edges: distance = point.getPoint().distance(edge.getGeometry()) if distance < dist: dist = distance result = edge return result def getNearestNode(self, point): """Returns the closest node to a GPS point. """ nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y))) return self.node_counter__node.get(nodes[0]) def find_all_paths(self, graph, start, end, path=[]): path = path + [start] if start == end: return [path] if not graph.has_key(start): return [] paths = [] for node in graph[start]: if node not in path: newpaths = self.find_all_paths(graph, node, end, path) for newpath in newpaths: paths.append(newpath) return paths def findRoutes2(self): start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) graph = {} print "preparing python graph" for node in self.node_counter__node.values(): graph[node.getNodeID()] = [n.getOutNode2(node).getNodeID() for n in node.getOutEdges()] import pprint pprint.pprint( graph ) print "From ", start_node.getNodeID(), " to ", end_node.getNodeID(), "." results = self.find_all_paths(graph, start_node.getNodeID(), end_node.getNodeID()) # let us find the edges route_list = [] for result in results: i=1 edges = [] while i<len(result): node_from = self.node_counter__node.get(result[i-1] ) node_to = self.node_counter__node.get(result[i] ) i = i +1 edge_dict = self.G[node_from][node_to] if edge_dict.keys()>1: lngth = 9E99999 for k in edge_dict.keys(): if edge_dict[k]['edge'].getLength() < lngth: lngth = edge_dict[k]['edge'].getLength() edge = edge_dict[k]['edge'] else: edge = edge_dict[0]['edge'] edges.append(edge) route_list.append(Route(edges)) factor__selected_route = {} for route in route_list: number_of_points = 0 length = 0 for edge in route.getEdges(): # import pdb;pdb.set_trace() length = length + edge.getLength() edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) if number_of_points > 1: factor__selected_route[number_of_points/length] = route keys = factor__selected_route.keys() keys.sort() return factor__selected_route.get(keys[0]) def findRoute(self, returnNonSelection=False): """Finds a route from the node closest to the first GPS point to the node closest to the latest GPS point. """ # pick the start and end GPS points # TODO: sort GPS Points first start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) # the start and endnodes returns by the index are not in the graph, # therefore we need to look them up .... start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter")) end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter")) self.routfinder = RouteFinder(self.G, euclideanOD=self.distance_matrix) label_list = self.routefinder.findroutes(start_node, end_node) label_scores = [] # import pdb;pdb.set_trace() # let us loop through the label list for label in label_list: number_of_points = 0 # we sum up the number of points and relate them to the length of the route print label for edge in label.getEdges(): edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) print " ", number_of_points #we add the scores to a dict if number_of_points > 1: label_scores.append((label, number_of_points/label.getLength())) # print label_scores # and extract the maximum score score = 0 selected = None for ls in label_scores: if ls[1] > score: selected = ls[0] score = ls[1] if returnNonSelection: pass else: return selected def eliminiateEmptyEdges(self, distance = 100): """Loops through the GPS pointset and selects edges within a boundary of <distance> meters """ print "Edge elimination started" selected_edge_ids = [] # let us for point in self.gps_points: results = self.idx.nearest(((point.getPoint().x-distance/2), (point.getPoint().y-distance/2), (point.getPoint().x+distance/2), (point.getPoint().y+distance/2)), objects=True) for result in results: from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter")) to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter")) edge_counter = self.G.edge[from_node][to_node].get("edgecounter") if edge_counter not in selected_edge_ids: selected_edge_ids.append(edge_counter) print str(len(selected_edge_ids)) + " edges found to keep." elimination_counter = 0 for edge in self.G.edges(): edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter") if edgecounter not in selected_edge_ids: edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node) self.G.remove_edge(*edge_tuple) elimination_counter = elimination_counter + 1 print str(elimination_counter) + " edges eliminated." def dumpPointShape(self, filename, original_coverage=None): if filename: driverName = "ESRI Shapefile" drv = ogr.GetDriverByName( driverName ) if drv: drv.DeleteDataSource(filename) if drv is None: print "%s driver not available.\n" % driverName ds = drv.CreateDataSource( filename) lyr = ds.CreateLayer( "blabla", None, ogr.wkbPoint ) if lyr is None: print "Layer creation failed.\n" field_defn = ogr.FieldDefn( "node_count", ogr.OFTInteger ) if lyr.CreateField ( field_defn ) != 0: print "Creating Name field failed.\n" sys.exit( 1 ) field_defn = ogr.FieldDefn( "edge_list", ogr.OFTString ) field_defn.SetWidth( 1024) if lyr.CreateField ( field_defn ) != 0: print "Creating Name field failed.\n" sys.exit( 1 ) for node in self.node_counter__node.values(): feat = ogr.Feature( lyr.GetLayerDefn() ) nc = node.getAttributes().get("nodecounter") print nc # import pdb;pdb.set_trace() feat.SetField( "node_count", nc ) s = "" for edge in node.getOutEdges(): s = s + str(int(edge.getAttributes().get("ID_NR"))) + ", " feat.SetField( "edge_list", s ) node_entity = ogr.Geometry(ogr.wkbPoint) # wkb = edge.getGeometry().to_wkb() node_entity.SetPoint_2D(0,node.getGeometry().x, node.getGeometry().y) feat.SetGeometry(node_entity) lyr.CreateFeature(feat) feat.Destroy() print "Shapefile (%s) written." % filename
class IntRtreeIndex(BaseIndex): """Avoids the slower Rtree query object=True interface """ _v_nextuid = None family = BTrees.family32 def clear(self): self.fwd = Rtree() self.bwd = self.family.OO.BTree() self.keys = self.family.IO.BTree() self.intids = self.family.OI.BTree() self.ids = self.family.OO.BTree() def __init__(self): self.clear() def key(self, item): try: return item['id'], tuple(self.bbox(item)) except: return tuple(item.items()) def fid(self, item): return item['id'] def intid(self, item): # Get and track next available key using zope.intid algorithm # Item might be already registered uid = self.intids.get(self.key(item)) if uid is not None: return uid # But if not registered nextuid = getattr(self, '_v_nextuid', None) while True: if nextuid is None: nextuid = random.randrange(0, self.family.maxint) uid = nextuid if uid not in self.keys: nextuid += 1 if nextuid > self.family.maxint: nextuid = None self._v_nextuid = nextuid return uid nextuid = None def intersection(self, bbox): """Return an iterator over Items that intersect with the bbox""" for hit in self.fwd.intersection(bbox, objects=False): yield self.bwd[int(hit)] def nearest(self, bbox, limit=1): """Return an iterator over the nearest N=limit Items to the bbox""" for hit in self.fwd.nearest(bbox, num_results=limit, objects=False): yield self.bwd[int(hit)] def item(self, fid, bbox): return self.bwd[self.intids[(fid, bbox)]] def items(self, fid): return [self.bwd[intid] for intid in self.ids[fid]] def index_item(self, itemid, bbox, item): """Add an Item to the index""" if itemid in self.bwd: self.unindex_item(itemid, bbox) # Store an id for the item if it has None try: item.update(id=item.get('id') or str(uuid.uuid4())) key = self.key(item) sid = self.fid(item) # Map keys <-> intids intid = self.intid(item) self.keys[intid] = key self.intids[key] = intid if sid not in self.ids: self.ids[sid] = IISet([]) self.ids[sid].add(intid) self.bwd[intid] = item self.fwd.add(intid, bbox) except: import pdb; pdb.set_trace() raise def unindex_item(self, itemid, bbox): """Remove an Item from the index""" intid = int(itemid) key = self.keys.get(intid) if key is None: return self.ids[key[0]].remove(intid) del self.keys[intid] del self.intids[key] del self.bwd[intid] self.fwd.delete(intid, bbox) def batch(self, changeset): BaseIndex.batch(self, changeset) def commit(self): transaction.commit() rtree_storage = self.fwd.properties.filename self.fwd.close() self.fwd = Rtree(rtree_storage) def close(self): self.fwd.close()
class MapMatcher(): """A very nice MapMatching class """ def __init__(self): self.GPS = GPS() self.idx = Rtree() self.nodeidx = Rtree() self.G = None self.edgeindex_edge = {} self.edgecounter = 0 self.nodecounter = 0 self.gps_points = []; self.edge_id__count = {} self.node_counter__node = {} self.result_counter = 0 def saveGraph(self, filename): """Saves the graph as a YAML file """ nx.write_yaml(self.G,filename) def readGraphFromYAMLFile(self, filename): """Loads a graph from an YAML file. """ self.G = nx.read_yaml(filename) # TODO: buiild up the indexes !!! def addNodeToIndex(self, node): """Adds a node to the node index (RTree) """ # self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node) self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1])) self.node_counter__node[self.nodecounter] = node def addEdgeToIndex(self, edge): """Add an edge to the edhe index. """ self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge) # print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()) self.edgeindex_edge[self.edgecounter] = edge self.edgecounter = self.edgecounter + 1 def openShape(self, inFile, index=0): self.shapeFile = ogr.Open(inFile) if self.shapeFile is None: print "Failed to open " + inFile + ".\n" sys.exit( 1 ) else: print "SHP file successfully read" def getfieldinfo(self, lyr, feature, flds): f = feature return [f.GetField(f.GetFieldIndex(x)) for x in flds] def addlyr(self, G,lyr, fields): point_coords__nodes = {} for findex in xrange(lyr.GetFeatureCount()): f = lyr.GetFeature(findex) flddata = self.getfieldinfo(lyr, f, fields) g = f.geometry() attributes = dict(zip(fields, flddata)) attributes["ShpName"] = lyr.GetName() if g.GetGeometryType() == 2: #linestring last = g.GetPointCount() - 1 p_from = g.GetPoint_2D(0) p_to = g.GetPoint_2D(last) # check whether we have a node in the index intersection_mask = (p_from[0]-INTERSECTION_MASK/2, p_from[1]-INTERSECTION_MASK/2, p_from[0]+INTERSECTION_MASK/2, p_from[1]+INTERSECTION_MASK/2) results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pfrom self.nodeidx.add(self.nodecounter, (p_from[0], p_from[1], p_from[0], p_from[1])) # print p_from self.nodecounter = self.nodecounter + 1 else: print len(results) print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pfrom = self.node_counter__node[results[0]] intersection_mask = (p_to[0]-INTERSECTION_MASK/2, p_to[1]-INTERSECTION_MASK/2, p_to[0]+INTERSECTION_MASK/2, p_to[1]+INTERSECTION_MASK/2) # print intersection_mask results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pto self.nodeidx.add(self.nodecounter, (p_to[0], p_to[1], p_to[0], p_to[1])) self.nodecounter = self.nodecounter + 1 else: print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pto = self.node_counter__node[results[0]] shly_geom = shapely.wkt.loads(g.ExportToWkt()) e = Edge(pfrom, pto, attributes, geometry = shly_geom) # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter) self.addEdgeToIndex(e) #if g.GetGeometryType() == 1: #point # G.add_node((g.GetPoint_2D(0)), attributes) # if g.GetGeometryType() == 2: #linestring # last = g.GetPointCount() - 1 # # p_from = g.GetPoint_2D(0) # p_to = g.GetPoint_2D(last) # # if point_coords__nodes.get(p_from): # # pfrom = point_coords__nodes.get(p_from) # print "node " + str(pfrom.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR')) # # else: # # pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) # self.nodecounter = self.nodecounter + 1 # point_coords__nodes[p_from] = pfrom # # if point_coords__nodes.get(p_to): # # pto = point_coords__nodes.get(p_to) # print "node " + str(pto.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR')) # # else: # # pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) # self.nodecounter = self.nodecounter + 1 # point_coords__nodes[p_to] = pto # # shly_geom = shapely.wkt.loads(g.ExportToWkt()) # e = Edge(pfrom, pto, attributes, geometry = shly_geom) # # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) # # # we pull the nodes out of the graph again to index them # edges_dict = nx.get_edge_attributes(G,"edgecounter") # # # import pdb;pdb.set_trace() # # edges_keys = edges_dict.keys() # for k in edges_keys: # if self.edgecounter == edges_dict[k]: # self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0] # self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1] # self.addNodeToIndex(k[0]) # self.addNodeToIndex(k[1]) # # # let us throw the Edge into the index # self.addEdgeToIndex(e) # ## # add an edge in the other direction ## e2 = Edge(pto, pfrom, attributes, geometry = shly_geom) ## G.add_edge(pto, pfrom, {"edge": e2, "edgecounter" : self.edgecounter}) ## ## edges_dict = nx.get_edge_attributes(G,"edgecounter") ## ## # import pdb;pdb.set_trace() ## ## edges_keys = edges_dict.keys() ## for k in edges_keys: ## if self.edgecounter == edges_dict[k]: ## self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0] ## self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1] ## self.addNodeToIndex(k[0]) ## self.addNodeToIndex(k[1]) ## ## # let us throw the Edge into the index ## self.addEdgeToIndex(e2) return G def shapeToGraph(self, inFile, uniqueId="FID"): """Loads a shapefile and builds the graph. uniqueId is the name of a unique field in the shape file. """ # self.G = nx.readwrite.nx_shp.read_shp(inFile) self.G = nx.MultiGraph() self.shapeFileUniqueId = uniqueId lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory for lyrindex in xrange(lyrcount): lyr = self.shapeFile.GetLayerByIndex(lyrindex) flds = [x.GetName() for x in lyr.schema] self.G=self.addlyr(self.G, lyr, flds) self.routefinder = RouteFinder(self.G) def readGPS(self, inFile): """Parses a shapefile and build the GPS object """ self.GPS.readFromShapeFile(inFile) self.gps_points = self.GPS.getGPSPoints() def maxGPSDistance(self): """Calculate the maximum distance of two consecutive GPS Points """ # TODO check whether GPS points are already there # TODO: move into sl.gps.GPS() maxDistance = 0 gps_point = self.gps_points[0] for gpspoint in self.gps_points: distance = gpspoint.getGeometry().distance(gps_point.getGeometry()) gps_point = gps_point if distance > maxDistance: maxDistance = distance return maxDistance def nearPoints(self): """Sums up the gps point per edge segment. Stores in self.edge_id__count """ # initialize the edge counter for edge in self.G.edges(): self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0 for point in self.gps_points: nearest_edge = self.getNearestEdge(point) # print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id')) self.addPointCountToEdge(nearest_edge) def addPointCountToEdge(self, edge): """Increments the point counter for the given edge by one. """ attributes = edge.getAttributes() if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)): self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1 else: self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1 edge.setAttributes(attributes) def getNearestEdge(self, point): """Returns the edge closes to a Shapely entity given (point) """ edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True) edges = [e.object for e in edge] if len(edges) == 1: result = edges[0] else: dist = 99999999999999999999999999999999999999999 for edge in edges: distance = point.getPoint().distance(edge.getGeometry()) if distance < dist: dist = distance result = edge return result def getNearestNode(self, point): """Returns the closest node to a GPS point. """ nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y))) return self.node_counter__node.get(nodes[0]) def findRoute(self, returnNonSelection=False): """Finds a route from the node closest to the first GPS point to the node closest to the latest GPS point. """ # pick the start and end GPS points # TODO: sort GPS Points first start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) # the start and endnodes returnes by the index are not in the graph, # therefore we need to look them up .... start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter")) end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter")) self.routfinder = RouteFinder(self.G) label_list = self.routefinder.findroutes(start_node, end_node) import pdb;pdb.set_trace() label_scores = [] # let us loop through the label list for label in label_list: number_of_points = 0 # we sum up the number of points and relate them to the length of the route for edge in label.getEdges(): edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) #we add the scores to a dict label_scores.append((label, number_of_points/label.getLength())) # print label_scores # and extract the maximum score score = 0 selected = None for ls in label_scores: if ls[1] > score: selected = ls[0] score = ls[1] if returnNonSelection: pass else: return selected def eliminiateEmptyEdges(self, distance = 100): """Loops through the GPS pointset and selects edges within a boundary of <distance> meters """ print "Edge elimination started" selected_edge_ids = [] # let us for point in self.gps_points: results = self.idx.nearest(((point.getPoint().x-distance/2), (point.getPoint().y-distance/2), (point.getPoint().x+distance/2), (point.getPoint().y+distance/2)), objects=True) for result in results: from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter")) to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter")) edge_counter = self.G.edge[from_node][to_node].get("edgecounter") if edge_counter not in selected_edge_ids: selected_edge_ids.append(edge_counter) print str(len(selected_edge_ids)) + " edges found to keep." elimination_counter = 0 for edge in self.G.edges(): edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter") if edgecounter not in selected_edge_ids: edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node) self.G.remove_edge(*edge_tuple) elimination_counter = elimination_counter + 1 print str(elimination_counter) + " edges eliminated."
class Graph: def __init__(self, all_trips): # trips self.all_trips = all_trips # cluster seeds self.cluster_seeds = {} self.cluster_seed_id = 0 self.cluster_seed_index = Rtree() # graph edges self.graph_edges = {} # indexed by "edge id" self.graph_edge_id = 0 self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" def cluster_traces(self): self._create_all_trip_edges() self._generate_cluster_seeds() self._cluster_seeds_with_traces() self._generate_graph_edges() self._output_graph_to_db() def _generate_graph_edges(self): sys.stdout.write("Generating graph edges... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # grab trip edges trip_edges = trip.edges.values() # put trip edges in order trip_edges.sort(key=lambda x: x.id) # storage for previous cluster prev_cluster = None # iterate through trip edges for trip_edge in trip_edges: # if the current trip edge is clustered if trip_edge.cluster is not None: # create a graph edge between the previous cluster and the current cluster self._create_graph_edge(prev_cluster, trip_edge.cluster) # update previous cluster with current cluster prev_cluster = trip_edge.cluster # output graph edges self._write_graph_edges_to_file() print "done." def _create_graph_edge(self, in_node, out_node): # if in_node or out_node is None if (in_node is None) or (out_node is None): # return without doing anything return # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if existing_graph_edge is None: # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # increment graph edge id self.graph_edge_id += 1 def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if edge_lookup_key in self.graph_edge_lookup.keys(): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _cluster_seeds_with_traces(self): # storage for total cluster distance moved total_cluster_distance_moved = float("infinity") # iterate until total cluster distance moved below threshold while total_cluster_distance_moved >= cluster_distance_moved_threshold: # find all points on traces and move clusters total_cluster_distance_moved = self._find_points_on_traces() # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_clustered.txt") def _find_points_on_traces(self): # counter for cluster seeds seed_counter = 1 # storage for total cluster distance moved total_cluster_distance_moved = 0.0 # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # clear current trace points from cluster cluster_seed.clear_trace_points() sys.stdout.write( "\rFinding intersecting points with cluster " + str(seed_counter) + "/" + str(len(self.cluster_seeds)) + "... " ) sys.stdout.flush() # increment seed counter seed_counter += 1 # determine leftward cluster bearing leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0, 360.0) # determine rightward cluster bearing rightward_bearing = math.fmod((cluster_seed.bearing + 90.0) + 360.0, 360.0) # storage for candidate trace points candidate_trace_points = [] # iterate through all trips for trip in self.all_trips: # find leftward intersection points with trip candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, leftward_bearing)) # find rightward intersection points with trip candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, rightward_bearing)) # add candidate trace points to cluster cluster_seed.add_trace_points(candidate_trace_points) # recompute cluster centroid total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid() # clear current trace points from cluster cluster_seed.clear_trace_points() # add candidate trace points to cluster, again cluster_seed.add_trace_points(candidate_trace_points) # normalize total cluster distance moved by number of seeds total_cluster_distance_moved = total_cluster_distance_moved / len(self.cluster_seeds.values()) # and we're done! print "done (clusters moved an average of " + str(total_cluster_distance_moved) + " meters)." # return total cluster distance moved return total_cluster_distance_moved def _find_intersection_points(self, trip, cluster, cluster_bearing): # find all nearby trip edge id's nearby_trip_edge_ids = self._find_nearby_trip_edge_ids(cluster, edge_bounding_box_size, trip.edge_index) # storage for intersection points intersection_points = [] # iterate through all nearby edge id's for edge_id in nearby_trip_edge_ids: # grab current edge edge = trip.edges[edge_id] # determine intersection point between edge and cluster intersection_point = self._intersection_point(edge.in_node, edge.bearing, cluster, cluster_bearing) # if there is an intersection point if intersection_point is not None: # determine distance from edge in_node to intersection point intersection_distance = self._distance_coords( edge.in_node.latitude, edge.in_node.longitude, intersection_point[0], intersection_point[1] ) # if intersection distance is less than edge length if intersection_distance <= edge.length: # this edge has a valid intersection point intersection_points.append( TracePoint(intersection_point[0], intersection_point[1], edge.bearing, edge) ) # return all intersection points for this trip return intersection_points def _generate_cluster_seeds(self): # iterate through all trips for i in range(0, len(self.all_trips)): sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" + str(len(self.all_trips)) + "... ") sys.stdout.flush() # grab current trip trip = self.all_trips[i] # set last cluster seed distance to zero for first trip location trip.locations[0].last_cluster_seed_distance = 0.0 # iterate through all trip locations for j in range(1, len(trip.locations)): # drop cluster seeds along current edge every 50 meters self._drop_cluster_seeds_along_edge(trip.locations[j - 1], trip.locations[j]) print "done (generated " + str(len(self.cluster_seeds)) + " cluster seeds)." # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt") def _drop_cluster_seeds_along_edge(self, in_node, out_node): # determine edge length edge_length = self._distance(in_node, out_node) # determine distance along edge for first cluster seed first_cluster_seed_distance = cluster_seed_interval - in_node.last_cluster_seed_distance # storage for relative cluster seed intervals rel_cluster_seed_intervals = [] # storage for current cluster seed distance along this edge curr_cluster_seed_distance = first_cluster_seed_distance # determine the relative cluster seed intervals needed for this edge while curr_cluster_seed_distance <= edge_length: # append current cluster seed distance to relative cluster seed interval list rel_cluster_seed_intervals.append(curr_cluster_seed_distance) # increment current cluster seed distance curr_cluster_seed_distance += cluster_seed_interval # determine bearing of current edge edge_bearing = self._path_bearing(in_node, out_node) # create cluster seeds for edge for i in range(0, len(rel_cluster_seed_intervals)): # determine fraction along current edge to drop cluster seed fraction_along = rel_cluster_seed_intervals[i] / edge_length # determine point along line to drop cluster seed (new_cluster_seed_latitude, new_cluster_seed_longitude) = self._point_along_line( in_node, out_node, fraction_along ) # locate nearest existing cluster seeds closest_cluster_seeds = list( self.cluster_seed_index.nearest((new_cluster_seed_longitude, new_cluster_seed_latitude), 25) ) # if there does not exist a closest existing cluster seed if len(closest_cluster_seeds) == 0: # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing ) # else, if there exists a closest existing cluster seed elif len(closest_cluster_seeds) > 0: # storage for matched cluster seed matched_cluster_seed = None # iterate through closest existing cluster seeds for curr_cluster_seed_id in closest_cluster_seeds: # grab current cluster seed curr_cluster_seed = self.cluster_seeds[curr_cluster_seed_id] # compute distance to current cluster seed distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, curr_cluster_seed.latitude, curr_cluster_seed.longitude, ) # determine bearing difference between edge and current cluster seed bearing_difference = math.cos(math.radians(edge_bearing - curr_cluster_seed.bearing)) # if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees if (distance <= cluster_seed_interval) and (bearing_difference >= cluster_bearing_difference_limit): # store current cluster seed as matched cluster seed matched_cluster_seed = curr_cluster_seed # stop searching break # if there was not a matched cluster seed if matched_cluster_seed is None: # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing ) # update last cluster seed distance out_node.last_cluster_seed_distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, out_node.latitude, out_node.longitude ) # if no cluster seeds were generated along this edge if len(rel_cluster_seed_intervals) == 0: # update last cluster seed distance out_node.last_cluster_seed_distance = in_node.last_cluster_seed_distance + edge_length def _create_new_cluster_seed(self, latitude, longitude, bearing): # create a new cluster seed new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude, longitude, bearing) # add new cluster seed to the cluster seeds dictionary self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed # insert new cluster seed into spatial index self.cluster_seed_index.insert(new_cluster_seed.id, (new_cluster_seed.longitude, new_cluster_seed.latitude)) # increment cluster seed id self.cluster_seed_id += 1 # return new cluster seed return new_cluster_seed def _create_all_trip_edges(self): sys.stdout.write("Creating and indexing edges for all trips... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # add edge storage to trip trip.edges = {} # add edge index to trip trip.edge_index = Rtree() # storage for edge id trip_edge_id = 0 # iterate through all trip locations for i in range(1, len(trip.locations)): # create new edge new_edge = Edge(trip_edge_id, trip.locations[i - 1], trip.locations[i]) # insert edge into dictionary trip.edges[trip_edge_id] = new_edge # insert edge into index self._index_trip_edge(new_edge, trip.edge_index) # increment trip edge id trip_edge_id += 1 # done print "done." def _index_trip_edge(self, edge, edge_index): # determine edge minx, miny, maxx, maxy values edge_minx = min(edge.in_node.longitude, edge.out_node.longitude) edge_miny = min(edge.in_node.latitude, edge.out_node.latitude) edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude) edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude) # insert edge into spatial index edge_index.insert(edge.id, (edge_minx, edge_miny, edge_maxx, edge_maxy)) def _find_nearby_trip_edge_ids(self, location, distance, edge_index): # define longitude/latitude offset lon_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE lat_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE # create bounding box bounding_box = ( location.longitude - lon_offset, location.latitude - lat_offset, location.longitude + lon_offset, location.latitude + lat_offset, ) # return nearby edge id's inside bounding box return list(edge_index.intersection(bounding_box)) def _intersection_point(self, location1, location1_bearing, location2, location2_bearing): return spatialfunclib.intersection_point( location1.latitude, location1.longitude, location1_bearing, location2.latitude, location2.longitude, location2_bearing, ) def _point_along_line(self, location1, location2, fraction_along): return spatialfunclib.point_along_line( location1.latitude, location1.longitude, location2.latitude, location2.longitude, fraction_along ) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing( location1.latitude, location1.longitude, location2.latitude, location2.longitude ) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance_coords(self, location1_latitude, location1_longitude, location2_latitude, location2_longitude): return spatialfunclib.distance(location1_latitude, location1_longitude, location2_latitude, location2_longitude) def _write_cluster_seeds_to_file(self, filename="edelkamp_cluster_seeds.txt"): # open graph file graph_file = open(filename, "w") # iterate through all cluster_seeds for cluster_seed in self.cluster_seeds.values(): # output cluster seed to file graph_file.write( str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) + "\n" ) # close graph file graph_file.close() def _write_graph_edges_to_file(self): # open graph file graph_file = open("edelkamp_cluster_edges.txt", "w") # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # output edge to file graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "\n\n") # close graph file graph_file.close() def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("edelkamp_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)") # remove values from nodes table # cur.execute("DELETE FROM nodes") # remove values from edges table # cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # insert cluster seed into nodes table cur.execute( "INSERT INTO nodes VALUES (" + str(cluster_seed.id) + "," + str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + ")" ) # iterate through all graph edges for graph_edge in self.graph_edges.values(): # insert graph edge into edges table cur.execute( "INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")" ) # commit inserts conn.commit() # close database connection conn.close() print "done."
class OSMDB: def __init__(self, dbname, overwrite=False, rtree_index=True): self.dbname = dbname if overwrite: try: os.remove(dbname) except OSError: pass self.conn = sqlite3.connect(dbname) if rtree_index: self.index = Rtree(dbname) else: self.index = None if overwrite: self.setup() def get_cursor(self): # Attempts to get a cursor using the current connection to the db. If we've found ourselves in a different thread # than that which the connection was made in, re-make the connection. try: ret = self.conn.cursor() except sqlite3.ProgrammingError: self.conn = sqlite3.connect(self.dbname) ret = self.conn.cursor() return ret def setup(self): c = self.get_cursor() c.execute( "CREATE TABLE nodes (id TEXT UNIQUE, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" ) c.execute("CREATE TABLE ways (id TEXT UNIQUE, tags TEXT, nds TEXT)") self.conn.commit() c.close() def create_indexes(self): c = self.get_cursor() c.execute("CREATE INDEX nodes_id ON nodes (id)") c.execute("CREATE INDEX nodes_lon ON nodes (lon)") c.execute("CREATE INDEX nodes_lat ON nodes (lat)") c.execute("CREATE INDEX ways_id ON ways (id)") self.conn.commit() c.close() def populate(self, osm_filename, dryrun=False, accept=lambda tags: True, reporter=None, create_indexes=True): print "importing %s osm from XML to sqlite database" % osm_filename c = self.get_cursor() self.n_nodes = 0 self.n_ways = 0 superself = self class OSMHandler(xml.sax.ContentHandler): @classmethod def setDocumentLocator(self, loc): pass @classmethod def startDocument(self): pass @classmethod def endDocument(self): pass @classmethod def startElement(self, name, attrs): if name == 'node': self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat'])) elif name == 'way': self.currElem = Way(attrs['id']) elif name == 'tag': self.currElem.tags[attrs['k']] = attrs['v'] elif name == 'nd': self.currElem.nd_ids.append(attrs['ref']) @classmethod def endElement(self, name): if name == 'node': if superself.n_nodes % 5000 == 0: print "node %d" % superself.n_nodes superself.n_nodes += 1 if not dryrun: superself.add_node(self.currElem, c) elif name == 'way': if superself.n_ways % 5000 == 0: print "way %d" % superself.n_ways superself.n_ways += 1 if not dryrun and accept(self.currElem.tags): superself.add_way(self.currElem, c) @classmethod def characters(self, chars): pass xml.sax.parse(osm_filename, OSMHandler) self.conn.commit() c.close() if not dryrun and create_indexes: print "indexing primary tables...", self.create_indexes() print "done" def set_endnode_ref_counts(self): """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways""" print "counting end-node references to find way split-points" c = self.get_cursor() endnode_ref_counts = {} c.execute("SELECT nds from ways") print "...counting" for i, (nds_str, ) in enumerate(c): if i % 5000 == 0: print i nds = json.loads(nds_str) for nd in nds: endnode_ref_counts[nd] = endnode_ref_counts.get(nd, 0) + 1 print "...updating nodes table" for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()): if i % 5000 == 0: print i if ref_count > 1: c.execute("UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id)) self.conn.commit() c.close() def index_endnodes(self): print "indexing endpoint nodes into rtree" c = self.get_cursor() #TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes c.execute("SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1") for id, lat, lon in c: self.index.add(int(id), (lon, lat, lon, lat)) c.close() def create_and_populate_edges_table(self, tolerant=False): self.set_endnode_ref_counts() self.index_endnodes() print "splitting ways and inserting into edge table" c = self.get_cursor() c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" ) for i, way in enumerate(self.ways()): try: if i % 5000 == 0: print i subways = [] curr_subway = [way.nds[0] ] # add first node to the current subway for nd in way.nds[1:-1]: # for every internal node of the way curr_subway.append(nd) if self.node( nd )[4] > 1: # node reference count is greater than one, node is shared by two ways subways.append(curr_subway) curr_subway = [nd] curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway subways.append(curr_subway) #insert into edge table for i, subway in enumerate(subways): coords = [(lambda x: (x[3], x[2]))(self.node(nd)) for nd in subway] packt = pack_coords(coords) dist = sum([ vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords) ]) c.execute("INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s" % (way.id, i), way.id, subway[0], subway[-1], dist, packt)) except IndexError: if tolerant: continue else: raise print "indexing edges...", c.execute("CREATE INDEX edges_id ON edges (id)") c.execute("CREATE INDEX edges_parent_id ON edges (parent_id)") print "done" self.conn.commit() c.close() def edge(self, id): c = self.get_cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id, )) try: ret = c.next() way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) except StopIteration: c.close() raise IndexError("Database does not have an edge with id '%s'" % id) c.close() return ret def edges(self): c = self.get_cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" ) for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c: yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) c.close() def add_way(self, way, curs=None): if curs is None: curs = self.get_cursor() close_cursor = True else: close_cursor = False curs.execute( "INSERT OR IGNORE INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids))) if close_cursor: self.conn.commit() curs.close() def add_node(self, node, curs=None): if curs is None: curs = self.get_cursor() close_cursor = True else: close_cursor = False curs.execute( "INSERT OR IGNORE INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", (node.id, json.dumps(node.tags), node.lat, node.lon)) if close_cursor: self.conn.commit() curs.close() def nodes(self): c = self.get_cursor() c.execute("SELECT * FROM nodes") for node_row in c: yield node_row c.close() def node(self, id): c = self.get_cursor() c.execute("SELECT * FROM nodes WHERE id = ?", (id, )) try: ret = c.next() except StopIteration: c.close() raise IndexError("Database does not have node with id '%s'" % id) c.close() return ret def nearest_node(self, lat, lon, range=0.005): c = self.get_cursor() if self.index: #print "YOUR'RE USING THE INDEX" id = list(self.index.nearest((lon, lat), 1))[0] #print "THE ID IS %d"%id c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id, )) else: c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat - range, lat + range, lon - range, lon + range)) dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists) == 0: return (None, None, None, None) return min(dists, key=lambda x: x[3]) def nearest_of(self, lat, lon, nodes): c = self.get_cursor() c.execute("SELECT id, lat, lon FROM nodes WHERE id IN (%s)" % ",".join([str(x) for x in nodes])) dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists) == 0: return (None, None, None, None) return min(dists, key=lambda x: x[3]) def way(self, id): c = self.get_cursor() c.execute("SELECT id, tags, nds FROM ways WHERE id = ?", (id, )) try: id, tags_str, nds_str = c.next() ret = WayRecord(id, tags_str, nds_str) except StopIteration: raise Exception("OSMDB has no way with id '%s'" % id) finally: c.close() return ret def way_nds(self, id): c = self.get_cursor() c.execute("SELECT nds FROM ways WHERE id = ?", (id, )) (nds_str, ) = c.next() c.close() return json.loads(nds_str) def ways(self): c = self.get_cursor() c.execute("SELECT id, tags, nds FROM ways") for id, tags_str, nds_str in c: yield WayRecord(id, tags_str, nds_str) c.close() def count_ways(self): c = self.get_cursor() c.execute("SELECT count(*) FROM ways") ret = c.next()[0] c.close() return ret def count_edges(self): c = self.get_cursor() c.execute("SELECT count(*) FROM edges") ret = c.next()[0] c.close() return ret def delete_way(self, id): c = self.get_cursor() c.execute("DELETE FROM ways WHERE id = ?", (id, )) c.close() def bounds(self): c = self.get_cursor() c.execute("SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes") ret = c.next() c.close() return ret def execute(self, sql, args=None): c = self.get_cursor() if args: for row in c.execute(sql, args): yield row else: for row in c.execute(sql): yield row c.close() def cursor(self): return self.get_cursor()
class Graph: def __init__(self, all_trips): # trips self.all_trips = all_trips # cluster seeds self.cluster_seeds = {} self.cluster_seed_id = 0 self.cluster_seed_index = Rtree() # graph edges self.graph_edges = {} # indexed by "edge id" self.graph_edge_id = 0 self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" def cluster_traces(self): self._create_all_trip_edges() self._generate_cluster_seeds() self._cluster_seeds_with_traces() self._generate_graph_edges() self._output_graph_to_db() def _generate_graph_edges(self): sys.stdout.write("Generating graph edges... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # grab trip edges trip_edges = trip.edges.values() # put trip edges in order trip_edges.sort(key=lambda x: x.id) # storage for previous cluster prev_cluster = None # iterate through trip edges for trip_edge in trip_edges: # if the current trip edge is clustered if (trip_edge.cluster is not None): # create a graph edge between the previous cluster and the current cluster self._create_graph_edge(prev_cluster, trip_edge.cluster) # update previous cluster with current cluster prev_cluster = trip_edge.cluster # output graph edges self._write_graph_edges_to_file() print "done." def _create_graph_edge(self, in_node, out_node): # if in_node or out_node is None if ((in_node is None) or (out_node is None)): # return without doing anything return # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # increment graph edge id self.graph_edge_id += 1 def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _cluster_seeds_with_traces(self): # storage for total cluster distance moved total_cluster_distance_moved = float('infinity') # iterate until total cluster distance moved below threshold while (total_cluster_distance_moved >= cluster_distance_moved_threshold): # find all points on traces and move clusters total_cluster_distance_moved = self._find_points_on_traces() # write cluster seeds to file self._write_cluster_seeds_to_file( "edelkamp_cluster_seeds_clustered.txt") def _find_points_on_traces(self): # counter for cluster seeds seed_counter = 1 # storage for total cluster distance moved total_cluster_distance_moved = 0.0 # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # clear current trace points from cluster cluster_seed.clear_trace_points() sys.stdout.write("\rFinding intersecting points with cluster " + str(seed_counter) + "/" + str(len(self.cluster_seeds)) + "... ") sys.stdout.flush() # increment seed counter seed_counter += 1 # determine leftward cluster bearing leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0, 360.0) # determine rightward cluster bearing rightward_bearing = math.fmod( (cluster_seed.bearing + 90.0) + 360.0, 360.0) # storage for candidate trace points candidate_trace_points = [] # iterate through all trips for trip in self.all_trips: # find leftward intersection points with trip candidate_trace_points.extend( self._find_intersection_points(trip, cluster_seed, leftward_bearing)) # find rightward intersection points with trip candidate_trace_points.extend( self._find_intersection_points(trip, cluster_seed, rightward_bearing)) # add candidate trace points to cluster cluster_seed.add_trace_points(candidate_trace_points) # recompute cluster centroid total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid( ) # clear current trace points from cluster cluster_seed.clear_trace_points() # add candidate trace points to cluster, again cluster_seed.add_trace_points(candidate_trace_points) # normalize total cluster distance moved by number of seeds total_cluster_distance_moved = (total_cluster_distance_moved / len(self.cluster_seeds.values())) # and we're done! print "done (clusters moved an average of " + str( total_cluster_distance_moved) + " meters)." # return total cluster distance moved return total_cluster_distance_moved def _find_intersection_points(self, trip, cluster, cluster_bearing): # find all nearby trip edge id's nearby_trip_edge_ids = self._find_nearby_trip_edge_ids( cluster, edge_bounding_box_size, trip.edge_index) # storage for intersection points intersection_points = [] # iterate through all nearby edge id's for edge_id in nearby_trip_edge_ids: # grab current edge edge = trip.edges[edge_id] # determine intersection point between edge and cluster intersection_point = self._intersection_point( edge.in_node, edge.bearing, cluster, cluster_bearing) # if there is an intersection point if (intersection_point is not None): # determine distance from edge in_node to intersection point intersection_distance = self._distance_coords( edge.in_node.latitude, edge.in_node.longitude, intersection_point[0], intersection_point[1]) # if intersection distance is less than edge length if (intersection_distance <= edge.length): # this edge has a valid intersection point intersection_points.append( TracePoint(intersection_point[0], intersection_point[1], edge.bearing, edge)) # return all intersection points for this trip return intersection_points def _generate_cluster_seeds(self): # iterate through all trips for i in range(0, len(self.all_trips)): sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" + str(len(self.all_trips)) + "... ") sys.stdout.flush() # grab current trip trip = self.all_trips[i] # set last cluster seed distance to zero for first trip location trip.locations[0].last_cluster_seed_distance = 0.0 # iterate through all trip locations for j in range(1, len(trip.locations)): # drop cluster seeds along current edge every 50 meters self._drop_cluster_seeds_along_edge(trip.locations[j - 1], trip.locations[j]) print "done (generated " + str(len( self.cluster_seeds)) + " cluster seeds)." # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt") def _drop_cluster_seeds_along_edge(self, in_node, out_node): # determine edge length edge_length = self._distance(in_node, out_node) # determine distance along edge for first cluster seed first_cluster_seed_distance = (cluster_seed_interval - in_node.last_cluster_seed_distance) # storage for relative cluster seed intervals rel_cluster_seed_intervals = [] # storage for current cluster seed distance along this edge curr_cluster_seed_distance = first_cluster_seed_distance # determine the relative cluster seed intervals needed for this edge while (curr_cluster_seed_distance <= edge_length): # append current cluster seed distance to relative cluster seed interval list rel_cluster_seed_intervals.append(curr_cluster_seed_distance) # increment current cluster seed distance curr_cluster_seed_distance += cluster_seed_interval # determine bearing of current edge edge_bearing = self._path_bearing(in_node, out_node) # create cluster seeds for edge for i in range(0, len(rel_cluster_seed_intervals)): # determine fraction along current edge to drop cluster seed fraction_along = (rel_cluster_seed_intervals[i] / edge_length) # determine point along line to drop cluster seed (new_cluster_seed_latitude, new_cluster_seed_longitude) = self._point_along_line( in_node, out_node, fraction_along) # locate nearest existing cluster seeds closest_cluster_seeds = list( self.cluster_seed_index.nearest( (new_cluster_seed_longitude, new_cluster_seed_latitude), 25)) # if there does not exist a closest existing cluster seed if (len(closest_cluster_seeds) == 0): # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing) # else, if there exists a closest existing cluster seed elif (len(closest_cluster_seeds) > 0): # storage for matched cluster seed matched_cluster_seed = None # iterate through closest existing cluster seeds for curr_cluster_seed_id in closest_cluster_seeds: # grab current cluster seed curr_cluster_seed = self.cluster_seeds[ curr_cluster_seed_id] # compute distance to current cluster seed distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, curr_cluster_seed.latitude, curr_cluster_seed.longitude) # determine bearing difference between edge and current cluster seed bearing_difference = math.cos( math.radians(edge_bearing - curr_cluster_seed.bearing)) # if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees if ((distance <= cluster_seed_interval) and (bearing_difference >= cluster_bearing_difference_limit)): # store current cluster seed as matched cluster seed matched_cluster_seed = curr_cluster_seed # stop searching break # if there was not a matched cluster seed if (matched_cluster_seed is None): # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing) # update last cluster seed distance out_node.last_cluster_seed_distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, out_node.latitude, out_node.longitude) # if no cluster seeds were generated along this edge if (len(rel_cluster_seed_intervals) == 0): # update last cluster seed distance out_node.last_cluster_seed_distance = ( in_node.last_cluster_seed_distance + edge_length) def _create_new_cluster_seed(self, latitude, longitude, bearing): # create a new cluster seed new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude, longitude, bearing) # add new cluster seed to the cluster seeds dictionary self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed # insert new cluster seed into spatial index self.cluster_seed_index.insert( new_cluster_seed.id, (new_cluster_seed.longitude, new_cluster_seed.latitude)) # increment cluster seed id self.cluster_seed_id += 1 # return new cluster seed return new_cluster_seed def _create_all_trip_edges(self): sys.stdout.write("Creating and indexing edges for all trips... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # add edge storage to trip trip.edges = {} # add edge index to trip trip.edge_index = Rtree() # storage for edge id trip_edge_id = 0 # iterate through all trip locations for i in range(1, len(trip.locations)): # create new edge new_edge = Edge(trip_edge_id, trip.locations[i - 1], trip.locations[i]) # insert edge into dictionary trip.edges[trip_edge_id] = new_edge # insert edge into index self._index_trip_edge(new_edge, trip.edge_index) # increment trip edge id trip_edge_id += 1 # done print "done." def _index_trip_edge(self, edge, edge_index): # determine edge minx, miny, maxx, maxy values edge_minx = min(edge.in_node.longitude, edge.out_node.longitude) edge_miny = min(edge.in_node.latitude, edge.out_node.latitude) edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude) edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude) # insert edge into spatial index edge_index.insert(edge.id, (edge_minx, edge_miny, edge_maxx, edge_maxy)) def _find_nearby_trip_edge_ids(self, location, distance, edge_index): # define longitude/latitude offset lon_offset = ((distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE) lat_offset = ((distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE) # create bounding box bounding_box = (location.longitude - lon_offset, location.latitude - lat_offset, location.longitude + lon_offset, location.latitude + lat_offset) # return nearby edge id's inside bounding box return list(edge_index.intersection(bounding_box)) def _intersection_point(self, location1, location1_bearing, location2, location2_bearing): return spatialfunclib.intersection_point( location1.latitude, location1.longitude, location1_bearing, location2.latitude, location2.longitude, location2_bearing) def _point_along_line(self, location1, location2, fraction_along): return spatialfunclib.point_along_line(location1.latitude, location1.longitude, location2.latitude, location2.longitude, fraction_along) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance_coords(self, location1_latitude, location1_longitude, location2_latitude, location2_longitude): return spatialfunclib.distance(location1_latitude, location1_longitude, location2_latitude, location2_longitude) def _write_cluster_seeds_to_file(self, filename="edelkamp_cluster_seeds.txt"): # open graph file graph_file = open(filename, 'w') # iterate through all cluster_seeds for cluster_seed in self.cluster_seeds.values(): # output cluster seed to file graph_file.write( str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) + "\n") # close graph file graph_file.close() def _write_graph_edges_to_file(self): # open graph file graph_file = open('edelkamp_cluster_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # output edge to file graph_file.write( str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write( str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "\n\n") # close graph file graph_file.close() def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("edelkamp_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute( "CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute( "CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)" ) # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # insert cluster seed into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(cluster_seed.id) + "," + str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done."
class OSMDB: def __init__(self, dbname,overwrite=False,rtree_index=True): if overwrite: try: os.remove( dbname ) except OSError: pass self.conn = sqlite3.connect(dbname) if rtree_index: self.index = Rtree( dbname ) else: self.index = None if overwrite: self.setup() def setup(self): c = self.conn.cursor() c.execute( "CREATE TABLE nodes (id TEXT, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" ) c.execute( "CREATE TABLE ways (id TEXT, tags TEXT, nds TEXT)" ) self.conn.commit() c.close() def create_indexes(self): c = self.conn.cursor() c.execute( "CREATE INDEX nodes_id ON nodes (id)" ) c.execute( "CREATE INDEX nodes_lon ON nodes (lon)" ) c.execute( "CREATE INDEX nodes_lat ON nodes (lat)" ) c.execute( "CREATE INDEX ways_id ON ways (id)" ) self.conn.commit() c.close() def populate(self, osm_filename, accept=lambda tags: True, reporter=None): print "importing osm from XML to sqlite database" c = self.conn.cursor() self.n_nodes = 0 self.n_ways = 0 superself = self class OSMHandler(xml.sax.ContentHandler): @classmethod def setDocumentLocator(self,loc): pass @classmethod def startDocument(self): pass @classmethod def endDocument(self): pass @classmethod def startElement(self, name, attrs): if name=='node': self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat'])) elif name=='way': self.currElem = Way(attrs['id']) elif name=='tag': self.currElem.tags[attrs['k']] = attrs['v'] elif name=='nd': self.currElem.nd_ids.append( attrs['ref'] ) @classmethod def endElement(self,name): if name=='node': if superself.n_nodes%5000==0: print "node %d"%superself.n_nodes superself.n_nodes += 1 superself.add_node( self.currElem, c ) elif name=='way': if superself.n_ways%5000==0: print "way %d"%superself.n_ways superself.n_ways += 1 superself.add_way( self.currElem, c ) @classmethod def characters(self, chars): pass xml.sax.parse(osm_filename, OSMHandler) self.conn.commit() c.close() print "indexing primary tables...", self.create_indexes() print "done" def set_endnode_ref_counts( self ): """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways""" print "counting end-node references to find way split-points" c = self.conn.cursor() endnode_ref_counts = {} c.execute( "SELECT nds from ways" ) print "...counting" for i, (nds_str,) in enumerate(c): if i%5000==0: print i nds = json.loads( nds_str ) for nd in nds: endnode_ref_counts[ nd ] = endnode_ref_counts.get( nd, 0 )+1 print "...updating nodes table" for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()): if i%5000==0: print i if ref_count > 1: c.execute( "UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id) ) self.conn.commit() c.close() def index_endnodes( self ): print "indexing endpoint nodes into rtree" c = self.conn.cursor() #TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1" ) for id, lat, lon in c: self.index.add( int(id), (lon, lat, lon, lat) ) c.close() def create_and_populate_edges_table( self, tolerant=False ): self.set_endnode_ref_counts() self.index_endnodes() print "splitting ways and inserting into edge table" c = self.conn.cursor() c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" ) for i, way in enumerate(self.ways()): try: if i%5000==0: print i subways = [] curr_subway = [ way.nds[0] ] # add first node to the current subway for nd in way.nds[1:-1]: # for every internal node of the way curr_subway.append( nd ) if self.node(nd)[4] > 1: # node reference count is greater than one, node is shared by two ways subways.append( curr_subway ) curr_subway = [ nd ] curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway subways.append( curr_subway ); #insert into edge table for i, subway in enumerate(subways): coords = [(lambda x:(x[3],x[2]))(self.node(nd)) for nd in subway] packt = pack_coords( coords ) dist = sum([vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords)]) c.execute( "INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s"%(way.id, i), way.id, subway[0], subway[-1], dist, packt) ) except IndexError: if tolerant: continue else: raise print "indexing edges...", c.execute( "CREATE INDEX edges_id ON edges (id)" ) c.execute( "CREATE INDEX edges_parent_id ON edges (parent_id)" ) print "done" self.conn.commit() c.close() def edge(self, id): c = self.conn.cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id,) ) try: ret = c.next() way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords( geom ), json.loads(tags)) except StopIteration: c.close() raise IndexError( "Database does not have an edge with id '%s'"%id ) c.close() return ret def edges(self): c = self.conn.cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" ) for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c: yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) c.close() def add_way( self, way, curs=None ): if curs is None: curs = self.conn.cursor() close_cursor = True else: close_cursor = False curs.execute("INSERT INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids) )) if close_cursor: self.conn.commit() curs.close() def add_node( self, node, curs=None ): if curs is None: curs = self.conn.cursor() close_cursor = True else: close_cursor = False curs.execute("INSERT INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", ( node.id, json.dumps(node.tags), node.lat, node.lon ) ) if close_cursor: self.conn.commit() curs.close() def nodes(self): c = self.conn.cursor() c.execute( "SELECT * FROM nodes" ) for node_row in c: yield node_row c.close() def node(self, id): c = self.conn.cursor() c.execute( "SELECT * FROM nodes WHERE id = ?", (id,) ) try: ret = c.next() except StopIteration: c.close() raise IndexError( "Database does not have node with id '%s'"%id ) c.close() return ret def nearest_node(self, lat, lon, range=0.005): c = self.conn.cursor() if self.index: print "YOU'RE USING THE INDEX" id = list(self.index.nearest( (lon, lat), 1 ))[0] print "THE ID IS %d"%id c.execute( "SELECT id, lat, lon FROM nodes WHERE id = ?", (id,) ) else: c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat-range, lat+range, lon-range, lon+range) ) dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists)==0: return (None, None, None, None) return min( dists, key = lambda x:x[3] ) def nearest_of( self, lat, lon, nodes ): c = self.conn.cursor() c.execute( "SELECT id, lat, lon FROM nodes WHERE id IN (%s)"%",".join([str(x) for x in nodes]) ) dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists)==0: return (None, None, None, None) return min( dists, key = lambda x:x[3] ) def way(self, id): c = self.conn.cursor() c.execute( "SELECT id, tags, nds FROM ways WHERE id = ?", (id,) ) try: id, tags_str, nds_str = c.next() ret = WayRecord(id, tags_str, nds_str) except StopIteration: raise Exception( "OSMDB has no way with id '%s'"%id ) finally: c.close() return ret def way_nds(self, id): c = self.conn.cursor() c.execute( "SELECT nds FROM ways WHERE id = ?", (id,) ) (nds_str,) = c.next() c.close() return json.loads( nds_str ) def ways(self): c = self.conn.cursor() c.execute( "SELECT id, tags, nds FROM ways" ) for id, tags_str, nds_str in c: yield WayRecord( id, tags_str, nds_str ) c.close() def count_ways(self): c = self.conn.cursor() c.execute( "SELECT count(*) FROM ways" ) ret = c.next()[0] c.close() return ret def count_edges(self): c = self.conn.cursor() c.execute( "SELECT count(*) FROM edges" ) ret = c.next()[0] c.close() return ret def delete_way(self, id): c = self.conn.cursor() c.execute("DELETE FROM ways WHERE id = ?", (id,)) c.close() def bounds(self): c = self.conn.cursor() c.execute( "SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes" ) ret = c.next() c.close() return ret def execute(self,sql,args=None): c = self.conn.cursor() if args: for row in c.execute(sql,args): yield row else: for row in c.execute(sql): yield row c.close() def cursor(self): return self.conn.cursor()
class Graph: def __init__(self, bus_trips): self.bus_trips = bus_trips self.graph_nodes = {} # indexed by "location_id" self.graph_edge_id = 0 self.graph_edges = {} # indexed by "edge id" self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" self.graph_edge_index = Rtree() def generate_graph(self): print "Running graph generation algorithm..." # initialize trip counter trip_count = 1 for trip in self.bus_trips: # initialize location counter location_count = 1 # storage for previous node prev_node = None for location in trip.locations: sys.stdout.write("\rAnalyzing location " + str(location_count) + "/" + str(len(trip.locations)) + " for trip " + str(trip_count) + "/" + str(len(self.bus_trips)) + "... ") sys.stdout.flush() # find closest edges in graph to location closest_edges = self._find_closest_edges_in_graph( location, 100) # flag variable for whether we merged location did_merge_location = False # iterate through candidate edge ids for candidate_edge_id in closest_edges: # grab candidate edge from graph edge dictionary candidate_edge = self.graph_edges[candidate_edge_id] # determine whether we should merge with candidate edge if (self._should_merge_location_with_edge( location, candidate_edge) is True): # merge location with edge, update previous node prev_node = self._merge_location_with_edge( location, candidate_edge, prev_node) # update merge flag variable did_merge_location = True # no need to look at further edges, break out of candidate edges loop break # if we did not merge the location with any edge if (did_merge_location is False): # add location to graph self._add_location_to_graph(location, prev_node) # update previous node with current location prev_node = location # increment location counter location_count += 1 # done with current trip locations print "done." # increment trip counter trip_count += 1 # write graph edges to file self._write_graph_edges_to_file() # create graph database self._output_graph_to_db() def _merge_location_with_edge(self, location, edge, prev_node): # get the edge node closest to the location edge_node = self._get_closest_edge_node(location, edge) # if prev_node is None if (prev_node is None): # increase volume of just this edge edge.volume += 1 # if prev_node is not None else: # find path from prev_node to edge_node path = self._find_path(prev_node, edge_node, max_path_length) # if there was a path from prev_node to edge_node if (path is not None): # iterate through nodes in path for i in range(1, len(path)): # grab in_node in_node = path[i - 1] # grab out_node out_node = path[i] # find corresponding graph edge graph_edge = self._find_graph_edge(in_node, out_node) # increment volume on edge graph_edge.volume += 1 # if there is no path from prev_node to edge_node else: # create a new graph edge between prev_node and edge_node self._create_graph_edge(prev_node, edge_node) # return the edge_node return edge_node def _get_closest_edge_node(self, location, edge): # if in_node distance is less than out_node distance if (self._distance(location, edge.in_node) < self._distance( location, edge.out_node)): # return the edge in_node return edge.in_node # otherwise, return the edge out_node return edge.out_node def _find_path(self, source, destination, max_length): # reset all node visited flags self._reset_node_visited_flags() # get a breath-first search path from source to destination path = self._bfs_path(source, destination) # if there is a path from source to destination if (path is not None): # and if the path length is less than or equal to the maximum length if (len(path) <= max_length): # return the path return path # otherwise, return None return None def _bfs_path(self, source, destination): # storage for breadth-first search parents bfs_parent = {} # key is current node, value is parent node # source node has no breadth-first search parent bfs_parent[source] = None # node queue for breadth-first search bfs_queue = [] # enqueue source node bfs_queue.append(source) # mark source node as visited source.visited = True # while the queue is not empty while (len(bfs_queue) > 0): # dequeue the first node in the queue curr_node = bfs_queue.pop(0) # if the current node is the destination if (curr_node is destination): # create storage for breadth-first search path bfs_path = [] # add the current node to the breadth-first search path bfs_path.insert(0, curr_node) # grab the parent of the current node parent = bfs_parent[curr_node] # iterate through breadth-first search parents while (parent is not None): # add the parent to the breadth-first search path bfs_path.insert(0, parent) # grab the next parent parent = bfs_parent[parent] # return the breadth-first search path return bfs_path # if the current node is not the destination else: # iterate through the current node's out_nodes for out_node in curr_node.out_nodes: # if the out_node has not been visited if (out_node.visited is False): # mark the out_node as visited out_node.visited = True # enqueue the out_node bfs_queue.append(out_node) # store curr_node as out_node's breadth-first search parent bfs_parent[out_node] = curr_node # if we reached here, no path was found return None def _should_merge_location_with_edge(self, location, edge): # project location onto edge (location_projection, location_projection_fraction, location_projection_distance) = self._projection_onto_line( edge.in_node, edge.out_node, location) # if projection is not onto edge if (location_projection_fraction < 0.0 or location_projection_fraction > 1.0): # we cannot merge location with edge return False # determine bearing difference between edge and location bearing_difference = math.cos( math.radians( self._path_bearing(edge.in_node, edge.out_node) - self._location_bearing(location))) # if location projection distance is less than 20 meters if (location_projection_distance < location_projection_distance_limit): # if bearing difference is less than 45 degrees if (bearing_difference > location_bearing_difference_limit): # merge location with edge return True # otherwise, do not merge location with edge return False def _add_location_to_graph(self, location, prev_node): # add an out_nodes list to location location.out_nodes = [] # add an in_nodes list to location location.in_nodes = [] # add a visited flag to location location.visited = False # add location to graph nodes list self.graph_nodes[location.id] = location # if prev_node is not None if (prev_node is not None): # create a new graph edge between prev_node and location self._create_graph_edge(prev_node, location) def _create_graph_edge(self, in_node, out_node): # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # add new graph edge to graph edges spatial index self._add_graph_edge_to_index(new_graph_edge) # increment graph edge id self.graph_edge_id += 1 # store out_node in in_nodes's out_nodes list in_node.out_nodes.append(out_node) # store in_node in out_node's in_nodes list out_node.in_nodes.append(in_node) def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _add_graph_edge_to_index(self, graph_edge): # determine graph edge minx, miny, maxx, maxy values graph_edge_minx = min(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_miny = min(graph_edge.in_node.latitude, graph_edge.out_node.latitude) graph_edge_maxx = max(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_maxy = max(graph_edge.in_node.latitude, graph_edge.out_node.latitude) # insert graph edge into spatial index self.graph_edge_index.insert(graph_edge.id, (graph_edge_minx, graph_edge_miny, graph_edge_maxx, graph_edge_maxy)) def _location_bearing(self, location): # if location has a previous neighbor and a next neighbor if ((location.prev_location is not None) and (location.next_location is not None)): # determine bearing using previous and next neighbors return self._path_bearing(location.prev_location, location.next_location) # if location has no previous neighbor, but has a next neighbor elif ((location.prev_location is None) and (location.next_location is not None)): # determine bearing using current location and next neighbor return self._path_bearing(location, location.next_location) # if location has a previous neighbor, but not a next neighbor elif ((location.prev_location is not None) and (location.next_location is None)): # determine bearing using previous neighbor and current location return self._path_bearing(location.prev_location, location) # if we reach here, there is an error return None def _find_closest_edges_in_graph(self, location, number_of_edges): return self.graph_edge_index.nearest( (location.longitude, location.latitude), number_of_edges) def _projection_onto_line(self, location1, location2, location3): return spatialfunclib.projection_onto_line( location1.latitude, location1.longitude, location2.latitude, location2.longitude, location3.latitude, location3.longitude) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("cao_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute( "CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute( "CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)" ) # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # insert graph node into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) + "," + str(graph_node.latitude) + "," + str(graph_node.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done." def _write_graph_edges_to_file(self): # output that we are starting the writing process sys.stdout.write("\nWriting graph edges to file... ") sys.stdout.flush() # open graph file graph_file = open('cao_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # output edge to file graph_file.write( str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write( str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "," + str(graph_edge.volume) + "\n\n") # close graph file graph_file.close() print "done." def _reset_node_visited_flags(self): # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # set visited flag to False graph_node.visited = False
class Graph: def __init__(self, bus_trips): self.bus_trips = bus_trips self.graph_nodes = {} # indexed by "location_id" self.graph_edge_id = 0 self.graph_edges = {} # indexed by "edge id" self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" self.graph_edge_index = Rtree() def generate_graph(self): print "Running graph generation algorithm..." # initialize trip counter trip_count = 1 for trip in self.bus_trips: # initialize location counter location_count = 1 # storage for previous node prev_node = None for location in trip.locations: sys.stdout.write("\rAnalyzing location " + str(location_count) + "/" + str(len(trip.locations)) + " for trip " + str(trip_count) + "/" + str(len(self.bus_trips)) + "... ") sys.stdout.flush() # find closest edges in graph to location closest_edges = self._find_closest_edges_in_graph(location, 100) # flag variable for whether we merged location did_merge_location = False # iterate through candidate edge ids for candidate_edge_id in closest_edges: # grab candidate edge from graph edge dictionary candidate_edge = self.graph_edges[candidate_edge_id] # determine whether we should merge with candidate edge if (self._should_merge_location_with_edge(location, candidate_edge) is True): # merge location with edge, update previous node prev_node = self._merge_location_with_edge(location, candidate_edge, prev_node) # update merge flag variable did_merge_location = True # no need to look at further edges, break out of candidate edges loop break # if we did not merge the location with any edge if (did_merge_location is False): # add location to graph self._add_location_to_graph(location, prev_node) # update previous node with current location prev_node = location # increment location counter location_count += 1 # done with current trip locations print "done." # increment trip counter trip_count += 1 # write graph edges to file self._write_graph_edges_to_file() # create graph database self._output_graph_to_db() def _merge_location_with_edge(self, location, edge, prev_node): # get the edge node closest to the location edge_node = self._get_closest_edge_node(location, edge) # if prev_node is None if (prev_node is None): # increase volume of just this edge edge.volume += 1 # if prev_node is not None else: # find path from prev_node to edge_node path = self._find_path(prev_node, edge_node, max_path_length) # if there was a path from prev_node to edge_node if (path is not None): # iterate through nodes in path for i in range(1, len(path)): # grab in_node in_node = path[i - 1] # grab out_node out_node = path[i] # find corresponding graph edge graph_edge = self._find_graph_edge(in_node, out_node) # increment volume on edge graph_edge.volume += 1 # if there is no path from prev_node to edge_node else: # create a new graph edge between prev_node and edge_node self._create_graph_edge(prev_node, edge_node) # return the edge_node return edge_node def _get_closest_edge_node(self, location, edge): # if in_node distance is less than out_node distance if (self._distance(location, edge.in_node) < self._distance(location, edge.out_node)): # return the edge in_node return edge.in_node # otherwise, return the edge out_node return edge.out_node def _find_path(self, source, destination, max_length): # reset all node visited flags self._reset_node_visited_flags() # get a breath-first search path from source to destination path = self._bfs_path(source, destination) # if there is a path from source to destination if (path is not None): # and if the path length is less than or equal to the maximum length if (len(path) <= max_length): # return the path return path # otherwise, return None return None def _bfs_path(self, source, destination): # storage for breadth-first search parents bfs_parent = {} # key is current node, value is parent node # source node has no breadth-first search parent bfs_parent[source] = None # node queue for breadth-first search bfs_queue = [] # enqueue source node bfs_queue.append(source) # mark source node as visited source.visited = True # while the queue is not empty while (len(bfs_queue) > 0): # dequeue the first node in the queue curr_node = bfs_queue.pop(0) # if the current node is the destination if (curr_node is destination): # create storage for breadth-first search path bfs_path = [] # add the current node to the breadth-first search path bfs_path.insert(0, curr_node) # grab the parent of the current node parent = bfs_parent[curr_node] # iterate through breadth-first search parents while (parent is not None): # add the parent to the breadth-first search path bfs_path.insert(0, parent) # grab the next parent parent = bfs_parent[parent] # return the breadth-first search path return bfs_path # if the current node is not the destination else: # iterate through the current node's out_nodes for out_node in curr_node.out_nodes: # if the out_node has not been visited if (out_node.visited is False): # mark the out_node as visited out_node.visited = True # enqueue the out_node bfs_queue.append(out_node) # store curr_node as out_node's breadth-first search parent bfs_parent[out_node] = curr_node # if we reached here, no path was found return None def _should_merge_location_with_edge(self, location, edge): # project location onto edge (location_projection, location_projection_fraction, location_projection_distance) = self._projection_onto_line(edge.in_node, edge.out_node, location) # if projection is not onto edge if (location_projection_fraction < 0.0 or location_projection_fraction > 1.0): # we cannot merge location with edge return False # determine bearing difference between edge and location bearing_difference = math.cos(math.radians(self._path_bearing(edge.in_node, edge.out_node) - self._location_bearing(location))) # if location projection distance is less than 20 meters if (location_projection_distance < location_projection_distance_limit): # if bearing difference is less than 45 degrees if (bearing_difference > location_bearing_difference_limit): # merge location with edge return True # otherwise, do not merge location with edge return False def _add_location_to_graph(self, location, prev_node): # add an out_nodes list to location location.out_nodes = [] # add an in_nodes list to location location.in_nodes = [] # add a visited flag to location location.visited = False # add location to graph nodes list self.graph_nodes[location.id] = location # if prev_node is not None if (prev_node is not None): # create a new graph edge between prev_node and location self._create_graph_edge(prev_node, location) def _create_graph_edge(self, in_node, out_node): # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # add new graph edge to graph edges spatial index self._add_graph_edge_to_index(new_graph_edge) # increment graph edge id self.graph_edge_id += 1 # store out_node in in_nodes's out_nodes list in_node.out_nodes.append(out_node) # store in_node in out_node's in_nodes list out_node.in_nodes.append(in_node) def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _add_graph_edge_to_index(self, graph_edge): # determine graph edge minx, miny, maxx, maxy values graph_edge_minx = min(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_miny = min(graph_edge.in_node.latitude, graph_edge.out_node.latitude) graph_edge_maxx = max(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_maxy = max(graph_edge.in_node.latitude, graph_edge.out_node.latitude) # insert graph edge into spatial index self.graph_edge_index.insert(graph_edge.id, (graph_edge_minx, graph_edge_miny, graph_edge_maxx, graph_edge_maxy)) def _location_bearing(self, location): # if location has a previous neighbor and a next neighbor if ((location.prev_location is not None) and (location.next_location is not None)): # determine bearing using previous and next neighbors return self._path_bearing(location.prev_location, location.next_location) # if location has no previous neighbor, but has a next neighbor elif ((location.prev_location is None) and (location.next_location is not None)): # determine bearing using current location and next neighbor return self._path_bearing(location, location.next_location) # if location has a previous neighbor, but not a next neighbor elif ((location.prev_location is not None) and (location.next_location is None)): # determine bearing using previous neighbor and current location return self._path_bearing(location.prev_location, location) # if we reach here, there is an error return None def _find_closest_edges_in_graph(self, location, number_of_edges): return self.graph_edge_index.nearest((location.longitude, location.latitude), number_of_edges) def _projection_onto_line(self, location1, location2, location3): return spatialfunclib.projection_onto_line(location1.latitude, location1.longitude, location2.latitude, location2.longitude, location3.latitude, location3.longitude) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("cao_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)") # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # insert graph node into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) + "," + str(graph_node.latitude) + "," + str(graph_node.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done." def _write_graph_edges_to_file(self): # output that we are starting the writing process sys.stdout.write("\nWriting graph edges to file... ") sys.stdout.flush() # open graph file graph_file = open('cao_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # output edge to file graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "," + str(graph_edge.volume) + "\n\n") # close graph file graph_file.close() print "done." def _reset_node_visited_flags(self): # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # set visited flag to False graph_node.visited = False