def segment_spatial_index(linestring): coords = list(linestring.coords) spatial_index = Rtree() for i in range(len(coords)-1): x = map(lambda xy: xy[0],coords[i:i+2]) y = map(lambda xy: xy[1],coords[i:i+2]) spatial_index.add(i,(min(x),min(y),max(x),max(y))) return spatial_index
class Dispatch: def __init__(self, config_filename): self.index = Rtree() # get children from yaml file config_str = open(config_filename).read() print config_str self.children = yaml.load( config_str ) # index children according to their bounding box for i, child in enumerate(self.children): self.index.add(i, child['bounds']) def register(self, baseurl, left, bottom, right, top): if baseurl in self.children.values(): raise Exception( "Child with baseurl '%s' already registered"%baseurl ) childindex = len(self.children) self.children[childindex] = (baseurl, (left,bottom,right,top)) self.index.add( childindex, (left, bottom, right, top) ) return json.dumps(self.children) def children(self): return json.dumps( self.children ) def _over(self, lat, lon): return [self.children[x] for x in self.index.intersection( (lon, lat, lon, lat) )] def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85): child_servers = self._over( lat, lon ) if len(child_servers) == 0: return "NO SHEDS HERE" child_server = child_servers[0] args = {'lat':lat, 'lon':lon, 'year':year, 'month':month, 'day':day, 'hour':hour, 'minute':minute, 'second':second, 'cutoff':cutoff, 'step':step, 'speed':speed} contoururl = "http://%s/contour?%s&encoded=true"%(child_server['url'], urlencode(args)) return urlopen(contoururl).read()
class WayIndex: def __init__(self, ways): self.max_way_id = 0 self.way_idx_mapping = {} self.idx = Rtree() for way in ways: self.add(way) def add(self, way): self.idx.add(id=self.max_way_id, coordinates=way.get_bbox(), obj=way.id) self.way_idx_mapping[way.id] = self.max_way_id self.max_way_id += 1 def delete(self, way): self.idx.delete(id=self.way_idx_mapping[way.id], coordinates=way.get_bbox()) way_ids = map(lambda way: way.object, wayidx.idx.intersection(way.get_bbox(), objects=True))
class SpatialIndex(Persistent): def __init__(self, *args): self.rtree_args = args self.rtree = Rtree(*args) self.backward = IOBTree() def index_doc(self, docid, value): if docid in self.backward: self.unindex_doc(docid) self.backward[docid] = value self.rtree.add(docid, value, obj=docid) def unindex_doc(self, docid): value = self.backward.get(docid) if value is None: return self.rtree.delete(docid, value) del self.backward[docid] def apply(self, value): return [x.object for x in self.rtree.intersection(value, objects=True)] def clear(self): self.backward.clear() props = self.rtree.properties if props.storage == RT_Disk: self.rtree.close() fname = props.filename try: os.unlink('%s.%s' % (fname, props.dat_extension)) except OSError: pass try: os.unlink('%s.%s' % (fname, props.idx_extension)) except OSError: pass self.rtree = Rtree(*self.rtree_args) def count(self, value): return self.rtree.count(value)
class SampleSpace(object): def __init__(self): self._index = Rtree() self._locations = [] self._values = [] def __setitem__(self, location, value): i = len(self._locations) self._locations.append(location) self._values.append(value) self._index.add(i, self._locations[i]) def __getitem__(self, location): js = list(self._index.nearest(location, 3)) if len(js) == 0: return 0 if len(js) == 1: return self._values[js[0]] ds = [sqrt(sum([(self._locations[j][i]-location[i])**2 for i in range(2)])) for j in js] for i in range(len(js)): if ds[i] == 0: return self._values[js[i]] R = max(ds) nums = [((R - d)/(R * d))**2 for d in ds] den = sum(nums) ws = [num/den for num in nums] return sum([self._values[js[i]] * ws[i] for i in range(len(js))]) def __iter__(self): for i in range(len(self._values)): yield self._locations[i], self._values[i]
class Datasource: """ Store an exploded representation of a data source, so it can be simplified. """ def __init__(self, srs, geom_type, fields, values, shapes): """ Use load() to call this constructor. """ self.srs = srs self.fields = fields self.geom_type = geom_type self.values = values self.shapes = shapes # this will be changed later self.tolerance = 0 # guid, src1_id, src2_id, line_id, x1, y1, x2, y2 db = connect(':memory:').cursor() db.execute("""CREATE table segments ( -- global identifier for this segment guid INTEGER PRIMARY KEY AUTOINCREMENT, -- identifiers for source shape or shapes for shared borders src1_id INTEGER, src2_id INTEGER, -- global identifier for this line line_id INTEGER, -- start and end coordinates for this segment x1 REAL, y1 REAL, x2 REAL, y2 REAL, -- flag removed INTEGER )""") db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)') db.execute('CREATE INDEX shape1_parts ON segments (src1_id)') db.execute('CREATE INDEX shape2_parts ON segments (src2_id)') self.db = db self.rtree = Rtree() self.memo_line = make_memo_line() def _indexes(self): return range(len(self.values)) def simplify(self, tolerance, verbose=False): """ Simplify the polygonal linework. This method can be called multiple times, but the process is destructive so it must be called with progressively increasing tolerance values. """ if tolerance < self.tolerance: raise Exception( 'Repeat calls to simplify must have increasing tolerances.') self.tolerance = tolerance q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC' line_ids = [line_id for (line_id, count) in self.db.execute(q)] stable_lines = set() while True: was = self.db.execute( 'SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0] preserved, popped = set(), False for line_id in line_ids: if line_id in stable_lines: continue # For each coordinate that forms the apex of a two-segment # triangle, find the area of that triangle and put it into a list # along with the segment identifier and the resulting line if the # triangle were flattened, ordered from smallest to largest. rows = self.db.execute( """SELECT guid, x1, y1, x2, y2 FROM segments WHERE line_id = ? AND removed = 0 ORDER BY guid""", (line_id, )) segs = [(guid, (x1, y1), (x2, y2)) for (guid, x1, y1, x2, y2) in rows] triples = [(segs[i][0], segs[i + 1][0], segs[i][1], segs[i][2], segs[i + 1][2]) for i in range(len(segs) - 1)] triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3) for (guid1, guid2, c1, c2, c3) in triples] areas = sorted([(triangle.area, guid1, guid2, c1, c3) for (guid1, guid2, triangle, c1, c3) in triangles]) min_area = self.tolerance**2 if not areas or areas[0][0] > min_area: # there's nothing to be done stable_lines.add(line_id) if verbose: stderr.write('-') continue # Reduce any segments that makes a triangle whose area is below # the minimum threshold, starting with the smallest and working up. # Mark segments to be preserved until the next iteration. for (area, guid1, guid2, ca, cb) in areas: if area > min_area: # there won't be any more points to remove. break if guid1 in preserved or guid2 in preserved: # the current segment is too close to a previously-preserved one. continue # Check the resulting flattened line against the rest # any of the original shapefile, to determine if it would # cross any existing line segment. (x1, y1), (x2, y2) = ca, cb new_line = self.memo_line(x1, y1, x2, y2) old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2)) old_rows = self.db.execute( 'SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0' % ','.join(map(str, old_guids))) old_lines = [ self.memo_line(x1, y1, x2, y2) for (x1, y1, x2, y2) in old_rows ] if True in [ new_line.crosses(old_line) for old_line in old_lines ]: if verbose: stderr.write('x%d' % line_id) continue preserved.add(guid1) preserved.add(guid2) popped = True x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1] self.db.execute( 'UPDATE segments SET removed=1 WHERE guid=%d' % guid2) self.db.execute( 'UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?', (x1, y1, x2, y2, guid1)) self.rtree.add(guid1, bbox(x1, y1, x2, y2)) if verbose: stderr.write('.') if verbose: print >> stderr, ' reduced from', was, 'to', print >> stderr, self.db.execute( 'SELECT COUNT(guid) FROM segments WHERE removed=0' ).fetchone()[0], self.rtree = Rtree() for (guid, x1, y1, x2, y2) in self.db.execute( 'SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0' ): self.rtree.add(guid1, bbox(x1, y1, x2, y2)) if verbose: print >> stderr, '.' if not popped: break
class MapMatcher(): """A very nice MapMatching class """ def __init__(self): self.GPS = GPS() self.idx = Rtree() self.nodeidx = Rtree() self.G = None self.edgeindex_edge = {} self.edgecounter = 0 self.nodecounter = 0 self.gps_points = []; self.edge_id__count = {} self.node_counter__node = {} self.result_counter = 0 def saveGraph(self, filename): """Saves the graph as a YAML file """ nx.write_yaml(self.G,filename) def readGraphFromYAMLFile(self, filename): """Loads a graph from an YAML file. """ self.G = nx.read_yaml(filename) # TODO: buiild up the indexes !!! def addNodeToIndex(self, node): """Adds a node to the node index (RTree) """ # self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node) self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1])) self.node_counter__node[self.nodecounter] = node def addEdgeToIndex(self, edge): """Add an edge to the edhe index. """ self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge) # print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()) self.edgeindex_edge[self.edgecounter] = edge self.edgecounter = self.edgecounter + 1 def openShape(self, inFile, index=0): self.shapeFile = ogr.Open(inFile) if self.shapeFile is None: print "Failed to open " + inFile + ".\n" sys.exit( 1 ) else: print "SHP file successfully read" def getfieldinfo(self, lyr, feature, flds): f = feature return [f.GetField(f.GetFieldIndex(x)) for x in flds] def addlyr(self, G,lyr, fields): point_coords__nodes = {} for findex in xrange(lyr.GetFeatureCount()): f = lyr.GetFeature(findex) flddata = self.getfieldinfo(lyr, f, fields) g = f.geometry() attributes = dict(zip(fields, flddata)) attributes["ShpName"] = lyr.GetName() if g.GetGeometryType() == 2: #linestring last = g.GetPointCount() - 1 p_from = g.GetPoint_2D(0) p_to = g.GetPoint_2D(last) # check whether we have a node in the index intersection_mask = (p_from[0]-INTERSECTION_MASK/2, p_from[1]-INTERSECTION_MASK/2, p_from[0]+INTERSECTION_MASK/2, p_from[1]+INTERSECTION_MASK/2) results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pfrom self.nodeidx.add(self.nodecounter, (p_from[0], p_from[1], p_from[0], p_from[1])) # print p_from self.nodecounter = self.nodecounter + 1 else: print len(results) print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pfrom = self.node_counter__node[results[0]] intersection_mask = (p_to[0]-INTERSECTION_MASK/2, p_to[1]-INTERSECTION_MASK/2, p_to[0]+INTERSECTION_MASK/2, p_to[1]+INTERSECTION_MASK/2) # print intersection_mask results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pto self.nodeidx.add(self.nodecounter, (p_to[0], p_to[1], p_to[0], p_to[1])) self.nodecounter = self.nodecounter + 1 else: print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pto = self.node_counter__node[results[0]] shly_geom = shapely.wkt.loads(g.ExportToWkt()) e = Edge(pfrom, pto, attributes, geometry = shly_geom) # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter) self.addEdgeToIndex(e) #if g.GetGeometryType() == 1: #point # G.add_node((g.GetPoint_2D(0)), attributes) # if g.GetGeometryType() == 2: #linestring # last = g.GetPointCount() - 1 # # p_from = g.GetPoint_2D(0) # p_to = g.GetPoint_2D(last) # # if point_coords__nodes.get(p_from): # # pfrom = point_coords__nodes.get(p_from) # print "node " + str(pfrom.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR')) # # else: # # pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) # self.nodecounter = self.nodecounter + 1 # point_coords__nodes[p_from] = pfrom # # if point_coords__nodes.get(p_to): # # pto = point_coords__nodes.get(p_to) # print "node " + str(pto.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR')) # # else: # # pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) # self.nodecounter = self.nodecounter + 1 # point_coords__nodes[p_to] = pto # # shly_geom = shapely.wkt.loads(g.ExportToWkt()) # e = Edge(pfrom, pto, attributes, geometry = shly_geom) # # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) # # # we pull the nodes out of the graph again to index them # edges_dict = nx.get_edge_attributes(G,"edgecounter") # # # import pdb;pdb.set_trace() # # edges_keys = edges_dict.keys() # for k in edges_keys: # if self.edgecounter == edges_dict[k]: # self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0] # self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1] # self.addNodeToIndex(k[0]) # self.addNodeToIndex(k[1]) # # # let us throw the Edge into the index # self.addEdgeToIndex(e) # ## # add an edge in the other direction ## e2 = Edge(pto, pfrom, attributes, geometry = shly_geom) ## G.add_edge(pto, pfrom, {"edge": e2, "edgecounter" : self.edgecounter}) ## ## edges_dict = nx.get_edge_attributes(G,"edgecounter") ## ## # import pdb;pdb.set_trace() ## ## edges_keys = edges_dict.keys() ## for k in edges_keys: ## if self.edgecounter == edges_dict[k]: ## self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0] ## self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1] ## self.addNodeToIndex(k[0]) ## self.addNodeToIndex(k[1]) ## ## # let us throw the Edge into the index ## self.addEdgeToIndex(e2) return G def shapeToGraph(self, inFile, uniqueId="FID"): """Loads a shapefile and builds the graph. uniqueId is the name of a unique field in the shape file. """ # self.G = nx.readwrite.nx_shp.read_shp(inFile) self.G = nx.MultiGraph() self.shapeFileUniqueId = uniqueId lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory for lyrindex in xrange(lyrcount): lyr = self.shapeFile.GetLayerByIndex(lyrindex) flds = [x.GetName() for x in lyr.schema] self.G=self.addlyr(self.G, lyr, flds) self.routefinder = RouteFinder(self.G) def readGPS(self, inFile): """Parses a shapefile and build the GPS object """ self.GPS.readFromShapeFile(inFile) self.gps_points = self.GPS.getGPSPoints() def maxGPSDistance(self): """Calculate the maximum distance of two consecutive GPS Points """ # TODO check whether GPS points are already there # TODO: move into sl.gps.GPS() maxDistance = 0 gps_point = self.gps_points[0] for gpspoint in self.gps_points: distance = gpspoint.getGeometry().distance(gps_point.getGeometry()) gps_point = gps_point if distance > maxDistance: maxDistance = distance return maxDistance def nearPoints(self): """Sums up the gps point per edge segment. Stores in self.edge_id__count """ # initialize the edge counter for edge in self.G.edges(): self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0 for point in self.gps_points: nearest_edge = self.getNearestEdge(point) # print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id')) self.addPointCountToEdge(nearest_edge) def addPointCountToEdge(self, edge): """Increments the point counter for the given edge by one. """ attributes = edge.getAttributes() if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)): self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1 else: self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1 edge.setAttributes(attributes) def getNearestEdge(self, point): """Returns the edge closes to a Shapely entity given (point) """ edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True) edges = [e.object for e in edge] if len(edges) == 1: result = edges[0] else: dist = 99999999999999999999999999999999999999999 for edge in edges: distance = point.getPoint().distance(edge.getGeometry()) if distance < dist: dist = distance result = edge return result def getNearestNode(self, point): """Returns the closest node to a GPS point. """ nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y))) return self.node_counter__node.get(nodes[0]) def findRoute(self, returnNonSelection=False): """Finds a route from the node closest to the first GPS point to the node closest to the latest GPS point. """ # pick the start and end GPS points # TODO: sort GPS Points first start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) # the start and endnodes returnes by the index are not in the graph, # therefore we need to look them up .... start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter")) end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter")) self.routfinder = RouteFinder(self.G) label_list = self.routefinder.findroutes(start_node, end_node) import pdb;pdb.set_trace() label_scores = [] # let us loop through the label list for label in label_list: number_of_points = 0 # we sum up the number of points and relate them to the length of the route for edge in label.getEdges(): edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) #we add the scores to a dict label_scores.append((label, number_of_points/label.getLength())) # print label_scores # and extract the maximum score score = 0 selected = None for ls in label_scores: if ls[1] > score: selected = ls[0] score = ls[1] if returnNonSelection: pass else: return selected def eliminiateEmptyEdges(self, distance = 100): """Loops through the GPS pointset and selects edges within a boundary of <distance> meters """ print "Edge elimination started" selected_edge_ids = [] # let us for point in self.gps_points: results = self.idx.nearest(((point.getPoint().x-distance/2), (point.getPoint().y-distance/2), (point.getPoint().x+distance/2), (point.getPoint().y+distance/2)), objects=True) for result in results: from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter")) to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter")) edge_counter = self.G.edge[from_node][to_node].get("edgecounter") if edge_counter not in selected_edge_ids: selected_edge_ids.append(edge_counter) print str(len(selected_edge_ids)) + " edges found to keep." elimination_counter = 0 for edge in self.G.edges(): edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter") if edgecounter not in selected_edge_ids: edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node) self.G.remove_edge(*edge_tuple) elimination_counter = elimination_counter + 1 print str(elimination_counter) + " edges eliminated."
class OSMDB: def __init__(self, dbname, overwrite=False, rtree_index=True): self.dbname = dbname if overwrite: try: os.remove(dbname) except OSError: pass self.conn = sqlite3.connect(dbname) if rtree_index: self.index = Rtree(dbname) else: self.index = None if overwrite: self.setup() def get_cursor(self): # Attempts to get a cursor using the current connection to the db. If we've found ourselves in a different thread # than that which the connection was made in, re-make the connection. try: ret = self.conn.cursor() except sqlite3.ProgrammingError: self.conn = sqlite3.connect(self.dbname) ret = self.conn.cursor() return ret def setup(self): c = self.get_cursor() c.execute( "CREATE TABLE nodes (id TEXT UNIQUE, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" ) c.execute("CREATE TABLE ways (id TEXT UNIQUE, tags TEXT, nds TEXT)") self.conn.commit() c.close() def create_indexes(self): c = self.get_cursor() c.execute("CREATE INDEX nodes_id ON nodes (id)") c.execute("CREATE INDEX nodes_lon ON nodes (lon)") c.execute("CREATE INDEX nodes_lat ON nodes (lat)") c.execute("CREATE INDEX ways_id ON ways (id)") self.conn.commit() c.close() def populate(self, osm_filename, dryrun=False, accept=lambda tags: True, reporter=None, create_indexes=True): print "importing %s osm from XML to sqlite database" % osm_filename c = self.get_cursor() self.n_nodes = 0 self.n_ways = 0 superself = self class OSMHandler(xml.sax.ContentHandler): @classmethod def setDocumentLocator(self, loc): pass @classmethod def startDocument(self): pass @classmethod def endDocument(self): pass @classmethod def startElement(self, name, attrs): if name == 'node': self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat'])) elif name == 'way': self.currElem = Way(attrs['id']) elif name == 'tag': self.currElem.tags[attrs['k']] = attrs['v'] elif name == 'nd': self.currElem.nd_ids.append(attrs['ref']) @classmethod def endElement(self, name): if name == 'node': if superself.n_nodes % 5000 == 0: print "node %d" % superself.n_nodes superself.n_nodes += 1 if not dryrun: superself.add_node(self.currElem, c) elif name == 'way': if superself.n_ways % 5000 == 0: print "way %d" % superself.n_ways superself.n_ways += 1 if not dryrun and accept(self.currElem.tags): superself.add_way(self.currElem, c) @classmethod def characters(self, chars): pass xml.sax.parse(osm_filename, OSMHandler) self.conn.commit() c.close() if not dryrun and create_indexes: print "indexing primary tables...", self.create_indexes() print "done" def set_endnode_ref_counts(self): """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways""" print "counting end-node references to find way split-points" c = self.get_cursor() endnode_ref_counts = {} c.execute("SELECT nds from ways") print "...counting" for i, (nds_str, ) in enumerate(c): if i % 5000 == 0: print i nds = json.loads(nds_str) for nd in nds: endnode_ref_counts[nd] = endnode_ref_counts.get(nd, 0) + 1 print "...updating nodes table" for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()): if i % 5000 == 0: print i if ref_count > 1: c.execute("UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id)) self.conn.commit() c.close() def index_endnodes(self): print "indexing endpoint nodes into rtree" c = self.get_cursor() #TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes c.execute("SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1") for id, lat, lon in c: self.index.add(int(id), (lon, lat, lon, lat)) c.close() def create_and_populate_edges_table(self, tolerant=False): self.set_endnode_ref_counts() self.index_endnodes() print "splitting ways and inserting into edge table" c = self.get_cursor() c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" ) for i, way in enumerate(self.ways()): try: if i % 5000 == 0: print i subways = [] curr_subway = [way.nds[0] ] # add first node to the current subway for nd in way.nds[1:-1]: # for every internal node of the way curr_subway.append(nd) if self.node( nd )[4] > 1: # node reference count is greater than one, node is shared by two ways subways.append(curr_subway) curr_subway = [nd] curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway subways.append(curr_subway) #insert into edge table for i, subway in enumerate(subways): coords = [(lambda x: (x[3], x[2]))(self.node(nd)) for nd in subway] packt = pack_coords(coords) dist = sum([ vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords) ]) c.execute("INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s" % (way.id, i), way.id, subway[0], subway[-1], dist, packt)) except IndexError: if tolerant: continue else: raise print "indexing edges...", c.execute("CREATE INDEX edges_id ON edges (id)") c.execute("CREATE INDEX edges_parent_id ON edges (parent_id)") print "done" self.conn.commit() c.close() def edge(self, id): c = self.get_cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id, )) try: ret = c.next() way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) except StopIteration: c.close() raise IndexError("Database does not have an edge with id '%s'" % id) c.close() return ret def edges(self): c = self.get_cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" ) for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c: yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) c.close() def add_way(self, way, curs=None): if curs is None: curs = self.get_cursor() close_cursor = True else: close_cursor = False curs.execute( "INSERT OR IGNORE INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids))) if close_cursor: self.conn.commit() curs.close() def add_node(self, node, curs=None): if curs is None: curs = self.get_cursor() close_cursor = True else: close_cursor = False curs.execute( "INSERT OR IGNORE INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", (node.id, json.dumps(node.tags), node.lat, node.lon)) if close_cursor: self.conn.commit() curs.close() def nodes(self): c = self.get_cursor() c.execute("SELECT * FROM nodes") for node_row in c: yield node_row c.close() def node(self, id): c = self.get_cursor() c.execute("SELECT * FROM nodes WHERE id = ?", (id, )) try: ret = c.next() except StopIteration: c.close() raise IndexError("Database does not have node with id '%s'" % id) c.close() return ret def nearest_node(self, lat, lon, range=0.005): c = self.get_cursor() if self.index: #print "YOUR'RE USING THE INDEX" id = list(self.index.nearest((lon, lat), 1))[0] #print "THE ID IS %d"%id c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id, )) else: c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat - range, lat + range, lon - range, lon + range)) dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists) == 0: return (None, None, None, None) return min(dists, key=lambda x: x[3]) def nearest_of(self, lat, lon, nodes): c = self.get_cursor() c.execute("SELECT id, lat, lon FROM nodes WHERE id IN (%s)" % ",".join([str(x) for x in nodes])) dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists) == 0: return (None, None, None, None) return min(dists, key=lambda x: x[3]) def way(self, id): c = self.get_cursor() c.execute("SELECT id, tags, nds FROM ways WHERE id = ?", (id, )) try: id, tags_str, nds_str = c.next() ret = WayRecord(id, tags_str, nds_str) except StopIteration: raise Exception("OSMDB has no way with id '%s'" % id) finally: c.close() return ret def way_nds(self, id): c = self.get_cursor() c.execute("SELECT nds FROM ways WHERE id = ?", (id, )) (nds_str, ) = c.next() c.close() return json.loads(nds_str) def ways(self): c = self.get_cursor() c.execute("SELECT id, tags, nds FROM ways") for id, tags_str, nds_str in c: yield WayRecord(id, tags_str, nds_str) c.close() def count_ways(self): c = self.get_cursor() c.execute("SELECT count(*) FROM ways") ret = c.next()[0] c.close() return ret def count_edges(self): c = self.get_cursor() c.execute("SELECT count(*) FROM edges") ret = c.next()[0] c.close() return ret def delete_way(self, id): c = self.get_cursor() c.execute("DELETE FROM ways WHERE id = ?", (id, )) c.close() def bounds(self): c = self.get_cursor() c.execute("SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes") ret = c.next() c.close() return ret def execute(self, sql, args=None): c = self.get_cursor() if args: for row in c.execute(sql, args): yield row else: for row in c.execute(sql): yield row c.close() def cursor(self): return self.get_cursor()
def populate_shared_segments_by_rtree(datasource, verbose=False): """ """ rtree = Rtree() indexes = datasource._indexes() for i in indexes: xmin, ymin, xmax, ymax = datasource.shapes[i].bounds xbuf = (xmax - xmin) * .001 ybuf = (ymax - ymin) * .001 bounds = (xmin - xbuf, ymin - ybuf, xmax + xbuf, ymax + ybuf) rtree.add(i, bounds) shared = [[] for i in indexes] for i in indexes: for j in rtree.intersection(datasource.shapes[i].bounds): if i >= j: continue shape1 = datasource.shapes[i] shape2 = datasource.shapes[j] if not shape1.intersects(shape2): continue if verbose: print >> stderr, 'Features %d and %d:' % (i, j), 'of', len( indexes), border = linemerge(shape1.intersection(shape2)) geoms = hasattr(border, 'geoms') and border.geoms or [border] for geom in geoms: try: line_id = datasource.rtree.count( datasource.rtree.get_bounds()) except RTreeError: line_id = 0 coords = list(geom.coords) segments = [coords[k:k + 2] for k in range(len(coords) - 1)] for ((x1, y1), (x2, y2)) in segments: datasource.db.execute( """INSERT INTO segments (src1_id, src2_id, line_id, x1, y1, x2, y2, removed) VALUES (?, ?, ?, ?, ?, ?, ?, 0)""", (i, j, line_id, x1, y1, x2, y2)) bbox = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2) datasource.rtree.add(datasource.db.lastrowid, bbox) if verbose: print >> stderr, len(coords), '-', shared[i].append(border) shared[j].append(border) if verbose: print >> stderr, border.type return shared
class ContourServer(Servable): def __init__(self, settings_filename): settings = yaml.load( open( settings_filename ) ) self.home_point = settings['center'] # create cache of osm-node positions self.osmdb = OSMDB( settings['osmdb_filename'] ) self.gtfsdb = GTFSDatabase( settings['gtfsdb_filename'] ) self.port = settings['port'] self.node_positions = {} self.index = Rtree() for node_id, tags, lat, lon in self.osmdb.nodes(): self.node_positions[node_id] = (lon,lat) self.index.add( int(node_id), (lon,lat,lon,lat) ) # incarnate graph from graphdb graphdb = GraphDatabase( settings['graphdb_filename'] ) self.graph = graphdb.incarnate() def strgraph(self): return str(self.graph) def vertices(self): return "\n".join( [vv.label for vv in self.graph.vertices] ) def _get_important_routes(self, spt): # set edge thicknesses t0 = time.time() print "setting thicknesses" spt.set_thicknesses( vertex_label ) t1 = time.time() print "took %s"%(t1-t0) t0 = time.time() print "finding gateway boardings" # use thicknesses to determine important boardings origin = spt.get_vertex( vertex_label ) sum_thickness = sum( [edge.thickness for edge in origin.outgoing] ) important_boardings = sorted( filter(lambda x: x.payload.__class__==core.TripBoard and \ x.thickness/float(sum_thickness) > 0.01, spt.edges), key = lambda x:x.thickness ) for edge in important_boardings: print "gateway to %f%% vertices"%(100*edge.thickness/float(sum_thickness)) print "hop onto trip '%s' at stop '%s', time '%s'"%(edge.to_v.payload.trip_id, edge.from_v.label, edge.to_v.payload.time) print "took %ss"%(time.time()-t0) def _points(self, vertex_label, starttime, cutoff, speed): starttime = starttime or time.time() #=== find shortest path tree === print "Finding shortest path tree" t0 = time.time() wo = WalkOptions() wo.walking_speed = speed spt = self.graph.shortest_path_tree( vertex_label, None, State(1,starttime), wo, maxtime=starttime+int(cutoff*1.25) ) wo.destroy() t1 = time.time() print "took %s s"%(t1-t0) #=== cobble together ETA surface === print "Creating ETA surface from OSM points..." points = [] t0 = time.time() for vertex in spt.vertices: if "osm" in vertex.label: x, y = self.node_positions[vertex.label[3:]] points.append( (x, y, vertex.payload.time-starttime) ) t1 = time.time() print "Took %s s"%(t1-t0) spt.destroy() return points def _contour(self, vertex_label, starttime, cutoff, step=None, speed=0.85): points = self._points( vertex_label, starttime, cutoff, speed ) #=== create contour === print "creating contour...", t0 = time.time() contours = travel_time_contour( points, cutoff=cutoff, cellsize=0.004, fudge=1.7, step=step ) print "%s sec"%(time.time()-t0) print "done. here you go..." return contours def label_contour(self, vertex_label, starttime=None, cutoff=1800): starttime = starttime or time.time() return json.dumps( self._contour( vertex_label, starttime, cutoff ) ) def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85): if step is not None and step < 600: raise Exception( "Step cannot be less than 600 seconds" ) starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" ) #=== get osm vertex == print( "getting nearest vertex" ) #find osmid of origin intersection t0 = time.time() range = 0.001 bbox = (lon-range, lat-range, lon+range, lat+range) candidates = self.index.intersection( bbox ) vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates ) t1 = time.time() print( "done, took %s seconds"%(t1-t0) ) #vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon ) if vlabel is None: return json.dumps( "NO NEARBY INTERSECTION" ) print( "found - %s"%vlabel ) contours = self._contour( "osm"+vlabel, starttime, cutoff, step, speed ) if encoded: encoded_contours = [] for contour in contours: encoded_contour = [] for ring in contour: encoded_contour.append( encode_pairs( [(lat,lon) for lon,lat in ring] ) ) encoded_contours.append( encoded_contour ) contours = encoded_contours return json.dumps( contours ) def _surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed, cellsize=0.004): starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" ) #=== get osm vertex == print( "getting nearest vertex" ) #find osmid of origin intersection t0 = time.time() range = 0.001 bbox = (lon-range, lat-range, lon+range, lat+range) candidates = self.index.intersection( bbox ) vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates ) t1 = time.time() print( "done, took %s seconds"%(t1-t0) ) #vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon ) if vlabel is None: return json.dumps( "NO NEARBY INTERSECTION" ) print( "found - %s"%vlabel ) #=== get points which comprise ETA surface === points = self._points( "osm"+vlabel, starttime, cutoff, speed ) #=== create regular grid from ETA surface === print "creating surface...", t0 = time.time() ret = points_to_surface_grid( points, cutoff=cutoff, fudge=1.1, margin=2, closure_tolerance=0.05, cellsize=cellsize ) #ret = travel_time_surface( points, cutoff=cutoff, cellsize=0.004, fudge=1.7 ) print "%s sec"%(time.time()-t0) print "done. here you go..." return ret def surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85): return json.dumps( self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed).to_matrix() ) def transitability(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85): grid = self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed) if type(grid) == str: return None ret = 0 for i in range(10): contourslice=sum( [len(filter(lambda x:x[2]<=(cutoff/10.0)*(i+1),col)) for col in grid] ) print contourslice ret += contourslice return ret def transitability_surface(self, left, bottom, right, top, res, year, month, day, hour, minute, second, cutoff, speed=0.85): step = (right-left)/res return json.dumps([[(lon,lat,self.transitability(lat,lon,year,month,day,hour,minute,second,cutoff,speed)) for lat in frange(bottom,top,step)] for lon in frange(left,right,step)]) def nodes(self): return "\n".join( ["%s-%s"%(k,v) for k,v in self.node_positions.items()] ) def nearest_node(self, lat, lon): range = 0.005 bbox = (lon-range, lat-range, lon+range, lat+range) print bbox print self.index.intersection( bbox ) return json.dumps(self.osmdb.nearest_node( lat, lon )) def bounds(self): return json.dumps(self.osmdb.bounds()) def run_test_server(self): Servable.run_test_server(self, self.port)
class GisCanvas: """ Top-level drawing class that contains a list of all the objects to draw. """ def __init__(self): """Initialise the class by creating an instance of each object.""" self._ratio_index = (0.05, 0.1, 0.2, 0.5, 1, 2, 4, 8, 16, 32, 64) # pagal sita dydi turi kisti tasko atstumas nuo zoominimo centro self._zoom_level = self._ratio_index.index(4) # 0-dirbame su realiomis koordinatemis - kitur koordinates yra gaunamos atliekant dalyba... todel nera labai tikslios self._device_area = None # gtk.gdk.Rectangle() self._shapes = [] # shapes data self._index_rtree = Rtree() # shapes indexes in self._shapes self._offset = Point(0, 0) #- naudojamas ctx.translate() #device koordinates self._prj = GisProjection() self._styler = Styler(self) self._painter = Painter(self._styler) self._drag_to_center = True # jeigu norime kad resize metu (bus iskviestas set_device_area) butu iskvietsas center() def load(self, name): self._styler.load_css_file("%s.css" % name) self.load_fcad_file("%s.fcad" % name) def save(self, file_name): #self.create_fcad_file("%s.fcad" % file_name) ShapeFile.create_fcad_file("%s.fcad" % file_name, self._shapes) StyleFile.create_css_file("%s.css" % file_name, self._styler.get_shapes_style(), self._styler.get_symbols_style(), prettyprint=True) #self._styler.create_css_file("%s.css" % file_name, prettyprint=True) def clear(self): print "clear canvas!" self._zoom_level = self._ratio_index.index(1) self._offset = Point(0, 0) self._shapes = [] #self._cairo_paths = {} self._index_rtree = Rtree() self._styler.load_default_style() def load_pickle(self, file_path): timer.start("loading shapes.p") if os.path.exists(file_path): self._shapes = pickle.load(open(file_path, "rb")) if len(self._shapes): def generator_function(points): for i, obj in enumerate(points): if obj == None: continue yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj) self._index_rtree = Rtree(generator_function(self._shapes)) timer.end("loading shapes.p") def save_pickle(self, file_path): pickle.dump(self._shapes, open(file_path, "wb")) def load_fcad_file(self, file_path): timer.start("loading shapes.fcad") self._shapes = ShapeFile.read_fcad_file(file_path) if len(self._shapes): def generator_function(points): for i, obj in enumerate(points): if obj == None: continue yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj) self._index_rtree = Rtree(generator_function(self._shapes)) timer.end("loading shapes.fcad") def set_device_area(self, area): # atnaujina screen.resize() self._device_area = area if self._drag_to_center: self.center() self._drag_to_center = False def get_shape_by_id(self, id): compressed_shape = self._shapes[id] if compressed_shape == None: return None return fshape.Shape.decompress(compressed_shape) def get_object_by_id(self, id): return self._shapes[id] def draw_100(self, ctx, area): """100% draw for printing, no scale, area in user coordinates""" page_area = Area(0, 0, area.width, area.height) #ctx.rectangle(0, 0, area.width, area.height) #ctx.clip() # tam kad nepaisytu uzh sito staciakampio ribu (tada gaunasi dubliuotos linijos) #self.background(ctx, page_area) # paint white background self._painter.background(ctx, page_area, color=(1,1,1), clip=True) # paint white background self._painter.setup(ctx, transform={"translate":(-area.x, -area.y)}) radius = 0 #self.pixel_radius + self.line_width # ne cia reikia prideti radiusa, o ten kur dedame i cavas'a shape'us elements = self._index_rtree.intersection((area.x-radius, area.y-radius, area.x+area.width+radius, area.y+area.height+radius)) elements_zindex = self._styler.create_zindex(elements) # jis yra pilnas visu galimu zindex'u sarasas - pradzioje dalis ju gali buti ir tusti [] timer.start("draw100") for zindex in sorted(elements_zindex.keys()): for element in elements_zindex[zindex]: self._painter.draw(element, update=elements_zindex) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u # tie elementai bus nupaisomi veliau. timer.end("draw100") def draw_object(self, ctx, id, fill_or_stroke=True): #self.apply_transforms(ctx) # drag and scale self._painter.setup(ctx, transform={"translate":self.get_offset(), "scale": self.get_ratio()}) elements_zindex = self._styler.create_zindex([id]) timer.start("draw single object") for zindex in sorted(elements_zindex.keys()): for element in elements_zindex[zindex]: self._painter.draw(element, update=elements_zindex, fill_or_stroke=fill_or_stroke) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u timer.end("draw single object") def draw(self, ctx, area, fill_or_stroke=True): """Draw the complete drawing by drawing each object in turn.""" self._painter.background(ctx, area, color=(1,1,1), clip=True) # paint white background self._painter.setup(ctx, transform={"translate":self.get_offset(), "scale": self.get_ratio()}) # paishysime tik tuos tashkus kurie pakliuna i vartotojo langa # reikia device_area konvertuoti i user koordinates x, y = self.device_to_user(Point(area.x, area.y)) x2, y2 = self.device_to_user(Point(area.x + area.width, area.y + area.height)) radius = 0 #self.pixel_radius + self.line_width # ne cia reikia prideti radiusa, o ten kur dedame i cavas'a shape'us # geriau cia, nes paprasciau yra iskviesti nupaisyti didesni gabala nei paskaiciuoti tikslu pvz linijo simboliu dydi... elements = self._index_rtree.intersection((x-radius, y-radius, x2+radius, y2+radius)) elements_zindex = self._styler.create_zindex(elements) # jis yra pilnas visu galimu zindex'u sarasas - pradzioje dalis ju gali buti ir tusti [] timer.start("draw") for zindex in sorted(elements_zindex.keys()): for element in elements_zindex[zindex]: #print "element: ", element[0][0] self._painter.draw(element, update=elements_zindex, fill_or_stroke=fill_or_stroke) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u # tie elementai bus nupaisomi veliau. timer.end("draw") def get_ratio(self, level=None): if level == None: level = self._zoom_level return self._ratio_index[level] def drag2(self, drag_offset): self._offset = Point(self._offset.x + drag_offset.x, self._offset.y + drag_offset.y) def get_offset(self): return self._offset def find_objects_at_position(self, point): #ctx = self.get_context(transform=False) # naujas kontekstas kur paisysime, transformuoti nereikia - nes tai padarys .draw() ctx = cairo.Context(cairo.ImageSurface(cairo.FORMAT_ARGB32, 0, 0)) # naujas kontekstas kur paisysime, transformuoti nereikia - nes tai padarys .draw() area = Area(point.x, point.y, 1, 1) listener = ContextObjectsListener(point) listener_id = self._painter.addContextListener(listener) try: self.draw(ctx, area, fill_or_stroke=False) # nepaisysime tik sukursime path'us context'e ir su jais kazka atliks ContextListeneris finally: self._painter.removeContextListener(listener_id) return listener.get_objects() # rastu elementu indeksai def get_shape_redraw_area(self, id): #ctx = self.get_context(transform=False) ctx = cairo.Context(cairo.ImageSurface(cairo.FORMAT_ARGB32, 0, 0)) shape = self.get_object_by_id(id) if shape == None: return None listener = ContextBoundsListener() listener_id = self._painter.addContextListener(listener) try: self.draw_object(ctx, id, fill_or_stroke=False) finally: self._painter.removeContextListener(listener_id) area = listener.get_area() #print "Area: ", area return area def device_to_user(self, point): #x, y = self.ctx.device_to_user(point.x, point.y) #x, y = self.get_context().device_to_user(point.x, point.y) ratio = self.get_ratio() offset = self.get_offset() x, y = (point.x - offset.x)/ratio, (point.y - offset.y)/ratio return Point(x, y) def user_to_device(self, point, offset=(0, 0)): #x, y = self.ctx.user_to_device(point.x, point.y) #x, y = self.get_context().user_to_device(point.x, point.y) ratio = self.get_ratio() drag_offset = self.get_offset() x, y = (point.x * ratio) + drag_offset.x, (point.y * ratio) + drag_offset.y return Point(x + offset[0], y + offset[1]) def add(self, shape): id = len(self._shapes) # top element index self._shapes.append(shape.compress()) self._index_rtree.add(id, self._styler.get_bbox(shape)) return id def remove(self, id): shape = self.get_shape_by_id(id) self._index_rtree.delete(id, shape.bbox()) # po sito as jau niekada tokio id negausiu (nes viskas eina per rtree) self._shapes[id] = None def replace(self, id, shape): old = self.get_shape_by_id(id) #print "before :", list(self._index_rtree.intersection(old.bbox())) if old != None: self._index_rtree.delete(id, old.bbox()) #print "after :", list(self._index_rtree.intersection(old.bbox())) self._shapes[id] = shape.compress() self._index_rtree.add(id, self._styler.get_bbox(shape)) def zoom(self, direction, center=None): """center cia yra device koordinatemis - jeigu butu paspausta su zoom irankiu peles pagalba""" new_zoom_level = self._zoom_level+direction if new_zoom_level in range(0, len(self._ratio_index)): #esamas centro taskas atlikus zooma turi islikti toje pacioje vietoje center = center or Point(self._device_area.width/2, self._device_area.height/2) # vartotojo parinktas tashkas arba tiesiog centras center_user = self.device_to_user(center) # gauname priesh tai buvusio centro koordinates naujame zoom lygyje new_ratio = self.get_ratio(new_zoom_level) new_center = Point(center_user.x * new_ratio, center_user.y * new_ratio) # gauname centro poslinki (per tiek reikia perstumti visa vaizdeli) self._offset = Point(center.x - new_center.x, center.y - new_center.y) # naujas poslinkis #print "zoom: ", self._offset self._zoom_level = new_zoom_level return True return False def center(self, point=None): """center to user point""" if not point: bounds = self._index_rtree.bounds point = Point((bounds[0]+bounds[2])/2.0, (bounds[1]+bounds[3])/2.0) hand = self.user_to_device(point) to = Point(self._device_area.width/2, self._device_area.height/2) self.drag2(Point(to.x-hand.x, to.y-hand.y)) def get_projection(self): return self._prj def get_styler(self): return self._styler def load_ocad_file(self, file_path, generator=True): #import ocadfile #self.add(Shape(1, Point(0, 0), symbol="graphics")) # koordinaciu centras self._prj = GisProjection() self._zoom_level = self._ratio_index.index(4) of = OcadFile(file_path, self._prj) #self._styler.load_ocad_symbols(of, prj) timer.start("Adding ocad symbols") self._styler.set_symbols_style(of.get_symbols_style()) timer.end("Adding ocad symbols") timer.start("Adding ocad elements") shapes = of.get_shapes() print "Shapes: ", len(shapes) for shape in shapes: self.add(shape) timer.end("Adding ocad elements") self.center() def load_shape_file(self, file_path, generator=True): import shapefile from random import randint sf = shapefile.Reader(file_path) print "Number of shapes: ", sf.numRecords self.add(fshape.Shape(1, Point(0, 0), style={"z-index":99})) # koordinaciu centras if self.prj.scale == 1: # pirmas kartas prj = GisProjection(self, sf.bbox) #po sito ciklo jau turesime zemelapio ribas center = prj.map_to_user(prj.get_center()) self.center(center) self.prj = prj timer.start("Adding shapes") symbol = sf.shapeName.split("/")[-1] self._styler.set_symbol_style(symbol, {"color": (randint(0,255),randint(0,255),randint(0,255))}) for shape in sf.ogis_shapes(self.prj): self.add(fshape.Shape(shape.shapeType, shape.points, symbol=symbol)) timer.end("Adding shapes") def add_random_points(self, number, area, generator=True): """Kai generator=False - sunaudoja maziau atminties ikrovimo metu, bet trunka gerokai leciau """ self._shapes = [] timer.start("Adding random data") from random import randint for x in range(0, number): color = 65536 * randint(0,255) + 256 * randint(0,255) + randint(0,255) # RGBint x, y = randint(2, area.width), randint(2, area.height) if not generator: # darysime rtree.add kiekvienam taskui atskirai self.add(fshape.Shape(1, Point(x, y), color=color)) else: self._shapes.append((1, color, x, y)) if generator: def generator_function(points): for i, obj in enumerate(points): yield (i, (obj[2], obj[3], obj[2], obj[3]), obj) self._index_rtree = Rtree(generator_function(self._shapes)) timer.end("Adding random data")
def add(self, id_, coords): with threading.RLock(): Rtree.add(self, id_, coords)
from shapely.geometry import Point, Polygon from shapely import wkt from rtree import Rtree import sys import struct output = sys.argv[1] data_filename = output + '.data' index_filename = output + '.data.idx' data = open(data_filename, 'wb') index = open(index_filename, 'wb') tree = Rtree(output, overwrite=1) id = 0 offset = 0 for l in sys.stdin: x = wkt.loads(l) tree.add(id, x.bounds) s = len(x.wkb) data.write(x.wkb) index.write(struct.pack('L', offset)) offset += s id += 1 data.close() index.close()
def geometry_spatial_index(geometries): spatial_index = Rtree() for g,geom in enumerate(geometries): spatial_index.add(g,geom.bounds) return spatial_index
# TODO: path hackery. if __name__ == "__main__": import sys, os mypath = os.path.dirname(sys.argv[0]) sys.path.append(os.path.abspath(os.path.join(mypath, "../../"))) from pyrtree.bench.bench_rtree import ITER, INTERVAL from tests import RectangleGen import time from rtree import Rtree if __name__ == "__main__": G = RectangleGen() idx = Rtree() # this is a libspatialindex one. start = time.clock() interval_start = time.clock() for v in range(ITER): if 0 == (v % INTERVAL): # interval time taken, total time taken, # rects, cur max depth t = time.clock() dt = t - interval_start print("%d,%s,%f" % (v, "itime_t", dt)) print("%d,%s,%f" % (v, "avg_insert_t", (dt / float(INTERVAL)))) #print("%d,%s,%d" % (v, "max_depth", rt.node.max_depth())) #print("%d,%s,%d" % (v, "mean_depth", rt.node.mean_depth())) interval_start = time.clock() rect = G.rect(0.000001) idx.add(v, rect.coords())
from rtree import Rtree idx = Rtree() idx.add(5, (1,2,1,2)) idx.add(1, (2,0,2,0)) idx.add(2, (3,0,3,0)) idx.add(3, (4,0,4,0)) idx.add(4, (5,0,5,0)) idx2 = Rtree() idx2.add(10, (0,0,0,0)) idx2.add(11, (1,1,1,1)) print 'Dimensions: %d' % (idx.properties.dimension,) h = idx.hausdorff(idx2) id1 = h[1].contents.value id2 = h[2].contents.value print 'Hausdorff distance: %.2f, %d, %d' % (h[0], id1, id2)
def main(): # You can edit these vars: # ----------------------- filename = 'matrix_plot3.png' numberOfPoints = 200 gridWidth = 10000 gridHeight = 10000 edgesMinimum = 2 edgesMaximum = 4 # The maximum number of edges (connecting lines) a vertex can have # ----------------------- edgesMaxDistance = (gridWidth / numberOfPoints) * 2 # instantiate the Rtree class RtreeIndex = Rtree() # CREATE THE RANDOM POINTS: shapelyPointIndex = [ ] # Stores index = shapely object - We will reference this point from its Index from this point on. for idx in range( numberOfPoints): # Each point gets its own index defined by i x = random.randint(0, gridWidth) y = random.randint(0, gridHeight) iPoint = Point(x, y) # Create a new shapely POINT object shapelyPointIndex.append(iPoint) RtreeIndex.add(idx, (x, y, x, y)) # Add the point to the RTREE Index del idx #unset temp variable # -------------------------------------------------------------------------- #MultiPoint([(0, 0), (1, 1)]) #point.coords #point.y point.x lineMatrices = [] ComplexLines = [] graphIndex = { } # This is a dictionary we will build into a graph/matrix - Each key will be an integer representing an index # and each value will be a dictionary of dictionaries representing edges. # iterate over all points to calculate each points nearest points # build line segments for each point to near points. # add all edges to a master complex line to compute intersection. for idx, point in enumerate(shapelyPointIndex): # Find nearby points quickly rtreeMatches = list() sRadiusX = gridWidth / 10 sRadiusY = gridHeight / 10 tries = 0 while (len(rtreeMatches) < edgesMinimum + 1): # Add 1 because we can assume it will always find itself # Coords is a bounding-box to search within. coords = (point.x - sRadiusX, point.y - sRadiusY, point.x + sRadiusX, point.y + sRadiusY) print "searching coords" print coords rtreeMatches = list(RtreeIndex.intersection(coords)) print "found: ", rtreeMatches, " ", len(rtreeMatches) tries += 1 sRadiusX *= 1.5 sRadiusY *= 1.5 if (tries > 6): #Don't run forever if there is a logic problem. break del tries, sRadiusX, sRadiusY # FIND POSSIBLE EDGES TO ADD BY SEARCHING REGION RECURSIVELY ----------- # AND CALCULATE DISTANCES TO POSSIBLE POINTS --------------------------- # This dictionary will store our distances to each other point from this point. edgeDistances = {} # Calculate distances to matched points for pointIdx in rtreeMatches: if (pointIdx == idx): continue # Don't record a distance to itself mPoint = shapelyPointIndex[pointIdx] edgeDistances[pointIdx] = round( mPoint.distance(point), 2) # keep the distance accuracy reasonable del mPoint, pointIdx, rtreeMatches # create a list that references the distances index for the closest indexes and is sorted by close to far. edgeDistancesIdx = sorted(edgeDistances, key=edgeDistances.get) #----------------------------------------------------------------------- # DECIDE WHICH EDGES TO ADD -------------------------------------------- # Now add a minimum number of points, and try to keep them within a logic distance edgesAdded = 0 edgesIdx = [] # this will contain indexes to our added edges for pointIdx in edgeDistancesIdx: distance = edgeDistances[pointIdx] # Stop adding edges if we've meant the minimum number of edges needed and the rest of the edges are too far away, or we have reach the maximum number of allowed edges if ((edgesAdded >= edgesMinimum and distance > edgesMaxDistance) or edgesAdded >= edgesMaximum): break edgesIdx.append(pointIdx) del pointIdx #----------------------------------------------------------------------- # Initialize the graphIndex as a dict() for the current vertex if it hasn't been. if (idx not in graphIndex): graphIndex[idx] = {} #ADD THE EDGES TO THE GRID STRUCTURE ----------------------------------- for pointIdx in edgesIdx: # Add the edge graphIndex[idx][pointIdx] = edgeDistances[pointIdx] # Add the reverse edge (so both points share an edge) if pointIdx not in graphIndex: graphIndex[pointIdx] = {} graphIndex[pointIdx][idx] = edgeDistances[pointIdx] #----------------------------------------------------------------------- # Print out the graph of vertices pprint.pprint(graphIndex) # Randomely select a Start Vertex, and an End Vertex for Shortest Path # calculation startIdx = random.randint(0, len(graphIndex) - 1) endIdx = startIdx # Calculate a random end index - and make sure its not the same as the start while (endIdx == startIdx): endIdx = random.randint(0, len(graphIndex) - 1) # Setup shortest path calculation! print "Calculating Shortest Path:\n" # (8894.6959143222557, [11, 5, 8L, 7, 9]) # GraphIndex is a dictionary of dictionaries, startIdx is the INDEX of the startpoint, endIdx is the index of the endpoint vertex. shortestPath = dijkstra.shortestpath(graphIndex, startIdx, endIdx) shortestPathVertices = shortestPath[1] shortestPathDistance = shortestPath[0] print "Shortest Path Vertices:" print shortestPathVertices, "\n" del shortestPath #_------------------------------------------------------------------------------- # DRAW A PLOT FOR THE LINES CALCULATED ABOVE: import matplotlib matplotlib.use('Agg') # Do NOT attempt to open X11 instance from pylab import * from matplotlib.patches import Circle import matplotlib.pyplot as pyplot from matplotlib.text import * mylist = lineMatrices matplotlib.rcParams['lines.linewidth'] = 2 pyplot.axis([0, gridWidth, 0, gridHeight]) pyplot.grid(True) # Setting the axis labels. pyplot.xlabel('X Space') pyplot.ylabel('Y Space') figure = pyplot.figure() labels = figure.add_subplot(111, projection='rectilinear') #Give the plot a title pyplot.title('Shapely Shortest Path Simulation') del idx if not isinstance(mylist, list): print "The matrix is not a list!" print type(mylist).__name__ for idx, edges in graphIndex.iteritems(): vertex = shapelyPointIndex[idx] for edgeIdx, edgeDistance in edges.iteritems(): # This shouldn't happen, but lets just make sure. # DONT PLOT A PATH TO OURSELF if (edgeIdx == idx): continue # Get the edges point-coordinates edgePoint = shapelyPointIndex[edgeIdx] # Delete the reverse edge so we don't plot the line twice. if (edgeIdx in graphIndex): del graphIndex[edgeIdx][idx] # Print Debuggin Information print "PLOTTING EDGE: [%s] to [%s] X:[%s] Y:[%s] DIS:[%s] " % ( idx, edgeIdx, edgePoint.x, edgePoint.y, edgeDistance) # Plot the edge! pyplot.plot([vertex.x, edgePoint.x], [vertex.y, edgePoint.y], 'b-', alpha=0.3, linewidth=.2, markersize=1) # FIND THE MID-POINT OF THE LINE: xMidpoint = (vertex.x + edgePoint.x) / 2 yMidpoint = (vertex.y + edgePoint.y) / 2 #print "Plotting Text [%s] [%s]" % (xMidpoint, yMidpoint) # Figure out arrows later... arrowprops=dict(facecolor='black', linewidth=2, alpha=0.4), # ANNOTATE THE LENGTH OF THE LINE AT ITS MIDPOINT labels.annotate(edgeDistance, xy=(xMidpoint, yMidpoint), xytext=(xMidpoint + 80, yMidpoint), alpha=0.4, size='4', color='green') # end for ---------------------------------------------------------- # Plot the VERTEX as a Red Circle pyplot.plot(vertex.x, vertex.y, 'ro', linewidth=2, markersize=2) # Annotate the VERTEX by its INDEX labels.annotate(idx, xy=(vertex.x, vertex.y), xytext=(vertex.x + 50, vertex.y), alpha=0.6, size='6', color='red') print "Graphing Shortest Path..." print "START: %s END: %s" % (startIdx, endIdx) #shortestPathVertices for idx, vertexIdx in enumerate(shortestPathVertices): if (vertexIdx == endIdx): break # We are done xy1 = shapelyPointIndex[vertexIdx] # this is the start of the line vertex2 = shortestPathVertices[ idx + 1] # The end of the line is the next item in the list shortest path list of indexes xy2 = shapelyPointIndex[vertex2] #this is the end of the line pyplot.plot([xy1.x, xy2.x], [xy1.y, xy2.y], 'r--', alpha=0.7, linewidth=2, markersize=1) print "PLOT -> SHORTEST PATH: [%s] to [%s]" % (vertexIdx, vertex2) print "Saving Plot Image %s" % (filename) pyplot.savefig(os.getcwd() + '/' + str(filename), dpi=240) print "\n\n" """
if __name__ == "__main__": import sys, os mypath = os.path.dirname(sys.argv[0]) sys.path.append(os.path.abspath(os.path.join(mypath, "../../"))) from pyrtree.bench.bench_rtree import ITER,INTERVAL from pyrtree.tests.test_rtree import RectangleGen,TstO import time from rtree import Rtree if __name__ == "__main__": G = RectangleGen() idx = Rtree() # this is a libspatialindex one. start = time.clock() interval_start = time.clock() for v in range(ITER): if 0 == (v % INTERVAL): # interval time taken, total time taken, # rects, cur max depth t = time.clock() dt = t - interval_start print("%d,%s,%f" % (v, "itime_t", dt)) print("%d,%s,%f" % (v, "avg_insert_t", (dt/float(INTERVAL)))) #print("%d,%s,%d" % (v, "max_depth", rt.node.max_depth())) #print("%d,%s,%d" % (v, "mean_depth", rt.node.mean_depth())) interval_start = time.clock() rect = G.rect(0.000001) idx.add(v,rect.coords())
def populate_shared_segments_by_rtree(datasource, verbose=False): """ """ rtree = Rtree() indexes = datasource._indexes() for i in indexes: xmin, ymin, xmax, ymax = datasource.shapes[i].bounds xbuf = (xmax - xmin) * .001 ybuf = (ymax - ymin) * .001 bounds = (xmin-xbuf, ymin-ybuf, xmax+xbuf, ymax+ybuf) rtree.add(i, bounds) shared = [[] for i in indexes] for i in indexes: for j in rtree.intersection(datasource.shapes[i].bounds): if i >= j: continue shape1 = datasource.shapes[i] shape2 = datasource.shapes[j] if not shape1.intersects(shape2): continue if verbose: print >> stderr, 'Features %d and %d:' % (i, j), 'of', len(indexes), border = linemerge(shape1.intersection(shape2)) geoms = hasattr(border, 'geoms') and border.geoms or [border] for geom in geoms: try: line_id = datasource.rtree.count(datasource.rtree.get_bounds()) except RTreeError: line_id = 0 coords = list(geom.coords) segments = [coords[k:k+2] for k in range(len(coords) - 1)] for ((x1, y1), (x2, y2)) in segments: datasource.db.execute("""INSERT INTO segments (src1_id, src2_id, line_id, x1, y1, x2, y2, removed) VALUES (?, ?, ?, ?, ?, ?, ?, 0)""", (i, j, line_id, x1, y1, x2, y2)) bbox = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2) datasource.rtree.add(datasource.db.lastrowid, bbox) if verbose: print >> stderr, len(coords), '-', shared[i].append(border) shared[j].append(border) if verbose: print >> stderr, border.type return shared
shapePoints = [] pointList = [] for i in range(numberOfPoints): x = random.randint(0, gridWidth) y = random.randint(0, gridHeight) # Add the points to a python list of lists pointList.append((x, y)) # Create a Shapely point object iPoint = Point(x, y) iPoint.idx = i # set our custom attribute. (if this doesnt work I have other ways) shapePoints.append(iPoint) # Add the point to an RTREE index # index.add(id=id, (left, bottom, right, top)) # Note that this would work if the RtreeIndex.add(i, (x, y, x, y)) matchingPoints = [] searchBench = time.time() for idx, point in enumerate(shapePoints): if sCircle.contains(point): matchingPoints.append(idx) searchBench = time.time() - searchBench #------------------------------------------------------------------------------- # NOW TEST RTREE RtreeBench = time.time() RtreeIndex rtreeMatches = list(RtreeIndex.intersection((boxLeft, boxBottom, boxRight, boxTop)))
class Datasource: """ Store an exploded representation of a data source, so it can be simplified. """ def __init__(self, srs, geom_type, fields, values, shapes): """ Use load() to call this constructor. """ self.srs = srs self.fields = fields self.geom_type = geom_type self.values = values self.shapes = shapes # this will be changed later self.tolerance = 0 # guid, src1_id, src2_id, line_id, x1, y1, x2, y2 db = connect(':memory:').cursor() db.execute("""CREATE table segments ( -- global identifier for this segment guid INTEGER PRIMARY KEY AUTOINCREMENT, -- identifiers for source shape or shapes for shared borders src1_id INTEGER, src2_id INTEGER, -- global identifier for this line line_id INTEGER, -- start and end coordinates for this segment x1 REAL, y1 REAL, x2 REAL, y2 REAL, -- flag removed INTEGER )""") db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)') db.execute('CREATE INDEX shape1_parts ON segments (src1_id)') db.execute('CREATE INDEX shape2_parts ON segments (src2_id)') self.db = db self.rtree = Rtree() self.memo_line = make_memo_line() def _indexes(self): return range(len(self.values)) def simplify(self, tolerance, verbose=False): """ Simplify the polygonal linework. This method can be called multiple times, but the process is destructive so it must be called with progressively increasing tolerance values. """ if tolerance < self.tolerance: raise Exception('Repeat calls to simplify must have increasing tolerances.') self.tolerance = tolerance q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC' line_ids = [line_id for (line_id, count) in self.db.execute(q)] stable_lines = set() while True: was = self.db.execute('SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0] preserved, popped = set(), False for line_id in line_ids: if line_id in stable_lines: continue # For each coordinate that forms the apex of a two-segment # triangle, find the area of that triangle and put it into a list # along with the segment identifier and the resulting line if the # triangle were flattened, ordered from smallest to largest. rows = self.db.execute("""SELECT guid, x1, y1, x2, y2 FROM segments WHERE line_id = ? AND removed = 0 ORDER BY guid""", (line_id, )) segs = [(guid, (x1, y1), (x2, y2)) for (guid, x1, y1, x2, y2) in rows] triples = [(segs[i][0], segs[i+1][0], segs[i][1], segs[i][2], segs[i+1][2]) for i in range(len(segs) - 1)] triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3) for (guid1, guid2, c1, c2, c3) in triples] areas = sorted( [(triangle.area, guid1, guid2, c1, c3) for (guid1, guid2, triangle, c1, c3) in triangles] ) min_area = self.tolerance ** 2 if not areas or areas[0][0] > min_area: # there's nothing to be done stable_lines.add(line_id) if verbose: stderr.write('-') continue # Reduce any segments that makes a triangle whose area is below # the minimum threshold, starting with the smallest and working up. # Mark segments to be preserved until the next iteration. for (area, guid1, guid2, ca, cb) in areas: if area > min_area: # there won't be any more points to remove. break if guid1 in preserved or guid2 in preserved: # the current segment is too close to a previously-preserved one. continue # Check the resulting flattened line against the rest # any of the original shapefile, to determine if it would # cross any existing line segment. (x1, y1), (x2, y2) = ca, cb new_line = self.memo_line(x1, y1, x2, y2) old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2)) old_rows = self.db.execute('SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0' % ','.join(map(str, old_guids))) old_lines = [self.memo_line(x1, y1, x2, y2) for (x1, y1, x2, y2) in old_rows] if True in [new_line.crosses(old_line) for old_line in old_lines]: if verbose: stderr.write('x%d' % line_id) continue preserved.add(guid1) preserved.add(guid2) popped = True x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1] self.db.execute('UPDATE segments SET removed=1 WHERE guid=%d' % guid2) self.db.execute('UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?', (x1, y1, x2, y2, guid1)) self.rtree.add(guid1, bbox(x1, y1, x2, y2)) if verbose: stderr.write('.') if verbose: print >> stderr, ' reduced from', was, 'to', print >> stderr, self.db.execute('SELECT COUNT(guid) FROM segments WHERE removed=0').fetchone()[0], self.rtree = Rtree() for (guid, x1, y1, x2, y2) in self.db.execute('SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0'): self.rtree.add(guid1, bbox(x1, y1, x2, y2)) if verbose: print >> stderr, '.' if not popped: break
class GraphicScene(object): ITEM_SELECTION_RADIUS = 2 # px _logger = logging.getLogger(__name__) ############################################## def __init__(self, glortho2d): self._glortho2d = glortho2d # Fixme: used for window_to_scene_coordinate window_to_scene_distance self._items = {} self._rtree = Rtree() self._selected_item = None ############################################## def _window_to_scene_coordinate(self, event): location = np.array((event.x(), event.y()), dtype=np.float) return list(self._glortho2d.window_to_gl_coordinate(location)) ############################################## def _window_to_scene_distance(self, x): return self._glortho2d.window_to_gl_distance(x) ############################################## def add_item(self, item): item_id = hash(item) self._items[item_id] = item self._rtree.add(item_id, item.bounding_box) ############################################## def remove_item(self, item): if self._selected_item is item: self.deselect_item(item) item_id = hash(item) del self._items[item_id] self._rtree.delete(item_id, item.bounding_box) ############################################## def select_item(self, item): item.is_selected = True self._selected_item = item ############################################## def deselect_item(self, item): item.is_selected = False self._selected_item = None ############################################## def items_in(self, interval): self._logger.debug(str( interval)) items = [self._items[x] for x in self._rtree.intersection(interval.bounding_box())] items.sort() # accordind to z value return items ############################################## def items_at(self, x, y): return self.items_in(point_interval(x, y)) ############################################## def items_around(self, x, y, radius): return self.items_in(centred_interval(x, y, radius)) ############################################## def item_under_mouse(self, event): x, y = self._window_to_scene_coordinate(event) radius = self._window_to_scene_distance(GraphicScene.ITEM_SELECTION_RADIUS) items = self.items_around(x, y, radius) if items: # return the highest z value # Fixme: ok ? return items[-1] else: return None ############################################## def keyPressEvent(self, event): return False ############################################## def keyReleaseEvent(self, event): return False ############################################## def mousePressEvent(self, event): item = self.item_under_mouse(event) if item is not None: self.select_item(item) item.mousePressEvent(event) return True else: return False ############################################## def mouseReleaseEvent(self, event): if self._selected_item is not None: self._selected_item.mouseReleaseEvent(event) self.deselect_item(self._selected_item) return True else: return False ############################################## def mouseDoubleClickEvent(self, event): return False ############################################## def wheelEvent(self, event): return False ############################################## def mouseMoveEvent(self, event): if self._selected_item is not None: self._selected_item.mouseMoveEvent(event) return True else: return False
class IntRtreeIndex(BaseIndex): """Avoids the slower Rtree query object=True interface """ _v_nextuid = None family = BTrees.family32 def clear(self): self.fwd = Rtree() self.bwd = self.family.OO.BTree() self.keys = self.family.IO.BTree() self.intids = self.family.OI.BTree() self.ids = self.family.OO.BTree() def __init__(self): self.clear() def key(self, item): try: return item['id'], tuple(self.bbox(item)) except: return tuple(item.items()) def fid(self, item): return item['id'] def intid(self, item): # Get and track next available key using zope.intid algorithm # Item might be already registered uid = self.intids.get(self.key(item)) if uid is not None: return uid # But if not registered nextuid = getattr(self, '_v_nextuid', None) while True: if nextuid is None: nextuid = random.randrange(0, self.family.maxint) uid = nextuid if uid not in self.keys: nextuid += 1 if nextuid > self.family.maxint: nextuid = None self._v_nextuid = nextuid return uid nextuid = None def intersection(self, bbox): """Return an iterator over Items that intersect with the bbox""" for hit in self.fwd.intersection(bbox, objects=False): yield self.bwd[int(hit)] def nearest(self, bbox, limit=1): """Return an iterator over the nearest N=limit Items to the bbox""" for hit in self.fwd.nearest(bbox, num_results=limit, objects=False): yield self.bwd[int(hit)] def item(self, fid, bbox): return self.bwd[self.intids[(fid, bbox)]] def items(self, fid): return [self.bwd[intid] for intid in self.ids[fid]] def index_item(self, itemid, bbox, item): """Add an Item to the index""" if itemid in self.bwd: self.unindex_item(itemid, bbox) # Store an id for the item if it has None try: item.update(id=item.get('id') or str(uuid.uuid4())) key = self.key(item) sid = self.fid(item) # Map keys <-> intids intid = self.intid(item) self.keys[intid] = key self.intids[key] = intid if sid not in self.ids: self.ids[sid] = IISet([]) self.ids[sid].add(intid) self.bwd[intid] = item self.fwd.add(intid, bbox) except: import pdb; pdb.set_trace() raise def unindex_item(self, itemid, bbox): """Remove an Item from the index""" intid = int(itemid) key = self.keys.get(intid) if key is None: return self.ids[key[0]].remove(intid) del self.keys[intid] del self.intids[key] del self.bwd[intid] self.fwd.delete(intid, bbox) def batch(self, changeset): BaseIndex.batch(self, changeset) def commit(self): transaction.commit() rtree_storage = self.fwd.properties.filename self.fwd.close() self.fwd = Rtree(rtree_storage) def close(self): self.fwd.close()
class OSMDB: def __init__(self, dbname,overwrite=False,rtree_index=True): if overwrite: try: os.remove( dbname ) except OSError: pass self.conn = sqlite3.connect(dbname) if rtree_index: self.index = Rtree( dbname ) else: self.index = None if overwrite: self.setup() def setup(self): c = self.conn.cursor() c.execute( "CREATE TABLE nodes (id TEXT, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" ) c.execute( "CREATE TABLE ways (id TEXT, tags TEXT, nds TEXT)" ) self.conn.commit() c.close() def create_indexes(self): c = self.conn.cursor() c.execute( "CREATE INDEX nodes_id ON nodes (id)" ) c.execute( "CREATE INDEX nodes_lon ON nodes (lon)" ) c.execute( "CREATE INDEX nodes_lat ON nodes (lat)" ) c.execute( "CREATE INDEX ways_id ON ways (id)" ) self.conn.commit() c.close() def populate(self, osm_filename, accept=lambda tags: True, reporter=None): print "importing osm from XML to sqlite database" c = self.conn.cursor() self.n_nodes = 0 self.n_ways = 0 superself = self class OSMHandler(xml.sax.ContentHandler): @classmethod def setDocumentLocator(self,loc): pass @classmethod def startDocument(self): pass @classmethod def endDocument(self): pass @classmethod def startElement(self, name, attrs): if name=='node': self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat'])) elif name=='way': self.currElem = Way(attrs['id']) elif name=='tag': self.currElem.tags[attrs['k']] = attrs['v'] elif name=='nd': self.currElem.nd_ids.append( attrs['ref'] ) @classmethod def endElement(self,name): if name=='node': if superself.n_nodes%5000==0: print "node %d"%superself.n_nodes superself.n_nodes += 1 superself.add_node( self.currElem, c ) elif name=='way': if superself.n_ways%5000==0: print "way %d"%superself.n_ways superself.n_ways += 1 superself.add_way( self.currElem, c ) @classmethod def characters(self, chars): pass xml.sax.parse(osm_filename, OSMHandler) self.conn.commit() c.close() print "indexing primary tables...", self.create_indexes() print "done" def set_endnode_ref_counts( self ): """Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways""" print "counting end-node references to find way split-points" c = self.conn.cursor() endnode_ref_counts = {} c.execute( "SELECT nds from ways" ) print "...counting" for i, (nds_str,) in enumerate(c): if i%5000==0: print i nds = json.loads( nds_str ) for nd in nds: endnode_ref_counts[ nd ] = endnode_ref_counts.get( nd, 0 )+1 print "...updating nodes table" for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()): if i%5000==0: print i if ref_count > 1: c.execute( "UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id) ) self.conn.commit() c.close() def index_endnodes( self ): print "indexing endpoint nodes into rtree" c = self.conn.cursor() #TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1" ) for id, lat, lon in c: self.index.add( int(id), (lon, lat, lon, lat) ) c.close() def create_and_populate_edges_table( self, tolerant=False ): self.set_endnode_ref_counts() self.index_endnodes() print "splitting ways and inserting into edge table" c = self.conn.cursor() c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" ) for i, way in enumerate(self.ways()): try: if i%5000==0: print i subways = [] curr_subway = [ way.nds[0] ] # add first node to the current subway for nd in way.nds[1:-1]: # for every internal node of the way curr_subway.append( nd ) if self.node(nd)[4] > 1: # node reference count is greater than one, node is shared by two ways subways.append( curr_subway ) curr_subway = [ nd ] curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway subways.append( curr_subway ); #insert into edge table for i, subway in enumerate(subways): coords = [(lambda x:(x[3],x[2]))(self.node(nd)) for nd in subway] packt = pack_coords( coords ) dist = sum([vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords)]) c.execute( "INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s"%(way.id, i), way.id, subway[0], subway[-1], dist, packt) ) except IndexError: if tolerant: continue else: raise print "indexing edges...", c.execute( "CREATE INDEX edges_id ON edges (id)" ) c.execute( "CREATE INDEX edges_parent_id ON edges (parent_id)" ) print "done" self.conn.commit() c.close() def edge(self, id): c = self.conn.cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id,) ) try: ret = c.next() way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords( geom ), json.loads(tags)) except StopIteration: c.close() raise IndexError( "Database does not have an edge with id '%s'"%id ) c.close() return ret def edges(self): c = self.conn.cursor() c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" ) for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c: yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags)) c.close() def add_way( self, way, curs=None ): if curs is None: curs = self.conn.cursor() close_cursor = True else: close_cursor = False curs.execute("INSERT INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids) )) if close_cursor: self.conn.commit() curs.close() def add_node( self, node, curs=None ): if curs is None: curs = self.conn.cursor() close_cursor = True else: close_cursor = False curs.execute("INSERT INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", ( node.id, json.dumps(node.tags), node.lat, node.lon ) ) if close_cursor: self.conn.commit() curs.close() def nodes(self): c = self.conn.cursor() c.execute( "SELECT * FROM nodes" ) for node_row in c: yield node_row c.close() def node(self, id): c = self.conn.cursor() c.execute( "SELECT * FROM nodes WHERE id = ?", (id,) ) try: ret = c.next() except StopIteration: c.close() raise IndexError( "Database does not have node with id '%s'"%id ) c.close() return ret def nearest_node(self, lat, lon, range=0.005): c = self.conn.cursor() if self.index: print "YOU'RE USING THE INDEX" id = list(self.index.nearest( (lon, lat), 1 ))[0] print "THE ID IS %d"%id c.execute( "SELECT id, lat, lon FROM nodes WHERE id = ?", (id,) ) else: c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat-range, lat+range, lon-range, lon+range) ) dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists)==0: return (None, None, None, None) return min( dists, key = lambda x:x[3] ) def nearest_of( self, lat, lon, nodes ): c = self.conn.cursor() c.execute( "SELECT id, lat, lon FROM nodes WHERE id IN (%s)"%",".join([str(x) for x in nodes]) ) dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c] if len(dists)==0: return (None, None, None, None) return min( dists, key = lambda x:x[3] ) def way(self, id): c = self.conn.cursor() c.execute( "SELECT id, tags, nds FROM ways WHERE id = ?", (id,) ) try: id, tags_str, nds_str = c.next() ret = WayRecord(id, tags_str, nds_str) except StopIteration: raise Exception( "OSMDB has no way with id '%s'"%id ) finally: c.close() return ret def way_nds(self, id): c = self.conn.cursor() c.execute( "SELECT nds FROM ways WHERE id = ?", (id,) ) (nds_str,) = c.next() c.close() return json.loads( nds_str ) def ways(self): c = self.conn.cursor() c.execute( "SELECT id, tags, nds FROM ways" ) for id, tags_str, nds_str in c: yield WayRecord( id, tags_str, nds_str ) c.close() def count_ways(self): c = self.conn.cursor() c.execute( "SELECT count(*) FROM ways" ) ret = c.next()[0] c.close() return ret def count_edges(self): c = self.conn.cursor() c.execute( "SELECT count(*) FROM edges" ) ret = c.next()[0] c.close() return ret def delete_way(self, id): c = self.conn.cursor() c.execute("DELETE FROM ways WHERE id = ?", (id,)) c.close() def bounds(self): c = self.conn.cursor() c.execute( "SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes" ) ret = c.next() c.close() return ret def execute(self,sql,args=None): c = self.conn.cursor() if args: for row in c.execute(sql,args): yield row else: for row in c.execute(sql): yield row c.close() def cursor(self): return self.conn.cursor()
class MapMatcher(): """A very nice MapMatching class """ def __init__(self): self.GPS = GPS() self.idx = Rtree() self.nodeidx = Rtree() self.G = None self.edgeindex_edge = {} self.edgecounter = 0 self.nodecounter = 0 self.gps_points = []; self.edge_id__count = {} self.node_counter__node = {} self.distance_matrix = {} def saveGraph(self, filename): """Saves the graph as a YAML file """ nx.write_yaml(self.G,filename) def readGraphFromYAMLFile(self, filename): """Loads a graph from an YAML file. """ self.G = nx.read_yaml(filename) # TODO: buiild up the indexes !!! def addNodeToIndex(self, node): """Adds a node to the node index (RTree) """ # self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node) self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1])) self.node_counter__node[self.nodecounter] = node def addEdgeToIndex(self, edge): """Add an edge to the edhe index. """ self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge) # print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()) self.edgeindex_edge[self.edgecounter] = edge self.edgecounter = self.edgecounter + 1 def openShape(self, inFile, index=0): self.shapeFile = ogr.Open(inFile) if self.shapeFile is None: print "Failed to open " + inFile + ".\n" sys.exit( 1 ) else: print "SHP file successfully read" def getfieldinfo(self, lyr, feature, flds): f = feature return [f.GetField(f.GetFieldIndex(x)) for x in flds] def addlyr(self, G,lyr, fields): point_coords__nodes = {} for findex in xrange(lyr.GetFeatureCount()): f = lyr.GetFeature(findex) flddata = self.getfieldinfo(lyr, f, fields) g = f.geometry() attributes = dict(zip(fields, flddata)) attributes["ShpName"] = lyr.GetName() if g.GetGeometryType() == 2: #linestring last = g.GetPointCount() - 1 p_from = g.GetPoint_2D(0) p_to = g.GetPoint_2D(last) # check whether we have a node in the index intersection_mask = (p_from[0]-INTERSECTION_MASK/2, p_from[1]-INTERSECTION_MASK/2, p_from[0]+INTERSECTION_MASK/2, p_from[1]+INTERSECTION_MASK/2) results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: #print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pfrom self.nodeidx.add(self.nodecounter, (p_from[0], p_from[1], p_from[0], p_from[1])) # print p_from self.nodecounter = self.nodecounter + 1 else: #print len(results) #print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pfrom = self.node_counter__node[results[0]] intersection_mask = (p_to[0]-INTERSECTION_MASK/2, p_to[1]-INTERSECTION_MASK/2, p_to[0]+INTERSECTION_MASK/2, p_to[1]+INTERSECTION_MASK/2) # print intersection_mask results = list(self.nodeidx.intersection(intersection_mask)) if len(results)==0: #print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "." pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter}) self.node_counter__node[self.nodecounter] = pto self.nodeidx.add(self.nodecounter, (p_to[0], p_to[1], p_to[0], p_to[1])) self.nodecounter = self.nodecounter + 1 else: #print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "." pto = self.node_counter__node[results[0]] shly_geom = shapely.wkt.loads(g.ExportToWkt()) e = Edge(pfrom, pto, attributes, geometry = shly_geom) # G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter}) G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter) self.addEdgeToIndex(e) return G def shapeToGraph(self, inFile, uniqueId="FID"): """Loads a shapefile and builds the graph. uniqueId is the name of a unique field in the shape file. """ # self.G = nx.readwrite.nx_shp.read_shp(inFile) self.G = nx.MultiGraph() self.shapeFileUniqueId = uniqueId lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory for lyrindex in xrange(lyrcount): lyr = self.shapeFile.GetLayerByIndex(lyrindex) flds = [x.GetName() for x in lyr.schema] self.G=self.addlyr(self.G, lyr, flds) self.routefinder = RouteFinder(self.G, euclideanOD=self.distance_matrix) def readGPS(self, inFile): """Parses a shapefile and build the GPS object """ self.GPS.readFromShapeFile(inFile) self.gps_points = self.GPS.getGPSPoints() def maxGPSDistance(self): """Calculate the maximum distance of two consecutive GPS Points """ # TODO check whether GPS points are already there # TODO: move into sl.gps.GPS() maxDistance = 0 gps_point = self.gps_points[0] for gpspoint in self.gps_points: distance = gpspoint.getGeometry().distance(gps_point.getGeometry()) gps_point = gps_point if distance > maxDistance: maxDistance = distance return maxDistance def nearPoints(self): """Sums up the gps point per edge segment. Stores in self.edge_id__count """ # initialize the edge counter for edge in self.G.edges(): self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0 for point in self.gps_points: nearest_edge = self.getNearestEdge(point) # print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id')) self.addPointCountToEdge(nearest_edge) def addPointCountToEdge(self, edge): """Increments the point counter for the given edge by one. """ attributes = edge.getAttributes() if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)): self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1 else: self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1 edge.setAttributes(attributes) def getNearestEdge(self, point): """Returns the edge closes to a Shapely entity given (point) """ edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True) edges = [e.object for e in edge] if len(edges) == 1: result = edges[0] else: dist = 99999999999999999999999999999999999999999 for edge in edges: distance = point.getPoint().distance(edge.getGeometry()) if distance < dist: dist = distance result = edge return result def getNearestNode(self, point): """Returns the closest node to a GPS point. """ nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y))) return self.node_counter__node.get(nodes[0]) def find_all_paths(self, graph, start, end, path=[]): path = path + [start] if start == end: return [path] if not graph.has_key(start): return [] paths = [] for node in graph[start]: if node not in path: newpaths = self.find_all_paths(graph, node, end, path) for newpath in newpaths: paths.append(newpath) return paths def findRoutes2(self): start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) graph = {} print "preparing python graph" for node in self.node_counter__node.values(): graph[node.getNodeID()] = [n.getOutNode2(node).getNodeID() for n in node.getOutEdges()] import pprint pprint.pprint( graph ) print "From ", start_node.getNodeID(), " to ", end_node.getNodeID(), "." results = self.find_all_paths(graph, start_node.getNodeID(), end_node.getNodeID()) # let us find the edges route_list = [] for result in results: i=1 edges = [] while i<len(result): node_from = self.node_counter__node.get(result[i-1] ) node_to = self.node_counter__node.get(result[i] ) i = i +1 edge_dict = self.G[node_from][node_to] if edge_dict.keys()>1: lngth = 9E99999 for k in edge_dict.keys(): if edge_dict[k]['edge'].getLength() < lngth: lngth = edge_dict[k]['edge'].getLength() edge = edge_dict[k]['edge'] else: edge = edge_dict[0]['edge'] edges.append(edge) route_list.append(Route(edges)) factor__selected_route = {} for route in route_list: number_of_points = 0 length = 0 for edge in route.getEdges(): # import pdb;pdb.set_trace() length = length + edge.getLength() edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) if number_of_points > 1: factor__selected_route[number_of_points/length] = route keys = factor__selected_route.keys() keys.sort() return factor__selected_route.get(keys[0]) def findRoute(self, returnNonSelection=False): """Finds a route from the node closest to the first GPS point to the node closest to the latest GPS point. """ # pick the start and end GPS points # TODO: sort GPS Points first start_point = self.gps_points[0] end_point = self.gps_points[-1] start_node = self.getNearestNode(start_point) end_node = self.getNearestNode(end_point) # the start and endnodes returns by the index are not in the graph, # therefore we need to look them up .... start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter")) end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter")) self.routfinder = RouteFinder(self.G, euclideanOD=self.distance_matrix) label_list = self.routefinder.findroutes(start_node, end_node) label_scores = [] # import pdb;pdb.set_trace() # let us loop through the label list for label in label_list: number_of_points = 0 # we sum up the number of points and relate them to the length of the route print label for edge in label.getEdges(): edge_id = edge.getAttributes().get(self.shapeFileUniqueId) number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0) print " ", number_of_points #we add the scores to a dict if number_of_points > 1: label_scores.append((label, number_of_points/label.getLength())) # print label_scores # and extract the maximum score score = 0 selected = None for ls in label_scores: if ls[1] > score: selected = ls[0] score = ls[1] if returnNonSelection: pass else: return selected def eliminiateEmptyEdges(self, distance = 100): """Loops through the GPS pointset and selects edges within a boundary of <distance> meters """ print "Edge elimination started" selected_edge_ids = [] # let us for point in self.gps_points: results = self.idx.nearest(((point.getPoint().x-distance/2), (point.getPoint().y-distance/2), (point.getPoint().x+distance/2), (point.getPoint().y+distance/2)), objects=True) for result in results: from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter")) to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter")) edge_counter = self.G.edge[from_node][to_node].get("edgecounter") if edge_counter not in selected_edge_ids: selected_edge_ids.append(edge_counter) print str(len(selected_edge_ids)) + " edges found to keep." elimination_counter = 0 for edge in self.G.edges(): edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter") if edgecounter not in selected_edge_ids: edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node) self.G.remove_edge(*edge_tuple) elimination_counter = elimination_counter + 1 print str(elimination_counter) + " edges eliminated." def dumpPointShape(self, filename, original_coverage=None): if filename: driverName = "ESRI Shapefile" drv = ogr.GetDriverByName( driverName ) if drv: drv.DeleteDataSource(filename) if drv is None: print "%s driver not available.\n" % driverName ds = drv.CreateDataSource( filename) lyr = ds.CreateLayer( "blabla", None, ogr.wkbPoint ) if lyr is None: print "Layer creation failed.\n" field_defn = ogr.FieldDefn( "node_count", ogr.OFTInteger ) if lyr.CreateField ( field_defn ) != 0: print "Creating Name field failed.\n" sys.exit( 1 ) field_defn = ogr.FieldDefn( "edge_list", ogr.OFTString ) field_defn.SetWidth( 1024) if lyr.CreateField ( field_defn ) != 0: print "Creating Name field failed.\n" sys.exit( 1 ) for node in self.node_counter__node.values(): feat = ogr.Feature( lyr.GetLayerDefn() ) nc = node.getAttributes().get("nodecounter") print nc # import pdb;pdb.set_trace() feat.SetField( "node_count", nc ) s = "" for edge in node.getOutEdges(): s = s + str(int(edge.getAttributes().get("ID_NR"))) + ", " feat.SetField( "edge_list", s ) node_entity = ogr.Geometry(ogr.wkbPoint) # wkb = edge.getGeometry().to_wkb() node_entity.SetPoint_2D(0,node.getGeometry().x, node.getGeometry().y) feat.SetGeometry(node_entity) lyr.CreateFeature(feat) feat.Destroy() print "Shapefile (%s) written." % filename
shapePoints = [] pointList = [] for i in range(numberOfPoints): x = random.randint(0, gridWidth) y = random.randint(0, gridHeight) # Add the points to a python list of lists pointList.append((x, y)) # Create a Shapely point object iPoint = Point(x, y) iPoint.idx = i # set our custom attribute. (if this doesnt work I have other ways) shapePoints.append(iPoint) # Add the point to an RTREE index # index.add(id=id, (left, bottom, right, top)) # Note that this would work if the RtreeIndex.add(i, (x, y, x, y)) matchingPoints = [] searchBench = time.time() for idx, point in enumerate(shapePoints): if sCircle.contains(point): matchingPoints.append(idx) searchBench = time.time() - searchBench #------------------------------------------------------------------------------- # NOW TEST RTREE RtreeBench = time.time() RtreeIndex rtreeMatches = list( RtreeIndex.intersection((boxLeft, boxBottom, boxRight, boxTop)))
''' RTREE is a spatial index that answers the question -> What points are within this 2-dimensional box. RTREE helps us speed up other calculations by only considering legitimate objects for more complex comparisons. ''' from rtree import Rtree # instantiate the Rtree class idx = Rtree() # create a selection boundary (2d bounding box) minx, miny, maxx, maxy = (0.0, 0.0, 1.0, 1.0) # add an item to the index idx.add(0, (minx, miny, maxx, maxy)) # Intersect another bounding box with the original bounding box. list(idx.intersection((1.0, 1.0, 2.0, 2.0))) #[0L] # Point out the accuracy of the spatial calculation list(idx.intersection((1.0000001, 1.0000001, 2.0, 2.0))) #[] #add another item to the index. - show EXACT dimensiion matching index.add(id=id, (left, bottom, right, top)) object = [n for n in index.intersection((left, bottom, right, top))] #[id] # Find nearest object / bounding box idx.add(1, (minx, miny, maxx, maxy)) list(idx.nearest((1.0000001, 1.0000001, 2.0, 2.0), 1)) #[0L, 1L]
from shapely.geometry import Point, Polygon from shapely import wkt from rtree import Rtree import sys import struct output = sys.argv[1] data_filename = output + '.data' index_filename = output + '.data.idx' data = open(data_filename, 'wb') index = open(index_filename, 'wb') tree = Rtree(output, overwrite = 1) id = 0 offset = 0 for l in sys.stdin: x = wkt.loads(l) tree.add(id, x.bounds) s = len(x.wkb) data.write(x.wkb) index.write(struct.pack('L', offset)) offset += s id += 1 data.close() index.close()
class GraphicScene(object): ITEM_SELECTION_RADIUS = 2 # px _logger = _module_logger.getChild('GraphicScene') ############################################## def __init__(self): self._items = OrderedDict() self._rtree = Rtree() self._selected_items = OrderedDict() ############################################## def add_item(self, item): item_id = item._id self._items[item_id] = item self._rtree.add(item_id, item.bounding_box) item._scene = self ############################################## def remove_item(self, item): if item in self._selected_items: item.deselect() item_id = item._id del self._items[item_id] self._rtree.delete(item_id, item.bounding_box) item._scene = None ############################################## def add_items(self, items): for item in items: self.add_item(item) ############################################## def remove_items(self, items): for item in items: self.remove_item(item) ############################################## def items_in(self, interval): items = [self._items[x] for x in self._rtree.intersection(interval.bounding_box())] items.sort() # accordind to z value return items ############################################## def items_at(self, x, y): # Fixme: vector or x, y return self.items_in(point_interval(x, y)) ############################################## def items_around(self, x, y, radius): return self.items_in(centred_interval(x, y, radius)) ############################################## def __iter__(self): return iter(self._items.values()) ############################################## def _select_item(self, item): self._selected_items[item._id] = self ############################################## def _deselect_item(self, item): del self._selected_items[item._id] ############################################## def iter_on_selected_items(self): return self._selected_items.values() ############################################## def erase(self, position, radius): removed_items = [] new_items = [] for item in self: # for path in self.items_around(position.x, position.y, radius): # erase return None, the item or an iterable of new items items = item.erase(position, radius) if items is not item: self._logger.info("Erase item {}".format(item.id)) removed_items.append(item) if items is not None: new_items.extend(items) self.remove_items(removed_items) self.add_items(new_items) return removed_items, new_items
def main(): # You can edit these vars: # ----------------------- filename='matrix_plot3.png' numberOfPoints = 200 gridWidth = 10000 gridHeight = 10000 edgesMinimum = 2 edgesMaximum = 4 # The maximum number of edges (connecting lines) a vertex can have # ----------------------- edgesMaxDistance = (gridWidth / numberOfPoints) * 2 # instantiate the Rtree class RtreeIndex = Rtree() # CREATE THE RANDOM POINTS: shapelyPointIndex = [] # Stores index = shapely object - We will reference this point from its Index from this point on. for idx in range(numberOfPoints): # Each point gets its own index defined by i x = random.randint(0, gridWidth) y = random.randint(0, gridHeight) iPoint = Point(x, y) # Create a new shapely POINT object shapelyPointIndex.append(iPoint) RtreeIndex.add(idx, (x, y, x, y)) # Add the point to the RTREE Index del idx #unset temp variable # -------------------------------------------------------------------------- #MultiPoint([(0, 0), (1, 1)]) #point.coords #point.y point.x lineMatrices = [] ComplexLines = [] graphIndex = {} # This is a dictionary we will build into a graph/matrix - Each key will be an integer representing an index # and each value will be a dictionary of dictionaries representing edges. # iterate over all points to calculate each points nearest points # build line segments for each point to near points. # add all edges to a master complex line to compute intersection. for idx, point in enumerate(shapelyPointIndex): # Find nearby points quickly rtreeMatches = list() sRadiusX = gridWidth / 10 sRadiusY = gridHeight / 10 tries = 0 while(len(rtreeMatches) < edgesMinimum+1): # Add 1 because we can assume it will always find itself # Coords is a bounding-box to search within. coords = (point.x-sRadiusX, point.y-sRadiusY, point.x+sRadiusX, point.y+sRadiusY) print "searching coords" print coords rtreeMatches = list(RtreeIndex.intersection(coords)) print "found: ", rtreeMatches, " ", len(rtreeMatches) tries +=1 sRadiusX *= 1.5 sRadiusY *= 1.5 if(tries > 6): #Don't run forever if there is a logic problem. break del tries, sRadiusX, sRadiusY # FIND POSSIBLE EDGES TO ADD BY SEARCHING REGION RECURSIVELY ----------- # AND CALCULATE DISTANCES TO POSSIBLE POINTS --------------------------- # This dictionary will store our distances to each other point from this point. edgeDistances={} # Calculate distances to matched points for pointIdx in rtreeMatches: if(pointIdx == idx): continue # Don't record a distance to itself mPoint = shapelyPointIndex[pointIdx] edgeDistances[pointIdx] = round(mPoint.distance(point), 2) # keep the distance accuracy reasonable del mPoint, pointIdx, rtreeMatches # create a list that references the distances index for the closest indexes and is sorted by close to far. edgeDistancesIdx = sorted(edgeDistances, key=edgeDistances.get) #----------------------------------------------------------------------- # DECIDE WHICH EDGES TO ADD -------------------------------------------- # Now add a minimum number of points, and try to keep them within a logic distance edgesAdded = 0 edgesIdx=[] # this will contain indexes to our added edges for pointIdx in edgeDistancesIdx: distance = edgeDistances[pointIdx] # Stop adding edges if we've meant the minimum number of edges needed and the rest of the edges are too far away, or we have reach the maximum number of allowed edges if((edgesAdded >= edgesMinimum and distance > edgesMaxDistance) or edgesAdded >= edgesMaximum): break edgesIdx.append(pointIdx) del pointIdx #----------------------------------------------------------------------- # Initialize the graphIndex as a dict() for the current vertex if it hasn't been. if(idx not in graphIndex): graphIndex[idx] = {} #ADD THE EDGES TO THE GRID STRUCTURE ----------------------------------- for pointIdx in edgesIdx: # Add the edge graphIndex[idx][pointIdx] = edgeDistances[pointIdx] # Add the reverse edge (so both points share an edge) if pointIdx not in graphIndex: graphIndex[pointIdx] = {} graphIndex[pointIdx][idx] = edgeDistances[pointIdx] #----------------------------------------------------------------------- # Print out the graph of vertices pprint.pprint(graphIndex) # Randomely select a Start Vertex, and an End Vertex for Shortest Path # calculation startIdx = random.randint(0, len(graphIndex)-1) endIdx = startIdx # Calculate a random end index - and make sure its not the same as the start while(endIdx == startIdx): endIdx = random.randint(0, len(graphIndex)-1) # Setup shortest path calculation! print "Calculating Shortest Path:\n" # (8894.6959143222557, [11, 5, 8L, 7, 9]) # GraphIndex is a dictionary of dictionaries, startIdx is the INDEX of the startpoint, endIdx is the index of the endpoint vertex. shortestPath = dijkstra.shortestpath(graphIndex, startIdx, endIdx) shortestPathVertices = shortestPath[1] shortestPathDistance = shortestPath[0] print "Shortest Path Vertices:" print shortestPathVertices, "\n" del shortestPath #_------------------------------------------------------------------------------- # DRAW A PLOT FOR THE LINES CALCULATED ABOVE: import matplotlib matplotlib.use('Agg') # Do NOT attempt to open X11 instance from pylab import * from matplotlib.patches import Circle import matplotlib.pyplot as pyplot from matplotlib.text import * mylist = lineMatrices matplotlib.rcParams['lines.linewidth'] = 2 pyplot.axis([0, gridWidth, 0, gridHeight]) pyplot.grid(True) # Setting the axis labels. pyplot.xlabel('X Space') pyplot.ylabel('Y Space') figure = pyplot.figure() labels = figure.add_subplot(111, projection='rectilinear') #Give the plot a title pyplot.title('Shapely Shortest Path Simulation') del idx if not isinstance(mylist, list): print "The matrix is not a list!" print type(mylist).__name__ for idx, edges in graphIndex.iteritems(): vertex = shapelyPointIndex[idx] for edgeIdx, edgeDistance in edges.iteritems(): # This shouldn't happen, but lets just make sure. # DONT PLOT A PATH TO OURSELF if(edgeIdx == idx): continue # Get the edges point-coordinates edgePoint = shapelyPointIndex[edgeIdx] # Delete the reverse edge so we don't plot the line twice. if(edgeIdx in graphIndex): del graphIndex[edgeIdx][idx] # Print Debuggin Information print "PLOTTING EDGE: [%s] to [%s] X:[%s] Y:[%s] DIS:[%s] " % (idx, edgeIdx, edgePoint.x, edgePoint.y, edgeDistance) # Plot the edge! pyplot.plot([vertex.x, edgePoint.x], [vertex.y, edgePoint.y], 'b-', alpha=0.3, linewidth=.2, markersize=1) # FIND THE MID-POINT OF THE LINE: xMidpoint = (vertex.x + edgePoint.x) / 2 yMidpoint = (vertex.y + edgePoint.y) / 2 #print "Plotting Text [%s] [%s]" % (xMidpoint, yMidpoint) # Figure out arrows later... arrowprops=dict(facecolor='black', linewidth=2, alpha=0.4), # ANNOTATE THE LENGTH OF THE LINE AT ITS MIDPOINT labels.annotate(edgeDistance, xy=(xMidpoint, yMidpoint), xytext=(xMidpoint+80, yMidpoint), alpha=0.4, size='4', color='green') # end for ---------------------------------------------------------- # Plot the VERTEX as a Red Circle pyplot.plot(vertex.x, vertex.y, 'ro', linewidth=2, markersize=2) # Annotate the VERTEX by its INDEX labels.annotate(idx, xy=(vertex.x, vertex.y), xytext=(vertex.x+50, vertex.y), alpha=0.6, size='6', color='red') print "Graphing Shortest Path..." print "START: %s END: %s" % (startIdx, endIdx) #shortestPathVertices for idx, vertexIdx in enumerate(shortestPathVertices): if(vertexIdx == endIdx): break # We are done xy1 = shapelyPointIndex[vertexIdx] # this is the start of the line vertex2 = shortestPathVertices[idx+1] # The end of the line is the next item in the list shortest path list of indexes xy2 = shapelyPointIndex[vertex2] #this is the end of the line pyplot.plot([xy1.x, xy2.x], [xy1.y, xy2.y], 'r--', alpha=0.7, linewidth=2, markersize=1) print "PLOT -> SHORTEST PATH: [%s] to [%s]" % (vertexIdx, vertex2) print "Saving Plot Image %s" % (filename) pyplot.savefig(os.getcwd()+'/'+str(filename), dpi=240) print "\n\n" """