def _build_trip_edge_index(self): sys.stdout.write( "\nBuilding trip edge index for clarification algorithm... ") sys.stdout.flush() # storage for trip edge index trip_edge_index = Rtree() # iterate through all trip edges for trip_edge in self.trip_edges.values(): # determine trip edge minx, miny, maxx, maxy values trip_edge_minx = min(trip_edge.in_node.longitude, trip_edge.out_node.longitude) trip_edge_miny = min(trip_edge.in_node.latitude, trip_edge.out_node.latitude) trip_edge_maxx = max(trip_edge.in_node.longitude, trip_edge.out_node.longitude) trip_edge_maxy = max(trip_edge.in_node.latitude, trip_edge.out_node.latitude) # insert trip edge into spatial index trip_edge_index.insert(trip_edge.id, (trip_edge_minx, trip_edge_miny, trip_edge_maxx, trip_edge_maxy)) print "done." # return the trip edge index return trip_edge_index
def polygons_enclosure_tree(polygons): ''' Given a list of shapely polygons, which are the root (aka outermost) polygons, and which represent the holes which penetrate the root curve. We do this by creating an R-tree for rough collision detection, and then do polygon queries for a final result ''' tree = Rtree() for i, polygon in enumerate(polygons): tree.insert(i, polygon.bounds) count = len(polygons) g = nx.DiGraph() g.add_nodes_from(np.arange(count)) for i in range(count): if polygons[i] is None: continue # we first query for bounding box intersections from the R-tree for j in tree.intersection(polygons[i].bounds): if (i == j): continue # we then do a more accurate polygon in polygon test to generate # the enclosure tree information if polygons[i].contains(polygons[j]): g.add_edge(i, j) elif polygons[j].contains(polygons[i]): g.add_edge(j, i) roots = [n for n, deg in dict(g.in_degree()).items() if deg == 0] return roots, g
def polygons_enclosure_tree(polygons): ''' Given a list of shapely polygons, which are the root (aka outermost) polygons, and which represent the holes which penetrate the root curve. We do this by creating an R-tree for rough collision detection, and then do polygon queries for a final result ''' tree = Rtree() for i, polygon in enumerate(polygons): tree.insert(i, polygon.bounds) count = len(polygons) g = nx.DiGraph() g.add_nodes_from(np.arange(count)) for i in range(count): if polygons[i] is None: continue #we first query for bounding box intersections from the R-tree for j in tree.intersection(polygons[i].bounds): if (i==j): continue #we then do a more accurate polygon in polygon test to generate #the enclosure tree information if polygons[i].contains(polygons[j]): g.add_edge(i,j) elif polygons[j].contains(polygons[i]): g.add_edge(j,i) roots = [n for n, deg in list(g.in_degree().items()) if deg==0] return roots, g
def get_rtree_index(self): """ Get an rtree index of the edges within this GeoGraph Returns: rtree: rtree of edges indexed by bounding_box (4 coordinates) allowing lookup of the edge (node1, node2) and it's segment """ def edge_generator(): edges_bounds = map(lambda tup: (tup, gm.make_bounding_box( *map(lambda n: self.coords[n], tup))), self.edges()) for edge, box in edges_bounds: # Object is in form of (u.label, v.label), (u.coord, v.coord) yield (hash(edge), box, (edge, map(lambda ep: np.array(self.coords[ep]), edge))) # something's not working right with latest version of rtree/spatialib # index where we need to insert the objects in a loop rather than # via a generator to their constructor # Init rtree and store grid edges rtree = Rtree() for e in edge_generator(): # takes id, bbox, and segment/edge rtree.insert(e[0], e[1], e[2]) return rtree
def get_rtree_index(self): """ Get an rtree index of the edges within this GeoGraph Returns: rtree: rtree of edges indexed by bounding_box (4 coordinates) allowing lookup of the edge (node1, node2) and it's segment """ def edge_generator(): edges_bounds = map(lambda tup: (tup, gm.make_bounding_box( *map(lambda n: self.coords[n], tup))), self.edges()) for edge, box in edges_bounds: # Object is in form of (u.label, v.label), (u.coord, v.coord) yield (hash(edge), box, (edge, map(lambda ep: np.array(self.coords[ep]), edge))) # something's not working right with latest version of rtree/spatial lib # index where we need to insert the objects in a loop rather than # via a generator to their constructor # Init rtree and store grid edges rtree = Rtree() for e in edge_generator(): rtree.insert(e[0], e[1], e[2]) return rtree
def enclosure_tree(polygons): """ Given a list of shapely polygons with only exteriors, find which curves represent the exterior shell or root curve and which represent holes which penetrate the exterior. This is done with an R-tree for rough overlap detection, and then exact polygon queries for a final result. Parameters ----------- polygons : (n,) shapely.geometry.Polygon Polygons which only have exteriors and may overlap Returns ----------- roots : (m,) int Index of polygons which are root contains : networkx.DiGraph Edges indicate a polygon is contained by another polygon """ tree = Rtree() # nodes are indexes in polygons contains = nx.DiGraph() for i, polygon in enumerate(polygons): # if a polygon is None it means creation # failed due to weird geometry so ignore it if polygon is None: continue # insert polygon bounds into rtree tree.insert(i, polygon.bounds) # make sure every valid polygon has a node contains.add_node(i) # loop through every polygon for i, polygon in enumerate(polygons): # if polygon creation failed ignore it if polygon is None: continue # we first query for bounding box intersections from the R-tree for j in tree.intersection(polygon.bounds): # if we are checking a polygon against itself continue if (i == j): continue # do a more accurate polygon in polygon test # for the enclosure tree information if polygons[i].contains(polygons[j]): contains.add_edge(i, j) elif polygons[j].contains(polygons[i]): contains.add_edge(j, i) # a root or exterior curve has no parents # wrap in dict call to avoid networkx view in_degree = dict(contains.in_degree()) roots = [n for n, deg in in_degree.items() if deg == 0] return roots, contains
def __init__(self, hmm, emission_probability, constraint_length=10, MAX_DIST=500, priors=None, smallV=0.00000000001): # initialize spatial index self.previous_obs = None if priors == None: priors = dict([(state, 1.0 / len(hmm)) for state in hmm]) state_spatial_index = Rtree() unlocated_states = [] id_to_state = {} id = 0 for state in hmm: geom = self.geometry_of_state(state) if not geom: unlocated_states.append(state) else: ((lat1, lon1), (lat2, lon2)) = geom state_spatial_index.insert(id, (min(lon1, lon2), min( lat1, lat2), max(lon1, lon2), max(lat1, lat2))) id_to_state[id] = state id = id + 1 def candidate_states(obs): #was (lat,lon) in place of obs geom = self.geometry_of_observation(obs) if geom == None: return hmm.keys() else: (lat, lon) = geom nearby_states = state_spatial_index.intersection( (lon - MAX_DIST / METERS_PER_DEGREE_LONGITUDE, lat - MAX_DIST / METERS_PER_DEGREE_LATITUDE, lon + MAX_DIST / METERS_PER_DEGREE_LONGITUDE, lat + MAX_DIST / METERS_PER_DEGREE_LATITUDE)) candidates = [id_to_state[id] for id in nearby_states] + unlocated_states return candidates self.viterbi = Viterbi(hmm, emission_probability, constraint_length=constraint_length, priors=priors, candidate_states=candidate_states, smallV=smallV)
def points_in_mesh(points, mesh): from rtree import Rtree points = np.array(points) tri_3D = mesh.vertices[mesh.faces] tri_2D = tri_3D[:,:,0:2] z_bounds = np.column_stack((np.min(tri_3D[:,:,2], axis=1), np.max(tri_3D[:,:,2], axis=1))) bounds = np.column_stack((np.min(tri_2D, axis=1), np.max(tri_2D, axis=1))) tree = Rtree() for i, bound in enumerate(bounds): tree.insert(i, bound) result = np.zeros(len(points), dtype=np.bool) for point_index, point in enumerate(points): intersections = np.array(list(tree.intersection(point[0:2].tolist()))) in_triangle = [point_in_triangle_2D(point[0:2], tri_2D[i]) for i in intersections] result[point_index] = np.int(np.mod(np.sum(in_triangle), 2)) == 0 return result
def __init__(self, hmm, emission_probability, constraint_length=10, MAX_DIST=500, priors=None, smallV=0.00000000001): # initialize spatial index self.previous_obs = None if priors == None: priors=dict([(state,1.0/len(hmm)) for state in hmm]) state_spatial_index = Rtree() unlocated_states = [] id_to_state = {} id = 0 for state in hmm: geom=self.geometry_of_state(state) if not geom: unlocated_states.append(state) else: ((lat1,lon1),(lat2,lon2))=geom state_spatial_index.insert(id, (min(lon1, lon2), min(lat1, lat2), max(lon1, lon2), max(lat1, lat2))) id_to_state[id]=state id=id+1 def candidate_states(obs): #was (lat,lon) in place of obs geom = self.geometry_of_observation(obs) if geom == None: return hmm.keys() else: (lat,lon)=geom nearby_states = state_spatial_index.intersection((lon-MAX_DIST/METERS_PER_DEGREE_LONGITUDE, lat-MAX_DIST/METERS_PER_DEGREE_LATITUDE, lon+MAX_DIST/METERS_PER_DEGREE_LONGITUDE, lat+MAX_DIST/METERS_PER_DEGREE_LATITUDE)) candidates = [id_to_state[id] for id in nearby_states]+unlocated_states return candidates self.viterbi = Viterbi(hmm,emission_probability, constraint_length=constraint_length, priors=priors, candidate_states=candidate_states, smallV=smallV)
def enclosure_tree(polygons): """ Given a list of shapely polygons, which are the root (aka outermost) polygons, and which represent the holes which penetrate the root curve. We do this by creating an R-tree for rough collision detection, and then do polygon queries for a final result Parameters ----------- polygons: (n,) list of shapely.geometry.Polygon objects Returns ----------- roots: (m,) int, index of polygons which are root contains: networkx.DiGraph, edges indicate a polygon contained by another polygon """ tree = Rtree() for i, polygon in enumerate(polygons): if polygon is not None: tree.insert(i, polygon.bounds) count = len(polygons) # nodes are indexes in polygons contains = nx.DiGraph() # make sure every polygon is included contains.add_nodes_from(np.arange(count)) for i in range(count): if polygons[i] is None: continue # we first query for bounding box intersections from the R-tree for j in tree.intersection(polygons[i].bounds): if (i == j): continue # we then do a more accurate polygon in polygon test to generate # the enclosure tree information if polygons[i].contains(polygons[j]): contains.add_edge(i, j) elif polygons[j].contains(polygons[i]): contains.add_edge(j, i) roots = [n for n, deg in dict(contains.in_degree()).items() if deg == 0] return roots, contains
def to_directed(self, as_view=False): """ Convert undirected road network to directed road network new edge will have new eid, and each original edge will have two edge with reversed coords :return: """ assert as_view is False, "as_view is not supported" avail_eid = max([eid for u, v, eid in self.edges.data(data='eid')]) + 1 g = nx.DiGraph() edge_spatial_idx = Rtree() edge_idx = {} # add nodes for n, data in self.nodes(data=True): # when data=True, it means will data=node's attributes new_data = copy.deepcopy(data) g.add_node(n, **new_data) # add edges for u, v, data in self.edges(data=True): mbr = MBR.cal_mbr(data['coords']) # add forward edge forward_data = copy.deepcopy(data) g.add_edge(u, v, **forward_data) edge_spatial_idx.insert( forward_data['eid'], (mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat)) edge_idx[forward_data['eid']] = (u, v) # add backward edge backward_data = copy.deepcopy(data) backward_data['eid'] = avail_eid avail_eid += 1 backward_data['coords'].reverse() g.add_edge(v, u, **backward_data) edge_spatial_idx.insert( backward_data['eid'], (mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat)) edge_idx[backward_data['eid']] = (v, u) print('# of nodes:{}'.format(g.number_of_nodes())) print('# of edges:{}'.format(g.number_of_edges())) return RoadNetwork(g, edge_spatial_idx, edge_idx)
def _find_intersection_nodes(self): # storage for intersection nodes intersection_nodes = [] # spatial index for intersection nodes intersection_nodes_index = Rtree() # iterate through all nodes in map for curr_node in self.nodes.values(): # set storage for current node's unique neighbors neighbors = set() # iterate through all in_nodes for in_node in curr_node.in_nodes: # add in_node to neighbors set neighbors.add(in_node) # iterate through all out_nodes for out_node in curr_node.out_nodes: # add out_node to neighbors set neighbors.add(out_node) # if current node has more than 2 neighbors if (len(neighbors) > 2): # add current node to intersection nodes list intersection_nodes.append(curr_node) # add current node to intersection nodes index intersection_nodes_index.insert( curr_node.id, (curr_node.longitude, curr_node.latitude)) # return intersection nodes and index return (intersection_nodes, intersection_nodes_index)
def _build_trip_edge_index(self): sys.stdout.write("\nBuilding trip edge index for clarification algorithm... ") sys.stdout.flush() # storage for trip edge index trip_edge_index = Rtree() # iterate through all trip edges for trip_edge in self.trip_edges.values(): # determine trip edge minx, miny, maxx, maxy values trip_edge_minx = min(trip_edge.in_node.longitude, trip_edge.out_node.longitude) trip_edge_miny = min(trip_edge.in_node.latitude, trip_edge.out_node.latitude) trip_edge_maxx = max(trip_edge.in_node.longitude, trip_edge.out_node.longitude) trip_edge_maxy = max(trip_edge.in_node.latitude, trip_edge.out_node.latitude) # insert trip edge into spatial index trip_edge_index.insert(trip_edge.id, (trip_edge_minx, trip_edge_miny, trip_edge_maxx, trip_edge_maxy)) print "done." # return the trip edge index return trip_edge_index
def load_rn_shp(path, is_directed=True): edge_spatial_idx = Rtree() edge_idx = {} # node uses coordinate as key # edge uses coordinate tuple as key g = nx.read_shp(path, simplify=True, strict=False) if not is_directed: g = g.to_undirected() # node attrs: nid, pt, ... for n, data in g.nodes(data=True): data['pt'] = SPoint(n[1], n[0]) if 'ShpName' in data: del data['ShpName'] # edge attrs: eid, length, coords, ... for u, v, data in g.edges(data=True): geom_line = ogr.CreateGeometryFromWkb(data['Wkb']) coords = [] for i in range(geom_line.GetPointCount()): geom_pt = geom_line.GetPoint(i) coords.append(SPoint(geom_pt[1], geom_pt[0])) data['coords'] = coords data['length'] = sum([ distance(coords[i], coords[i + 1]) for i in range(len(coords) - 1) ]) env = geom_line.GetEnvelope() edge_spatial_idx.insert(data['eid'], (env[0], env[2], env[1], env[3])) edge_idx[data['eid']] = (u, v) del data['ShpName'] del data['Json'] del data['Wkt'] del data['Wkb'] print('# of nodes:{}'.format(g.number_of_nodes())) print('# of edges:{}'.format(g.number_of_edges())) if not is_directed: return UndirRoadNetwork(g, edge_spatial_idx, edge_idx) else: return RoadNetwork(g, edge_spatial_idx, edge_idx)
def _find_intersection_nodes(self): # storage for intersection nodes intersection_nodes = [] # spatial index for intersection nodes intersection_nodes_index = Rtree() # iterate through all nodes in map for curr_node in self.nodes.values(): # set storage for current node's unique neighbors neighbors = set() # iterate through all in_nodes for in_node in curr_node.in_nodes: # add in_node to neighbors set neighbors.add(in_node) # iterate through all out_nodes for out_node in curr_node.out_nodes: # add out_node to neighbors set neighbors.add(out_node) # if current node has more than 2 neighbors if (len(neighbors) > 2): # add current node to intersection nodes list intersection_nodes.append(curr_node) # add current node to intersection nodes index intersection_nodes_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude)) # return intersection nodes and index return (intersection_nodes, intersection_nodes_index)
class Graph: def __init__(self, bus_trips): self.bus_trips = bus_trips self.graph_nodes = {} # indexed by "location_id" self.graph_edge_id = 0 self.graph_edges = {} # indexed by "edge id" self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" self.graph_edge_index = Rtree() def generate_graph(self): print "Running graph generation algorithm..." # initialize trip counter trip_count = 1 for trip in self.bus_trips: # initialize location counter location_count = 1 # storage for previous node prev_node = None for location in trip.locations: sys.stdout.write("\rAnalyzing location " + str(location_count) + "/" + str(len(trip.locations)) + " for trip " + str(trip_count) + "/" + str(len(self.bus_trips)) + "... ") sys.stdout.flush() # find closest edges in graph to location closest_edges = self._find_closest_edges_in_graph(location, 100) # flag variable for whether we merged location did_merge_location = False # iterate through candidate edge ids for candidate_edge_id in closest_edges: # grab candidate edge from graph edge dictionary candidate_edge = self.graph_edges[candidate_edge_id] # determine whether we should merge with candidate edge if (self._should_merge_location_with_edge(location, candidate_edge) is True): # merge location with edge, update previous node prev_node = self._merge_location_with_edge(location, candidate_edge, prev_node) # update merge flag variable did_merge_location = True # no need to look at further edges, break out of candidate edges loop break # if we did not merge the location with any edge if (did_merge_location is False): # add location to graph self._add_location_to_graph(location, prev_node) # update previous node with current location prev_node = location # increment location counter location_count += 1 # done with current trip locations print "done." # increment trip counter trip_count += 1 # write graph edges to file self._write_graph_edges_to_file() # create graph database self._output_graph_to_db() def _merge_location_with_edge(self, location, edge, prev_node): # get the edge node closest to the location edge_node = self._get_closest_edge_node(location, edge) # if prev_node is None if (prev_node is None): # increase volume of just this edge edge.volume += 1 # if prev_node is not None else: # find path from prev_node to edge_node path = self._find_path(prev_node, edge_node, max_path_length) # if there was a path from prev_node to edge_node if (path is not None): # iterate through nodes in path for i in range(1, len(path)): # grab in_node in_node = path[i - 1] # grab out_node out_node = path[i] # find corresponding graph edge graph_edge = self._find_graph_edge(in_node, out_node) # increment volume on edge graph_edge.volume += 1 # if there is no path from prev_node to edge_node else: # create a new graph edge between prev_node and edge_node self._create_graph_edge(prev_node, edge_node) # return the edge_node return edge_node def _get_closest_edge_node(self, location, edge): # if in_node distance is less than out_node distance if (self._distance(location, edge.in_node) < self._distance(location, edge.out_node)): # return the edge in_node return edge.in_node # otherwise, return the edge out_node return edge.out_node def _find_path(self, source, destination, max_length): # reset all node visited flags self._reset_node_visited_flags() # get a breath-first search path from source to destination path = self._bfs_path(source, destination) # if there is a path from source to destination if (path is not None): # and if the path length is less than or equal to the maximum length if (len(path) <= max_length): # return the path return path # otherwise, return None return None def _bfs_path(self, source, destination): # storage for breadth-first search parents bfs_parent = {} # key is current node, value is parent node # source node has no breadth-first search parent bfs_parent[source] = None # node queue for breadth-first search bfs_queue = [] # enqueue source node bfs_queue.append(source) # mark source node as visited source.visited = True # while the queue is not empty while (len(bfs_queue) > 0): # dequeue the first node in the queue curr_node = bfs_queue.pop(0) # if the current node is the destination if (curr_node is destination): # create storage for breadth-first search path bfs_path = [] # add the current node to the breadth-first search path bfs_path.insert(0, curr_node) # grab the parent of the current node parent = bfs_parent[curr_node] # iterate through breadth-first search parents while (parent is not None): # add the parent to the breadth-first search path bfs_path.insert(0, parent) # grab the next parent parent = bfs_parent[parent] # return the breadth-first search path return bfs_path # if the current node is not the destination else: # iterate through the current node's out_nodes for out_node in curr_node.out_nodes: # if the out_node has not been visited if (out_node.visited is False): # mark the out_node as visited out_node.visited = True # enqueue the out_node bfs_queue.append(out_node) # store curr_node as out_node's breadth-first search parent bfs_parent[out_node] = curr_node # if we reached here, no path was found return None def _should_merge_location_with_edge(self, location, edge): # project location onto edge (location_projection, location_projection_fraction, location_projection_distance) = self._projection_onto_line(edge.in_node, edge.out_node, location) # if projection is not onto edge if (location_projection_fraction < 0.0 or location_projection_fraction > 1.0): # we cannot merge location with edge return False # determine bearing difference between edge and location bearing_difference = math.cos(math.radians(self._path_bearing(edge.in_node, edge.out_node) - self._location_bearing(location))) # if location projection distance is less than 20 meters if (location_projection_distance < location_projection_distance_limit): # if bearing difference is less than 45 degrees if (bearing_difference > location_bearing_difference_limit): # merge location with edge return True # otherwise, do not merge location with edge return False def _add_location_to_graph(self, location, prev_node): # add an out_nodes list to location location.out_nodes = [] # add an in_nodes list to location location.in_nodes = [] # add a visited flag to location location.visited = False # add location to graph nodes list self.graph_nodes[location.id] = location # if prev_node is not None if (prev_node is not None): # create a new graph edge between prev_node and location self._create_graph_edge(prev_node, location) def _create_graph_edge(self, in_node, out_node): # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # add new graph edge to graph edges spatial index self._add_graph_edge_to_index(new_graph_edge) # increment graph edge id self.graph_edge_id += 1 # store out_node in in_nodes's out_nodes list in_node.out_nodes.append(out_node) # store in_node in out_node's in_nodes list out_node.in_nodes.append(in_node) def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _add_graph_edge_to_index(self, graph_edge): # determine graph edge minx, miny, maxx, maxy values graph_edge_minx = min(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_miny = min(graph_edge.in_node.latitude, graph_edge.out_node.latitude) graph_edge_maxx = max(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_maxy = max(graph_edge.in_node.latitude, graph_edge.out_node.latitude) # insert graph edge into spatial index self.graph_edge_index.insert(graph_edge.id, (graph_edge_minx, graph_edge_miny, graph_edge_maxx, graph_edge_maxy)) def _location_bearing(self, location): # if location has a previous neighbor and a next neighbor if ((location.prev_location is not None) and (location.next_location is not None)): # determine bearing using previous and next neighbors return self._path_bearing(location.prev_location, location.next_location) # if location has no previous neighbor, but has a next neighbor elif ((location.prev_location is None) and (location.next_location is not None)): # determine bearing using current location and next neighbor return self._path_bearing(location, location.next_location) # if location has a previous neighbor, but not a next neighbor elif ((location.prev_location is not None) and (location.next_location is None)): # determine bearing using previous neighbor and current location return self._path_bearing(location.prev_location, location) # if we reach here, there is an error return None def _find_closest_edges_in_graph(self, location, number_of_edges): return self.graph_edge_index.nearest((location.longitude, location.latitude), number_of_edges) def _projection_onto_line(self, location1, location2, location3): return spatialfunclib.projection_onto_line(location1.latitude, location1.longitude, location2.latitude, location2.longitude, location3.latitude, location3.longitude) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("cao_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)") # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # insert graph node into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) + "," + str(graph_node.latitude) + "," + str(graph_node.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done." def _write_graph_edges_to_file(self): # output that we are starting the writing process sys.stdout.write("\nWriting graph edges to file... ") sys.stdout.flush() # open graph file graph_file = open('cao_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # output edge to file graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "," + str(graph_edge.volume) + "\n\n") # close graph file graph_file.close() print "done." def _reset_node_visited_flags(self): # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # set visited flag to False graph_node.visited = False
def mod_kruskal(G, subgraphs=None, rtree=None): """ algorithm to compute the euclidean minimum spanning forest of nodes in GeoGraph G with 'budget' based restrictions on edges Uses a modified version of Kruskal's algorithm NOTE: subgraphs is modified as a side-effect...may remove in future (useful for testing right now) Args: G: GeoGraph of nodes to be connected if appropriate Nodes should have 'budget' attribute subgraphs: UnionFind data structure representing existing network's connected components AND the 'fake' nodes projected onto it. This is the basis for the agglomerative nearest neighbor approach in this algorithm. NOTE: ONLY the existing networks components are represented in the subgraphs argument. The nodes in G will be added within this function rtree: RTree based index of existing network Returns: GeoGraph: representing minimum spanning forest of G subject to the budget based restrictions """ # special case (already MST) if G.number_of_nodes() < 2: return G def is_fake(node): """ Tests whether the node is a projection on the existing grid, using its MV """ return subgraphs.budget[node] == np.inf # handy to have coords array coords = np.row_stack(G.coords.values()) if subgraphs is None: assert rtree is not None, \ "subgraphs (disjoint set) required when rtree is passed" rtree = Rtree() # modified to handle queues, children, mv subgraphs = UnionFind() # add nodes and budgets from G to subgraphs as components for node in G.nodes(): subgraphs.add_component(node, budget=G.node[node]['budget']) # get fully connected graph g = G.get_connected_weighted_graph() # edges in MSF Et = [] # connect the shortest safe edge until all edges have been tested # at which point, we have made all possible connections for u, v, w in sorted(g.edges(data=True), key=lambda x: x[2]['weight']): # if doesn't create cycle # and subgraphs have enough MV # and we're not connecting 2 fake nodes # then allow the connection w = w['weight'] if subgraphs[u] != subgraphs[v] and \ (subgraphs.budget[subgraphs[u]] >= w or is_fake(u)) and \ (subgraphs.budget[subgraphs[v]] >= w or is_fake(v)) and \ not (is_fake(u) and is_fake(v)): # doesn't create cycles from line segment intersection invalid_edge, intersections = \ line_subgraph_intersection(subgraphs, rtree, coords[u], coords[v]) if not invalid_edge: # edges should not intersect a subgraph more than once assert(filter(lambda n: n > 1, intersections.values()) == []) # merge the subgraphs subgraphs.union(u, v, w) # For all intersected subgraphs update the mv to that # created by the edge intersecting them map(lambda (n, _): subgraphs.union(u, n, 0), filter(lambda (n, i): i == 1 and subgraphs[n] != subgraphs[u], intersections.iteritems())) # index the newly added edge box = make_bounding_box(coords[u], coords[v]) # Object is (u.label, v.label), (u.coord, v.coord) rtree.insert(hash((u, v)), box, obj=((u, v), (coords[u], coords[v]))) Et += [(u, v, {'weight': w})] # create new GeoGraph with results result = G.copy() result.coords = G.coords result.remove_edges_from(result.edges()) result.add_edges_from(Et) return result
def detect_rn_component(self, status_matrix): node_pixels = np.where(status_matrix == GraphExtractor.NODE) nb_node_pixels = len(node_pixels[0]) print('\n# of node pixels:{}'.format(nb_node_pixels)) neighbor_deltas = [ dxdy for dxdy in itertools.product([-1, 0, 1], [-1, 0, 1]) if dxdy[0] != 0 or dxdy[1] != 0 ] # node pixel -> center node nodes = {} node_pixel_spatial_index = Rtree() # [node pixel sequence (start and end must be node pixel)] connected_segments = [] cnt = 1 center_nodes = [] node_pixel_id = 0 for i, j in zip(node_pixels[0], node_pixels[1]): if (cnt % 100 == 0) or (cnt == nb_node_pixels): sys.stdout.write("\r" + str(cnt) + "/" + str(nb_node_pixels) + "... ") sys.stdout.flush() cnt += 1 if status_matrix[i][j] == GraphExtractor.VISITED_NODE: continue # region merge neighbor node pixels status_matrix[i][j] = GraphExtractor.VISITED_NODE candidates = [(i, j)] node_pixels = [] while len(candidates) > 0: node_pixel = candidates.pop() node_pixels.append(node_pixel) m, n = node_pixel for dm, dn in neighbor_deltas: if status_matrix[m + dm][n + dn] == GraphExtractor.NODE: status_matrix[m + dm][n + dn] = GraphExtractor.VISITED_NODE candidates.append((m + dm, n + dn)) center_node = CenterNodePixel(node_pixels) center_nodes.append(center_node) for node_pixel in node_pixels: nodes[node_pixel] = center_node node_pixel_spatial_index.insert(node_pixel_id, node_pixel, obj=node_pixel) node_pixel_id += 1 # endregion # region find neighbor segments # mask current nodes, make sure the edge doesn't return to itself for m, n in node_pixels: status_matrix[m][n] = GraphExtractor.INVALID # find new road segment of the current node in each possible direction for node_pixel in node_pixels: connected_segment = self.detect_connected_segment( status_matrix, node_pixel) if connected_segment is not None: connected_segments.append(connected_segment) # restore masked nodes for m, n in node_pixels: status_matrix[m][n] = GraphExtractor.VISITED_NODE # endregion print('\n# of directly connected segments:{}'.format( len(connected_segments))) # there might be few edge pixels left, that should be fine nb_unprocessed_edge_pixels = np.sum( status_matrix[status_matrix == GraphExtractor.EDGE]) print('unprocessed edge pixels:{}'.format(nb_unprocessed_edge_pixels)) print('# of nodes:{}'.format(len(center_nodes))) print('# of segments:{}'.format(len(connected_segments))) return nodes, connected_segments
def p_mod_boruvka(G, subgraphs=None, rtree=None): V = set(T.nodes(data=False)) coords = np.row_stack(nx.get_node_attributes(T, 'coords').values()) projcoords = ang_to_vec_coords(coords) kdtree = KDTree(projcoords) if subgraphs is None: if rtree is not None: raise ValueError('RTree passed without UnionFind') rtree = Rtree() # modified to handle queues, children, mv subgraphs = UnionFind() # Tests whether the node is a projection on the existing grid, using its MV is_fake = lambda n: subgraphs.budget[n] == np.inf def find_nn(node_tuple): u, up = node_tuple v, _ = kdtree.query_subset(up, list(V - {u})) return u, v, spherical_distance([coords[u], coords[v]]) # find the nearest neigbor of all nodes p = mp.Pool(processes=6) neighbors = p.map(find_nn, enumerate(projcoords)) p.close() # push the results into their respective queues for u, v, d in neighbors: subgraphs.add_component(u, budget=T.node[u]['budget']) subgraphs.queues[u].push((u, v), d) # list to hold mst edges Et = [] last_state = None while Et != last_state: # consolidates top candidate edges from each subgraph Ep = PriorityQueue() def update_queues(component): q_top = subgraphs.queues[component].top() try: (u, v) = q_top except: return (None, None, None), np.inf component_set = subgraphs.component_set(u) disjointVC = list(V - set(component_set)) if not disjointVC: return (None, None, None), np.inf while v in component_set: subgraphs.queues[component].pop() vprime, _ = kdtree.query_subset(projcoords[u], disjointVC) dm = spherical_distance([coords[u], coords[vprime]]) subgraphs.queues[component].push((u, vprime), dm) (u, v) = subgraphs.queues[component].top() else: dm = spherical_distance([coords[u], coords[v]]) return (u, v, dm), dm p = mp.Pool(processes=6) foreign_neighbors = map(update_queues, subgraphs.connected_components(component_subset=V)) p.close() for neighbor in foreign_neighbors: obj, priority = neighbor if priority != np.inf: Ep.push(*neighbor) last_state = deepcopy(Et) # add all the edges in E' to Et so long as no cycles are created while Ep._queue: (um, vm, dm) = Ep.pop() # if doesn't create cycle and subgraph has enough MV if subgraphs[um] != subgraphs[vm] and \ (subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)): # test that the connecting subgraph can receive the MV if subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm): # doesn't create cycles from line segment intersection invalid_edge, intersections =\ line_subgraph_intersection(subgraphs, rtree, coords[um], coords[vm]) if not invalid_edge: # edges should not intersect a subgraph more than once assert(filter(lambda n: n > 1, intersections.values()) == []) # merge the subgraphs subgraphs.union(um, vm, dm) # Union all intersecting subgraphs # and update budgets (happens within union) map(lambda (n, _): subgraphs.union(um, n, 0), filter(lambda (n, i): i == 1 and subgraphs[n] != subgraphs[um], intersections.iteritems())) # index the newly added edge box = make_bounding_box(coords[um], coords[vm]) # Object is (u.label, v.label), (u.coord, v.coord) rtree.insert(hash((um, vm)), box, obj=((um, vm), (coords[um], coords[vm]))) Et += [(um, vm)] T.remove_edges_from(T.edges()) T.add_edges_from(Et) return T
class Graph: def __init__(self, all_trips): # trips self.all_trips = all_trips # cluster seeds self.cluster_seeds = {} self.cluster_seed_id = 0 self.cluster_seed_index = Rtree() # graph edges self.graph_edges = {} # indexed by "edge id" self.graph_edge_id = 0 self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" def cluster_traces(self): self._create_all_trip_edges() self._generate_cluster_seeds() self._cluster_seeds_with_traces() self._generate_graph_edges() self._output_graph_to_db() def _generate_graph_edges(self): sys.stdout.write("Generating graph edges... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # grab trip edges trip_edges = trip.edges.values() # put trip edges in order trip_edges.sort(key=lambda x: x.id) # storage for previous cluster prev_cluster = None # iterate through trip edges for trip_edge in trip_edges: # if the current trip edge is clustered if trip_edge.cluster is not None: # create a graph edge between the previous cluster and the current cluster self._create_graph_edge(prev_cluster, trip_edge.cluster) # update previous cluster with current cluster prev_cluster = trip_edge.cluster # output graph edges self._write_graph_edges_to_file() print "done." def _create_graph_edge(self, in_node, out_node): # if in_node or out_node is None if (in_node is None) or (out_node is None): # return without doing anything return # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if existing_graph_edge is None: # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # increment graph edge id self.graph_edge_id += 1 def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if edge_lookup_key in self.graph_edge_lookup.keys(): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _cluster_seeds_with_traces(self): # storage for total cluster distance moved total_cluster_distance_moved = float("infinity") # iterate until total cluster distance moved below threshold while total_cluster_distance_moved >= cluster_distance_moved_threshold: # find all points on traces and move clusters total_cluster_distance_moved = self._find_points_on_traces() # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_clustered.txt") def _find_points_on_traces(self): # counter for cluster seeds seed_counter = 1 # storage for total cluster distance moved total_cluster_distance_moved = 0.0 # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # clear current trace points from cluster cluster_seed.clear_trace_points() sys.stdout.write( "\rFinding intersecting points with cluster " + str(seed_counter) + "/" + str(len(self.cluster_seeds)) + "... " ) sys.stdout.flush() # increment seed counter seed_counter += 1 # determine leftward cluster bearing leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0, 360.0) # determine rightward cluster bearing rightward_bearing = math.fmod((cluster_seed.bearing + 90.0) + 360.0, 360.0) # storage for candidate trace points candidate_trace_points = [] # iterate through all trips for trip in self.all_trips: # find leftward intersection points with trip candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, leftward_bearing)) # find rightward intersection points with trip candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, rightward_bearing)) # add candidate trace points to cluster cluster_seed.add_trace_points(candidate_trace_points) # recompute cluster centroid total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid() # clear current trace points from cluster cluster_seed.clear_trace_points() # add candidate trace points to cluster, again cluster_seed.add_trace_points(candidate_trace_points) # normalize total cluster distance moved by number of seeds total_cluster_distance_moved = total_cluster_distance_moved / len(self.cluster_seeds.values()) # and we're done! print "done (clusters moved an average of " + str(total_cluster_distance_moved) + " meters)." # return total cluster distance moved return total_cluster_distance_moved def _find_intersection_points(self, trip, cluster, cluster_bearing): # find all nearby trip edge id's nearby_trip_edge_ids = self._find_nearby_trip_edge_ids(cluster, edge_bounding_box_size, trip.edge_index) # storage for intersection points intersection_points = [] # iterate through all nearby edge id's for edge_id in nearby_trip_edge_ids: # grab current edge edge = trip.edges[edge_id] # determine intersection point between edge and cluster intersection_point = self._intersection_point(edge.in_node, edge.bearing, cluster, cluster_bearing) # if there is an intersection point if intersection_point is not None: # determine distance from edge in_node to intersection point intersection_distance = self._distance_coords( edge.in_node.latitude, edge.in_node.longitude, intersection_point[0], intersection_point[1] ) # if intersection distance is less than edge length if intersection_distance <= edge.length: # this edge has a valid intersection point intersection_points.append( TracePoint(intersection_point[0], intersection_point[1], edge.bearing, edge) ) # return all intersection points for this trip return intersection_points def _generate_cluster_seeds(self): # iterate through all trips for i in range(0, len(self.all_trips)): sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" + str(len(self.all_trips)) + "... ") sys.stdout.flush() # grab current trip trip = self.all_trips[i] # set last cluster seed distance to zero for first trip location trip.locations[0].last_cluster_seed_distance = 0.0 # iterate through all trip locations for j in range(1, len(trip.locations)): # drop cluster seeds along current edge every 50 meters self._drop_cluster_seeds_along_edge(trip.locations[j - 1], trip.locations[j]) print "done (generated " + str(len(self.cluster_seeds)) + " cluster seeds)." # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt") def _drop_cluster_seeds_along_edge(self, in_node, out_node): # determine edge length edge_length = self._distance(in_node, out_node) # determine distance along edge for first cluster seed first_cluster_seed_distance = cluster_seed_interval - in_node.last_cluster_seed_distance # storage for relative cluster seed intervals rel_cluster_seed_intervals = [] # storage for current cluster seed distance along this edge curr_cluster_seed_distance = first_cluster_seed_distance # determine the relative cluster seed intervals needed for this edge while curr_cluster_seed_distance <= edge_length: # append current cluster seed distance to relative cluster seed interval list rel_cluster_seed_intervals.append(curr_cluster_seed_distance) # increment current cluster seed distance curr_cluster_seed_distance += cluster_seed_interval # determine bearing of current edge edge_bearing = self._path_bearing(in_node, out_node) # create cluster seeds for edge for i in range(0, len(rel_cluster_seed_intervals)): # determine fraction along current edge to drop cluster seed fraction_along = rel_cluster_seed_intervals[i] / edge_length # determine point along line to drop cluster seed (new_cluster_seed_latitude, new_cluster_seed_longitude) = self._point_along_line( in_node, out_node, fraction_along ) # locate nearest existing cluster seeds closest_cluster_seeds = list( self.cluster_seed_index.nearest((new_cluster_seed_longitude, new_cluster_seed_latitude), 25) ) # if there does not exist a closest existing cluster seed if len(closest_cluster_seeds) == 0: # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing ) # else, if there exists a closest existing cluster seed elif len(closest_cluster_seeds) > 0: # storage for matched cluster seed matched_cluster_seed = None # iterate through closest existing cluster seeds for curr_cluster_seed_id in closest_cluster_seeds: # grab current cluster seed curr_cluster_seed = self.cluster_seeds[curr_cluster_seed_id] # compute distance to current cluster seed distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, curr_cluster_seed.latitude, curr_cluster_seed.longitude, ) # determine bearing difference between edge and current cluster seed bearing_difference = math.cos(math.radians(edge_bearing - curr_cluster_seed.bearing)) # if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees if (distance <= cluster_seed_interval) and (bearing_difference >= cluster_bearing_difference_limit): # store current cluster seed as matched cluster seed matched_cluster_seed = curr_cluster_seed # stop searching break # if there was not a matched cluster seed if matched_cluster_seed is None: # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing ) # update last cluster seed distance out_node.last_cluster_seed_distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, out_node.latitude, out_node.longitude ) # if no cluster seeds were generated along this edge if len(rel_cluster_seed_intervals) == 0: # update last cluster seed distance out_node.last_cluster_seed_distance = in_node.last_cluster_seed_distance + edge_length def _create_new_cluster_seed(self, latitude, longitude, bearing): # create a new cluster seed new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude, longitude, bearing) # add new cluster seed to the cluster seeds dictionary self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed # insert new cluster seed into spatial index self.cluster_seed_index.insert(new_cluster_seed.id, (new_cluster_seed.longitude, new_cluster_seed.latitude)) # increment cluster seed id self.cluster_seed_id += 1 # return new cluster seed return new_cluster_seed def _create_all_trip_edges(self): sys.stdout.write("Creating and indexing edges for all trips... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # add edge storage to trip trip.edges = {} # add edge index to trip trip.edge_index = Rtree() # storage for edge id trip_edge_id = 0 # iterate through all trip locations for i in range(1, len(trip.locations)): # create new edge new_edge = Edge(trip_edge_id, trip.locations[i - 1], trip.locations[i]) # insert edge into dictionary trip.edges[trip_edge_id] = new_edge # insert edge into index self._index_trip_edge(new_edge, trip.edge_index) # increment trip edge id trip_edge_id += 1 # done print "done." def _index_trip_edge(self, edge, edge_index): # determine edge minx, miny, maxx, maxy values edge_minx = min(edge.in_node.longitude, edge.out_node.longitude) edge_miny = min(edge.in_node.latitude, edge.out_node.latitude) edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude) edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude) # insert edge into spatial index edge_index.insert(edge.id, (edge_minx, edge_miny, edge_maxx, edge_maxy)) def _find_nearby_trip_edge_ids(self, location, distance, edge_index): # define longitude/latitude offset lon_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE lat_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE # create bounding box bounding_box = ( location.longitude - lon_offset, location.latitude - lat_offset, location.longitude + lon_offset, location.latitude + lat_offset, ) # return nearby edge id's inside bounding box return list(edge_index.intersection(bounding_box)) def _intersection_point(self, location1, location1_bearing, location2, location2_bearing): return spatialfunclib.intersection_point( location1.latitude, location1.longitude, location1_bearing, location2.latitude, location2.longitude, location2_bearing, ) def _point_along_line(self, location1, location2, fraction_along): return spatialfunclib.point_along_line( location1.latitude, location1.longitude, location2.latitude, location2.longitude, fraction_along ) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing( location1.latitude, location1.longitude, location2.latitude, location2.longitude ) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance_coords(self, location1_latitude, location1_longitude, location2_latitude, location2_longitude): return spatialfunclib.distance(location1_latitude, location1_longitude, location2_latitude, location2_longitude) def _write_cluster_seeds_to_file(self, filename="edelkamp_cluster_seeds.txt"): # open graph file graph_file = open(filename, "w") # iterate through all cluster_seeds for cluster_seed in self.cluster_seeds.values(): # output cluster seed to file graph_file.write( str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) + "\n" ) # close graph file graph_file.close() def _write_graph_edges_to_file(self): # open graph file graph_file = open("edelkamp_cluster_edges.txt", "w") # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # output edge to file graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "\n\n") # close graph file graph_file.close() def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("edelkamp_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)") # remove values from nodes table # cur.execute("DELETE FROM nodes") # remove values from edges table # cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # insert cluster seed into nodes table cur.execute( "INSERT INTO nodes VALUES (" + str(cluster_seed.id) + "," + str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + ")" ) # iterate through all graph edges for graph_edge in self.graph_edges.values(): # insert graph edge into edges table cur.execute( "INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")" ) # commit inserts conn.commit() # close database connection conn.close() print "done."
def mod_boruvka(G, subgraphs=None, rtree=None): """ algorithm to calculate the minimum spanning forest of nodes in GeoGraph G with 'budget' based restrictions on edges. Uses a modified version of Boruvka's algorithm NOTE: subgraphs is modified as a side-effect...may remove in future (useful for testing right now) Args: G: GeoGraph of nodes to be connected if appropriate Nodes should have 'budget' attribute subgraphs: UnionFind data structure representing existing network's connected components AND the 'fake' nodes projected onto it. This is the basis for the agglomerative nearest neighbor approach in this algorithm. rtree: RTree based index of existing network Returns: GeoGraph: representing minimum spanning forest of G subject to the budget based restrictions """ # special case (already MST) if G.number_of_nodes() < 2: return G V = set(G.nodes()) # GeoGraph coords may be ndarray or dict if isinstance(G.coords, np.ndarray): coords = G.coords else: coords = np.row_stack(G.coords.values()) projcoords = ang_to_vec_coords(coords) if G.is_geographic() else coords kdtree = KDTree(projcoords) # Handle "dead" components D = set() if subgraphs is None: if rtree is not None: raise ValueError('RTree passed without UnionFind') rtree = Rtree() # modified to handle queues, children, mv subgraphs = UnionFind() # <helper_functions> def candidate_components(C): """ return the set of candidate nearest components for the connected component containing C. Do not consider those in C's connected component or those that are 'dead'. """ component_set = subgraphs.component_set(C) return list((V - set(component_set)) - D) def update_nn_component(C, candidates): """ find the nearest neighbor pair for the connected component represented by c. candidates represents the list of components from which to select. """ (v, vm) = subgraphs.queues[C].top() # vm ∈ C {not a foreign nearest neighbor} # go through the queue until an edge is found between this node # and the set of candidates, updating the neighbors in the connected # components queue in the process. while vm not in candidates: subgraphs.queues[C].pop() um, _ = kdtree.query_subset(projcoords[v], candidates) dm = square_distance(projcoords[v], projcoords[um]) subgraphs.push(subgraphs.queues[C], (v, um), dm) # Note: v will always be a vertex in this connected component # vm *may* be external (v, vm) = subgraphs.queues[C].top() return (v, vm) def is_fake(node): """ Tests whether the node is a projection on the existing grid """ return subgraphs.budget[node] == np.inf # Test whether the component is dead # i.e. can never connect to another node def is_dead(c, nn_dist): return not is_fake(c) and subgraphs.budget[c] < nn_dist # "true" distance between components def component_dist(c1, c2): dist = 0 if G.is_geographic(): dist = spherical_distance([coords[c1], coords[c2]]) else: dist = euclidean_distance([coords[c1], coords[c2]]) return dist # </helper_functions> # Initialize the connected components holding a single node # and push the nearest neighbor into its queue for v in V: vm, _ = kdtree.query_subset(projcoords[v], list(V - {v})) dm = square_distance(projcoords[v], projcoords[vm]) subgraphs.add_component(v, budget=G.node[v]['budget']) subgraphs.push(subgraphs.queues[v], (v, vm), dm) # Add to dead set if warranted nn_dist = component_dist(v, vm) if is_dead(v, nn_dist): # here components are single nodes # so no need to worry about adding children to dead set if v not in D: D.add(v) Et = [] # Initialize MST edges to empty list last_state = None # MST is complete when no progress was made in the prior iteration while Et != last_state: # This is a candidate list of edges that might be added to the MST Ep = PriorityQueue() # ∀ C of G; where c <- connected component for C in subgraphs.connected_components(component_subset=V): candidates = candidate_components(C) # Skip if no valid candidates if not candidates: continue (v, vm) = update_nn_component(C, candidates) # Add to dead set if warranted nn_dist = component_dist(v, vm) if is_dead(C, nn_dist): # add all dead components to the dead set D # (note that fake nodes can never be dead) for c in subgraphs.component_set(C): if c not in D and not is_fake(c): D.add(c) # One more round to root out connections to dead components # found in above iteration. # Need to do this BEFORE pushing candidate edges into Ep. # Otherwise we might be testing only 'dead' candidates # and therefore mistakenly think we were done (since # no new edges would have been added) for C in subgraphs.connected_components(component_subset=V): candidates = candidate_components(C) # Skip if no valid candidates if not candidates: continue (v, vm) = update_nn_component(C, candidates) # Calculate nn_dist for comparison to mv later nn_dist = component_dist(v, vm) # Append the top priority edge from the subgraph to the candidate # edge set Ep.push((v, vm, nn_dist), nn_dist) last_state = deepcopy(Et) # Candidate Test # At this point we have all of our nearest neighbor component edge # candidates defined for this "round" # # Now test all candidate edges in Ep for cycles and satisfaction of # custom criteria while Ep._queue: (um, vm, dm) = Ep.pop() # if doesn't create cycle # and subgraphs have enough MV # and we're not connecting 2 fake nodes # then allow the connection if subgraphs[um] != subgraphs[vm] and \ (subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)) and \ (subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm)) and \ not (is_fake(um) and is_fake(vm)): # doesn't create cycles from line segment intersection invalid_edge, intersections = \ line_subgraph_intersection(subgraphs, rtree, coords[um], coords[vm]) if not invalid_edge: # edges should not intersect a subgraph more than once assert(filter(lambda n: n > 1, intersections.values()) == []) # merge the subgraphs subgraphs.union(um, vm, dm) # For all intersected subgraphs update the mv to that # created by the edge intersecting them, # TODO: This should be updated in not such a naive method map(lambda (n, _): subgraphs.union(um, n, 0), filter(lambda (n, i): i == 1 and subgraphs[n] != subgraphs[um], intersections.iteritems())) # index the newly added edge box = make_bounding_box(coords[um], coords[vm]) # Object is (u.label, v.label), (u.coord, v.coord) rtree.insert(hash((um, vm)), box, obj=((um, vm), (coords[um], coords[vm]))) Et += [(um, vm, {'weight': dm})] # create new GeoGraph with results result = G.copy() result.coords = G.coords result.remove_edges_from(result.edges()) result.add_edges_from(Et) return result
class Graph: def __init__(self, all_trips): # trips self.all_trips = all_trips # cluster seeds self.cluster_seeds = {} self.cluster_seed_id = 0 self.cluster_seed_index = Rtree() # graph edges self.graph_edges = {} # indexed by "edge id" self.graph_edge_id = 0 self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" def cluster_traces(self): self._create_all_trip_edges() self._generate_cluster_seeds() self._cluster_seeds_with_traces() self._generate_graph_edges() self._output_graph_to_db() def _generate_graph_edges(self): sys.stdout.write("Generating graph edges... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # grab trip edges trip_edges = trip.edges.values() # put trip edges in order trip_edges.sort(key=lambda x: x.id) # storage for previous cluster prev_cluster = None # iterate through trip edges for trip_edge in trip_edges: # if the current trip edge is clustered if (trip_edge.cluster is not None): # create a graph edge between the previous cluster and the current cluster self._create_graph_edge(prev_cluster, trip_edge.cluster) # update previous cluster with current cluster prev_cluster = trip_edge.cluster # output graph edges self._write_graph_edges_to_file() print "done." def _create_graph_edge(self, in_node, out_node): # if in_node or out_node is None if ((in_node is None) or (out_node is None)): # return without doing anything return # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # increment graph edge id self.graph_edge_id += 1 def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _cluster_seeds_with_traces(self): # storage for total cluster distance moved total_cluster_distance_moved = float('infinity') # iterate until total cluster distance moved below threshold while (total_cluster_distance_moved >= cluster_distance_moved_threshold): # find all points on traces and move clusters total_cluster_distance_moved = self._find_points_on_traces() # write cluster seeds to file self._write_cluster_seeds_to_file( "edelkamp_cluster_seeds_clustered.txt") def _find_points_on_traces(self): # counter for cluster seeds seed_counter = 1 # storage for total cluster distance moved total_cluster_distance_moved = 0.0 # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # clear current trace points from cluster cluster_seed.clear_trace_points() sys.stdout.write("\rFinding intersecting points with cluster " + str(seed_counter) + "/" + str(len(self.cluster_seeds)) + "... ") sys.stdout.flush() # increment seed counter seed_counter += 1 # determine leftward cluster bearing leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0, 360.0) # determine rightward cluster bearing rightward_bearing = math.fmod( (cluster_seed.bearing + 90.0) + 360.0, 360.0) # storage for candidate trace points candidate_trace_points = [] # iterate through all trips for trip in self.all_trips: # find leftward intersection points with trip candidate_trace_points.extend( self._find_intersection_points(trip, cluster_seed, leftward_bearing)) # find rightward intersection points with trip candidate_trace_points.extend( self._find_intersection_points(trip, cluster_seed, rightward_bearing)) # add candidate trace points to cluster cluster_seed.add_trace_points(candidate_trace_points) # recompute cluster centroid total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid( ) # clear current trace points from cluster cluster_seed.clear_trace_points() # add candidate trace points to cluster, again cluster_seed.add_trace_points(candidate_trace_points) # normalize total cluster distance moved by number of seeds total_cluster_distance_moved = (total_cluster_distance_moved / len(self.cluster_seeds.values())) # and we're done! print "done (clusters moved an average of " + str( total_cluster_distance_moved) + " meters)." # return total cluster distance moved return total_cluster_distance_moved def _find_intersection_points(self, trip, cluster, cluster_bearing): # find all nearby trip edge id's nearby_trip_edge_ids = self._find_nearby_trip_edge_ids( cluster, edge_bounding_box_size, trip.edge_index) # storage for intersection points intersection_points = [] # iterate through all nearby edge id's for edge_id in nearby_trip_edge_ids: # grab current edge edge = trip.edges[edge_id] # determine intersection point between edge and cluster intersection_point = self._intersection_point( edge.in_node, edge.bearing, cluster, cluster_bearing) # if there is an intersection point if (intersection_point is not None): # determine distance from edge in_node to intersection point intersection_distance = self._distance_coords( edge.in_node.latitude, edge.in_node.longitude, intersection_point[0], intersection_point[1]) # if intersection distance is less than edge length if (intersection_distance <= edge.length): # this edge has a valid intersection point intersection_points.append( TracePoint(intersection_point[0], intersection_point[1], edge.bearing, edge)) # return all intersection points for this trip return intersection_points def _generate_cluster_seeds(self): # iterate through all trips for i in range(0, len(self.all_trips)): sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" + str(len(self.all_trips)) + "... ") sys.stdout.flush() # grab current trip trip = self.all_trips[i] # set last cluster seed distance to zero for first trip location trip.locations[0].last_cluster_seed_distance = 0.0 # iterate through all trip locations for j in range(1, len(trip.locations)): # drop cluster seeds along current edge every 50 meters self._drop_cluster_seeds_along_edge(trip.locations[j - 1], trip.locations[j]) print "done (generated " + str(len( self.cluster_seeds)) + " cluster seeds)." # write cluster seeds to file self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt") def _drop_cluster_seeds_along_edge(self, in_node, out_node): # determine edge length edge_length = self._distance(in_node, out_node) # determine distance along edge for first cluster seed first_cluster_seed_distance = (cluster_seed_interval - in_node.last_cluster_seed_distance) # storage for relative cluster seed intervals rel_cluster_seed_intervals = [] # storage for current cluster seed distance along this edge curr_cluster_seed_distance = first_cluster_seed_distance # determine the relative cluster seed intervals needed for this edge while (curr_cluster_seed_distance <= edge_length): # append current cluster seed distance to relative cluster seed interval list rel_cluster_seed_intervals.append(curr_cluster_seed_distance) # increment current cluster seed distance curr_cluster_seed_distance += cluster_seed_interval # determine bearing of current edge edge_bearing = self._path_bearing(in_node, out_node) # create cluster seeds for edge for i in range(0, len(rel_cluster_seed_intervals)): # determine fraction along current edge to drop cluster seed fraction_along = (rel_cluster_seed_intervals[i] / edge_length) # determine point along line to drop cluster seed (new_cluster_seed_latitude, new_cluster_seed_longitude) = self._point_along_line( in_node, out_node, fraction_along) # locate nearest existing cluster seeds closest_cluster_seeds = list( self.cluster_seed_index.nearest( (new_cluster_seed_longitude, new_cluster_seed_latitude), 25)) # if there does not exist a closest existing cluster seed if (len(closest_cluster_seeds) == 0): # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing) # else, if there exists a closest existing cluster seed elif (len(closest_cluster_seeds) > 0): # storage for matched cluster seed matched_cluster_seed = None # iterate through closest existing cluster seeds for curr_cluster_seed_id in closest_cluster_seeds: # grab current cluster seed curr_cluster_seed = self.cluster_seeds[ curr_cluster_seed_id] # compute distance to current cluster seed distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, curr_cluster_seed.latitude, curr_cluster_seed.longitude) # determine bearing difference between edge and current cluster seed bearing_difference = math.cos( math.radians(edge_bearing - curr_cluster_seed.bearing)) # if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees if ((distance <= cluster_seed_interval) and (bearing_difference >= cluster_bearing_difference_limit)): # store current cluster seed as matched cluster seed matched_cluster_seed = curr_cluster_seed # stop searching break # if there was not a matched cluster seed if (matched_cluster_seed is None): # create a new cluster seed new_cluster_seed = self._create_new_cluster_seed( new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing) # update last cluster seed distance out_node.last_cluster_seed_distance = self._distance_coords( new_cluster_seed_latitude, new_cluster_seed_longitude, out_node.latitude, out_node.longitude) # if no cluster seeds were generated along this edge if (len(rel_cluster_seed_intervals) == 0): # update last cluster seed distance out_node.last_cluster_seed_distance = ( in_node.last_cluster_seed_distance + edge_length) def _create_new_cluster_seed(self, latitude, longitude, bearing): # create a new cluster seed new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude, longitude, bearing) # add new cluster seed to the cluster seeds dictionary self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed # insert new cluster seed into spatial index self.cluster_seed_index.insert( new_cluster_seed.id, (new_cluster_seed.longitude, new_cluster_seed.latitude)) # increment cluster seed id self.cluster_seed_id += 1 # return new cluster seed return new_cluster_seed def _create_all_trip_edges(self): sys.stdout.write("Creating and indexing edges for all trips... ") sys.stdout.flush() # iterate through all trips for trip in self.all_trips: # add edge storage to trip trip.edges = {} # add edge index to trip trip.edge_index = Rtree() # storage for edge id trip_edge_id = 0 # iterate through all trip locations for i in range(1, len(trip.locations)): # create new edge new_edge = Edge(trip_edge_id, trip.locations[i - 1], trip.locations[i]) # insert edge into dictionary trip.edges[trip_edge_id] = new_edge # insert edge into index self._index_trip_edge(new_edge, trip.edge_index) # increment trip edge id trip_edge_id += 1 # done print "done." def _index_trip_edge(self, edge, edge_index): # determine edge minx, miny, maxx, maxy values edge_minx = min(edge.in_node.longitude, edge.out_node.longitude) edge_miny = min(edge.in_node.latitude, edge.out_node.latitude) edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude) edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude) # insert edge into spatial index edge_index.insert(edge.id, (edge_minx, edge_miny, edge_maxx, edge_maxy)) def _find_nearby_trip_edge_ids(self, location, distance, edge_index): # define longitude/latitude offset lon_offset = ((distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE) lat_offset = ((distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE) # create bounding box bounding_box = (location.longitude - lon_offset, location.latitude - lat_offset, location.longitude + lon_offset, location.latitude + lat_offset) # return nearby edge id's inside bounding box return list(edge_index.intersection(bounding_box)) def _intersection_point(self, location1, location1_bearing, location2, location2_bearing): return spatialfunclib.intersection_point( location1.latitude, location1.longitude, location1_bearing, location2.latitude, location2.longitude, location2_bearing) def _point_along_line(self, location1, location2, fraction_along): return spatialfunclib.point_along_line(location1.latitude, location1.longitude, location2.latitude, location2.longitude, fraction_along) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance_coords(self, location1_latitude, location1_longitude, location2_latitude, location2_longitude): return spatialfunclib.distance(location1_latitude, location1_longitude, location2_latitude, location2_longitude) def _write_cluster_seeds_to_file(self, filename="edelkamp_cluster_seeds.txt"): # open graph file graph_file = open(filename, 'w') # iterate through all cluster_seeds for cluster_seed in self.cluster_seeds.values(): # output cluster seed to file graph_file.write( str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) + "\n") # close graph file graph_file.close() def _write_graph_edges_to_file(self): # open graph file graph_file = open('edelkamp_cluster_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # output edge to file graph_file.write( str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write( str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "\n\n") # close graph file graph_file.close() def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("edelkamp_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute( "CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute( "CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)" ) # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all cluster seeds for cluster_seed in self.cluster_seeds.values(): # insert cluster seed into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(cluster_seed.id) + "," + str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done."
class Graph: def __init__(self, bus_trips): self.bus_trips = bus_trips self.graph_nodes = {} # indexed by "location_id" self.graph_edge_id = 0 self.graph_edges = {} # indexed by "edge id" self.graph_edge_lookup = {} # indexed by "location1_id,location2_id" self.graph_edge_index = Rtree() def generate_graph(self): print "Running graph generation algorithm..." # initialize trip counter trip_count = 1 for trip in self.bus_trips: # initialize location counter location_count = 1 # storage for previous node prev_node = None for location in trip.locations: sys.stdout.write("\rAnalyzing location " + str(location_count) + "/" + str(len(trip.locations)) + " for trip " + str(trip_count) + "/" + str(len(self.bus_trips)) + "... ") sys.stdout.flush() # find closest edges in graph to location closest_edges = self._find_closest_edges_in_graph( location, 100) # flag variable for whether we merged location did_merge_location = False # iterate through candidate edge ids for candidate_edge_id in closest_edges: # grab candidate edge from graph edge dictionary candidate_edge = self.graph_edges[candidate_edge_id] # determine whether we should merge with candidate edge if (self._should_merge_location_with_edge( location, candidate_edge) is True): # merge location with edge, update previous node prev_node = self._merge_location_with_edge( location, candidate_edge, prev_node) # update merge flag variable did_merge_location = True # no need to look at further edges, break out of candidate edges loop break # if we did not merge the location with any edge if (did_merge_location is False): # add location to graph self._add_location_to_graph(location, prev_node) # update previous node with current location prev_node = location # increment location counter location_count += 1 # done with current trip locations print "done." # increment trip counter trip_count += 1 # write graph edges to file self._write_graph_edges_to_file() # create graph database self._output_graph_to_db() def _merge_location_with_edge(self, location, edge, prev_node): # get the edge node closest to the location edge_node = self._get_closest_edge_node(location, edge) # if prev_node is None if (prev_node is None): # increase volume of just this edge edge.volume += 1 # if prev_node is not None else: # find path from prev_node to edge_node path = self._find_path(prev_node, edge_node, max_path_length) # if there was a path from prev_node to edge_node if (path is not None): # iterate through nodes in path for i in range(1, len(path)): # grab in_node in_node = path[i - 1] # grab out_node out_node = path[i] # find corresponding graph edge graph_edge = self._find_graph_edge(in_node, out_node) # increment volume on edge graph_edge.volume += 1 # if there is no path from prev_node to edge_node else: # create a new graph edge between prev_node and edge_node self._create_graph_edge(prev_node, edge_node) # return the edge_node return edge_node def _get_closest_edge_node(self, location, edge): # if in_node distance is less than out_node distance if (self._distance(location, edge.in_node) < self._distance( location, edge.out_node)): # return the edge in_node return edge.in_node # otherwise, return the edge out_node return edge.out_node def _find_path(self, source, destination, max_length): # reset all node visited flags self._reset_node_visited_flags() # get a breath-first search path from source to destination path = self._bfs_path(source, destination) # if there is a path from source to destination if (path is not None): # and if the path length is less than or equal to the maximum length if (len(path) <= max_length): # return the path return path # otherwise, return None return None def _bfs_path(self, source, destination): # storage for breadth-first search parents bfs_parent = {} # key is current node, value is parent node # source node has no breadth-first search parent bfs_parent[source] = None # node queue for breadth-first search bfs_queue = [] # enqueue source node bfs_queue.append(source) # mark source node as visited source.visited = True # while the queue is not empty while (len(bfs_queue) > 0): # dequeue the first node in the queue curr_node = bfs_queue.pop(0) # if the current node is the destination if (curr_node is destination): # create storage for breadth-first search path bfs_path = [] # add the current node to the breadth-first search path bfs_path.insert(0, curr_node) # grab the parent of the current node parent = bfs_parent[curr_node] # iterate through breadth-first search parents while (parent is not None): # add the parent to the breadth-first search path bfs_path.insert(0, parent) # grab the next parent parent = bfs_parent[parent] # return the breadth-first search path return bfs_path # if the current node is not the destination else: # iterate through the current node's out_nodes for out_node in curr_node.out_nodes: # if the out_node has not been visited if (out_node.visited is False): # mark the out_node as visited out_node.visited = True # enqueue the out_node bfs_queue.append(out_node) # store curr_node as out_node's breadth-first search parent bfs_parent[out_node] = curr_node # if we reached here, no path was found return None def _should_merge_location_with_edge(self, location, edge): # project location onto edge (location_projection, location_projection_fraction, location_projection_distance) = self._projection_onto_line( edge.in_node, edge.out_node, location) # if projection is not onto edge if (location_projection_fraction < 0.0 or location_projection_fraction > 1.0): # we cannot merge location with edge return False # determine bearing difference between edge and location bearing_difference = math.cos( math.radians( self._path_bearing(edge.in_node, edge.out_node) - self._location_bearing(location))) # if location projection distance is less than 20 meters if (location_projection_distance < location_projection_distance_limit): # if bearing difference is less than 45 degrees if (bearing_difference > location_bearing_difference_limit): # merge location with edge return True # otherwise, do not merge location with edge return False def _add_location_to_graph(self, location, prev_node): # add an out_nodes list to location location.out_nodes = [] # add an in_nodes list to location location.in_nodes = [] # add a visited flag to location location.visited = False # add location to graph nodes list self.graph_nodes[location.id] = location # if prev_node is not None if (prev_node is not None): # create a new graph edge between prev_node and location self._create_graph_edge(prev_node, location) def _create_graph_edge(self, in_node, out_node): # see if we can find an existing graph edge with the same nodes existing_graph_edge = self._find_graph_edge(in_node, out_node) # if there is no existing graph edge with the same nodes if (existing_graph_edge is None): # create new graph edge object new_graph_edge = Edge(self.graph_edge_id, in_node, out_node) # add new graph edge to graph edge dictionary self.graph_edges[new_graph_edge.id] = new_graph_edge # add new graph edge to graph edge lookup dictionary self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge # add new graph edge to graph edges spatial index self._add_graph_edge_to_index(new_graph_edge) # increment graph edge id self.graph_edge_id += 1 # store out_node in in_nodes's out_nodes list in_node.out_nodes.append(out_node) # store in_node in out_node's in_nodes list out_node.in_nodes.append(in_node) def _find_graph_edge(self, node1, node2): # generate edge lookup key edge_lookup_key = str(node1.id) + "," + str(node2.id) # if edge is in lookup table if (edge_lookup_key in self.graph_edge_lookup.keys()): # return the matching edge return self.graph_edge_lookup[edge_lookup_key] # if the edge wasn't in the lookup table return None def _add_graph_edge_to_index(self, graph_edge): # determine graph edge minx, miny, maxx, maxy values graph_edge_minx = min(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_miny = min(graph_edge.in_node.latitude, graph_edge.out_node.latitude) graph_edge_maxx = max(graph_edge.in_node.longitude, graph_edge.out_node.longitude) graph_edge_maxy = max(graph_edge.in_node.latitude, graph_edge.out_node.latitude) # insert graph edge into spatial index self.graph_edge_index.insert(graph_edge.id, (graph_edge_minx, graph_edge_miny, graph_edge_maxx, graph_edge_maxy)) def _location_bearing(self, location): # if location has a previous neighbor and a next neighbor if ((location.prev_location is not None) and (location.next_location is not None)): # determine bearing using previous and next neighbors return self._path_bearing(location.prev_location, location.next_location) # if location has no previous neighbor, but has a next neighbor elif ((location.prev_location is None) and (location.next_location is not None)): # determine bearing using current location and next neighbor return self._path_bearing(location, location.next_location) # if location has a previous neighbor, but not a next neighbor elif ((location.prev_location is not None) and (location.next_location is None)): # determine bearing using previous neighbor and current location return self._path_bearing(location.prev_location, location) # if we reach here, there is an error return None def _find_closest_edges_in_graph(self, location, number_of_edges): return self.graph_edge_index.nearest( (location.longitude, location.latitude), number_of_edges) def _projection_onto_line(self, location1, location2, location3): return spatialfunclib.projection_onto_line( location1.latitude, location1.longitude, location2.latitude, location2.longitude, location3.latitude, location3.longitude) def _path_bearing(self, location1, location2): return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude) def _output_graph_to_db(self): # output that we are starting the database writing process... sys.stdout.write("\nOutputting graph to database... ") sys.stdout.flush() # connect to database conn = sqlite3.connect("cao_graph.db") # grab cursor cur = conn.cursor() # create nodes table cur.execute( "CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)") # create edges table cur.execute( "CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)" ) # remove values from nodes table #cur.execute("DELETE FROM nodes") # remove values from edges table #cur.execute("DELETE FROM edges") # commit creates conn.commit() # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # insert graph node into nodes table cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) + "," + str(graph_node.latitude) + "," + str(graph_node.longitude) + ")") # iterate through all graph edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # insert graph edge into edges table cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")") # commit inserts conn.commit() # close database connection conn.close() print "done." def _write_graph_edges_to_file(self): # output that we are starting the writing process sys.stdout.write("\nWriting graph edges to file... ") sys.stdout.flush() # open graph file graph_file = open('cao_edges.txt', 'w') # iterate through all graph_edges for graph_edge in self.graph_edges.values(): # if the graph edge has volume greater than or equal to 3 if (graph_edge.volume >= min_graph_edge_volume): # output edge to file graph_file.write( str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n") graph_file.write( str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "," + str(graph_edge.volume) + "\n\n") # close graph file graph_file.close() print "done." def _reset_node_visited_flags(self): # iterate through all graph nodes for graph_node in self.graph_nodes.values(): # set visited flag to False graph_node.visited = False
print >> sys.stderr, "%d retried, %d unassigned." % (retries, len(unassigned)) hoodIndex = Rtree() print >> sys.stderr, "Buffering polygons." for place_id, polygon in polygons.items(): if type(polygon) is Polygon: polygon = Polygon(polygon.exterior.coords) else: bits = [] for p in polygon.geoms: if type(p) is Polygon: bits.append(Polygon(p.exterior.coords)) polygon = MultiPolygon(bits) polygons[place_id] = polygon.buffer(0) hoodIndex.insert(place_id, polygons[place_id].bounds) print >> sys.stderr, "Retconning blocks to shapes." cur.execute( """select geom, geoid10 FROM tabblock10 tb WHERE statefp10 = %s AND countyfp10 = %s AND blockce10 NOT LIKE '0%%'""", (statefp10, countyfp10)) for r in cur.fetchall(): poly = wkb.loads(r[0].decode('hex')) id = r[1] candidates = [i for i in hoodIndex.intersection(poly.bounds)] found = False for place_id in candidates: hood = polygons[place_id] if hood.contains(poly): cur.execute("""DELETE FROM votes WHERE source=%s AND id=%s""", ('blockr', id))
def mod_boruvka(G, subgraphs=None, rtree=None): """ algorithm to calculate the minimum spanning forest of nodes in GeoGraph G with 'budget' based restrictions on edges. Uses a modified version of Boruvka's algorithm NOTE: subgraphs is modified as a side-effect...may remove in future (useful for testing right now) Args: G: GeoGraph of nodes to be connected if appropriate Nodes should have 'budget' attribute subgraphs: UnionFind data structure representing existing network's connected components AND the 'fake' nodes projected onto it. This is the basis for the agglomerative nearest neighbor approach in this algorithm. rtree: RTree based index of existing network Returns: GeoGraph: representing minimum spanning forest of G subject to the budget based restrictions """ # special case (already MST) if G.number_of_nodes() < 2: return G V = set(G.nodes()) coords = np.row_stack(G.coords.values()) projcoords = ang_to_vec_coords(coords) if G.is_geographic() else coords kdtree = KDTree(projcoords) # Handle "dead" components D = set() if subgraphs is None: if rtree is not None: raise ValueError('RTree passed without UnionFind') rtree = Rtree() # modified to handle queues, children, mv subgraphs = UnionFind() # <helper_functions> def candidate_components(C): """ return the set of candidate nearest components for the connected component containing C. Do not consider those in C's connected component or those that are 'dead'. """ component_set = subgraphs.component_set(C) return list((V - set(component_set)) - D) def update_nn_component(C, candidates): """ find the nearest neighbor pair for the connected component represented by c. candidates represents the list of components from which to select. """ (v, vm) = subgraphs.queues[C].top() # vm ∈ C {not a foreign nearest neighbor} # go through the queue until an edge is found between this node # and the set of candidates, updating the neighbors in the connected # components queue in the process. while vm not in candidates: subgraphs.queues[C].pop() um, _ = kdtree.query_subset(projcoords[v], candidates) dm = square_distance(projcoords[v], projcoords[um]) subgraphs.push(subgraphs.queues[C], (v, um), dm) # Note: v will always be a vertex in this connected component # vm *may* be external (v, vm) = subgraphs.queues[C].top() return (v, vm) # Tests whether the node is a projection on the existing grid, using its MV is_fake = lambda n: subgraphs.budget[n] == np.inf # Test whether the component is dead # i.e. can never connect to another node def is_dead(c, nn_dist): return not is_fake(c) and subgraphs.budget[c] < nn_dist # "true" distance between components def component_dist(c1, c2): dist = 0 if G.is_geographic(): dist = spherical_distance([coords[c1], coords[c2]]) else: dist = euclidean_distance([coords[c1], coords[c2]]) return dist # </helper_functions> # Initialize the connected components holding a single node # and push the nearest neighbor into its queue for v in V: vm, _ = kdtree.query_subset(projcoords[v], list(V - {v})) dm = square_distance(projcoords[v], projcoords[vm]) subgraphs.add_component(v, budget=G.node[v]['budget']) subgraphs.push(subgraphs.queues[v], (v, vm), dm) # Add to dead set if warranted nn_dist = component_dist(v, vm) if is_dead(v, nn_dist): # here components are single nodes # so no need to worry about adding children to dead set if v not in D: D.add(v) Et = [] # Initialize MST edges to empty list last_state = None # MST is complete when no progress was made in the prior iteration while Et != last_state: # This is a candidate list of edges that might be added to the MST Ep = PriorityQueue() # ∀ C of G; where c <- connected component for C in subgraphs.connected_components(component_subset=V): candidates = candidate_components(C) # Skip if no valid candidates if not candidates: continue (v, vm) = update_nn_component(C, candidates) # Add to dead set if warranted nn_dist = component_dist(v, vm) if is_dead(C, nn_dist): # add all dead components to the dead set D # (note that fake nodes can never be dead) for c in subgraphs.component_set(C): if c not in D and not is_fake(c): D.add(c) # One more round to root out connections to dead components # found in above iteration. # Need to do this BEFORE pushing candidate edges into Ep. # Otherwise we might be testing only 'dead' candidates # and therefore mistakenly think we were done (since # no new edges would have been added) for C in subgraphs.connected_components(component_subset=V): candidates = candidate_components(C) # Skip if no valid candidates if not candidates: continue (v, vm) = update_nn_component(C, candidates) # Calculate nn_dist for comparison to mv later nn_dist = component_dist(v, vm) # Append the top priority edge from the subgraph to the candidate # edge set Ep.push((v, vm, nn_dist), nn_dist) last_state = deepcopy(Et) # Candidate Test # At this point we have all of our nearest neighbor component edge # candidates defined for this "round" # # Now test all candidate edges in Ep for cycles and satisfaction of # custom criteria while Ep._queue: (um, vm, dm) = Ep.pop() # if doesn't create cycle # and subgraphs have enough MV # and we're not connecting 2 fake nodes # then allow the connection if subgraphs[um] != subgraphs[vm] and \ (subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)) and \ (subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm)) and \ not (is_fake(um) and is_fake(vm)): # doesn't create cycles from line segment intersection invalid_edge, intersections = \ line_subgraph_intersection(subgraphs, rtree, coords[um], coords[vm]) if not invalid_edge: # edges should not intersect a subgraph more than once assert(filter(lambda n: n > 1, intersections.values()) == []) # merge the subgraphs subgraphs.union(um, vm, dm) # For all intersected subgraphs update the mv to that # created by the edge intersecting them, # TODO: This should be updated in not such a naive method map(lambda (n, _): subgraphs.union(um, n, 0), filter(lambda (n, i): i == 1 and subgraphs[n] != subgraphs[um], intersections.iteritems())) # index the newly added edge box = make_bounding_box(coords[um], coords[vm]) # Object is (u.label, v.label), (u.coord, v.coord) rtree.insert(hash((um, vm)), box, obj=((um, vm), (coords[um], coords[vm]))) Et += [(um, vm, {'weight': dm})] # create new GeoGraph with results result = G.copy() result.coords = G.coords result.remove_edges_from(result.edges()) result.add_edges_from(Et) return result
print >>sys.stderr, "%d retried, %d unassigned." % (retries, len(unassigned)) hoodIndex = Rtree() print >>sys.stderr, "Buffering polygons." for place_id, polygon in polygons.items(): if type(polygon) is Polygon: polygon = Polygon(polygon.exterior.coords) else: bits = [] for p in polygon.geoms: if type(p) is Polygon: bits.append(Polygon(p.exterior.coords)) polygon = MultiPolygon(bits) polygons[place_id] = polygon.buffer(0) hoodIndex.insert(place_id, polygons[place_id].bounds) print >>sys.stderr, "Retconning blocks to shapes." cur.execute("""select geom, geoid10 FROM tabblock10 tb WHERE statefp10 = %s AND countyfp10 = %s AND blockce10 NOT LIKE '0%%'""", (statefp10, countyfp10)) for r in cur.fetchall(): poly = wkb.loads(r[0].decode('hex')) id = r[1] candidates = [i for i in hoodIndex.intersection(poly.bounds)] found = False for place_id in candidates: hood = polygons[place_id] if hood.contains(poly): cur.execute("""DELETE FROM votes WHERE source=%s AND id=%s""", ('blockr', id)) cur.execute("""INSERT INTO votes (id, label, count, source) values (%s, %s, %s, 'blockr')""", ( id, place_id, 1)) conn.commit()
''' Created on Aug 5, 2016 @author: ionut ''' import fiona import logging from rtree import Rtree from shapely.geometry import shape logger = logging.getLogger('memory') logger.info('loading shp data') _countries = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm0.shp') _states = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm1.shp') _counties = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm2.shp') logger.info('building county index') _counties_idx = Rtree() for county in _counties: geom = shape(county['geometry']) gid = int(county['id']) _counties_idx.insert(gid, geom.bounds) logger.info('done')
class StreetMap: def __init__(self): self.nodes = {} # indexed by node id self.edges = {} # indexed by edge id self.intersections = {} # indexed by node id self.node_spatial_index = Rtree() self.edge_spatial_index = Rtree() self.intersection_spatial_index = Rtree() self.edge_lookup_table = {} # indexed by (in_node,out_node) self.edge_coords_lookup_table = { } # indexed by (in_node.coords, out_node.coords) self.segments = {} # indexed by segment id self.segment_lookup_table = { } # indexed by (head_edge.in_node, tail_edge.out_node) def load_osmdb(self, osmdb_filename): # connect to OSMDB conn = sqlite3.connect(osmdb_filename) # grab cursor cur = conn.cursor() # output that we are loading nodes sys.stdout.write("\nLoading nodes... ") sys.stdout.flush() # execute query on nodes table query_iter = cur.execute("select id, lat, lon from nodes") # query_result = cur.fetchall() # iterate through all query results for id, lat, lon in query_iter: # create and store node in nodes dictionary self.nodes[int(id)] = Node(float(lat), float(lon), int(id)) print "done." # output that we are loading edges sys.stdout.write("Loading edges... ") sys.stdout.flush() # execute query on ways table query_iter = cur.execute("select id, tags, nds from ways") # query_result = cur.fetchall() # storage for nodes used in valid edges valid_edge_nodes = {} # indexed by node id # iterate through all query results for id, tags, nodes in query_iter: # grab tags associated with current way way_tags_dict = eval(tags) # if current way is a valid highway if ('highway' in way_tags_dict.keys() and self._valid_highway_edge(way_tags_dict['highway'])): # grab all nodes that compose this way way_nodes_list = eval(nodes) # iterate through list of way nodes for i in range(1, len(way_nodes_list)): # grab in_node from nodes dictionary in_node = self.nodes[int(way_nodes_list[i - 1])] # grab out_node from nodes dictionary out_node = self.nodes[int(way_nodes_list[i])] # create edge_id based on way id edge_id = int(str(id) + str(i - 1) + "000000") # if either node on the edge is valid if ( True ): #self._valid_node(in_node) or self._valid_node(out_node)): # create and store edge in edges dictionary self.edges[int(edge_id)] = Edge( in_node, out_node, int(edge_id)) # store in_node in out_node's in_nodes list if (in_node not in out_node.in_nodes): out_node.in_nodes.append(in_node) # store out_node in in_node's out_nodes list if (out_node not in in_node.out_nodes): in_node.out_nodes.append(out_node) # if edge is bidirectional if ('oneway' not in way_tags_dict.keys()): # create new symmetric edge id symmetric_edge_id = int(str(edge_id / 10) + "1") # create and store symmetric edge in edges dictionary self.edges[int(symmetric_edge_id)] = Edge( out_node, in_node, int(symmetric_edge_id)) # store in_node in out_node's out_nodes list if (in_node not in out_node.out_nodes): out_node.out_nodes.append(in_node) # store out_node in in_node's in_nodes list if (out_node not in in_node.in_nodes): in_node.in_nodes.append(out_node) # store in_node in valid_edge_nodes dictionary if (in_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[in_node.id] = in_node # store out_node in valid_edge_nodes dictionary if (out_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[out_node.id] = out_node print "done." # close connection to OSMDB conn.close() # replace all nodes with valid edge nodes self.nodes = valid_edge_nodes # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str( len(self.edges)) + " edges and " + str(len( self.intersections)) + " intersections." def load_graphdb(self, grapdb_filename): # connect to graph database conn = sqlite3.connect(grapdb_filename) # grab cursor cur = conn.cursor() # output that we are loading nodes sys.stdout.write("\nLoading nodes... ") sys.stdout.flush() # execute query on nodes table cur.execute("select id, latitude, longitude, weight from nodes") query_result = cur.fetchall() # iterate through all query results for id, latitude, longitude, weight in query_result: # create and store node in nodes dictionary self.nodes[id] = Node(latitude, longitude, id, weight) print "done." # output that we are loading edges sys.stdout.write("Loading edges... ") sys.stdout.flush() # execute query on ways table cur.execute("select id, in_node, out_node, weight from edges") query_result = cur.fetchall() # storage for nodes used in valid edges valid_edge_nodes = {} # indexed by node id # iterate through all query results for id, in_node_id, out_node_id, weight in query_result: # grab in_node from nodes dictionary in_node = self.nodes[in_node_id] # grab out_node from nodes dictionary out_node = self.nodes[out_node_id] # if either node on the edge is valid if (True ): #self._valid_node(in_node) or self._valid_node(out_node)): # create and store edge in edges dictionary self.edges[id] = Edge(in_node, out_node, id, weight) # store in_node in out_node's in_nodes list if (in_node not in out_node.in_nodes): out_node.in_nodes.append(in_node) # store out_node in in_node's out_nodes list if (out_node not in in_node.out_nodes): in_node.out_nodes.append(out_node) # store in_node in valid_edge_nodes dictionary if (in_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[in_node.id] = in_node # store out_node in valid_edge_nodes dictionary if (out_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[out_node.id] = out_node # execute query on segments table cur.execute("select id, edge_ids from segments") query_result = cur.fetchall() for id, edge_ids in query_result: segment_edges = map(lambda edge_id: self.edges[edge_id], eval(edge_ids)) self.segments[id] = Segment(id, segment_edges) self.segment_lookup_table[( self.segments[id].head_edge.in_node, self.segments[id].tail_edge.out_node)] = self.segments[id] for segment_edge in segment_edges: segment_edge.segment = self.segments[id] # self.segment_lookup_table[segment_edge.id] = self.segments[id] # execute query on intersections table cur.execute("select node_id from intersections") query_result = cur.fetchall() for node_id in query_result: self.intersections[node_id[0]] = self.nodes[node_id[0]] try: cur.execute( "select transition_segment, from_segment, to_segment from transitions" ) query_result = cur.fetchall() self.transitions = {} for transition_segment, from_segment, to_segment in query_result: self.transitions[transition_segment] = (from_segment, to_segment) except: print "Got an error reading " print "done." # close connection to graph db conn.close() # replace all nodes with valid edge nodes self.nodes = valid_edge_nodes # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections #self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str( len(self.edges)) + " edges, " + str(len( self.segments)) + " segments and " + str( len(self.intersections)) + " intersections." def load_shapedb(self, shapedb_filename): # connect to graph database conn = sqlite3.connect(shapedb_filename) # grab cursor cur = conn.cursor() # execute query to find all shape ids cur.execute("select distinct shape_id from shapes") # output that we are loading nodes and edges sys.stdout.write("\nLoading nodes and edges... ") sys.stdout.flush() # storage for shape specific edges self.shape_edges = {} # indexed by shape_id # storage for node id node_id = 0 # iterate through all shape ids for shape_id in cur.fetchall(): # grab shape id shape_id = shape_id[0] # if route is a bus route if (shape_id == "0" or shape_id == "11" or shape_id == "15" or shape_id == "41" or shape_id == "65" or shape_id == "22"): # execute query to find all shape points cur.execute( "select shape_pt_lat, shape_pt_lon from shapes where shape_id='" + str(shape_id) + "' order by shape_pt_sequence asc") # amend shape id if (shape_id == "0"): shape_id = "10000000" elif (shape_id == "11"): shape_id = "10000011" elif (shape_id == "41"): shape_id = "10000041" elif (shape_id == "15"): shape_id = "10000015" elif (shape_id == "65"): shape_id = "10000065" elif (shape_id == "22"): shape_id = "10000022" # storage for first node first_node = None # storage for previous node prev_node = None # create list for this shape's edges self.shape_edges[shape_id] = [] # iterate through all shape points for shape_pt_lat, shape_pt_lon in cur.fetchall(): # create new node curr_node = Node(shape_pt_lat, shape_pt_lon, node_id) # store first node if (first_node is None): first_node = curr_node # increment node id node_id += 1 # add shape id to node curr_node.shape_id = shape_id # store new node in nodes dictionary self.nodes[node_id] = curr_node # if there exists a previous node if (prev_node is not None): # create edge id edge_id = int( str(shape_id) + str(prev_node.id) + str(curr_node.id)) # create new edge curr_edge = Edge(prev_node, curr_node, edge_id) # add shape id to edge curr_edge.shape_id = shape_id # store new edge in edges dictionary self.edges[edge_id] = curr_edge # store new edge in shape edges dictionary self.shape_edges[shape_id].append(curr_edge) # store previous node in current node's in_nodes list curr_node.in_nodes.append(prev_node) # store current node in previous node's out_nodes list prev_node.out_nodes.append(curr_node) # update previous node prev_node = curr_node # create edge id for last edge edge_id = int( str(shape_id) + str(prev_node.id) + str(first_node.id)) # create new edge curr_edge = Edge(prev_node, first_node, edge_id) # add shape id to edge curr_edge.shape_id = shape_id # store new edge in edges dictionary self.edges[edge_id] = curr_edge # store new edge in shape edges dictionary self.shape_edges[shape_id].append(curr_edge) # store previous node in first node's in_nodes list first_node.in_nodes.append(prev_node) # store first node in previous node's out_nodes list prev_node.out_nodes.append(first_node) print "done." # close connection to gtfs db conn.close() # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str( len(self.edges)) + " edges and " + str(len( self.intersections)) + " intersections." def _index_nodes(self): # output that we are indexing nodes sys.stdout.write("Indexing nodes... ") sys.stdout.flush() # iterate through all nodes for curr_node in self.nodes.values(): # insert node into spatial index self.node_spatial_index.insert( curr_node.id, (curr_node.longitude, curr_node.latitude)) print "done." def _index_edges(self): # output that we are indexing edges sys.stdout.write("Indexing edges... ") sys.stdout.flush() # iterate through all edges for curr_edge in self.edges.values(): # determine current edge minx, miny, maxx, maxy values curr_edge_minx = min(curr_edge.in_node.longitude, curr_edge.out_node.longitude) curr_edge_miny = min(curr_edge.in_node.latitude, curr_edge.out_node.latitude) curr_edge_maxx = max(curr_edge.in_node.longitude, curr_edge.out_node.longitude) curr_edge_maxy = max(curr_edge.in_node.latitude, curr_edge.out_node.latitude) # insert current edge into spatial index self.edge_spatial_index.insert(curr_edge.id, (curr_edge_minx, curr_edge_miny, curr_edge_maxx, curr_edge_maxy)) # insert current edge into lookup table self.edge_lookup_table[(curr_edge.in_node, curr_edge.out_node)] = curr_edge self.edge_coords_lookup_table[( curr_edge.in_node.coords(), curr_edge.out_node.coords())] = curr_edge # iterate through all edges for edge in self.edges.values(): # iterate through all out edges for out_node_neighbor in edge.out_node.out_nodes: # add out edge to out edges list edge.out_edges.append( self.edge_lookup_table[(edge.out_node, out_node_neighbor)]) # iterate through all in edges for in_node_neighbor in edge.in_node.in_nodes: # add in edge to in edges list edge.in_edges.append(self.edge_lookup_table[(in_node_neighbor, edge.in_node)]) print "done." def _find_and_index_intersections(self): # output that we are finding and indexing intersections sys.stdout.write("Finding and indexing intersections... ") sys.stdout.flush() # find intersection nodes and index (intersection_nodes, intersection_nodes_index) = self._find_intersection_nodes() # storage for intersection nodes already placed in intersections placed_intersection_nodes = set() # define longitude/latitude offset for bounding box lon_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LONGITUDE) lat_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LATITUDE) # storage for intersection id intersection_id = 0 # iterate through intersection nodes for intersection_node in intersection_nodes: # if the intersection node has not yet been placed if (intersection_node not in placed_intersection_nodes): # create bounding box bounding_box = (intersection_node.longitude - lon_offset, intersection_node.latitude - lat_offset, intersection_node.longitude + lon_offset, intersection_node.latitude + lat_offset) # find intersection node ids within bounding box intersection_node_ids = intersection_nodes_index.intersection( bounding_box) # get intersection nodes intersection_nodes = map(self._get_node, intersection_node_ids) # add intersection nodes to placed set placed_intersection_nodes.update(intersection_nodes) # create new intersection new_intersection = Intersection(intersection_id, intersection_nodes) # increment intersection id intersection_id += 1 # add new intersection to intersections list self.intersections[new_intersection.id] = new_intersection # insert new intersection into spatial index self.intersection_spatial_index.insert( new_intersection.id, (new_intersection.longitude, new_intersection.latitude)) print "done." def _get_node(self, node_id): # return node from dictionary return self.nodes[node_id] def _find_intersection_nodes(self): # storage for intersection nodes intersection_nodes = [] # spatial index for intersection nodes intersection_nodes_index = Rtree() # iterate through all nodes in map for curr_node in self.nodes.values(): # set storage for current node's unique neighbors neighbors = set() # iterate through all in_nodes for in_node in curr_node.in_nodes: # add in_node to neighbors set neighbors.add(in_node) # iterate through all out_nodes for out_node in curr_node.out_nodes: # add out_node to neighbors set neighbors.add(out_node) # if current node has more than 2 neighbors if (len(neighbors) > 2): # add current node to intersection nodes list intersection_nodes.append(curr_node) # add current node to intersection nodes index intersection_nodes_index.insert( curr_node.id, (curr_node.longitude, curr_node.latitude)) # return intersection nodes and index return (intersection_nodes, intersection_nodes_index) def _valid_node(self, node): # if node falls inside the designated bounding box if ((node.latitude >= 41.8619 and node.latitude <= 41.8842) and (node.longitude >= -87.6874 and node.longitude <= -87.6398)): return True else: return False def _valid_highway_edge(self, highway_tag_value): if ((highway_tag_value == 'primary') or (highway_tag_value == 'secondary') or (highway_tag_value == 'tertiary') or (highway_tag_value == 'residential')): return True else: return False def reset_node_visited_flags(self): # iterate through all nodes for node in self.nodes.values(): # set node visited flag to False node.visited = False def reset_edge_visited_flags(self): # iterate through all edges for edge in self.edges.values(): # set edge visited flag to False edge.visited = False def write_map_to_file(self, map_filename="map.txt"): # output that we are starting the writing process sys.stdout.write("\nWriting map to file... ") sys.stdout.flush() # open map file map_file = open(map_filename, 'w') # iterate through all map edges for curr_edge in self.edges.values(): # output current edge to file map_file.write( str(curr_edge.in_node.latitude) + "," + str(curr_edge.in_node.longitude) + "\n") map_file.write( str(curr_edge.out_node.latitude) + "," + str(curr_edge.out_node.longitude) + "\n\n") # close map file map_file.close() print "done." def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
class StreetMap: def __init__(self): self.nodes = {} # indexed by node id self.edges = {} # indexed by edge id self.intersections = {} # indexed by node id self.node_spatial_index = Rtree() self.edge_spatial_index = Rtree() self.intersection_spatial_index = Rtree() self.edge_lookup_table = {} # indexed by (in_node,out_node) self.edge_coords_lookup_table = {} # indexed by (in_node.coords, out_node.coords) self.segments = {} # indexed by segment id self.segment_lookup_table = {} # indexed by (head_edge.in_node, tail_edge.out_node) def load_osmdb(self, osmdb_filename): # connect to OSMDB conn = sqlite3.connect(osmdb_filename) # grab cursor cur = conn.cursor() # output that we are loading nodes sys.stdout.write("\nLoading nodes... ") sys.stdout.flush() # execute query on nodes table cur.execute("select id, lat, lon from nodes") query_result = cur.fetchall() # iterate through all query results for id, lat, lon in query_result: # create and store node in nodes dictionary self.nodes[int(id)] = Node(float(lat), float(lon), int(id)) print "done." # output that we are loading edges sys.stdout.write("Loading edges... ") sys.stdout.flush() # execute query on ways table cur.execute("select id, tags, nds from ways") query_result = cur.fetchall() # storage for nodes used in valid edges valid_edge_nodes = {} # indexed by node id # iterate through all query results for id, tags, nodes in query_result: # grab tags associated with current way way_tags_dict = eval(tags) # if current way is a valid highway if ('highway' in way_tags_dict.keys() and self._valid_highway_edge(way_tags_dict['highway'])): # grab all nodes that compose this way way_nodes_list = eval(nodes) # iterate through list of way nodes for i in range(1, len(way_nodes_list)): # grab in_node from nodes dictionary in_node = self.nodes[int(way_nodes_list[i - 1])] # grab out_node from nodes dictionary out_node = self.nodes[int(way_nodes_list[i])] # create edge_id based on way id edge_id = int(str(id) + str(i - 1) + "000000") # if either node on the edge is valid if (True): #self._valid_node(in_node) or self._valid_node(out_node)): # create and store edge in edges dictionary self.edges[int(edge_id)] = Edge(in_node, out_node,int(edge_id)) # store in_node in out_node's in_nodes list if (in_node not in out_node.in_nodes): out_node.in_nodes.append(in_node) # store out_node in in_node's out_nodes list if (out_node not in in_node.out_nodes): in_node.out_nodes.append(out_node) # if edge is bidirectional if ('oneway' not in way_tags_dict.keys()): # create new symmetric edge id symmetric_edge_id = int(str(edge_id / 10) + "1") # create and store symmetric edge in edges dictionary self.edges[int(symmetric_edge_id)] = Edge(out_node, in_node, int(symmetric_edge_id)) # store in_node in out_node's out_nodes list if (in_node not in out_node.out_nodes): out_node.out_nodes.append(in_node) # store out_node in in_node's in_nodes list if (out_node not in in_node.in_nodes): in_node.in_nodes.append(out_node) # store in_node in valid_edge_nodes dictionary if (in_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[in_node.id] = in_node # store out_node in valid_edge_nodes dictionary if (out_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[out_node.id] = out_node print "done." # close connection to OSMDB conn.close() # replace all nodes with valid edge nodes self.nodes = valid_edge_nodes # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges and " + str(len(self.intersections)) + " intersections." def load_graphdb(self, grapdb_filename): # connect to graph database conn = sqlite3.connect(grapdb_filename) # grab cursor cur = conn.cursor() # output that we are loading nodes sys.stdout.write("\nLoading nodes... ") sys.stdout.flush() # execute query on nodes table cur.execute("select id, latitude, longitude, weight from nodes") query_result = cur.fetchall() # iterate through all query results for id, latitude, longitude, weight in query_result: # create and store node in nodes dictionary self.nodes[id] = Node(latitude, longitude, id, weight) print "done." # output that we are loading edges sys.stdout.write("Loading edges... ") sys.stdout.flush() # execute query on ways table cur.execute("select id, in_node, out_node, weight from edges") query_result = cur.fetchall() # storage for nodes used in valid edges valid_edge_nodes = {} # indexed by node id # iterate through all query results for id, in_node_id, out_node_id, weight in query_result: # grab in_node from nodes dictionary in_node = self.nodes[in_node_id] # grab out_node from nodes dictionary out_node = self.nodes[out_node_id] # if either node on the edge is valid if (True): #self._valid_node(in_node) or self._valid_node(out_node)): # create and store edge in edges dictionary self.edges[id] = Edge(in_node, out_node, id, weight) # store in_node in out_node's in_nodes list if (in_node not in out_node.in_nodes): out_node.in_nodes.append(in_node) # store out_node in in_node's out_nodes list if (out_node not in in_node.out_nodes): in_node.out_nodes.append(out_node) # store in_node in valid_edge_nodes dictionary if (in_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[in_node.id] = in_node # store out_node in valid_edge_nodes dictionary if (out_node.id not in valid_edge_nodes.keys()): valid_edge_nodes[out_node.id] = out_node # execute query on segments table cur.execute("select id, edge_ids from segments") query_result = cur.fetchall() for id, edge_ids in query_result: segment_edges = map(lambda edge_id: self.edges[edge_id], eval(edge_ids)) self.segments[id] = Segment(id, segment_edges) self.segment_lookup_table[(self.segments[id].head_edge.in_node, self.segments[id].tail_edge.out_node)] = self.segments[id] for segment_edge in segment_edges: segment_edge.segment = self.segments[id] # self.segment_lookup_table[segment_edge.id] = self.segments[id] # execute query on intersections table cur.execute("select node_id from intersections") query_result = cur.fetchall() for node_id in query_result: self.intersections[node_id[0]] = self.nodes[node_id[0]] try: cur.execute("select transition_segment, from_segment, to_segment from transitions"); query_result = cur.fetchall() self.transitions={} for transition_segment, from_segment, to_segment in query_result: self.transitions[transition_segment]=(from_segment,to_segment) except: print "Got an error reading " print "done." # close connection to graph db conn.close() # replace all nodes with valid edge nodes self.nodes = valid_edge_nodes # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections #self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges, " + str(len(self.segments)) + " segments and " + str(len(self.intersections)) + " intersections." def load_shapedb(self, shapedb_filename): # connect to graph database conn = sqlite3.connect(shapedb_filename) # grab cursor cur = conn.cursor() # execute query to find all shape ids cur.execute("select distinct shape_id from shapes") # output that we are loading nodes and edges sys.stdout.write("\nLoading nodes and edges... ") sys.stdout.flush() # storage for shape specific edges self.shape_edges = {} # indexed by shape_id # storage for node id node_id = 0 # iterate through all shape ids for shape_id in cur.fetchall(): # grab shape id shape_id = shape_id[0] # if route is a bus route if (shape_id == "0" or shape_id == "11" or shape_id == "15" or shape_id == "41" or shape_id == "65" or shape_id == "22"): # execute query to find all shape points cur.execute("select shape_pt_lat, shape_pt_lon from shapes where shape_id='" + str(shape_id) + "' order by shape_pt_sequence asc") # amend shape id if (shape_id == "0"): shape_id = "10000000" elif (shape_id == "11"): shape_id = "10000011" elif (shape_id == "41"): shape_id = "10000041" elif (shape_id == "15"): shape_id = "10000015" elif (shape_id == "65"): shape_id = "10000065" elif (shape_id == "22"): shape_id = "10000022" # storage for first node first_node = None # storage for previous node prev_node = None # create list for this shape's edges self.shape_edges[shape_id] = [] # iterate through all shape points for shape_pt_lat, shape_pt_lon in cur.fetchall(): # create new node curr_node = Node(shape_pt_lat, shape_pt_lon, node_id) # store first node if (first_node is None): first_node = curr_node # increment node id node_id += 1 # add shape id to node curr_node.shape_id = shape_id # store new node in nodes dictionary self.nodes[node_id] = curr_node # if there exists a previous node if (prev_node is not None): # create edge id edge_id = int(str(shape_id) + str(prev_node.id) + str(curr_node.id)) # create new edge curr_edge = Edge(prev_node, curr_node, edge_id) # add shape id to edge curr_edge.shape_id = shape_id # store new edge in edges dictionary self.edges[edge_id] = curr_edge # store new edge in shape edges dictionary self.shape_edges[shape_id].append(curr_edge) # store previous node in current node's in_nodes list curr_node.in_nodes.append(prev_node) # store current node in previous node's out_nodes list prev_node.out_nodes.append(curr_node) # update previous node prev_node = curr_node # create edge id for last edge edge_id = int(str(shape_id) + str(prev_node.id) + str(first_node.id)) # create new edge curr_edge = Edge(prev_node, first_node, edge_id) # add shape id to edge curr_edge.shape_id = shape_id # store new edge in edges dictionary self.edges[edge_id] = curr_edge # store new edge in shape edges dictionary self.shape_edges[shape_id].append(curr_edge) # store previous node in first node's in_nodes list first_node.in_nodes.append(prev_node) # store first node in previous node's out_nodes list prev_node.out_nodes.append(first_node) print "done." # close connection to gtfs db conn.close() # index nodes self._index_nodes() # index edges self._index_edges() # find and index intersections self._find_and_index_intersections() # output map statistics print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges and " + str(len(self.intersections)) + " intersections." def _index_nodes(self): # output that we are indexing nodes sys.stdout.write("Indexing nodes... ") sys.stdout.flush() # iterate through all nodes for curr_node in self.nodes.values(): # insert node into spatial index self.node_spatial_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude)) print "done." def _index_edges(self): # output that we are indexing edges sys.stdout.write("Indexing edges... ") sys.stdout.flush() # iterate through all edges for curr_edge in self.edges.values(): # determine current edge minx, miny, maxx, maxy values curr_edge_minx = min(curr_edge.in_node.longitude, curr_edge.out_node.longitude) curr_edge_miny = min(curr_edge.in_node.latitude, curr_edge.out_node.latitude) curr_edge_maxx = max(curr_edge.in_node.longitude, curr_edge.out_node.longitude) curr_edge_maxy = max(curr_edge.in_node.latitude, curr_edge.out_node.latitude) # insert current edge into spatial index self.edge_spatial_index.insert(curr_edge.id, (curr_edge_minx, curr_edge_miny, curr_edge_maxx, curr_edge_maxy)) # insert current edge into lookup table self.edge_lookup_table[(curr_edge.in_node, curr_edge.out_node)] = curr_edge self.edge_coords_lookup_table[(curr_edge.in_node.coords(), curr_edge.out_node.coords())] = curr_edge # iterate through all edges for edge in self.edges.values(): # iterate through all out edges for out_node_neighbor in edge.out_node.out_nodes: # add out edge to out edges list edge.out_edges.append(self.edge_lookup_table[(edge.out_node, out_node_neighbor)]) # iterate through all in edges for in_node_neighbor in edge.in_node.in_nodes: # add in edge to in edges list edge.in_edges.append(self.edge_lookup_table[(in_node_neighbor, edge.in_node)]) print "done." def _find_and_index_intersections(self): # output that we are finding and indexing intersections sys.stdout.write("Finding and indexing intersections... ") sys.stdout.flush() # find intersection nodes and index (intersection_nodes, intersection_nodes_index) = self._find_intersection_nodes() # storage for intersection nodes already placed in intersections placed_intersection_nodes = set() # define longitude/latitude offset for bounding box lon_offset = ((intersection_size / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE) lat_offset = ((intersection_size / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE) # storage for intersection id intersection_id = 0 # iterate through intersection nodes for intersection_node in intersection_nodes: # if the intersection node has not yet been placed if (intersection_node not in placed_intersection_nodes): # create bounding box bounding_box = (intersection_node.longitude - lon_offset, intersection_node.latitude - lat_offset, intersection_node.longitude + lon_offset, intersection_node.latitude + lat_offset) # find intersection node ids within bounding box intersection_node_ids = intersection_nodes_index.intersection(bounding_box) # get intersection nodes intersection_nodes = map(self._get_node, intersection_node_ids) # add intersection nodes to placed set placed_intersection_nodes.update(intersection_nodes) # create new intersection new_intersection = Intersection(intersection_id, intersection_nodes) # increment intersection id intersection_id += 1 # add new intersection to intersections list self.intersections[new_intersection.id] = new_intersection # insert new intersection into spatial index self.intersection_spatial_index.insert(new_intersection.id, (new_intersection.longitude, new_intersection.latitude)) print "done." def _get_node(self, node_id): # return node from dictionary return self.nodes[node_id] def _find_intersection_nodes(self): # storage for intersection nodes intersection_nodes = [] # spatial index for intersection nodes intersection_nodes_index = Rtree() # iterate through all nodes in map for curr_node in self.nodes.values(): # set storage for current node's unique neighbors neighbors = set() # iterate through all in_nodes for in_node in curr_node.in_nodes: # add in_node to neighbors set neighbors.add(in_node) # iterate through all out_nodes for out_node in curr_node.out_nodes: # add out_node to neighbors set neighbors.add(out_node) # if current node has more than 2 neighbors if (len(neighbors) > 2): # add current node to intersection nodes list intersection_nodes.append(curr_node) # add current node to intersection nodes index intersection_nodes_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude)) # return intersection nodes and index return (intersection_nodes, intersection_nodes_index) def _valid_node(self, node): # if node falls inside the designated bounding box if ((node.latitude >= 41.8619 and node.latitude <= 41.8842) and (node.longitude >= -87.6874 and node.longitude <= -87.6398)): return True else: return False def _valid_highway_edge(self, highway_tag_value): if ((highway_tag_value == 'primary') or (highway_tag_value == 'secondary') or (highway_tag_value == 'tertiary') or (highway_tag_value == 'residential')): return True else: return False def reset_node_visited_flags(self): # iterate through all nodes for node in self.nodes.values(): # set node visited flag to False node.visited = False def reset_edge_visited_flags(self): # iterate through all edges for edge in self.edges.values(): # set edge visited flag to False edge.visited = False def write_map_to_file(self, map_filename="map.txt"): # output that we are starting the writing process sys.stdout.write("\nWriting map to file... ") sys.stdout.flush() # open map file map_file = open(map_filename, 'w') # iterate through all map edges for curr_edge in self.edges.values(): # output current edge to file map_file.write(str(curr_edge.in_node.latitude) + "," + str(curr_edge.in_node.longitude) + "\n") map_file.write(str(curr_edge.out_node.latitude) + "," + str(curr_edge.out_node.longitude) + "\n\n") # close map file map_file.close() print "done." def _distance(self, location1, location2): return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
def api_root(): ''' This is the root of the webservice, upon successful authentication a text will be displayed in the browser ''' try: projectid = request.args.get('projectid') diagramid = request.args.get('diagramid') apitoken = request.args.get('apitoken') except KeyError as ke: msg = json.dumps({ "message": "Could not parse Projectid, Diagram ID or API Token ID. One or more of these were not found in your JSON request." }) return Response(msg, status=400, mimetype='application/json') if projectid and diagramid and apitoken: myAPIHelper = GeodesignHub.GeodesignHubClient( url=config.apisettings['serviceurl'], project_id=projectid, token=apitoken) myGridGenerator = GridGenerator() myDiagramDecomposer = DiagramDecomposer() r = myAPIHelper.get_diagram(int(diagramid)) try: assert r.status_code == 200 except AssertionError as ae: print("Invalid reponse %s" % ae) else: op = json.loads(r.text) diaggeoms = op['geojson'] sysid = op['sysid'] desc = op['description'] projectorpolicy = op['type'] fundingtype = 'o' geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms) grid, allbounds = myGridGenerator.generateGrid(bounds) gridRTree = Rtree() for boundsid, bounds in allbounds.items(): gridRTree.insert(boundsid, bounds) choppedgeomsandareas = [] choppedgeoms = [] totalarea = 0 for curgeom in geoms: curbounds = curgeom.bounds igridids = list(gridRTree.intersection(curbounds)) for curintersectid in igridids: gridfeat = grid[curintersectid] ifeat = curgeom.intersection(gridfeat) ifeatarea = ifeat.area totalarea += ifeatarea ele = {'area': ifeatarea, 'feature': ifeat} choppedgeoms.append(ele) sortedgeomsandareas = sorted(choppedgeoms, key=lambda k: k['area']) tenpercent = ((totalarea * 10) / 100) thirtypercent = ((totalarea * 30) / 100) seventypercent = ((totalarea * 70) / 100) # print totalarea, tenpercent, thirtypercent, seventypercent a = 0 tenpercentfeats = {"type": "FeatureCollection", "features": []} twentypercentfeats = {"type": "FeatureCollection", "features": []} seventypercentfeats = {"type": "FeatureCollection", "features": []} for cursortedgeom in sortedgeomsandareas: cf = {} j = json.loads( shapelyHelper.export_to_JSON(cursortedgeom['feature'])) cf['type'] = 'Feature' cf['properties'] = {} cf['geometry'] = j if a < tenpercent: tenpercentfeats['features'].append(cf) elif a > tenpercent and a < thirtypercent: twentypercentfeats['features'].append(cf) else: seventypercentfeats['features'].append(cf) a += float(cursortedgeom['area']) opdata = [{ 'gj': tenpercentfeats, 'desc': "10% " + desc }, { 'gj': twentypercentfeats, 'desc': "20% " + desc }, { 'gj': seventypercentfeats, 'desc': "70% " + desc, "fundingtype": fundingtype }] alluploadmessages = [] for curopdata in opdata: print(json.dumps(curopdata['gj'])) # upload = myAPIHelper.post_as_diagram(geoms = json.dumps(curopdata['gj']), projectorpolicy= projectorpolicy,featuretype = 'polygon', description= curopdata['desc'], sysid = sysid) # alluploadmessages.append(json.loads(upload.text)) msg = json.dumps({ "message": "Diagrams have been uploaded", "uploadstatus": alluploadmessages }) return Response(msg, status=400, mimetype='application/json') else: msg = json.dumps({ "message": "Could not parse Project ID, Diagram ID or API Token ID. One or more of these were not found in your JSON request." }) return Response(msg, status=400, mimetype='application/json')
def snap_points_to_links(points, links): """Place points onto closest link in a set of links (arc/edges). Parameters ---------- points : dict Point ID as key and (x,y) coordinate as value. links : list Elements are of type ``libpysal.cg.shapes.Chain`` ** Note ** each element is a link represented as a chain with *one head and one tail vertex* in other words one link only. Returns ------- point2link : dict Key [point ID (see points in arguments)]; value [a 2-tuple ((head, tail), point) where (head, tail) is the target link, and point is the snapped location on the link. Examples -------- >>> import spaghetti >>> from libpysal.cg.shapes import Point, Chain >>> points = {0: Point((1,1))} >>> link = [Chain([Point((0,0)), Point((2,0))])] >>> spaghetti.util.snap_points_to_links(points, link) {0: ([(0.0, 0.0), (2.0, 0.0)], array([1., 0.]))} """ # instantiate an rtree rtree = Rtree() # set the smallest possible float epsilon on machine SMALL = numpy.finfo(float).eps # initialize network vertex to link lookup vertex_2_link = {} # iterate over network links for i, link in enumerate(links): # extract network link (x,y) vertex coordinates head, tail = link.vertices x0, y0 = head x1, y1 = tail if (x0, y0) not in vertex_2_link: vertex_2_link[(x0, y0)] = [] if (x1, y1) not in vertex_2_link: vertex_2_link[(x1, y1)] = [] vertex_2_link[(x0, y0)].append(link) vertex_2_link[(x1, y1)].append(link) # minimally increase the bounding box exterior bx0, by0, bx1, by1 = link.bounding_box bx0 -= SMALL by0 -= SMALL bx1 += SMALL by1 += SMALL # insert the network link and its associated # rectangle into the rtree rtree.insert(i, (bx0, by0, bx1, by1), obj=link) # build a KDtree on link vertices kdtree = cg.KDTree(list(vertex_2_link.keys())) point2link = {} for pt_idx, point in points.items(): # first, find nearest neighbor link vertices for the point dmin, vertex = kdtree.query(point, k=1) vertex = tuple(kdtree.data[vertex]) closest = vertex_2_link[vertex][0].vertices # Use this link as the candidate closest link: closest # Use the distance as the distance to beat: dmin point2link[pt_idx] = (closest, numpy.array(vertex)) x0 = point[0] - dmin y0 = point[1] - dmin x1 = point[0] + dmin y1 = point[1] + dmin # Find all links with bounding boxes that intersect # a query rectangle centered on the point with sides # of length dmin * dmin rtree_lookup = rtree.intersection([x0, y0, x1, y1], objects=True) candidates = [cand.object for cand in rtree_lookup] dmin += SMALL dmin2 = dmin * dmin # of the candidate arcs, find the nearest to the query point for candidate in candidates: dist2cand, nearp = squared_distance_point_link(point, candidate.vertices) if dist2cand <= dmin2: closest = candidate.vertices dmin2 = dist2cand point2link[pt_idx] = (closest, nearp) return point2link
def enclosure_tree(polygons): """ Given a list of shapely polygons with only exteriors, find which curves represent the exterior shell or root curve and which represent holes which penetrate the exterior. This is done with an R-tree for rough overlap detection, and then exact polygon queries for a final result. Parameters ----------- polygons : (n,) shapely.geometry.Polygon Polygons which only have exteriors and may overlap Returns ----------- roots : (m,) int Index of polygons which are root contains : networkx.DiGraph Edges indicate a polygon is contained by another polygon """ tree = Rtree() # nodes are indexes in polygons contains = nx.DiGraph() for i, polygon in enumerate(polygons): # if a polygon is None it means creation # failed due to weird geometry so ignore it if polygon is None or len(polygon.bounds) != 4: continue # insert polygon bounds into rtree tree.insert(i, polygon.bounds) # make sure every valid polygon has a node contains.add_node(i) # loop through every polygon for i in contains.nodes(): polygon = polygons[i] # we first query for bounding box intersections from the R-tree for j in tree.intersection(polygon.bounds): # if we are checking a polygon against itself continue if (i == j): continue # do a more accurate polygon in polygon test # for the enclosure tree information if polygons[i].contains(polygons[j]): contains.add_edge(i, j) elif polygons[j].contains(polygons[i]): contains.add_edge(j, i) # a root or exterior curve has an even number of parents # wrap in dict call to avoid networkx view degree = dict(contains.in_degree()) # convert keys and values to numpy arrays indexes = np.array(list(degree.keys())) degrees = np.array(list(degree.values())) # roots are curves with an even inward degree (parent count) roots = indexes[(degrees % 2) == 0] # if there are multiple nested polygons split the graph # so the contains logic returns the individual polygons if len(degrees) > 0 and degrees.max() > 1: # collect new edges for graph edges = [] # find edges of subgraph for each root and children for root in roots: children = indexes[degrees == degree[root] + 1] edges.extend(contains.subgraph(np.append(children, root)).edges()) # stack edges into new directed graph contains = nx.from_edgelist(edges, nx.DiGraph()) # if roots have no children add them anyway contains.add_nodes_from(roots) return roots, contains
try: assert r.status_code == 200 except AssertionError as ae: print("Invalid reponse %s" % ae) else: op = json.loads(r.text) diaggeoms = op['geojson'] sysid = op['sysid'] desc = op['description'] projectorpolicy = op['type'] geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms) grid, allbounds = myGridGenerator.generateGrid(bounds) gridRTree = Rtree() for boundsid, bounds in allbounds.iteritems(): gridRTree.insert(boundsid, bounds) choppedgeomsandareas = [] choppedgeoms = [] totalarea = 0 for curgeom in geoms: curbounds = curgeom.bounds igridids = list(gridRTree.intersection(curbounds)) for curintersectid in igridids: gridfeat = grid[curintersectid] ifeat = curgeom.intersection(gridfeat) ifeatarea = ifeat.area totalarea += ifeatarea ele = {'area': ifeatarea, 'feature': ifeat} choppedgeoms.append(ele)