def _build_trip_edge_index(self):
sys.stdout.write(
"\nBuilding trip edge index for clarification algorithm... ")
sys.stdout.flush()
# storage for trip edge index
trip_edge_index = Rtree()
# iterate through all trip edges
for trip_edge in self.trip_edges.values():
# determine trip edge minx, miny, maxx, maxy values
trip_edge_minx = min(trip_edge.in_node.longitude,
trip_edge.out_node.longitude)
trip_edge_miny = min(trip_edge.in_node.latitude,
trip_edge.out_node.latitude)
trip_edge_maxx = max(trip_edge.in_node.longitude,
trip_edge.out_node.longitude)
trip_edge_maxy = max(trip_edge.in_node.latitude,
trip_edge.out_node.latitude)
# insert trip edge into spatial index
trip_edge_index.insert(trip_edge.id,
(trip_edge_minx, trip_edge_miny,
trip_edge_maxx, trip_edge_maxy))
print "done."
# return the trip edge index
return trip_edge_index
def polygons_enclosure_tree(polygons):
'''
Given a list of shapely polygons, which are the root (aka outermost)
polygons, and which represent the holes which penetrate the root
curve. We do this by creating an R-tree for rough collision detection,
and then do polygon queries for a final result
'''
tree = Rtree()
for i, polygon in enumerate(polygons):
tree.insert(i, polygon.bounds)
count = len(polygons)
g = nx.DiGraph()
g.add_nodes_from(np.arange(count))
for i in range(count):
if polygons[i] is None:
continue
# we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygons[i].bounds):
if (i == j):
continue
# we then do a more accurate polygon in polygon test to generate
# the enclosure tree information
if polygons[i].contains(polygons[j]):
g.add_edge(i, j)
elif polygons[j].contains(polygons[i]):
g.add_edge(j, i)
roots = [n for n, deg in dict(g.in_degree()).items() if deg == 0]
return roots, g
def polygons_enclosure_tree(polygons):
'''
Given a list of shapely polygons, which are the root (aka outermost)
polygons, and which represent the holes which penetrate the root
curve. We do this by creating an R-tree for rough collision detection,
and then do polygon queries for a final result
'''
tree = Rtree()
for i, polygon in enumerate(polygons):
tree.insert(i, polygon.bounds)
count = len(polygons)
g = nx.DiGraph()
g.add_nodes_from(np.arange(count))
for i in range(count):
if polygons[i] is None: continue
#we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygons[i].bounds):
if (i==j): continue
#we then do a more accurate polygon in polygon test to generate
#the enclosure tree information
if polygons[i].contains(polygons[j]): g.add_edge(i,j)
elif polygons[j].contains(polygons[i]): g.add_edge(j,i)
roots = [n for n, deg in list(g.in_degree().items()) if deg==0]
return roots, g
def get_rtree_index(self):
"""
Get an rtree index of the edges within this GeoGraph
Returns:
rtree: rtree of edges indexed by bounding_box (4 coordinates)
allowing lookup of the edge (node1, node2) and it's segment
"""
def edge_generator():
edges_bounds = map(lambda tup: (tup, gm.make_bounding_box(
*map(lambda n: self.coords[n], tup))), self.edges())
for edge, box in edges_bounds:
# Object is in form of (u.label, v.label), (u.coord, v.coord)
yield (hash(edge), box, (edge,
map(lambda ep:
np.array(self.coords[ep]), edge)))
# something's not working right with latest version of rtree/spatialib
# index where we need to insert the objects in a loop rather than
# via a generator to their constructor
# Init rtree and store grid edges
rtree = Rtree()
for e in edge_generator():
# takes id, bbox, and segment/edge
rtree.insert(e[0], e[1], e[2])
return rtree
def get_rtree_index(self):
"""
Get an rtree index of the edges within this GeoGraph
Returns:
rtree: rtree of edges indexed by bounding_box (4 coordinates)
allowing lookup of the edge (node1, node2) and it's segment
"""
def edge_generator():
edges_bounds = map(lambda tup: (tup, gm.make_bounding_box(
*map(lambda n: self.coords[n], tup))), self.edges())
for edge, box in edges_bounds:
# Object is in form of (u.label, v.label), (u.coord, v.coord)
yield (hash(edge), box, (edge,
map(lambda ep: np.array(self.coords[ep]), edge)))
# something's not working right with latest version of rtree/spatial lib
# index where we need to insert the objects in a loop rather than
# via a generator to their constructor
# Init rtree and store grid edges
rtree = Rtree()
for e in edge_generator():
rtree.insert(e[0], e[1], e[2])
return rtree
def enclosure_tree(polygons):
"""
Given a list of shapely polygons with only exteriors,
find which curves represent the exterior shell or root curve
and which represent holes which penetrate the exterior.
This is done with an R-tree for rough overlap detection,
and then exact polygon queries for a final result.
Parameters
-----------
polygons : (n,) shapely.geometry.Polygon
Polygons which only have exteriors and may overlap
Returns
-----------
roots : (m,) int
Index of polygons which are root
contains : networkx.DiGraph
Edges indicate a polygon is
contained by another polygon
"""
tree = Rtree()
# nodes are indexes in polygons
contains = nx.DiGraph()
for i, polygon in enumerate(polygons):
# if a polygon is None it means creation
# failed due to weird geometry so ignore it
if polygon is None:
continue
# insert polygon bounds into rtree
tree.insert(i, polygon.bounds)
# make sure every valid polygon has a node
contains.add_node(i)
# loop through every polygon
for i, polygon in enumerate(polygons):
# if polygon creation failed ignore it
if polygon is None:
continue
# we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygon.bounds):
# if we are checking a polygon against itself continue
if (i == j):
continue
# do a more accurate polygon in polygon test
# for the enclosure tree information
if polygons[i].contains(polygons[j]):
contains.add_edge(i, j)
elif polygons[j].contains(polygons[i]):
contains.add_edge(j, i)
# a root or exterior curve has no parents
# wrap in dict call to avoid networkx view
in_degree = dict(contains.in_degree())
roots = [n for n, deg in in_degree.items() if deg == 0]
return roots, contains
def __init__(self,
hmm,
emission_probability,
constraint_length=10,
MAX_DIST=500,
priors=None,
smallV=0.00000000001):
# initialize spatial index
self.previous_obs = None
if priors == None:
priors = dict([(state, 1.0 / len(hmm)) for state in hmm])
state_spatial_index = Rtree()
unlocated_states = []
id_to_state = {}
id = 0
for state in hmm:
geom = self.geometry_of_state(state)
if not geom:
unlocated_states.append(state)
else:
((lat1, lon1), (lat2, lon2)) = geom
state_spatial_index.insert(id, (min(lon1, lon2), min(
lat1, lat2), max(lon1, lon2), max(lat1, lat2)))
id_to_state[id] = state
id = id + 1
def candidate_states(obs): #was (lat,lon) in place of obs
geom = self.geometry_of_observation(obs)
if geom == None:
return hmm.keys()
else:
(lat, lon) = geom
nearby_states = state_spatial_index.intersection(
(lon - MAX_DIST / METERS_PER_DEGREE_LONGITUDE,
lat - MAX_DIST / METERS_PER_DEGREE_LATITUDE,
lon + MAX_DIST / METERS_PER_DEGREE_LONGITUDE,
lat + MAX_DIST / METERS_PER_DEGREE_LATITUDE))
candidates = [id_to_state[id]
for id in nearby_states] + unlocated_states
return candidates
self.viterbi = Viterbi(hmm,
emission_probability,
constraint_length=constraint_length,
priors=priors,
candidate_states=candidate_states,
smallV=smallV)
def points_in_mesh(points, mesh):
from rtree import Rtree
points = np.array(points)
tri_3D = mesh.vertices[mesh.faces]
tri_2D = tri_3D[:,:,0:2]
z_bounds = np.column_stack((np.min(tri_3D[:,:,2], axis=1),
np.max(tri_3D[:,:,2], axis=1)))
bounds = np.column_stack((np.min(tri_2D, axis=1),
np.max(tri_2D, axis=1)))
tree = Rtree()
for i, bound in enumerate(bounds):
tree.insert(i, bound)
result = np.zeros(len(points), dtype=np.bool)
for point_index, point in enumerate(points):
intersections = np.array(list(tree.intersection(point[0:2].tolist())))
in_triangle = [point_in_triangle_2D(point[0:2], tri_2D[i]) for i in intersections]
result[point_index] = np.int(np.mod(np.sum(in_triangle), 2)) == 0
return result
def __init__(self, hmm, emission_probability, constraint_length=10, MAX_DIST=500, priors=None, smallV=0.00000000001):
# initialize spatial index
self.previous_obs = None
if priors == None:
priors=dict([(state,1.0/len(hmm)) for state in hmm])
state_spatial_index = Rtree()
unlocated_states = []
id_to_state = {}
id = 0
for state in hmm:
geom=self.geometry_of_state(state)
if not geom:
unlocated_states.append(state)
else:
((lat1,lon1),(lat2,lon2))=geom
state_spatial_index.insert(id,
(min(lon1, lon2), min(lat1, lat2),
max(lon1, lon2), max(lat1, lat2)))
id_to_state[id]=state
id=id+1
def candidate_states(obs): #was (lat,lon) in place of obs
geom = self.geometry_of_observation(obs)
if geom == None:
return hmm.keys()
else:
(lat,lon)=geom
nearby_states = state_spatial_index.intersection((lon-MAX_DIST/METERS_PER_DEGREE_LONGITUDE,
lat-MAX_DIST/METERS_PER_DEGREE_LATITUDE,
lon+MAX_DIST/METERS_PER_DEGREE_LONGITUDE,
lat+MAX_DIST/METERS_PER_DEGREE_LATITUDE))
candidates = [id_to_state[id] for id in nearby_states]+unlocated_states
return candidates
self.viterbi = Viterbi(hmm,emission_probability,
constraint_length=constraint_length,
priors=priors,
candidate_states=candidate_states,
smallV=smallV)
def enclosure_tree(polygons):
"""
Given a list of shapely polygons, which are the root (aka outermost)
polygons, and which represent the holes which penetrate the root
curve. We do this by creating an R-tree for rough collision detection,
and then do polygon queries for a final result
Parameters
-----------
polygons: (n,) list of shapely.geometry.Polygon objects
Returns
-----------
roots: (m,) int, index of polygons which are root
contains: networkx.DiGraph, edges indicate a polygon
contained by another polygon
"""
tree = Rtree()
for i, polygon in enumerate(polygons):
if polygon is not None:
tree.insert(i, polygon.bounds)
count = len(polygons)
# nodes are indexes in polygons
contains = nx.DiGraph()
# make sure every polygon is included
contains.add_nodes_from(np.arange(count))
for i in range(count):
if polygons[i] is None:
continue
# we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygons[i].bounds):
if (i == j):
continue
# we then do a more accurate polygon in polygon test to generate
# the enclosure tree information
if polygons[i].contains(polygons[j]):
contains.add_edge(i, j)
elif polygons[j].contains(polygons[i]):
contains.add_edge(j, i)
roots = [n for n, deg in dict(contains.in_degree()).items() if deg == 0]
return roots, contains
def to_directed(self, as_view=False):
"""
Convert undirected road network to directed road network
new edge will have new eid, and each original edge will have two edge with reversed coords
:return:
"""
assert as_view is False, "as_view is not supported"
avail_eid = max([eid for u, v, eid in self.edges.data(data='eid')]) + 1
g = nx.DiGraph()
edge_spatial_idx = Rtree()
edge_idx = {}
# add nodes
for n, data in self.nodes(data=True):
# when data=True, it means will data=node's attributes
new_data = copy.deepcopy(data)
g.add_node(n, **new_data)
# add edges
for u, v, data in self.edges(data=True):
mbr = MBR.cal_mbr(data['coords'])
# add forward edge
forward_data = copy.deepcopy(data)
g.add_edge(u, v, **forward_data)
edge_spatial_idx.insert(
forward_data['eid'],
(mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat))
edge_idx[forward_data['eid']] = (u, v)
# add backward edge
backward_data = copy.deepcopy(data)
backward_data['eid'] = avail_eid
avail_eid += 1
backward_data['coords'].reverse()
g.add_edge(v, u, **backward_data)
edge_spatial_idx.insert(
backward_data['eid'],
(mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat))
edge_idx[backward_data['eid']] = (v, u)
print('# of nodes:{}'.format(g.number_of_nodes()))
print('# of edges:{}'.format(g.number_of_edges()))
return RoadNetwork(g, edge_spatial_idx, edge_idx)
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(
curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
def _build_trip_edge_index(self):
sys.stdout.write("\nBuilding trip edge index for clarification algorithm... ")
sys.stdout.flush()
# storage for trip edge index
trip_edge_index = Rtree()
# iterate through all trip edges
for trip_edge in self.trip_edges.values():
# determine trip edge minx, miny, maxx, maxy values
trip_edge_minx = min(trip_edge.in_node.longitude, trip_edge.out_node.longitude)
trip_edge_miny = min(trip_edge.in_node.latitude, trip_edge.out_node.latitude)
trip_edge_maxx = max(trip_edge.in_node.longitude, trip_edge.out_node.longitude)
trip_edge_maxy = max(trip_edge.in_node.latitude, trip_edge.out_node.latitude)
# insert trip edge into spatial index
trip_edge_index.insert(trip_edge.id, (trip_edge_minx, trip_edge_miny, trip_edge_maxx, trip_edge_maxy))
print "done."
# return the trip edge index
return trip_edge_index
def load_rn_shp(path, is_directed=True):
edge_spatial_idx = Rtree()
edge_idx = {}
# node uses coordinate as key
# edge uses coordinate tuple as key
g = nx.read_shp(path, simplify=True, strict=False)
if not is_directed:
g = g.to_undirected()
# node attrs: nid, pt, ...
for n, data in g.nodes(data=True):
data['pt'] = SPoint(n[1], n[0])
if 'ShpName' in data:
del data['ShpName']
# edge attrs: eid, length, coords, ...
for u, v, data in g.edges(data=True):
geom_line = ogr.CreateGeometryFromWkb(data['Wkb'])
coords = []
for i in range(geom_line.GetPointCount()):
geom_pt = geom_line.GetPoint(i)
coords.append(SPoint(geom_pt[1], geom_pt[0]))
data['coords'] = coords
data['length'] = sum([
distance(coords[i], coords[i + 1]) for i in range(len(coords) - 1)
])
env = geom_line.GetEnvelope()
edge_spatial_idx.insert(data['eid'], (env[0], env[2], env[1], env[3]))
edge_idx[data['eid']] = (u, v)
del data['ShpName']
del data['Json']
del data['Wkt']
del data['Wkb']
print('# of nodes:{}'.format(g.number_of_nodes()))
print('# of edges:{}'.format(g.number_of_edges()))
if not is_directed:
return UndirRoadNetwork(g, edge_spatial_idx, edge_idx)
else:
return RoadNetwork(g, edge_spatial_idx, edge_idx)
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
class Graph:
def __init__(self, bus_trips):
self.bus_trips = bus_trips
self.graph_nodes = {} # indexed by "location_id"
self.graph_edge_id = 0
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
self.graph_edge_index = Rtree()
def generate_graph(self):
print "Running graph generation algorithm..."
# initialize trip counter
trip_count = 1
for trip in self.bus_trips:
# initialize location counter
location_count = 1
# storage for previous node
prev_node = None
for location in trip.locations:
sys.stdout.write("\rAnalyzing location " + str(location_count) + "/" + str(len(trip.locations)) + " for trip " + str(trip_count) + "/" + str(len(self.bus_trips)) + "... ")
sys.stdout.flush()
# find closest edges in graph to location
closest_edges = self._find_closest_edges_in_graph(location, 100)
# flag variable for whether we merged location
did_merge_location = False
# iterate through candidate edge ids
for candidate_edge_id in closest_edges:
# grab candidate edge from graph edge dictionary
candidate_edge = self.graph_edges[candidate_edge_id]
# determine whether we should merge with candidate edge
if (self._should_merge_location_with_edge(location, candidate_edge) is True):
# merge location with edge, update previous node
prev_node = self._merge_location_with_edge(location, candidate_edge, prev_node)
# update merge flag variable
did_merge_location = True
# no need to look at further edges, break out of candidate edges loop
break
# if we did not merge the location with any edge
if (did_merge_location is False):
# add location to graph
self._add_location_to_graph(location, prev_node)
# update previous node with current location
prev_node = location
# increment location counter
location_count += 1
# done with current trip locations
print "done."
# increment trip counter
trip_count += 1
# write graph edges to file
self._write_graph_edges_to_file()
# create graph database
self._output_graph_to_db()
def _merge_location_with_edge(self, location, edge, prev_node):
# get the edge node closest to the location
edge_node = self._get_closest_edge_node(location, edge)
# if prev_node is None
if (prev_node is None):
# increase volume of just this edge
edge.volume += 1
# if prev_node is not None
else:
# find path from prev_node to edge_node
path = self._find_path(prev_node, edge_node, max_path_length)
# if there was a path from prev_node to edge_node
if (path is not None):
# iterate through nodes in path
for i in range(1, len(path)):
# grab in_node
in_node = path[i - 1]
# grab out_node
out_node = path[i]
# find corresponding graph edge
graph_edge = self._find_graph_edge(in_node, out_node)
# increment volume on edge
graph_edge.volume += 1
# if there is no path from prev_node to edge_node
else:
# create a new graph edge between prev_node and edge_node
self._create_graph_edge(prev_node, edge_node)
# return the edge_node
return edge_node
def _get_closest_edge_node(self, location, edge):
# if in_node distance is less than out_node distance
if (self._distance(location, edge.in_node) < self._distance(location, edge.out_node)):
# return the edge in_node
return edge.in_node
# otherwise, return the edge out_node
return edge.out_node
def _find_path(self, source, destination, max_length):
# reset all node visited flags
self._reset_node_visited_flags()
# get a breath-first search path from source to destination
path = self._bfs_path(source, destination)
# if there is a path from source to destination
if (path is not None):
# and if the path length is less than or equal to the maximum length
if (len(path) <= max_length):
# return the path
return path
# otherwise, return None
return None
def _bfs_path(self, source, destination):
# storage for breadth-first search parents
bfs_parent = {} # key is current node, value is parent node
# source node has no breadth-first search parent
bfs_parent[source] = None
# node queue for breadth-first search
bfs_queue = []
# enqueue source node
bfs_queue.append(source)
# mark source node as visited
source.visited = True
# while the queue is not empty
while (len(bfs_queue) > 0):
# dequeue the first node in the queue
curr_node = bfs_queue.pop(0)
# if the current node is the destination
if (curr_node is destination):
# create storage for breadth-first search path
bfs_path = []
# add the current node to the breadth-first search path
bfs_path.insert(0, curr_node)
# grab the parent of the current node
parent = bfs_parent[curr_node]
# iterate through breadth-first search parents
while (parent is not None):
# add the parent to the breadth-first search path
bfs_path.insert(0, parent)
# grab the next parent
parent = bfs_parent[parent]
# return the breadth-first search path
return bfs_path
# if the current node is not the destination
else:
# iterate through the current node's out_nodes
for out_node in curr_node.out_nodes:
# if the out_node has not been visited
if (out_node.visited is False):
# mark the out_node as visited
out_node.visited = True
# enqueue the out_node
bfs_queue.append(out_node)
# store curr_node as out_node's breadth-first search parent
bfs_parent[out_node] = curr_node
# if we reached here, no path was found
return None
def _should_merge_location_with_edge(self, location, edge):
# project location onto edge
(location_projection, location_projection_fraction, location_projection_distance) = self._projection_onto_line(edge.in_node, edge.out_node, location)
# if projection is not onto edge
if (location_projection_fraction < 0.0 or location_projection_fraction > 1.0):
# we cannot merge location with edge
return False
# determine bearing difference between edge and location
bearing_difference = math.cos(math.radians(self._path_bearing(edge.in_node, edge.out_node) - self._location_bearing(location)))
# if location projection distance is less than 20 meters
if (location_projection_distance < location_projection_distance_limit):
# if bearing difference is less than 45 degrees
if (bearing_difference > location_bearing_difference_limit):
# merge location with edge
return True
# otherwise, do not merge location with edge
return False
def _add_location_to_graph(self, location, prev_node):
# add an out_nodes list to location
location.out_nodes = []
# add an in_nodes list to location
location.in_nodes = []
# add a visited flag to location
location.visited = False
# add location to graph nodes list
self.graph_nodes[location.id] = location
# if prev_node is not None
if (prev_node is not None):
# create a new graph edge between prev_node and location
self._create_graph_edge(prev_node, location)
def _create_graph_edge(self, in_node, out_node):
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if (existing_graph_edge is None):
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge
# add new graph edge to graph edges spatial index
self._add_graph_edge_to_index(new_graph_edge)
# increment graph edge id
self.graph_edge_id += 1
# store out_node in in_nodes's out_nodes list
in_node.out_nodes.append(out_node)
# store in_node in out_node's in_nodes list
out_node.in_nodes.append(in_node)
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if (edge_lookup_key in self.graph_edge_lookup.keys()):
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _add_graph_edge_to_index(self, graph_edge):
# determine graph edge minx, miny, maxx, maxy values
graph_edge_minx = min(graph_edge.in_node.longitude, graph_edge.out_node.longitude)
graph_edge_miny = min(graph_edge.in_node.latitude, graph_edge.out_node.latitude)
graph_edge_maxx = max(graph_edge.in_node.longitude, graph_edge.out_node.longitude)
graph_edge_maxy = max(graph_edge.in_node.latitude, graph_edge.out_node.latitude)
# insert graph edge into spatial index
self.graph_edge_index.insert(graph_edge.id, (graph_edge_minx, graph_edge_miny, graph_edge_maxx, graph_edge_maxy))
def _location_bearing(self, location):
# if location has a previous neighbor and a next neighbor
if ((location.prev_location is not None) and (location.next_location is not None)):
# determine bearing using previous and next neighbors
return self._path_bearing(location.prev_location, location.next_location)
# if location has no previous neighbor, but has a next neighbor
elif ((location.prev_location is None) and (location.next_location is not None)):
# determine bearing using current location and next neighbor
return self._path_bearing(location, location.next_location)
# if location has a previous neighbor, but not a next neighbor
elif ((location.prev_location is not None) and (location.next_location is None)):
# determine bearing using previous neighbor and current location
return self._path_bearing(location.prev_location, location)
# if we reach here, there is an error
return None
def _find_closest_edges_in_graph(self, location, number_of_edges):
return self.graph_edge_index.nearest((location.longitude, location.latitude), number_of_edges)
def _projection_onto_line(self, location1, location2, location3):
return spatialfunclib.projection_onto_line(location1.latitude, location1.longitude, location2.latitude, location2.longitude, location3.latitude, location3.longitude)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("cao_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)")
# remove values from nodes table
#cur.execute("DELETE FROM nodes")
# remove values from edges table
#cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# insert graph node into nodes table
cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) + "," + str(graph_node.latitude) + "," + str(graph_node.longitude) + ")")
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# insert graph edge into edges table
cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) + "," + str(graph_edge.in_node.id) + "," + str(graph_edge.out_node.id) + ")")
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
def _write_graph_edges_to_file(self):
# output that we are starting the writing process
sys.stdout.write("\nWriting graph edges to file... ")
sys.stdout.flush()
# open graph file
graph_file = open('cao_edges.txt', 'w')
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# output edge to file
graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n")
graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "," + str(graph_edge.volume) + "\n\n")
# close graph file
graph_file.close()
print "done."
def _reset_node_visited_flags(self):
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# set visited flag to False
graph_node.visited = False
def mod_kruskal(G, subgraphs=None, rtree=None):
"""
algorithm to compute the euclidean minimum spanning forest of nodes in
GeoGraph G with 'budget' based restrictions on edges
Uses a modified version of Kruskal's algorithm
NOTE: subgraphs is modified as a side-effect...may remove in future
(useful for testing right now)
Args:
G: GeoGraph of nodes to be connected if appropriate
Nodes should have 'budget' attribute
subgraphs: UnionFind data structure representing existing network's
connected components AND the 'fake' nodes projected onto it. This
is the basis for the agglomerative nearest neighbor approach in
this algorithm.
NOTE: ONLY the existing networks components are represented in
the subgraphs argument. The nodes in G will be added within
this function
rtree: RTree based index of existing network
Returns:
GeoGraph: representing minimum spanning forest of G subject to the
budget based restrictions
"""
# special case (already MST)
if G.number_of_nodes() < 2:
return G
def is_fake(node):
"""
Tests whether the node is a projection on the existing grid,
using its MV
"""
return subgraphs.budget[node] == np.inf
# handy to have coords array
coords = np.row_stack(G.coords.values())
if subgraphs is None:
assert rtree is not None, \
"subgraphs (disjoint set) required when rtree is passed"
rtree = Rtree()
# modified to handle queues, children, mv
subgraphs = UnionFind()
# add nodes and budgets from G to subgraphs as components
for node in G.nodes():
subgraphs.add_component(node, budget=G.node[node]['budget'])
# get fully connected graph
g = G.get_connected_weighted_graph()
# edges in MSF
Et = []
# connect the shortest safe edge until all edges have been tested
# at which point, we have made all possible connections
for u, v, w in sorted(g.edges(data=True), key=lambda x: x[2]['weight']):
# if doesn't create cycle
# and subgraphs have enough MV
# and we're not connecting 2 fake nodes
# then allow the connection
w = w['weight']
if subgraphs[u] != subgraphs[v] and \
(subgraphs.budget[subgraphs[u]] >= w or is_fake(u)) and \
(subgraphs.budget[subgraphs[v]] >= w or is_fake(v)) and \
not (is_fake(u) and is_fake(v)):
# doesn't create cycles from line segment intersection
invalid_edge, intersections = \
line_subgraph_intersection(subgraphs, rtree,
coords[u], coords[v])
if not invalid_edge:
# edges should not intersect a subgraph more than once
assert(filter(lambda n: n > 1,
intersections.values()) == [])
# merge the subgraphs
subgraphs.union(u, v, w)
# For all intersected subgraphs update the mv to that
# created by the edge intersecting them
map(lambda (n, _): subgraphs.union(u, n, 0),
filter(lambda (n, i): i == 1 and
subgraphs[n] != subgraphs[u],
intersections.iteritems()))
# index the newly added edge
box = make_bounding_box(coords[u], coords[v])
# Object is (u.label, v.label), (u.coord, v.coord)
rtree.insert(hash((u, v)), box,
obj=((u, v), (coords[u], coords[v])))
Et += [(u, v, {'weight': w})]
# create new GeoGraph with results
result = G.copy()
result.coords = G.coords
result.remove_edges_from(result.edges())
result.add_edges_from(Et)
return result
def detect_rn_component(self, status_matrix):
node_pixels = np.where(status_matrix == GraphExtractor.NODE)
nb_node_pixels = len(node_pixels[0])
print('\n# of node pixels:{}'.format(nb_node_pixels))
neighbor_deltas = [
dxdy for dxdy in itertools.product([-1, 0, 1], [-1, 0, 1])
if dxdy[0] != 0 or dxdy[1] != 0
]
# node pixel -> center node
nodes = {}
node_pixel_spatial_index = Rtree()
# [node pixel sequence (start and end must be node pixel)]
connected_segments = []
cnt = 1
center_nodes = []
node_pixel_id = 0
for i, j in zip(node_pixels[0], node_pixels[1]):
if (cnt % 100 == 0) or (cnt == nb_node_pixels):
sys.stdout.write("\r" + str(cnt) + "/" + str(nb_node_pixels) +
"... ")
sys.stdout.flush()
cnt += 1
if status_matrix[i][j] == GraphExtractor.VISITED_NODE:
continue
# region merge neighbor node pixels
status_matrix[i][j] = GraphExtractor.VISITED_NODE
candidates = [(i, j)]
node_pixels = []
while len(candidates) > 0:
node_pixel = candidates.pop()
node_pixels.append(node_pixel)
m, n = node_pixel
for dm, dn in neighbor_deltas:
if status_matrix[m + dm][n + dn] == GraphExtractor.NODE:
status_matrix[m + dm][n +
dn] = GraphExtractor.VISITED_NODE
candidates.append((m + dm, n + dn))
center_node = CenterNodePixel(node_pixels)
center_nodes.append(center_node)
for node_pixel in node_pixels:
nodes[node_pixel] = center_node
node_pixel_spatial_index.insert(node_pixel_id,
node_pixel,
obj=node_pixel)
node_pixel_id += 1
# endregion
# region find neighbor segments
# mask current nodes, make sure the edge doesn't return to itself
for m, n in node_pixels:
status_matrix[m][n] = GraphExtractor.INVALID
# find new road segment of the current node in each possible direction
for node_pixel in node_pixels:
connected_segment = self.detect_connected_segment(
status_matrix, node_pixel)
if connected_segment is not None:
connected_segments.append(connected_segment)
# restore masked nodes
for m, n in node_pixels:
status_matrix[m][n] = GraphExtractor.VISITED_NODE
# endregion
print('\n# of directly connected segments:{}'.format(
len(connected_segments)))
# there might be few edge pixels left, that should be fine
nb_unprocessed_edge_pixels = np.sum(
status_matrix[status_matrix == GraphExtractor.EDGE])
print('unprocessed edge pixels:{}'.format(nb_unprocessed_edge_pixels))
print('# of nodes:{}'.format(len(center_nodes)))
print('# of segments:{}'.format(len(connected_segments)))
return nodes, connected_segments
def p_mod_boruvka(G, subgraphs=None, rtree=None):
V = set(T.nodes(data=False))
coords = np.row_stack(nx.get_node_attributes(T, 'coords').values())
projcoords = ang_to_vec_coords(coords)
kdtree = KDTree(projcoords)
if subgraphs is None:
if rtree is not None: raise ValueError('RTree passed without UnionFind')
rtree = Rtree()
# modified to handle queues, children, mv
subgraphs = UnionFind()
# Tests whether the node is a projection on the existing grid, using its MV
is_fake = lambda n: subgraphs.budget[n] == np.inf
def find_nn(node_tuple):
u, up = node_tuple
v, _ = kdtree.query_subset(up, list(V - {u}))
return u, v, spherical_distance([coords[u], coords[v]])
# find the nearest neigbor of all nodes
p = mp.Pool(processes=6)
neighbors = p.map(find_nn, enumerate(projcoords))
p.close()
# push the results into their respective queues
for u, v, d in neighbors:
subgraphs.add_component(u, budget=T.node[u]['budget'])
subgraphs.queues[u].push((u, v), d)
# list to hold mst edges
Et = []
last_state = None
while Et != last_state:
# consolidates top candidate edges from each subgraph
Ep = PriorityQueue()
def update_queues(component):
q_top = subgraphs.queues[component].top()
try:
(u, v) = q_top
except:
return (None, None, None), np.inf
component_set = subgraphs.component_set(u)
disjointVC = list(V - set(component_set))
if not disjointVC:
return (None, None, None), np.inf
while v in component_set:
subgraphs.queues[component].pop()
vprime, _ = kdtree.query_subset(projcoords[u], disjointVC)
dm = spherical_distance([coords[u], coords[vprime]])
subgraphs.queues[component].push((u, vprime), dm)
(u, v) = subgraphs.queues[component].top()
else:
dm = spherical_distance([coords[u], coords[v]])
return (u, v, dm), dm
p = mp.Pool(processes=6)
foreign_neighbors = map(update_queues,
subgraphs.connected_components(component_subset=V))
p.close()
for neighbor in foreign_neighbors:
obj, priority = neighbor
if priority != np.inf:
Ep.push(*neighbor)
last_state = deepcopy(Et)
# add all the edges in E' to Et so long as no cycles are created
while Ep._queue:
(um, vm, dm) = Ep.pop()
# if doesn't create cycle and subgraph has enough MV
if subgraphs[um] != subgraphs[vm] and \
(subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)):
# test that the connecting subgraph can receive the MV
if subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm):
# doesn't create cycles from line segment intersection
invalid_edge, intersections =\
line_subgraph_intersection(subgraphs, rtree,
coords[um], coords[vm])
if not invalid_edge:
# edges should not intersect a subgraph more than once
assert(filter(lambda n: n > 1,
intersections.values()) == [])
# merge the subgraphs
subgraphs.union(um, vm, dm)
# Union all intersecting subgraphs
# and update budgets (happens within union)
map(lambda (n, _): subgraphs.union(um, n, 0),
filter(lambda (n, i): i == 1 and
subgraphs[n] != subgraphs[um],
intersections.iteritems()))
# index the newly added edge
box = make_bounding_box(coords[um], coords[vm])
# Object is (u.label, v.label), (u.coord, v.coord)
rtree.insert(hash((um, vm)), box,
obj=((um, vm), (coords[um], coords[vm])))
Et += [(um, vm)]
T.remove_edges_from(T.edges())
T.add_edges_from(Et)
return T
class Graph:
def __init__(self, all_trips):
# trips
self.all_trips = all_trips
# cluster seeds
self.cluster_seeds = {}
self.cluster_seed_id = 0
self.cluster_seed_index = Rtree()
# graph edges
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_id = 0
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
def cluster_traces(self):
self._create_all_trip_edges()
self._generate_cluster_seeds()
self._cluster_seeds_with_traces()
self._generate_graph_edges()
self._output_graph_to_db()
def _generate_graph_edges(self):
sys.stdout.write("Generating graph edges... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# grab trip edges
trip_edges = trip.edges.values()
# put trip edges in order
trip_edges.sort(key=lambda x: x.id)
# storage for previous cluster
prev_cluster = None
# iterate through trip edges
for trip_edge in trip_edges:
# if the current trip edge is clustered
if trip_edge.cluster is not None:
# create a graph edge between the previous cluster and the current cluster
self._create_graph_edge(prev_cluster, trip_edge.cluster)
# update previous cluster with current cluster
prev_cluster = trip_edge.cluster
# output graph edges
self._write_graph_edges_to_file()
print "done."
def _create_graph_edge(self, in_node, out_node):
# if in_node or out_node is None
if (in_node is None) or (out_node is None):
# return without doing anything
return
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if existing_graph_edge is None:
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," + str(out_node.id)] = new_graph_edge
# increment graph edge id
self.graph_edge_id += 1
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if edge_lookup_key in self.graph_edge_lookup.keys():
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _cluster_seeds_with_traces(self):
# storage for total cluster distance moved
total_cluster_distance_moved = float("infinity")
# iterate until total cluster distance moved below threshold
while total_cluster_distance_moved >= cluster_distance_moved_threshold:
# find all points on traces and move clusters
total_cluster_distance_moved = self._find_points_on_traces()
# write cluster seeds to file
self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_clustered.txt")
def _find_points_on_traces(self):
# counter for cluster seeds
seed_counter = 1
# storage for total cluster distance moved
total_cluster_distance_moved = 0.0
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# clear current trace points from cluster
cluster_seed.clear_trace_points()
sys.stdout.write(
"\rFinding intersecting points with cluster "
+ str(seed_counter)
+ "/"
+ str(len(self.cluster_seeds))
+ "... "
)
sys.stdout.flush()
# increment seed counter
seed_counter += 1
# determine leftward cluster bearing
leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0, 360.0)
# determine rightward cluster bearing
rightward_bearing = math.fmod((cluster_seed.bearing + 90.0) + 360.0, 360.0)
# storage for candidate trace points
candidate_trace_points = []
# iterate through all trips
for trip in self.all_trips:
# find leftward intersection points with trip
candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, leftward_bearing))
# find rightward intersection points with trip
candidate_trace_points.extend(self._find_intersection_points(trip, cluster_seed, rightward_bearing))
# add candidate trace points to cluster
cluster_seed.add_trace_points(candidate_trace_points)
# recompute cluster centroid
total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid()
# clear current trace points from cluster
cluster_seed.clear_trace_points()
# add candidate trace points to cluster, again
cluster_seed.add_trace_points(candidate_trace_points)
# normalize total cluster distance moved by number of seeds
total_cluster_distance_moved = total_cluster_distance_moved / len(self.cluster_seeds.values())
# and we're done!
print "done (clusters moved an average of " + str(total_cluster_distance_moved) + " meters)."
# return total cluster distance moved
return total_cluster_distance_moved
def _find_intersection_points(self, trip, cluster, cluster_bearing):
# find all nearby trip edge id's
nearby_trip_edge_ids = self._find_nearby_trip_edge_ids(cluster, edge_bounding_box_size, trip.edge_index)
# storage for intersection points
intersection_points = []
# iterate through all nearby edge id's
for edge_id in nearby_trip_edge_ids:
# grab current edge
edge = trip.edges[edge_id]
# determine intersection point between edge and cluster
intersection_point = self._intersection_point(edge.in_node, edge.bearing, cluster, cluster_bearing)
# if there is an intersection point
if intersection_point is not None:
# determine distance from edge in_node to intersection point
intersection_distance = self._distance_coords(
edge.in_node.latitude, edge.in_node.longitude, intersection_point[0], intersection_point[1]
)
# if intersection distance is less than edge length
if intersection_distance <= edge.length:
# this edge has a valid intersection point
intersection_points.append(
TracePoint(intersection_point[0], intersection_point[1], edge.bearing, edge)
)
# return all intersection points for this trip
return intersection_points
def _generate_cluster_seeds(self):
# iterate through all trips
for i in range(0, len(self.all_trips)):
sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" + str(len(self.all_trips)) + "... ")
sys.stdout.flush()
# grab current trip
trip = self.all_trips[i]
# set last cluster seed distance to zero for first trip location
trip.locations[0].last_cluster_seed_distance = 0.0
# iterate through all trip locations
for j in range(1, len(trip.locations)):
# drop cluster seeds along current edge every 50 meters
self._drop_cluster_seeds_along_edge(trip.locations[j - 1], trip.locations[j])
print "done (generated " + str(len(self.cluster_seeds)) + " cluster seeds)."
# write cluster seeds to file
self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt")
def _drop_cluster_seeds_along_edge(self, in_node, out_node):
# determine edge length
edge_length = self._distance(in_node, out_node)
# determine distance along edge for first cluster seed
first_cluster_seed_distance = cluster_seed_interval - in_node.last_cluster_seed_distance
# storage for relative cluster seed intervals
rel_cluster_seed_intervals = []
# storage for current cluster seed distance along this edge
curr_cluster_seed_distance = first_cluster_seed_distance
# determine the relative cluster seed intervals needed for this edge
while curr_cluster_seed_distance <= edge_length:
# append current cluster seed distance to relative cluster seed interval list
rel_cluster_seed_intervals.append(curr_cluster_seed_distance)
# increment current cluster seed distance
curr_cluster_seed_distance += cluster_seed_interval
# determine bearing of current edge
edge_bearing = self._path_bearing(in_node, out_node)
# create cluster seeds for edge
for i in range(0, len(rel_cluster_seed_intervals)):
# determine fraction along current edge to drop cluster seed
fraction_along = rel_cluster_seed_intervals[i] / edge_length
# determine point along line to drop cluster seed
(new_cluster_seed_latitude, new_cluster_seed_longitude) = self._point_along_line(
in_node, out_node, fraction_along
)
# locate nearest existing cluster seeds
closest_cluster_seeds = list(
self.cluster_seed_index.nearest((new_cluster_seed_longitude, new_cluster_seed_latitude), 25)
)
# if there does not exist a closest existing cluster seed
if len(closest_cluster_seeds) == 0:
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing
)
# else, if there exists a closest existing cluster seed
elif len(closest_cluster_seeds) > 0:
# storage for matched cluster seed
matched_cluster_seed = None
# iterate through closest existing cluster seeds
for curr_cluster_seed_id in closest_cluster_seeds:
# grab current cluster seed
curr_cluster_seed = self.cluster_seeds[curr_cluster_seed_id]
# compute distance to current cluster seed
distance = self._distance_coords(
new_cluster_seed_latitude,
new_cluster_seed_longitude,
curr_cluster_seed.latitude,
curr_cluster_seed.longitude,
)
# determine bearing difference between edge and current cluster seed
bearing_difference = math.cos(math.radians(edge_bearing - curr_cluster_seed.bearing))
# if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees
if (distance <= cluster_seed_interval) and (bearing_difference >= cluster_bearing_difference_limit):
# store current cluster seed as matched cluster seed
matched_cluster_seed = curr_cluster_seed
# stop searching
break
# if there was not a matched cluster seed
if matched_cluster_seed is None:
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude, edge_bearing
)
# update last cluster seed distance
out_node.last_cluster_seed_distance = self._distance_coords(
new_cluster_seed_latitude, new_cluster_seed_longitude, out_node.latitude, out_node.longitude
)
# if no cluster seeds were generated along this edge
if len(rel_cluster_seed_intervals) == 0:
# update last cluster seed distance
out_node.last_cluster_seed_distance = in_node.last_cluster_seed_distance + edge_length
def _create_new_cluster_seed(self, latitude, longitude, bearing):
# create a new cluster seed
new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude, longitude, bearing)
# add new cluster seed to the cluster seeds dictionary
self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed
# insert new cluster seed into spatial index
self.cluster_seed_index.insert(new_cluster_seed.id, (new_cluster_seed.longitude, new_cluster_seed.latitude))
# increment cluster seed id
self.cluster_seed_id += 1
# return new cluster seed
return new_cluster_seed
def _create_all_trip_edges(self):
sys.stdout.write("Creating and indexing edges for all trips... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# add edge storage to trip
trip.edges = {}
# add edge index to trip
trip.edge_index = Rtree()
# storage for edge id
trip_edge_id = 0
# iterate through all trip locations
for i in range(1, len(trip.locations)):
# create new edge
new_edge = Edge(trip_edge_id, trip.locations[i - 1], trip.locations[i])
# insert edge into dictionary
trip.edges[trip_edge_id] = new_edge
# insert edge into index
self._index_trip_edge(new_edge, trip.edge_index)
# increment trip edge id
trip_edge_id += 1
# done
print "done."
def _index_trip_edge(self, edge, edge_index):
# determine edge minx, miny, maxx, maxy values
edge_minx = min(edge.in_node.longitude, edge.out_node.longitude)
edge_miny = min(edge.in_node.latitude, edge.out_node.latitude)
edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude)
edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude)
# insert edge into spatial index
edge_index.insert(edge.id, (edge_minx, edge_miny, edge_maxx, edge_maxy))
def _find_nearby_trip_edge_ids(self, location, distance, edge_index):
# define longitude/latitude offset
lon_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE
lat_offset = (distance / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE
# create bounding box
bounding_box = (
location.longitude - lon_offset,
location.latitude - lat_offset,
location.longitude + lon_offset,
location.latitude + lat_offset,
)
# return nearby edge id's inside bounding box
return list(edge_index.intersection(bounding_box))
def _intersection_point(self, location1, location1_bearing, location2, location2_bearing):
return spatialfunclib.intersection_point(
location1.latitude,
location1.longitude,
location1_bearing,
location2.latitude,
location2.longitude,
location2_bearing,
)
def _point_along_line(self, location1, location2, fraction_along):
return spatialfunclib.point_along_line(
location1.latitude, location1.longitude, location2.latitude, location2.longitude, fraction_along
)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(
location1.latitude, location1.longitude, location2.latitude, location2.longitude
)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
def _distance_coords(self, location1_latitude, location1_longitude, location2_latitude, location2_longitude):
return spatialfunclib.distance(location1_latitude, location1_longitude, location2_latitude, location2_longitude)
def _write_cluster_seeds_to_file(self, filename="edelkamp_cluster_seeds.txt"):
# open graph file
graph_file = open(filename, "w")
# iterate through all cluster_seeds
for cluster_seed in self.cluster_seeds.values():
# output cluster seed to file
graph_file.write(
str(cluster_seed.latitude) + "," + str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) + "\n"
)
# close graph file
graph_file.close()
def _write_graph_edges_to_file(self):
# open graph file
graph_file = open("edelkamp_cluster_edges.txt", "w")
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# output edge to file
graph_file.write(str(graph_edge.in_node.latitude) + "," + str(graph_edge.in_node.longitude) + "\n")
graph_file.write(str(graph_edge.out_node.latitude) + "," + str(graph_edge.out_node.longitude) + "\n\n")
# close graph file
graph_file.close()
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("edelkamp_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute("CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute("CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)")
# remove values from nodes table
# cur.execute("DELETE FROM nodes")
# remove values from edges table
# cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# insert cluster seed into nodes table
cur.execute(
"INSERT INTO nodes VALUES ("
+ str(cluster_seed.id)
+ ","
+ str(cluster_seed.latitude)
+ ","
+ str(cluster_seed.longitude)
+ ")"
)
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# insert graph edge into edges table
cur.execute(
"INSERT INTO edges VALUES ("
+ str(graph_edge.id)
+ ","
+ str(graph_edge.in_node.id)
+ ","
+ str(graph_edge.out_node.id)
+ ")"
)
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
def mod_boruvka(G, subgraphs=None, rtree=None):
"""
algorithm to calculate the minimum spanning forest of nodes in GeoGraph G
with 'budget' based restrictions on edges.
Uses a modified version of Boruvka's algorithm
NOTE: subgraphs is modified as a side-effect...may remove in future
(useful for testing right now)
Args:
G: GeoGraph of nodes to be connected if appropriate
Nodes should have 'budget' attribute
subgraphs: UnionFind data structure representing existing network's
connected components AND the 'fake' nodes projected onto it. This
is the basis for the agglomerative nearest neighbor approach in
this algorithm.
rtree: RTree based index of existing network
Returns:
GeoGraph: representing minimum spanning forest of G subject to the
budget based restrictions
"""
# special case (already MST)
if G.number_of_nodes() < 2:
return G
V = set(G.nodes())
# GeoGraph coords may be ndarray or dict
if isinstance(G.coords, np.ndarray):
coords = G.coords
else:
coords = np.row_stack(G.coords.values())
projcoords = ang_to_vec_coords(coords) if G.is_geographic() else coords
kdtree = KDTree(projcoords)
# Handle "dead" components
D = set()
if subgraphs is None:
if rtree is not None:
raise ValueError('RTree passed without UnionFind')
rtree = Rtree()
# modified to handle queues, children, mv
subgraphs = UnionFind()
# <helper_functions>
def candidate_components(C):
"""
return the set of candidate nearest components for the connected
component containing C. Do not consider those in C's connected
component or those that are 'dead'.
"""
component_set = subgraphs.component_set(C)
return list((V - set(component_set)) - D)
def update_nn_component(C, candidates):
"""
find the nearest neighbor pair for the connected component
represented by c. candidates represents the list of
components from which to select.
"""
(v, vm) = subgraphs.queues[C].top()
# vm ∈ C {not a foreign nearest neighbor}
# go through the queue until an edge is found between this node
# and the set of candidates, updating the neighbors in the connected
# components queue in the process.
while vm not in candidates:
subgraphs.queues[C].pop()
um, _ = kdtree.query_subset(projcoords[v], candidates)
dm = square_distance(projcoords[v], projcoords[um])
subgraphs.push(subgraphs.queues[C], (v, um), dm)
# Note: v will always be a vertex in this connected component
# vm *may* be external
(v, vm) = subgraphs.queues[C].top()
return (v, vm)
def is_fake(node):
"""
Tests whether the node is a projection on the existing grid
"""
return subgraphs.budget[node] == np.inf
# Test whether the component is dead
# i.e. can never connect to another node
def is_dead(c, nn_dist):
return not is_fake(c) and subgraphs.budget[c] < nn_dist
# "true" distance between components
def component_dist(c1, c2):
dist = 0
if G.is_geographic():
dist = spherical_distance([coords[c1], coords[c2]])
else:
dist = euclidean_distance([coords[c1], coords[c2]])
return dist
# </helper_functions>
# Initialize the connected components holding a single node
# and push the nearest neighbor into its queue
for v in V:
vm, _ = kdtree.query_subset(projcoords[v], list(V - {v}))
dm = square_distance(projcoords[v], projcoords[vm])
subgraphs.add_component(v, budget=G.node[v]['budget'])
subgraphs.push(subgraphs.queues[v], (v, vm), dm)
# Add to dead set if warranted
nn_dist = component_dist(v, vm)
if is_dead(v, nn_dist):
# here components are single nodes
# so no need to worry about adding children to dead set
if v not in D:
D.add(v)
Et = [] # Initialize MST edges to empty list
last_state = None
# MST is complete when no progress was made in the prior iteration
while Et != last_state:
# This is a candidate list of edges that might be added to the MST
Ep = PriorityQueue()
# ∀ C of G; where c <- connected component
for C in subgraphs.connected_components(component_subset=V):
candidates = candidate_components(C)
# Skip if no valid candidates
if not candidates:
continue
(v, vm) = update_nn_component(C, candidates)
# Add to dead set if warranted
nn_dist = component_dist(v, vm)
if is_dead(C, nn_dist):
# add all dead components to the dead set D
# (note that fake nodes can never be dead)
for c in subgraphs.component_set(C):
if c not in D and not is_fake(c):
D.add(c)
# One more round to root out connections to dead components
# found in above iteration.
# Need to do this BEFORE pushing candidate edges into Ep.
# Otherwise we might be testing only 'dead' candidates
# and therefore mistakenly think we were done (since
# no new edges would have been added)
for C in subgraphs.connected_components(component_subset=V):
candidates = candidate_components(C)
# Skip if no valid candidates
if not candidates:
continue
(v, vm) = update_nn_component(C, candidates)
# Calculate nn_dist for comparison to mv later
nn_dist = component_dist(v, vm)
# Append the top priority edge from the subgraph to the candidate
# edge set
Ep.push((v, vm, nn_dist), nn_dist)
last_state = deepcopy(Et)
# Candidate Test
# At this point we have all of our nearest neighbor component edge
# candidates defined for this "round"
#
# Now test all candidate edges in Ep for cycles and satisfaction of
# custom criteria
while Ep._queue:
(um, vm, dm) = Ep.pop()
# if doesn't create cycle
# and subgraphs have enough MV
# and we're not connecting 2 fake nodes
# then allow the connection
if subgraphs[um] != subgraphs[vm] and \
(subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)) and \
(subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm)) and \
not (is_fake(um) and is_fake(vm)):
# doesn't create cycles from line segment intersection
invalid_edge, intersections = \
line_subgraph_intersection(subgraphs, rtree,
coords[um], coords[vm])
if not invalid_edge:
# edges should not intersect a subgraph more than once
assert(filter(lambda n: n > 1,
intersections.values()) == [])
# merge the subgraphs
subgraphs.union(um, vm, dm)
# For all intersected subgraphs update the mv to that
# created by the edge intersecting them,
# TODO: This should be updated in not such a naive method
map(lambda (n, _): subgraphs.union(um, n, 0),
filter(lambda (n, i): i == 1 and
subgraphs[n] != subgraphs[um],
intersections.iteritems()))
# index the newly added edge
box = make_bounding_box(coords[um], coords[vm])
# Object is (u.label, v.label), (u.coord, v.coord)
rtree.insert(hash((um, vm)), box,
obj=((um, vm), (coords[um], coords[vm])))
Et += [(um, vm, {'weight': dm})]
# create new GeoGraph with results
result = G.copy()
result.coords = G.coords
result.remove_edges_from(result.edges())
result.add_edges_from(Et)
return result
class Graph:
def __init__(self, all_trips):
# trips
self.all_trips = all_trips
# cluster seeds
self.cluster_seeds = {}
self.cluster_seed_id = 0
self.cluster_seed_index = Rtree()
# graph edges
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_id = 0
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
def cluster_traces(self):
self._create_all_trip_edges()
self._generate_cluster_seeds()
self._cluster_seeds_with_traces()
self._generate_graph_edges()
self._output_graph_to_db()
def _generate_graph_edges(self):
sys.stdout.write("Generating graph edges... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# grab trip edges
trip_edges = trip.edges.values()
# put trip edges in order
trip_edges.sort(key=lambda x: x.id)
# storage for previous cluster
prev_cluster = None
# iterate through trip edges
for trip_edge in trip_edges:
# if the current trip edge is clustered
if (trip_edge.cluster is not None):
# create a graph edge between the previous cluster and the current cluster
self._create_graph_edge(prev_cluster, trip_edge.cluster)
# update previous cluster with current cluster
prev_cluster = trip_edge.cluster
# output graph edges
self._write_graph_edges_to_file()
print "done."
def _create_graph_edge(self, in_node, out_node):
# if in_node or out_node is None
if ((in_node is None) or (out_node is None)):
# return without doing anything
return
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if (existing_graph_edge is None):
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," +
str(out_node.id)] = new_graph_edge
# increment graph edge id
self.graph_edge_id += 1
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if (edge_lookup_key in self.graph_edge_lookup.keys()):
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _cluster_seeds_with_traces(self):
# storage for total cluster distance moved
total_cluster_distance_moved = float('infinity')
# iterate until total cluster distance moved below threshold
while (total_cluster_distance_moved >=
cluster_distance_moved_threshold):
# find all points on traces and move clusters
total_cluster_distance_moved = self._find_points_on_traces()
# write cluster seeds to file
self._write_cluster_seeds_to_file(
"edelkamp_cluster_seeds_clustered.txt")
def _find_points_on_traces(self):
# counter for cluster seeds
seed_counter = 1
# storage for total cluster distance moved
total_cluster_distance_moved = 0.0
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# clear current trace points from cluster
cluster_seed.clear_trace_points()
sys.stdout.write("\rFinding intersecting points with cluster " +
str(seed_counter) + "/" +
str(len(self.cluster_seeds)) + "... ")
sys.stdout.flush()
# increment seed counter
seed_counter += 1
# determine leftward cluster bearing
leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0,
360.0)
# determine rightward cluster bearing
rightward_bearing = math.fmod(
(cluster_seed.bearing + 90.0) + 360.0, 360.0)
# storage for candidate trace points
candidate_trace_points = []
# iterate through all trips
for trip in self.all_trips:
# find leftward intersection points with trip
candidate_trace_points.extend(
self._find_intersection_points(trip, cluster_seed,
leftward_bearing))
# find rightward intersection points with trip
candidate_trace_points.extend(
self._find_intersection_points(trip, cluster_seed,
rightward_bearing))
# add candidate trace points to cluster
cluster_seed.add_trace_points(candidate_trace_points)
# recompute cluster centroid
total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid(
)
# clear current trace points from cluster
cluster_seed.clear_trace_points()
# add candidate trace points to cluster, again
cluster_seed.add_trace_points(candidate_trace_points)
# normalize total cluster distance moved by number of seeds
total_cluster_distance_moved = (total_cluster_distance_moved /
len(self.cluster_seeds.values()))
# and we're done!
print "done (clusters moved an average of " + str(
total_cluster_distance_moved) + " meters)."
# return total cluster distance moved
return total_cluster_distance_moved
def _find_intersection_points(self, trip, cluster, cluster_bearing):
# find all nearby trip edge id's
nearby_trip_edge_ids = self._find_nearby_trip_edge_ids(
cluster, edge_bounding_box_size, trip.edge_index)
# storage for intersection points
intersection_points = []
# iterate through all nearby edge id's
for edge_id in nearby_trip_edge_ids:
# grab current edge
edge = trip.edges[edge_id]
# determine intersection point between edge and cluster
intersection_point = self._intersection_point(
edge.in_node, edge.bearing, cluster, cluster_bearing)
# if there is an intersection point
if (intersection_point is not None):
# determine distance from edge in_node to intersection point
intersection_distance = self._distance_coords(
edge.in_node.latitude, edge.in_node.longitude,
intersection_point[0], intersection_point[1])
# if intersection distance is less than edge length
if (intersection_distance <= edge.length):
# this edge has a valid intersection point
intersection_points.append(
TracePoint(intersection_point[0],
intersection_point[1], edge.bearing, edge))
# return all intersection points for this trip
return intersection_points
def _generate_cluster_seeds(self):
# iterate through all trips
for i in range(0, len(self.all_trips)):
sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" +
str(len(self.all_trips)) + "... ")
sys.stdout.flush()
# grab current trip
trip = self.all_trips[i]
# set last cluster seed distance to zero for first trip location
trip.locations[0].last_cluster_seed_distance = 0.0
# iterate through all trip locations
for j in range(1, len(trip.locations)):
# drop cluster seeds along current edge every 50 meters
self._drop_cluster_seeds_along_edge(trip.locations[j - 1],
trip.locations[j])
print "done (generated " + str(len(
self.cluster_seeds)) + " cluster seeds)."
# write cluster seeds to file
self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt")
def _drop_cluster_seeds_along_edge(self, in_node, out_node):
# determine edge length
edge_length = self._distance(in_node, out_node)
# determine distance along edge for first cluster seed
first_cluster_seed_distance = (cluster_seed_interval -
in_node.last_cluster_seed_distance)
# storage for relative cluster seed intervals
rel_cluster_seed_intervals = []
# storage for current cluster seed distance along this edge
curr_cluster_seed_distance = first_cluster_seed_distance
# determine the relative cluster seed intervals needed for this edge
while (curr_cluster_seed_distance <= edge_length):
# append current cluster seed distance to relative cluster seed interval list
rel_cluster_seed_intervals.append(curr_cluster_seed_distance)
# increment current cluster seed distance
curr_cluster_seed_distance += cluster_seed_interval
# determine bearing of current edge
edge_bearing = self._path_bearing(in_node, out_node)
# create cluster seeds for edge
for i in range(0, len(rel_cluster_seed_intervals)):
# determine fraction along current edge to drop cluster seed
fraction_along = (rel_cluster_seed_intervals[i] / edge_length)
# determine point along line to drop cluster seed
(new_cluster_seed_latitude,
new_cluster_seed_longitude) = self._point_along_line(
in_node, out_node, fraction_along)
# locate nearest existing cluster seeds
closest_cluster_seeds = list(
self.cluster_seed_index.nearest(
(new_cluster_seed_longitude, new_cluster_seed_latitude),
25))
# if there does not exist a closest existing cluster seed
if (len(closest_cluster_seeds) == 0):
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude,
edge_bearing)
# else, if there exists a closest existing cluster seed
elif (len(closest_cluster_seeds) > 0):
# storage for matched cluster seed
matched_cluster_seed = None
# iterate through closest existing cluster seeds
for curr_cluster_seed_id in closest_cluster_seeds:
# grab current cluster seed
curr_cluster_seed = self.cluster_seeds[
curr_cluster_seed_id]
# compute distance to current cluster seed
distance = self._distance_coords(
new_cluster_seed_latitude, new_cluster_seed_longitude,
curr_cluster_seed.latitude,
curr_cluster_seed.longitude)
# determine bearing difference between edge and current cluster seed
bearing_difference = math.cos(
math.radians(edge_bearing - curr_cluster_seed.bearing))
# if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees
if ((distance <= cluster_seed_interval)
and (bearing_difference >=
cluster_bearing_difference_limit)):
# store current cluster seed as matched cluster seed
matched_cluster_seed = curr_cluster_seed
# stop searching
break
# if there was not a matched cluster seed
if (matched_cluster_seed is None):
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude,
edge_bearing)
# update last cluster seed distance
out_node.last_cluster_seed_distance = self._distance_coords(
new_cluster_seed_latitude, new_cluster_seed_longitude,
out_node.latitude, out_node.longitude)
# if no cluster seeds were generated along this edge
if (len(rel_cluster_seed_intervals) == 0):
# update last cluster seed distance
out_node.last_cluster_seed_distance = (
in_node.last_cluster_seed_distance + edge_length)
def _create_new_cluster_seed(self, latitude, longitude, bearing):
# create a new cluster seed
new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude,
longitude, bearing)
# add new cluster seed to the cluster seeds dictionary
self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed
# insert new cluster seed into spatial index
self.cluster_seed_index.insert(
new_cluster_seed.id,
(new_cluster_seed.longitude, new_cluster_seed.latitude))
# increment cluster seed id
self.cluster_seed_id += 1
# return new cluster seed
return new_cluster_seed
def _create_all_trip_edges(self):
sys.stdout.write("Creating and indexing edges for all trips... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# add edge storage to trip
trip.edges = {}
# add edge index to trip
trip.edge_index = Rtree()
# storage for edge id
trip_edge_id = 0
# iterate through all trip locations
for i in range(1, len(trip.locations)):
# create new edge
new_edge = Edge(trip_edge_id, trip.locations[i - 1],
trip.locations[i])
# insert edge into dictionary
trip.edges[trip_edge_id] = new_edge
# insert edge into index
self._index_trip_edge(new_edge, trip.edge_index)
# increment trip edge id
trip_edge_id += 1
# done
print "done."
def _index_trip_edge(self, edge, edge_index):
# determine edge minx, miny, maxx, maxy values
edge_minx = min(edge.in_node.longitude, edge.out_node.longitude)
edge_miny = min(edge.in_node.latitude, edge.out_node.latitude)
edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude)
edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude)
# insert edge into spatial index
edge_index.insert(edge.id,
(edge_minx, edge_miny, edge_maxx, edge_maxy))
def _find_nearby_trip_edge_ids(self, location, distance, edge_index):
# define longitude/latitude offset
lon_offset = ((distance / 2.0) /
spatialfunclib.METERS_PER_DEGREE_LONGITUDE)
lat_offset = ((distance / 2.0) /
spatialfunclib.METERS_PER_DEGREE_LATITUDE)
# create bounding box
bounding_box = (location.longitude - lon_offset,
location.latitude - lat_offset,
location.longitude + lon_offset,
location.latitude + lat_offset)
# return nearby edge id's inside bounding box
return list(edge_index.intersection(bounding_box))
def _intersection_point(self, location1, location1_bearing, location2,
location2_bearing):
return spatialfunclib.intersection_point(
location1.latitude, location1.longitude, location1_bearing,
location2.latitude, location2.longitude, location2_bearing)
def _point_along_line(self, location1, location2, fraction_along):
return spatialfunclib.point_along_line(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude,
fraction_along)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
def _distance_coords(self, location1_latitude, location1_longitude,
location2_latitude, location2_longitude):
return spatialfunclib.distance(location1_latitude, location1_longitude,
location2_latitude, location2_longitude)
def _write_cluster_seeds_to_file(self,
filename="edelkamp_cluster_seeds.txt"):
# open graph file
graph_file = open(filename, 'w')
# iterate through all cluster_seeds
for cluster_seed in self.cluster_seeds.values():
# output cluster seed to file
graph_file.write(
str(cluster_seed.latitude) + "," +
str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) +
"\n")
# close graph file
graph_file.close()
def _write_graph_edges_to_file(self):
# open graph file
graph_file = open('edelkamp_cluster_edges.txt', 'w')
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# output edge to file
graph_file.write(
str(graph_edge.in_node.latitude) + "," +
str(graph_edge.in_node.longitude) + "\n")
graph_file.write(
str(graph_edge.out_node.latitude) + "," +
str(graph_edge.out_node.longitude) + "\n\n")
# close graph file
graph_file.close()
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("edelkamp_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute(
"CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute(
"CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)"
)
# remove values from nodes table
#cur.execute("DELETE FROM nodes")
# remove values from edges table
#cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# insert cluster seed into nodes table
cur.execute("INSERT INTO nodes VALUES (" + str(cluster_seed.id) +
"," + str(cluster_seed.latitude) + "," +
str(cluster_seed.longitude) + ")")
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# insert graph edge into edges table
cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) +
"," + str(graph_edge.in_node.id) + "," +
str(graph_edge.out_node.id) + ")")
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
class Graph:
def __init__(self, bus_trips):
self.bus_trips = bus_trips
self.graph_nodes = {} # indexed by "location_id"
self.graph_edge_id = 0
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
self.graph_edge_index = Rtree()
def generate_graph(self):
print "Running graph generation algorithm..."
# initialize trip counter
trip_count = 1
for trip in self.bus_trips:
# initialize location counter
location_count = 1
# storage for previous node
prev_node = None
for location in trip.locations:
sys.stdout.write("\rAnalyzing location " +
str(location_count) + "/" +
str(len(trip.locations)) + " for trip " +
str(trip_count) + "/" +
str(len(self.bus_trips)) + "... ")
sys.stdout.flush()
# find closest edges in graph to location
closest_edges = self._find_closest_edges_in_graph(
location, 100)
# flag variable for whether we merged location
did_merge_location = False
# iterate through candidate edge ids
for candidate_edge_id in closest_edges:
# grab candidate edge from graph edge dictionary
candidate_edge = self.graph_edges[candidate_edge_id]
# determine whether we should merge with candidate edge
if (self._should_merge_location_with_edge(
location, candidate_edge) is True):
# merge location with edge, update previous node
prev_node = self._merge_location_with_edge(
location, candidate_edge, prev_node)
# update merge flag variable
did_merge_location = True
# no need to look at further edges, break out of candidate edges loop
break
# if we did not merge the location with any edge
if (did_merge_location is False):
# add location to graph
self._add_location_to_graph(location, prev_node)
# update previous node with current location
prev_node = location
# increment location counter
location_count += 1
# done with current trip locations
print "done."
# increment trip counter
trip_count += 1
# write graph edges to file
self._write_graph_edges_to_file()
# create graph database
self._output_graph_to_db()
def _merge_location_with_edge(self, location, edge, prev_node):
# get the edge node closest to the location
edge_node = self._get_closest_edge_node(location, edge)
# if prev_node is None
if (prev_node is None):
# increase volume of just this edge
edge.volume += 1
# if prev_node is not None
else:
# find path from prev_node to edge_node
path = self._find_path(prev_node, edge_node, max_path_length)
# if there was a path from prev_node to edge_node
if (path is not None):
# iterate through nodes in path
for i in range(1, len(path)):
# grab in_node
in_node = path[i - 1]
# grab out_node
out_node = path[i]
# find corresponding graph edge
graph_edge = self._find_graph_edge(in_node, out_node)
# increment volume on edge
graph_edge.volume += 1
# if there is no path from prev_node to edge_node
else:
# create a new graph edge between prev_node and edge_node
self._create_graph_edge(prev_node, edge_node)
# return the edge_node
return edge_node
def _get_closest_edge_node(self, location, edge):
# if in_node distance is less than out_node distance
if (self._distance(location, edge.in_node) < self._distance(
location, edge.out_node)):
# return the edge in_node
return edge.in_node
# otherwise, return the edge out_node
return edge.out_node
def _find_path(self, source, destination, max_length):
# reset all node visited flags
self._reset_node_visited_flags()
# get a breath-first search path from source to destination
path = self._bfs_path(source, destination)
# if there is a path from source to destination
if (path is not None):
# and if the path length is less than or equal to the maximum length
if (len(path) <= max_length):
# return the path
return path
# otherwise, return None
return None
def _bfs_path(self, source, destination):
# storage for breadth-first search parents
bfs_parent = {} # key is current node, value is parent node
# source node has no breadth-first search parent
bfs_parent[source] = None
# node queue for breadth-first search
bfs_queue = []
# enqueue source node
bfs_queue.append(source)
# mark source node as visited
source.visited = True
# while the queue is not empty
while (len(bfs_queue) > 0):
# dequeue the first node in the queue
curr_node = bfs_queue.pop(0)
# if the current node is the destination
if (curr_node is destination):
# create storage for breadth-first search path
bfs_path = []
# add the current node to the breadth-first search path
bfs_path.insert(0, curr_node)
# grab the parent of the current node
parent = bfs_parent[curr_node]
# iterate through breadth-first search parents
while (parent is not None):
# add the parent to the breadth-first search path
bfs_path.insert(0, parent)
# grab the next parent
parent = bfs_parent[parent]
# return the breadth-first search path
return bfs_path
# if the current node is not the destination
else:
# iterate through the current node's out_nodes
for out_node in curr_node.out_nodes:
# if the out_node has not been visited
if (out_node.visited is False):
# mark the out_node as visited
out_node.visited = True
# enqueue the out_node
bfs_queue.append(out_node)
# store curr_node as out_node's breadth-first search parent
bfs_parent[out_node] = curr_node
# if we reached here, no path was found
return None
def _should_merge_location_with_edge(self, location, edge):
# project location onto edge
(location_projection, location_projection_fraction,
location_projection_distance) = self._projection_onto_line(
edge.in_node, edge.out_node, location)
# if projection is not onto edge
if (location_projection_fraction < 0.0
or location_projection_fraction > 1.0):
# we cannot merge location with edge
return False
# determine bearing difference between edge and location
bearing_difference = math.cos(
math.radians(
self._path_bearing(edge.in_node, edge.out_node) -
self._location_bearing(location)))
# if location projection distance is less than 20 meters
if (location_projection_distance < location_projection_distance_limit):
# if bearing difference is less than 45 degrees
if (bearing_difference > location_bearing_difference_limit):
# merge location with edge
return True
# otherwise, do not merge location with edge
return False
def _add_location_to_graph(self, location, prev_node):
# add an out_nodes list to location
location.out_nodes = []
# add an in_nodes list to location
location.in_nodes = []
# add a visited flag to location
location.visited = False
# add location to graph nodes list
self.graph_nodes[location.id] = location
# if prev_node is not None
if (prev_node is not None):
# create a new graph edge between prev_node and location
self._create_graph_edge(prev_node, location)
def _create_graph_edge(self, in_node, out_node):
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if (existing_graph_edge is None):
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," +
str(out_node.id)] = new_graph_edge
# add new graph edge to graph edges spatial index
self._add_graph_edge_to_index(new_graph_edge)
# increment graph edge id
self.graph_edge_id += 1
# store out_node in in_nodes's out_nodes list
in_node.out_nodes.append(out_node)
# store in_node in out_node's in_nodes list
out_node.in_nodes.append(in_node)
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if (edge_lookup_key in self.graph_edge_lookup.keys()):
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _add_graph_edge_to_index(self, graph_edge):
# determine graph edge minx, miny, maxx, maxy values
graph_edge_minx = min(graph_edge.in_node.longitude,
graph_edge.out_node.longitude)
graph_edge_miny = min(graph_edge.in_node.latitude,
graph_edge.out_node.latitude)
graph_edge_maxx = max(graph_edge.in_node.longitude,
graph_edge.out_node.longitude)
graph_edge_maxy = max(graph_edge.in_node.latitude,
graph_edge.out_node.latitude)
# insert graph edge into spatial index
self.graph_edge_index.insert(graph_edge.id,
(graph_edge_minx, graph_edge_miny,
graph_edge_maxx, graph_edge_maxy))
def _location_bearing(self, location):
# if location has a previous neighbor and a next neighbor
if ((location.prev_location is not None)
and (location.next_location is not None)):
# determine bearing using previous and next neighbors
return self._path_bearing(location.prev_location,
location.next_location)
# if location has no previous neighbor, but has a next neighbor
elif ((location.prev_location is None)
and (location.next_location is not None)):
# determine bearing using current location and next neighbor
return self._path_bearing(location, location.next_location)
# if location has a previous neighbor, but not a next neighbor
elif ((location.prev_location is not None)
and (location.next_location is None)):
# determine bearing using previous neighbor and current location
return self._path_bearing(location.prev_location, location)
# if we reach here, there is an error
return None
def _find_closest_edges_in_graph(self, location, number_of_edges):
return self.graph_edge_index.nearest(
(location.longitude, location.latitude), number_of_edges)
def _projection_onto_line(self, location1, location2, location3):
return spatialfunclib.projection_onto_line(
location1.latitude, location1.longitude, location2.latitude,
location2.longitude, location3.latitude, location3.longitude)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("cao_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute(
"CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute(
"CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)"
)
# remove values from nodes table
#cur.execute("DELETE FROM nodes")
# remove values from edges table
#cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# insert graph node into nodes table
cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) +
"," + str(graph_node.latitude) + "," +
str(graph_node.longitude) + ")")
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# insert graph edge into edges table
cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) +
"," + str(graph_edge.in_node.id) + "," +
str(graph_edge.out_node.id) + ")")
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
def _write_graph_edges_to_file(self):
# output that we are starting the writing process
sys.stdout.write("\nWriting graph edges to file... ")
sys.stdout.flush()
# open graph file
graph_file = open('cao_edges.txt', 'w')
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# output edge to file
graph_file.write(
str(graph_edge.in_node.latitude) + "," +
str(graph_edge.in_node.longitude) + "\n")
graph_file.write(
str(graph_edge.out_node.latitude) + "," +
str(graph_edge.out_node.longitude) + "," +
str(graph_edge.volume) + "\n\n")
# close graph file
graph_file.close()
print "done."
def _reset_node_visited_flags(self):
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# set visited flag to False
graph_node.visited = False
print >> sys.stderr, "%d retried, %d unassigned." % (retries, len(unassigned))
hoodIndex = Rtree()
print >> sys.stderr, "Buffering polygons."
for place_id, polygon in polygons.items():
if type(polygon) is Polygon:
polygon = Polygon(polygon.exterior.coords)
else:
bits = []
for p in polygon.geoms:
if type(p) is Polygon:
bits.append(Polygon(p.exterior.coords))
polygon = MultiPolygon(bits)
polygons[place_id] = polygon.buffer(0)
hoodIndex.insert(place_id, polygons[place_id].bounds)
print >> sys.stderr, "Retconning blocks to shapes."
cur.execute(
"""select geom, geoid10 FROM tabblock10 tb WHERE statefp10 = %s AND countyfp10 = %s AND blockce10 NOT LIKE '0%%'""",
(statefp10, countyfp10))
for r in cur.fetchall():
poly = wkb.loads(r[0].decode('hex'))
id = r[1]
candidates = [i for i in hoodIndex.intersection(poly.bounds)]
found = False
for place_id in candidates:
hood = polygons[place_id]
if hood.contains(poly):
cur.execute("""DELETE FROM votes WHERE source=%s AND id=%s""",
('blockr', id))
def mod_boruvka(G, subgraphs=None, rtree=None):
"""
algorithm to calculate the minimum spanning forest of nodes in GeoGraph G
with 'budget' based restrictions on edges.
Uses a modified version of Boruvka's algorithm
NOTE: subgraphs is modified as a side-effect...may remove in future
(useful for testing right now)
Args:
G: GeoGraph of nodes to be connected if appropriate
Nodes should have 'budget' attribute
subgraphs: UnionFind data structure representing existing network's
connected components AND the 'fake' nodes projected onto it. This
is the basis for the agglomerative nearest neighbor approach in
this algorithm.
rtree: RTree based index of existing network
Returns:
GeoGraph: representing minimum spanning forest of G subject to the
budget based restrictions
"""
# special case (already MST)
if G.number_of_nodes() < 2:
return G
V = set(G.nodes())
coords = np.row_stack(G.coords.values())
projcoords = ang_to_vec_coords(coords) if G.is_geographic() else coords
kdtree = KDTree(projcoords)
# Handle "dead" components
D = set()
if subgraphs is None:
if rtree is not None: raise ValueError('RTree passed without UnionFind')
rtree = Rtree()
# modified to handle queues, children, mv
subgraphs = UnionFind()
# <helper_functions>
def candidate_components(C):
"""
return the set of candidate nearest components for the connected
component containing C. Do not consider those in C's connected
component or those that are 'dead'.
"""
component_set = subgraphs.component_set(C)
return list((V - set(component_set)) - D)
def update_nn_component(C, candidates):
"""
find the nearest neighbor pair for the connected component
represented by c. candidates represents the list of
components from which to select.
"""
(v, vm) = subgraphs.queues[C].top()
# vm ∈ C {not a foreign nearest neighbor}
# go through the queue until an edge is found between this node
# and the set of candidates, updating the neighbors in the connected
# components queue in the process.
while vm not in candidates:
subgraphs.queues[C].pop()
um, _ = kdtree.query_subset(projcoords[v], candidates)
dm = square_distance(projcoords[v], projcoords[um])
subgraphs.push(subgraphs.queues[C], (v, um), dm)
# Note: v will always be a vertex in this connected component
# vm *may* be external
(v, vm) = subgraphs.queues[C].top()
return (v, vm)
# Tests whether the node is a projection on the existing grid, using its MV
is_fake = lambda n: subgraphs.budget[n] == np.inf
# Test whether the component is dead
# i.e. can never connect to another node
def is_dead(c, nn_dist):
return not is_fake(c) and subgraphs.budget[c] < nn_dist
# "true" distance between components
def component_dist(c1, c2):
dist = 0
if G.is_geographic():
dist = spherical_distance([coords[c1], coords[c2]])
else:
dist = euclidean_distance([coords[c1], coords[c2]])
return dist
# </helper_functions>
# Initialize the connected components holding a single node
# and push the nearest neighbor into its queue
for v in V:
vm, _ = kdtree.query_subset(projcoords[v], list(V - {v}))
dm = square_distance(projcoords[v], projcoords[vm])
subgraphs.add_component(v, budget=G.node[v]['budget'])
subgraphs.push(subgraphs.queues[v], (v, vm), dm)
# Add to dead set if warranted
nn_dist = component_dist(v, vm)
if is_dead(v, nn_dist):
# here components are single nodes
# so no need to worry about adding children to dead set
if v not in D: D.add(v)
Et = [] # Initialize MST edges to empty list
last_state = None
# MST is complete when no progress was made in the prior iteration
while Et != last_state:
# This is a candidate list of edges that might be added to the MST
Ep = PriorityQueue()
# ∀ C of G; where c <- connected component
for C in subgraphs.connected_components(component_subset=V):
candidates = candidate_components(C)
# Skip if no valid candidates
if not candidates:
continue
(v, vm) = update_nn_component(C, candidates)
# Add to dead set if warranted
nn_dist = component_dist(v, vm)
if is_dead(C, nn_dist):
# add all dead components to the dead set D
# (note that fake nodes can never be dead)
for c in subgraphs.component_set(C):
if c not in D and not is_fake(c): D.add(c)
# One more round to root out connections to dead components
# found in above iteration.
# Need to do this BEFORE pushing candidate edges into Ep.
# Otherwise we might be testing only 'dead' candidates
# and therefore mistakenly think we were done (since
# no new edges would have been added)
for C in subgraphs.connected_components(component_subset=V):
candidates = candidate_components(C)
# Skip if no valid candidates
if not candidates:
continue
(v, vm) = update_nn_component(C, candidates)
# Calculate nn_dist for comparison to mv later
nn_dist = component_dist(v, vm)
# Append the top priority edge from the subgraph to the candidate
# edge set
Ep.push((v, vm, nn_dist), nn_dist)
last_state = deepcopy(Et)
# Candidate Test
# At this point we have all of our nearest neighbor component edge
# candidates defined for this "round"
#
# Now test all candidate edges in Ep for cycles and satisfaction of
# custom criteria
while Ep._queue:
(um, vm, dm) = Ep.pop()
# if doesn't create cycle
# and subgraphs have enough MV
# and we're not connecting 2 fake nodes
# then allow the connection
if subgraphs[um] != subgraphs[vm] and \
(subgraphs.budget[subgraphs[um]] >= dm or is_fake(um)) and \
(subgraphs.budget[subgraphs[vm]] >= dm or is_fake(vm)) and \
not (is_fake(um) and is_fake(vm)):
# doesn't create cycles from line segment intersection
invalid_edge, intersections = \
line_subgraph_intersection(subgraphs, rtree,
coords[um], coords[vm])
if not invalid_edge:
# edges should not intersect a subgraph more than once
assert(filter(lambda n: n > 1,
intersections.values()) == [])
# merge the subgraphs
subgraphs.union(um, vm, dm)
# For all intersected subgraphs update the mv to that
# created by the edge intersecting them,
# TODO: This should be updated in not such a naive method
map(lambda (n, _): subgraphs.union(um, n, 0),
filter(lambda (n, i): i == 1 and
subgraphs[n] != subgraphs[um],
intersections.iteritems()))
# index the newly added edge
box = make_bounding_box(coords[um], coords[vm])
# Object is (u.label, v.label), (u.coord, v.coord)
rtree.insert(hash((um, vm)), box,
obj=((um, vm), (coords[um], coords[vm])))
Et += [(um, vm, {'weight': dm})]
# create new GeoGraph with results
result = G.copy()
result.coords = G.coords
result.remove_edges_from(result.edges())
result.add_edges_from(Et)
return result
print >>sys.stderr, "%d retried, %d unassigned." % (retries, len(unassigned))
hoodIndex = Rtree()
print >>sys.stderr, "Buffering polygons."
for place_id, polygon in polygons.items():
if type(polygon) is Polygon:
polygon = Polygon(polygon.exterior.coords)
else:
bits = []
for p in polygon.geoms:
if type(p) is Polygon:
bits.append(Polygon(p.exterior.coords))
polygon = MultiPolygon(bits)
polygons[place_id] = polygon.buffer(0)
hoodIndex.insert(place_id, polygons[place_id].bounds)
print >>sys.stderr, "Retconning blocks to shapes."
cur.execute("""select geom, geoid10 FROM tabblock10 tb WHERE statefp10 = %s AND countyfp10 = %s AND blockce10 NOT LIKE '0%%'""", (statefp10, countyfp10))
for r in cur.fetchall():
poly = wkb.loads(r[0].decode('hex'))
id = r[1]
candidates = [i for i in hoodIndex.intersection(poly.bounds)]
found = False
for place_id in candidates:
hood = polygons[place_id]
if hood.contains(poly):
cur.execute("""DELETE FROM votes WHERE source=%s AND id=%s""", ('blockr', id))
cur.execute("""INSERT INTO votes (id, label, count, source) values (%s, %s, %s, 'blockr')""", (
id, place_id, 1))
conn.commit()
'''
Created on Aug 5, 2016
@author: ionut
'''
import fiona
import logging
from rtree import Rtree
from shapely.geometry import shape
logger = logging.getLogger('memory')
logger.info('loading shp data')
_countries = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm0.shp')
_states = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm1.shp')
_counties = fiona.open('/media/ionut/Work/gis/gadm28_levels.shp/gadm28_adm2.shp')
logger.info('building county index')
_counties_idx = Rtree()
for county in _counties:
geom = shape(county['geometry'])
gid = int(county['id'])
_counties_idx.insert(gid, geom.bounds)
logger.info('done')
class StreetMap:
def __init__(self):
self.nodes = {} # indexed by node id
self.edges = {} # indexed by edge id
self.intersections = {} # indexed by node id
self.node_spatial_index = Rtree()
self.edge_spatial_index = Rtree()
self.intersection_spatial_index = Rtree()
self.edge_lookup_table = {} # indexed by (in_node,out_node)
self.edge_coords_lookup_table = {
} # indexed by (in_node.coords, out_node.coords)
self.segments = {} # indexed by segment id
self.segment_lookup_table = {
} # indexed by (head_edge.in_node, tail_edge.out_node)
def load_osmdb(self, osmdb_filename):
# connect to OSMDB
conn = sqlite3.connect(osmdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
query_iter = cur.execute("select id, lat, lon from nodes")
# query_result = cur.fetchall()
# iterate through all query results
for id, lat, lon in query_iter:
# create and store node in nodes dictionary
self.nodes[int(id)] = Node(float(lat), float(lon), int(id))
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
query_iter = cur.execute("select id, tags, nds from ways")
# query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, tags, nodes in query_iter:
# grab tags associated with current way
way_tags_dict = eval(tags)
# if current way is a valid highway
if ('highway' in way_tags_dict.keys()
and self._valid_highway_edge(way_tags_dict['highway'])):
# grab all nodes that compose this way
way_nodes_list = eval(nodes)
# iterate through list of way nodes
for i in range(1, len(way_nodes_list)):
# grab in_node from nodes dictionary
in_node = self.nodes[int(way_nodes_list[i - 1])]
# grab out_node from nodes dictionary
out_node = self.nodes[int(way_nodes_list[i])]
# create edge_id based on way id
edge_id = int(str(id) + str(i - 1) + "000000")
# if either node on the edge is valid
if (
True
): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[int(edge_id)] = Edge(
in_node, out_node, int(edge_id))
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# if edge is bidirectional
if ('oneway' not in way_tags_dict.keys()):
# create new symmetric edge id
symmetric_edge_id = int(str(edge_id / 10) + "1")
# create and store symmetric edge in edges dictionary
self.edges[int(symmetric_edge_id)] = Edge(
out_node, in_node, int(symmetric_edge_id))
# store in_node in out_node's out_nodes list
if (in_node not in out_node.out_nodes):
out_node.out_nodes.append(in_node)
# store out_node in in_node's in_nodes list
if (out_node not in in_node.in_nodes):
in_node.in_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
print "done."
# close connection to OSMDB
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges and " + str(len(
self.intersections)) + " intersections."
def load_graphdb(self, grapdb_filename):
# connect to graph database
conn = sqlite3.connect(grapdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
cur.execute("select id, latitude, longitude, weight from nodes")
query_result = cur.fetchall()
# iterate through all query results
for id, latitude, longitude, weight in query_result:
# create and store node in nodes dictionary
self.nodes[id] = Node(latitude, longitude, id, weight)
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
cur.execute("select id, in_node, out_node, weight from edges")
query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, in_node_id, out_node_id, weight in query_result:
# grab in_node from nodes dictionary
in_node = self.nodes[in_node_id]
# grab out_node from nodes dictionary
out_node = self.nodes[out_node_id]
# if either node on the edge is valid
if (True
): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[id] = Edge(in_node, out_node, id, weight)
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
# execute query on segments table
cur.execute("select id, edge_ids from segments")
query_result = cur.fetchall()
for id, edge_ids in query_result:
segment_edges = map(lambda edge_id: self.edges[edge_id],
eval(edge_ids))
self.segments[id] = Segment(id, segment_edges)
self.segment_lookup_table[(
self.segments[id].head_edge.in_node,
self.segments[id].tail_edge.out_node)] = self.segments[id]
for segment_edge in segment_edges:
segment_edge.segment = self.segments[id]
# self.segment_lookup_table[segment_edge.id] = self.segments[id]
# execute query on intersections table
cur.execute("select node_id from intersections")
query_result = cur.fetchall()
for node_id in query_result:
self.intersections[node_id[0]] = self.nodes[node_id[0]]
try:
cur.execute(
"select transition_segment, from_segment, to_segment from transitions"
)
query_result = cur.fetchall()
self.transitions = {}
for transition_segment, from_segment, to_segment in query_result:
self.transitions[transition_segment] = (from_segment,
to_segment)
except:
print "Got an error reading "
print "done."
# close connection to graph db
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
#self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges, " + str(len(
self.segments)) + " segments and " + str(
len(self.intersections)) + " intersections."
def load_shapedb(self, shapedb_filename):
# connect to graph database
conn = sqlite3.connect(shapedb_filename)
# grab cursor
cur = conn.cursor()
# execute query to find all shape ids
cur.execute("select distinct shape_id from shapes")
# output that we are loading nodes and edges
sys.stdout.write("\nLoading nodes and edges... ")
sys.stdout.flush()
# storage for shape specific edges
self.shape_edges = {} # indexed by shape_id
# storage for node id
node_id = 0
# iterate through all shape ids
for shape_id in cur.fetchall():
# grab shape id
shape_id = shape_id[0]
# if route is a bus route
if (shape_id == "0" or shape_id == "11" or shape_id == "15"
or shape_id == "41" or shape_id == "65"
or shape_id == "22"):
# execute query to find all shape points
cur.execute(
"select shape_pt_lat, shape_pt_lon from shapes where shape_id='"
+ str(shape_id) + "' order by shape_pt_sequence asc")
# amend shape id
if (shape_id == "0"):
shape_id = "10000000"
elif (shape_id == "11"):
shape_id = "10000011"
elif (shape_id == "41"):
shape_id = "10000041"
elif (shape_id == "15"):
shape_id = "10000015"
elif (shape_id == "65"):
shape_id = "10000065"
elif (shape_id == "22"):
shape_id = "10000022"
# storage for first node
first_node = None
# storage for previous node
prev_node = None
# create list for this shape's edges
self.shape_edges[shape_id] = []
# iterate through all shape points
for shape_pt_lat, shape_pt_lon in cur.fetchall():
# create new node
curr_node = Node(shape_pt_lat, shape_pt_lon, node_id)
# store first node
if (first_node is None):
first_node = curr_node
# increment node id
node_id += 1
# add shape id to node
curr_node.shape_id = shape_id
# store new node in nodes dictionary
self.nodes[node_id] = curr_node
# if there exists a previous node
if (prev_node is not None):
# create edge id
edge_id = int(
str(shape_id) + str(prev_node.id) +
str(curr_node.id))
# create new edge
curr_edge = Edge(prev_node, curr_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in current node's in_nodes list
curr_node.in_nodes.append(prev_node)
# store current node in previous node's out_nodes list
prev_node.out_nodes.append(curr_node)
# update previous node
prev_node = curr_node
# create edge id for last edge
edge_id = int(
str(shape_id) + str(prev_node.id) + str(first_node.id))
# create new edge
curr_edge = Edge(prev_node, first_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in first node's in_nodes list
first_node.in_nodes.append(prev_node)
# store first node in previous node's out_nodes list
prev_node.out_nodes.append(first_node)
print "done."
# close connection to gtfs db
conn.close()
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges and " + str(len(
self.intersections)) + " intersections."
def _index_nodes(self):
# output that we are indexing nodes
sys.stdout.write("Indexing nodes... ")
sys.stdout.flush()
# iterate through all nodes
for curr_node in self.nodes.values():
# insert node into spatial index
self.node_spatial_index.insert(
curr_node.id, (curr_node.longitude, curr_node.latitude))
print "done."
def _index_edges(self):
# output that we are indexing edges
sys.stdout.write("Indexing edges... ")
sys.stdout.flush()
# iterate through all edges
for curr_edge in self.edges.values():
# determine current edge minx, miny, maxx, maxy values
curr_edge_minx = min(curr_edge.in_node.longitude,
curr_edge.out_node.longitude)
curr_edge_miny = min(curr_edge.in_node.latitude,
curr_edge.out_node.latitude)
curr_edge_maxx = max(curr_edge.in_node.longitude,
curr_edge.out_node.longitude)
curr_edge_maxy = max(curr_edge.in_node.latitude,
curr_edge.out_node.latitude)
# insert current edge into spatial index
self.edge_spatial_index.insert(curr_edge.id,
(curr_edge_minx, curr_edge_miny,
curr_edge_maxx, curr_edge_maxy))
# insert current edge into lookup table
self.edge_lookup_table[(curr_edge.in_node,
curr_edge.out_node)] = curr_edge
self.edge_coords_lookup_table[(
curr_edge.in_node.coords(),
curr_edge.out_node.coords())] = curr_edge
# iterate through all edges
for edge in self.edges.values():
# iterate through all out edges
for out_node_neighbor in edge.out_node.out_nodes:
# add out edge to out edges list
edge.out_edges.append(
self.edge_lookup_table[(edge.out_node, out_node_neighbor)])
# iterate through all in edges
for in_node_neighbor in edge.in_node.in_nodes:
# add in edge to in edges list
edge.in_edges.append(self.edge_lookup_table[(in_node_neighbor,
edge.in_node)])
print "done."
def _find_and_index_intersections(self):
# output that we are finding and indexing intersections
sys.stdout.write("Finding and indexing intersections... ")
sys.stdout.flush()
# find intersection nodes and index
(intersection_nodes,
intersection_nodes_index) = self._find_intersection_nodes()
# storage for intersection nodes already placed in intersections
placed_intersection_nodes = set()
# define longitude/latitude offset for bounding box
lon_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LONGITUDE)
lat_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LATITUDE)
# storage for intersection id
intersection_id = 0
# iterate through intersection nodes
for intersection_node in intersection_nodes:
# if the intersection node has not yet been placed
if (intersection_node not in placed_intersection_nodes):
# create bounding box
bounding_box = (intersection_node.longitude - lon_offset,
intersection_node.latitude - lat_offset,
intersection_node.longitude + lon_offset,
intersection_node.latitude + lat_offset)
# find intersection node ids within bounding box
intersection_node_ids = intersection_nodes_index.intersection(
bounding_box)
# get intersection nodes
intersection_nodes = map(self._get_node, intersection_node_ids)
# add intersection nodes to placed set
placed_intersection_nodes.update(intersection_nodes)
# create new intersection
new_intersection = Intersection(intersection_id,
intersection_nodes)
# increment intersection id
intersection_id += 1
# add new intersection to intersections list
self.intersections[new_intersection.id] = new_intersection
# insert new intersection into spatial index
self.intersection_spatial_index.insert(
new_intersection.id,
(new_intersection.longitude, new_intersection.latitude))
print "done."
def _get_node(self, node_id):
# return node from dictionary
return self.nodes[node_id]
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(
curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
def _valid_node(self, node):
# if node falls inside the designated bounding box
if ((node.latitude >= 41.8619 and node.latitude <= 41.8842) and
(node.longitude >= -87.6874 and node.longitude <= -87.6398)):
return True
else:
return False
def _valid_highway_edge(self, highway_tag_value):
if ((highway_tag_value == 'primary')
or (highway_tag_value == 'secondary')
or (highway_tag_value == 'tertiary')
or (highway_tag_value == 'residential')):
return True
else:
return False
def reset_node_visited_flags(self):
# iterate through all nodes
for node in self.nodes.values():
# set node visited flag to False
node.visited = False
def reset_edge_visited_flags(self):
# iterate through all edges
for edge in self.edges.values():
# set edge visited flag to False
edge.visited = False
def write_map_to_file(self, map_filename="map.txt"):
# output that we are starting the writing process
sys.stdout.write("\nWriting map to file... ")
sys.stdout.flush()
# open map file
map_file = open(map_filename, 'w')
# iterate through all map edges
for curr_edge in self.edges.values():
# output current edge to file
map_file.write(
str(curr_edge.in_node.latitude) + "," +
str(curr_edge.in_node.longitude) + "\n")
map_file.write(
str(curr_edge.out_node.latitude) + "," +
str(curr_edge.out_node.longitude) + "\n\n")
# close map file
map_file.close()
print "done."
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
class StreetMap:
def __init__(self):
self.nodes = {} # indexed by node id
self.edges = {} # indexed by edge id
self.intersections = {} # indexed by node id
self.node_spatial_index = Rtree()
self.edge_spatial_index = Rtree()
self.intersection_spatial_index = Rtree()
self.edge_lookup_table = {} # indexed by (in_node,out_node)
self.edge_coords_lookup_table = {} # indexed by (in_node.coords, out_node.coords)
self.segments = {} # indexed by segment id
self.segment_lookup_table = {} # indexed by (head_edge.in_node, tail_edge.out_node)
def load_osmdb(self, osmdb_filename):
# connect to OSMDB
conn = sqlite3.connect(osmdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
cur.execute("select id, lat, lon from nodes")
query_result = cur.fetchall()
# iterate through all query results
for id, lat, lon in query_result:
# create and store node in nodes dictionary
self.nodes[int(id)] = Node(float(lat), float(lon), int(id))
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
cur.execute("select id, tags, nds from ways")
query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, tags, nodes in query_result:
# grab tags associated with current way
way_tags_dict = eval(tags)
# if current way is a valid highway
if ('highway' in way_tags_dict.keys() and self._valid_highway_edge(way_tags_dict['highway'])):
# grab all nodes that compose this way
way_nodes_list = eval(nodes)
# iterate through list of way nodes
for i in range(1, len(way_nodes_list)):
# grab in_node from nodes dictionary
in_node = self.nodes[int(way_nodes_list[i - 1])]
# grab out_node from nodes dictionary
out_node = self.nodes[int(way_nodes_list[i])]
# create edge_id based on way id
edge_id = int(str(id) + str(i - 1) + "000000")
# if either node on the edge is valid
if (True): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[int(edge_id)] = Edge(in_node, out_node,int(edge_id))
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# if edge is bidirectional
if ('oneway' not in way_tags_dict.keys()):
# create new symmetric edge id
symmetric_edge_id = int(str(edge_id / 10) + "1")
# create and store symmetric edge in edges dictionary
self.edges[int(symmetric_edge_id)] = Edge(out_node, in_node, int(symmetric_edge_id))
# store in_node in out_node's out_nodes list
if (in_node not in out_node.out_nodes):
out_node.out_nodes.append(in_node)
# store out_node in in_node's in_nodes list
if (out_node not in in_node.in_nodes):
in_node.in_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
print "done."
# close connection to OSMDB
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges and " + str(len(self.intersections)) + " intersections."
def load_graphdb(self, grapdb_filename):
# connect to graph database
conn = sqlite3.connect(grapdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
cur.execute("select id, latitude, longitude, weight from nodes")
query_result = cur.fetchall()
# iterate through all query results
for id, latitude, longitude, weight in query_result:
# create and store node in nodes dictionary
self.nodes[id] = Node(latitude, longitude, id, weight)
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
cur.execute("select id, in_node, out_node, weight from edges")
query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, in_node_id, out_node_id, weight in query_result:
# grab in_node from nodes dictionary
in_node = self.nodes[in_node_id]
# grab out_node from nodes dictionary
out_node = self.nodes[out_node_id]
# if either node on the edge is valid
if (True): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[id] = Edge(in_node, out_node, id, weight)
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
# execute query on segments table
cur.execute("select id, edge_ids from segments")
query_result = cur.fetchall()
for id, edge_ids in query_result:
segment_edges = map(lambda edge_id: self.edges[edge_id], eval(edge_ids))
self.segments[id] = Segment(id, segment_edges)
self.segment_lookup_table[(self.segments[id].head_edge.in_node, self.segments[id].tail_edge.out_node)] = self.segments[id]
for segment_edge in segment_edges:
segment_edge.segment = self.segments[id]
# self.segment_lookup_table[segment_edge.id] = self.segments[id]
# execute query on intersections table
cur.execute("select node_id from intersections")
query_result = cur.fetchall()
for node_id in query_result:
self.intersections[node_id[0]] = self.nodes[node_id[0]]
try:
cur.execute("select transition_segment, from_segment, to_segment from transitions");
query_result = cur.fetchall()
self.transitions={}
for transition_segment, from_segment, to_segment in query_result:
self.transitions[transition_segment]=(from_segment,to_segment)
except:
print "Got an error reading "
print "done."
# close connection to graph db
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
#self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges, " + str(len(self.segments)) + " segments and " + str(len(self.intersections)) + " intersections."
def load_shapedb(self, shapedb_filename):
# connect to graph database
conn = sqlite3.connect(shapedb_filename)
# grab cursor
cur = conn.cursor()
# execute query to find all shape ids
cur.execute("select distinct shape_id from shapes")
# output that we are loading nodes and edges
sys.stdout.write("\nLoading nodes and edges... ")
sys.stdout.flush()
# storage for shape specific edges
self.shape_edges = {} # indexed by shape_id
# storage for node id
node_id = 0
# iterate through all shape ids
for shape_id in cur.fetchall():
# grab shape id
shape_id = shape_id[0]
# if route is a bus route
if (shape_id == "0" or shape_id == "11" or shape_id == "15" or shape_id == "41" or shape_id == "65" or shape_id == "22"):
# execute query to find all shape points
cur.execute("select shape_pt_lat, shape_pt_lon from shapes where shape_id='" + str(shape_id) + "' order by shape_pt_sequence asc")
# amend shape id
if (shape_id == "0"):
shape_id = "10000000"
elif (shape_id == "11"):
shape_id = "10000011"
elif (shape_id == "41"):
shape_id = "10000041"
elif (shape_id == "15"):
shape_id = "10000015"
elif (shape_id == "65"):
shape_id = "10000065"
elif (shape_id == "22"):
shape_id = "10000022"
# storage for first node
first_node = None
# storage for previous node
prev_node = None
# create list for this shape's edges
self.shape_edges[shape_id] = []
# iterate through all shape points
for shape_pt_lat, shape_pt_lon in cur.fetchall():
# create new node
curr_node = Node(shape_pt_lat, shape_pt_lon, node_id)
# store first node
if (first_node is None):
first_node = curr_node
# increment node id
node_id += 1
# add shape id to node
curr_node.shape_id = shape_id
# store new node in nodes dictionary
self.nodes[node_id] = curr_node
# if there exists a previous node
if (prev_node is not None):
# create edge id
edge_id = int(str(shape_id) + str(prev_node.id) + str(curr_node.id))
# create new edge
curr_edge = Edge(prev_node, curr_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in current node's in_nodes list
curr_node.in_nodes.append(prev_node)
# store current node in previous node's out_nodes list
prev_node.out_nodes.append(curr_node)
# update previous node
prev_node = curr_node
# create edge id for last edge
edge_id = int(str(shape_id) + str(prev_node.id) + str(first_node.id))
# create new edge
curr_edge = Edge(prev_node, first_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in first node's in_nodes list
first_node.in_nodes.append(prev_node)
# store first node in previous node's out_nodes list
prev_node.out_nodes.append(first_node)
print "done."
# close connection to gtfs db
conn.close()
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(len(self.edges)) + " edges and " + str(len(self.intersections)) + " intersections."
def _index_nodes(self):
# output that we are indexing nodes
sys.stdout.write("Indexing nodes... ")
sys.stdout.flush()
# iterate through all nodes
for curr_node in self.nodes.values():
# insert node into spatial index
self.node_spatial_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude))
print "done."
def _index_edges(self):
# output that we are indexing edges
sys.stdout.write("Indexing edges... ")
sys.stdout.flush()
# iterate through all edges
for curr_edge in self.edges.values():
# determine current edge minx, miny, maxx, maxy values
curr_edge_minx = min(curr_edge.in_node.longitude, curr_edge.out_node.longitude)
curr_edge_miny = min(curr_edge.in_node.latitude, curr_edge.out_node.latitude)
curr_edge_maxx = max(curr_edge.in_node.longitude, curr_edge.out_node.longitude)
curr_edge_maxy = max(curr_edge.in_node.latitude, curr_edge.out_node.latitude)
# insert current edge into spatial index
self.edge_spatial_index.insert(curr_edge.id, (curr_edge_minx, curr_edge_miny, curr_edge_maxx, curr_edge_maxy))
# insert current edge into lookup table
self.edge_lookup_table[(curr_edge.in_node, curr_edge.out_node)] = curr_edge
self.edge_coords_lookup_table[(curr_edge.in_node.coords(), curr_edge.out_node.coords())] = curr_edge
# iterate through all edges
for edge in self.edges.values():
# iterate through all out edges
for out_node_neighbor in edge.out_node.out_nodes:
# add out edge to out edges list
edge.out_edges.append(self.edge_lookup_table[(edge.out_node, out_node_neighbor)])
# iterate through all in edges
for in_node_neighbor in edge.in_node.in_nodes:
# add in edge to in edges list
edge.in_edges.append(self.edge_lookup_table[(in_node_neighbor, edge.in_node)])
print "done."
def _find_and_index_intersections(self):
# output that we are finding and indexing intersections
sys.stdout.write("Finding and indexing intersections... ")
sys.stdout.flush()
# find intersection nodes and index
(intersection_nodes, intersection_nodes_index) = self._find_intersection_nodes()
# storage for intersection nodes already placed in intersections
placed_intersection_nodes = set()
# define longitude/latitude offset for bounding box
lon_offset = ((intersection_size / 2.0) / spatialfunclib.METERS_PER_DEGREE_LONGITUDE)
lat_offset = ((intersection_size / 2.0) / spatialfunclib.METERS_PER_DEGREE_LATITUDE)
# storage for intersection id
intersection_id = 0
# iterate through intersection nodes
for intersection_node in intersection_nodes:
# if the intersection node has not yet been placed
if (intersection_node not in placed_intersection_nodes):
# create bounding box
bounding_box = (intersection_node.longitude - lon_offset, intersection_node.latitude - lat_offset, intersection_node.longitude + lon_offset, intersection_node.latitude + lat_offset)
# find intersection node ids within bounding box
intersection_node_ids = intersection_nodes_index.intersection(bounding_box)
# get intersection nodes
intersection_nodes = map(self._get_node, intersection_node_ids)
# add intersection nodes to placed set
placed_intersection_nodes.update(intersection_nodes)
# create new intersection
new_intersection = Intersection(intersection_id, intersection_nodes)
# increment intersection id
intersection_id += 1
# add new intersection to intersections list
self.intersections[new_intersection.id] = new_intersection
# insert new intersection into spatial index
self.intersection_spatial_index.insert(new_intersection.id, (new_intersection.longitude, new_intersection.latitude))
print "done."
def _get_node(self, node_id):
# return node from dictionary
return self.nodes[node_id]
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
def _valid_node(self, node):
# if node falls inside the designated bounding box
if ((node.latitude >= 41.8619 and node.latitude <= 41.8842) and
(node.longitude >= -87.6874 and node.longitude <= -87.6398)):
return True
else:
return False
def _valid_highway_edge(self, highway_tag_value):
if ((highway_tag_value == 'primary') or
(highway_tag_value == 'secondary') or
(highway_tag_value == 'tertiary') or
(highway_tag_value == 'residential')):
return True
else:
return False
def reset_node_visited_flags(self):
# iterate through all nodes
for node in self.nodes.values():
# set node visited flag to False
node.visited = False
def reset_edge_visited_flags(self):
# iterate through all edges
for edge in self.edges.values():
# set edge visited flag to False
edge.visited = False
def write_map_to_file(self, map_filename="map.txt"):
# output that we are starting the writing process
sys.stdout.write("\nWriting map to file... ")
sys.stdout.flush()
# open map file
map_file = open(map_filename, 'w')
# iterate through all map edges
for curr_edge in self.edges.values():
# output current edge to file
map_file.write(str(curr_edge.in_node.latitude) + "," + str(curr_edge.in_node.longitude) + "\n")
map_file.write(str(curr_edge.out_node.latitude) + "," + str(curr_edge.out_node.longitude) + "\n\n")
# close map file
map_file.close()
print "done."
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude, location2.latitude, location2.longitude)
def api_root():
''' This is the root of the webservice, upon successful authentication a text will be displayed in the browser '''
try:
projectid = request.args.get('projectid')
diagramid = request.args.get('diagramid')
apitoken = request.args.get('apitoken')
except KeyError as ke:
msg = json.dumps({
"message":
"Could not parse Projectid, Diagram ID or API Token ID. One or more of these were not found in your JSON request."
})
return Response(msg, status=400, mimetype='application/json')
if projectid and diagramid and apitoken:
myAPIHelper = GeodesignHub.GeodesignHubClient(
url=config.apisettings['serviceurl'],
project_id=projectid,
token=apitoken)
myGridGenerator = GridGenerator()
myDiagramDecomposer = DiagramDecomposer()
r = myAPIHelper.get_diagram(int(diagramid))
try:
assert r.status_code == 200
except AssertionError as ae:
print("Invalid reponse %s" % ae)
else:
op = json.loads(r.text)
diaggeoms = op['geojson']
sysid = op['sysid']
desc = op['description']
projectorpolicy = op['type']
fundingtype = 'o'
geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms)
grid, allbounds = myGridGenerator.generateGrid(bounds)
gridRTree = Rtree()
for boundsid, bounds in allbounds.items():
gridRTree.insert(boundsid, bounds)
choppedgeomsandareas = []
choppedgeoms = []
totalarea = 0
for curgeom in geoms:
curbounds = curgeom.bounds
igridids = list(gridRTree.intersection(curbounds))
for curintersectid in igridids:
gridfeat = grid[curintersectid]
ifeat = curgeom.intersection(gridfeat)
ifeatarea = ifeat.area
totalarea += ifeatarea
ele = {'area': ifeatarea, 'feature': ifeat}
choppedgeoms.append(ele)
sortedgeomsandareas = sorted(choppedgeoms, key=lambda k: k['area'])
tenpercent = ((totalarea * 10) / 100)
thirtypercent = ((totalarea * 30) / 100)
seventypercent = ((totalarea * 70) / 100)
# print totalarea, tenpercent, thirtypercent, seventypercent
a = 0
tenpercentfeats = {"type": "FeatureCollection", "features": []}
twentypercentfeats = {"type": "FeatureCollection", "features": []}
seventypercentfeats = {"type": "FeatureCollection", "features": []}
for cursortedgeom in sortedgeomsandareas:
cf = {}
j = json.loads(
shapelyHelper.export_to_JSON(cursortedgeom['feature']))
cf['type'] = 'Feature'
cf['properties'] = {}
cf['geometry'] = j
if a < tenpercent:
tenpercentfeats['features'].append(cf)
elif a > tenpercent and a < thirtypercent:
twentypercentfeats['features'].append(cf)
else:
seventypercentfeats['features'].append(cf)
a += float(cursortedgeom['area'])
opdata = [{
'gj': tenpercentfeats,
'desc': "10% " + desc
}, {
'gj': twentypercentfeats,
'desc': "20% " + desc
}, {
'gj': seventypercentfeats,
'desc': "70% " + desc,
"fundingtype": fundingtype
}]
alluploadmessages = []
for curopdata in opdata:
print(json.dumps(curopdata['gj']))
# upload = myAPIHelper.post_as_diagram(geoms = json.dumps(curopdata['gj']), projectorpolicy= projectorpolicy,featuretype = 'polygon', description= curopdata['desc'], sysid = sysid)
# alluploadmessages.append(json.loads(upload.text))
msg = json.dumps({
"message": "Diagrams have been uploaded",
"uploadstatus": alluploadmessages
})
return Response(msg, status=400, mimetype='application/json')
else:
msg = json.dumps({
"message":
"Could not parse Project ID, Diagram ID or API Token ID. One or more of these were not found in your JSON request."
})
return Response(msg, status=400, mimetype='application/json')
def snap_points_to_links(points, links):
"""Place points onto closest link in a set of links (arc/edges).
Parameters
----------
points : dict
Point ID as key and (x,y) coordinate as value.
links : list
Elements are of type ``libpysal.cg.shapes.Chain``
** Note ** each element is a link represented as a chain with
*one head and one tail vertex* in other words one link only.
Returns
-------
point2link : dict
Key [point ID (see points in arguments)]; value [a 2-tuple
((head, tail), point) where (head, tail) is the target link,
and point is the snapped location on the link.
Examples
--------
>>> import spaghetti
>>> from libpysal.cg.shapes import Point, Chain
>>> points = {0: Point((1,1))}
>>> link = [Chain([Point((0,0)), Point((2,0))])]
>>> spaghetti.util.snap_points_to_links(points, link)
{0: ([(0.0, 0.0), (2.0, 0.0)], array([1., 0.]))}
"""
# instantiate an rtree
rtree = Rtree()
# set the smallest possible float epsilon on machine
SMALL = numpy.finfo(float).eps
# initialize network vertex to link lookup
vertex_2_link = {}
# iterate over network links
for i, link in enumerate(links):
# extract network link (x,y) vertex coordinates
head, tail = link.vertices
x0, y0 = head
x1, y1 = tail
if (x0, y0) not in vertex_2_link:
vertex_2_link[(x0, y0)] = []
if (x1, y1) not in vertex_2_link:
vertex_2_link[(x1, y1)] = []
vertex_2_link[(x0, y0)].append(link)
vertex_2_link[(x1, y1)].append(link)
# minimally increase the bounding box exterior
bx0, by0, bx1, by1 = link.bounding_box
bx0 -= SMALL
by0 -= SMALL
bx1 += SMALL
by1 += SMALL
# insert the network link and its associated
# rectangle into the rtree
rtree.insert(i, (bx0, by0, bx1, by1), obj=link)
# build a KDtree on link vertices
kdtree = cg.KDTree(list(vertex_2_link.keys()))
point2link = {}
for pt_idx, point in points.items():
# first, find nearest neighbor link vertices for the point
dmin, vertex = kdtree.query(point, k=1)
vertex = tuple(kdtree.data[vertex])
closest = vertex_2_link[vertex][0].vertices
# Use this link as the candidate closest link: closest
# Use the distance as the distance to beat: dmin
point2link[pt_idx] = (closest, numpy.array(vertex))
x0 = point[0] - dmin
y0 = point[1] - dmin
x1 = point[0] + dmin
y1 = point[1] + dmin
# Find all links with bounding boxes that intersect
# a query rectangle centered on the point with sides
# of length dmin * dmin
rtree_lookup = rtree.intersection([x0, y0, x1, y1], objects=True)
candidates = [cand.object for cand in rtree_lookup]
dmin += SMALL
dmin2 = dmin * dmin
# of the candidate arcs, find the nearest to the query point
for candidate in candidates:
dist2cand, nearp = squared_distance_point_link(point, candidate.vertices)
if dist2cand <= dmin2:
closest = candidate.vertices
dmin2 = dist2cand
point2link[pt_idx] = (closest, nearp)
return point2link
def enclosure_tree(polygons):
"""
Given a list of shapely polygons with only exteriors,
find which curves represent the exterior shell or root curve
and which represent holes which penetrate the exterior.
This is done with an R-tree for rough overlap detection,
and then exact polygon queries for a final result.
Parameters
-----------
polygons : (n,) shapely.geometry.Polygon
Polygons which only have exteriors and may overlap
Returns
-----------
roots : (m,) int
Index of polygons which are root
contains : networkx.DiGraph
Edges indicate a polygon is
contained by another polygon
"""
tree = Rtree()
# nodes are indexes in polygons
contains = nx.DiGraph()
for i, polygon in enumerate(polygons):
# if a polygon is None it means creation
# failed due to weird geometry so ignore it
if polygon is None or len(polygon.bounds) != 4:
continue
# insert polygon bounds into rtree
tree.insert(i, polygon.bounds)
# make sure every valid polygon has a node
contains.add_node(i)
# loop through every polygon
for i in contains.nodes():
polygon = polygons[i]
# we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygon.bounds):
# if we are checking a polygon against itself continue
if (i == j):
continue
# do a more accurate polygon in polygon test
# for the enclosure tree information
if polygons[i].contains(polygons[j]):
contains.add_edge(i, j)
elif polygons[j].contains(polygons[i]):
contains.add_edge(j, i)
# a root or exterior curve has an even number of parents
# wrap in dict call to avoid networkx view
degree = dict(contains.in_degree())
# convert keys and values to numpy arrays
indexes = np.array(list(degree.keys()))
degrees = np.array(list(degree.values()))
# roots are curves with an even inward degree (parent count)
roots = indexes[(degrees % 2) == 0]
# if there are multiple nested polygons split the graph
# so the contains logic returns the individual polygons
if len(degrees) > 0 and degrees.max() > 1:
# collect new edges for graph
edges = []
# find edges of subgraph for each root and children
for root in roots:
children = indexes[degrees == degree[root] + 1]
edges.extend(contains.subgraph(np.append(children, root)).edges())
# stack edges into new directed graph
contains = nx.from_edgelist(edges, nx.DiGraph())
# if roots have no children add them anyway
contains.add_nodes_from(roots)
return roots, contains
try:
assert r.status_code == 200
except AssertionError as ae:
print("Invalid reponse %s" % ae)
else:
op = json.loads(r.text)
diaggeoms = op['geojson']
sysid = op['sysid']
desc = op['description']
projectorpolicy = op['type']
geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms)
grid, allbounds = myGridGenerator.generateGrid(bounds)
gridRTree = Rtree()
for boundsid, bounds in allbounds.iteritems():
gridRTree.insert(boundsid, bounds)
choppedgeomsandareas = []
choppedgeoms = []
totalarea = 0
for curgeom in geoms:
curbounds = curgeom.bounds
igridids = list(gridRTree.intersection(curbounds))
for curintersectid in igridids:
gridfeat = grid[curintersectid]
ifeat = curgeom.intersection(gridfeat)
ifeatarea = ifeat.area
totalarea += ifeatarea
ele = {'area': ifeatarea, 'feature': ifeat}
choppedgeoms.append(ele)
