def get_rtree_index(self):
"""
Get an rtree index of the edges within this GeoGraph
Returns:
rtree: rtree of edges indexed by bounding_box (4 coordinates)
allowing lookup of the edge (node1, node2) and it's segment
"""
def edge_generator():
edges_bounds = map(lambda tup: (tup, gm.make_bounding_box(
*map(lambda n: self.coords[n], tup))), self.edges())
for edge, box in edges_bounds:
# Object is in form of (u.label, v.label), (u.coord, v.coord)
yield (hash(edge), box, (edge,
map(lambda ep: np.array(self.coords[ep]), edge)))
# something's not working right with latest version of rtree/spatial lib
# index where we need to insert the objects in a loop rather than
# via a generator to their constructor
# Init rtree and store grid edges
rtree = Rtree()
for e in edge_generator():
rtree.insert(e[0], e[1], e[2])
return rtree
def polygons_enclosure_tree(polygons):
'''
Given a list of shapely polygons, which are the root (aka outermost)
polygons, and which represent the holes which penetrate the root
curve. We do this by creating an R-tree for rough collision detection,
and then do polygon queries for a final result
'''
tree = Rtree()
for i, polygon in enumerate(polygons):
tree.insert(i, polygon.bounds)
count = len(polygons)
g = nx.DiGraph()
g.add_nodes_from(np.arange(count))
for i in range(count):
if polygons[i] is None: continue
#we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygons[i].bounds):
if (i==j): continue
#we then do a more accurate polygon in polygon test to generate
#the enclosure tree information
if polygons[i].contains(polygons[j]): g.add_edge(i,j)
elif polygons[j].contains(polygons[i]): g.add_edge(j,i)
roots = [n for n, deg in list(g.in_degree().items()) if deg==0]
return roots, g
def segment_spatial_index(linestring):
coords = list(linestring.coords)
spatial_index = Rtree()
for i in range(len(coords)-1):
x = map(lambda xy: xy[0],coords[i:i+2])
y = map(lambda xy: xy[1],coords[i:i+2])
spatial_index.add(i,(min(x),min(y),max(x),max(y)))
return spatial_index
def __call__(self, container, name, args):
copts, cargs = self.parser.parse_args(args)
data = os.path.join(container.opts.data, name)
# First, pack the ZODB
storage = FileStorage.FileStorage("%s/var/Data.fs" % data)
db = DB(storage)
db.pack()
# Can't pack an Rtree's storage in-place, so we move it away and
# recreate from the contents of the ZODB
rtree = None
rtree_filename = '%s/var/vrt1' % data
try:
shutil.move(rtree_filename + ".dat", rtree_filename + ".bkup.dat")
shutil.move(rtree_filename + ".idx", rtree_filename + ".bkup.idx")
conn = db.open()
root = conn.root()
keys = root['index'].keys
bkup = Rtree('%s/var/vrt1.bkup' % data)
pagesize = bkup.properties.pagesize
if len(keys) == 0:
fwd = Rtree(
'%s/var/vrt1' % data,
# Passing in copied properties doesn't work,
# leading to errors involving page ids
# properties=new_properties,
pagesize=pagesize
)
else:
gen = ((intid, bbox, None) for intid, (uid, bbox) \
in keys.items())
fwd = Rtree(
'%s/var/vrt1' % data,
gen,
# Passing in copied properties doesn't work,
# leading to errors involving page ids
# properties=new_properties,
pagesize=pagesize
)
conn.close()
db.close()
storage.close()
except:
# Restore backups
shutil.copy(rtree_filename + ".bkup.dat", rtree_filename + ".dat")
shutil.copy(rtree_filename + ".bkup.idx", rtree_filename + ".idx")
raise
finally:
if fwd is not None:
fwd.close()
def __init__(self):
self.nodes = {} # indexed by node id
self.edges = {} # indexed by edge id
self.intersections = {} # indexed by node id
self.node_spatial_index = Rtree()
self.edge_spatial_index = Rtree()
self.intersection_spatial_index = Rtree()
self.edge_lookup_table = {} # indexed by (in_node,out_node)
self.edge_coords_lookup_table = {} # indexed by (in_node.coords, out_node.coords)
self.segments = {} # indexed by segment id
self.segment_lookup_table = {} # indexed by (head_edge.in_node, tail_edge.out_node)
def __init__(self):
self.GPS = GPS()
self.idx = Rtree()
self.nodeidx = Rtree()
self.G = None
self.edgeindex_edge = {}
self.edgecounter = 0
self.nodecounter = 0
self.gps_points = [];
self.edge_id__count = {}
self.node_counter__node = {}
class Dispatch:
def __init__(self, config_filename):
self.index = Rtree()
# get children from yaml file
config_str = open(config_filename).read()
print config_str
self.children = yaml.load( config_str )
# index children according to their bounding box
for i, child in enumerate(self.children):
self.index.add(i, child['bounds'])
def register(self, baseurl, left, bottom, right, top):
if baseurl in self.children.values():
raise Exception( "Child with baseurl '%s' already registered"%baseurl )
childindex = len(self.children)
self.children[childindex] = (baseurl, (left,bottom,right,top))
self.index.add( childindex, (left, bottom, right, top) )
return json.dumps(self.children)
def children(self):
return json.dumps( self.children )
def _over(self, lat, lon):
return [self.children[x] for x in self.index.intersection( (lon, lat, lon, lat) )]
def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85):
child_servers = self._over( lat, lon )
if len(child_servers) == 0:
return "NO SHEDS HERE"
child_server = child_servers[0]
args = {'lat':lat,
'lon':lon,
'year':year,
'month':month,
'day':day,
'hour':hour,
'minute':minute,
'second':second,
'cutoff':cutoff,
'step':step,
'speed':speed}
contoururl = "http://%s/contour?%s&encoded=true"%(child_server['url'], urlencode(args))
return urlopen(contoururl).read()
def __init__(self, bus_trips):
self.bus_trips = bus_trips
self.graph_nodes = {} # indexed by "location_id"
self.graph_edge_id = 0
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
self.graph_edge_index = Rtree()
def getRGsForBox(request):
l = float( request.GET['x1'] )
b = float( request.GET['y1'] )
r = float( request.GET['x2'] )
t = float( request.GET['y2'] )
index = Rtree('/home/renci/geoanalytics/geoanalytics-lib/rg')
ids = list(index.intersection((l,b,r,t), objects=True))
rgs = []
if len(ids) > 0:
rgs = [{
'name' : r.name,
'filepath' : r.filepath,
'bounds' : r.bbox4326,
'seriescount' : Series.objects(record_group=r).count()
} for r in [RecordGroup.objects(id=i.object).first() for i in ids]]
return HttpResponse(json.dumps(rgs), mimetype='application/json')
def clear(self):
print "clear canvas!"
self._zoom_level = self._ratio_index.index(1)
self._offset = Point(0, 0)
self._shapes = []
#self._cairo_paths = {}
self._index_rtree = Rtree()
self._styler.load_default_style()
def points_in_mesh(points, mesh):
from rtree import Rtree
points = np.array(points)
tri_3D = mesh.vertices[mesh.faces]
tri_2D = tri_3D[:,:,0:2]
z_bounds = np.column_stack((np.min(tri_3D[:,:,2], axis=1),
np.max(tri_3D[:,:,2], axis=1)))
bounds = np.column_stack((np.min(tri_2D, axis=1),
np.max(tri_2D, axis=1)))
tree = Rtree()
for i, bound in enumerate(bounds):
tree.insert(i, bound)
result = np.zeros(len(points), dtype=np.bool)
for point_index, point in enumerate(points):
intersections = np.array(list(tree.intersection(point[0:2].tolist())))
in_triangle = [point_in_triangle_2D(point[0:2], tri_2D[i]) for i in intersections]
result[point_index] = np.int(np.mod(np.sum(in_triangle), 2)) == 0
return result
def getSeriesForBox(request):
l = float( request.GET['x1'] )
b = float( request.GET['y1'] )
r = float( request.GET['x2'] )
t = float( request.GET['y2'] )
rg = request.GET['rg']
index = Rtree('/home/renci/geoanalytics/geoanalytics-lib/sr')
ids = list(index.intersection((l,b,r,t), objects=True))
if len(ids)> 0:
sers = [{
'name' : s.name,
'filepath' : s.filepath,
'bounds' : s.bbox4326,
'filecount' : VectorFile.objects(series=s).count() + RasterFile.objects(series=s).count()
} for s in [Series.objects(id=i.object[0]).first() for i in filter(lambda x:x.object[1] == rg, ids)]]
return HttpResponse(json.dumps(sers), mimetype='application/json')
else:
return HttpResponse("[]", mimetype='application/json')
def __init__(self, hmm, emission_probability, constraint_length=10, MAX_DIST=500, priors=None, smallV=0.00000000001):
# initialize spatial index
self.previous_obs = None
if priors == None:
priors=dict([(state,1.0/len(hmm)) for state in hmm])
state_spatial_index = Rtree()
unlocated_states = []
id_to_state = {}
id = 0
for state in hmm:
geom=self.geometry_of_state(state)
if not geom:
unlocated_states.append(state)
else:
((lat1,lon1),(lat2,lon2))=geom
state_spatial_index.insert(id,
(min(lon1, lon2), min(lat1, lat2),
max(lon1, lon2), max(lat1, lat2)))
id_to_state[id]=state
id=id+1
def candidate_states(obs): #was (lat,lon) in place of obs
geom = self.geometry_of_observation(obs)
if geom == None:
return hmm.keys()
else:
(lat,lon)=geom
nearby_states = state_spatial_index.intersection((lon-MAX_DIST/METERS_PER_DEGREE_LONGITUDE,
lat-MAX_DIST/METERS_PER_DEGREE_LATITUDE,
lon+MAX_DIST/METERS_PER_DEGREE_LONGITUDE,
lat+MAX_DIST/METERS_PER_DEGREE_LATITUDE))
candidates = [id_to_state[id] for id in nearby_states]+unlocated_states
return candidates
self.viterbi = Viterbi(hmm,emission_probability,
constraint_length=constraint_length,
priors=priors,
candidate_states=candidate_states,
smallV=smallV)
def load_fcad_file(self, file_path):
timer.start("loading shapes.fcad")
self._shapes = ShapeFile.read_fcad_file(file_path)
if len(self._shapes):
def generator_function(points):
for i, obj in enumerate(points):
if obj == None: continue
yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("loading shapes.fcad")
class WayIndex:
def __init__(self, ways):
self.max_way_id = 0
self.way_idx_mapping = {}
self.idx = Rtree()
for way in ways:
self.add(way)
def add(self, way):
self.idx.add(id=self.max_way_id, coordinates=way.get_bbox(),
obj=way.id)
self.way_idx_mapping[way.id] = self.max_way_id
self.max_way_id += 1
def delete(self, way):
self.idx.delete(id=self.way_idx_mapping[way.id],
coordinates=way.get_bbox())
way_ids = map(lambda way: way.object,
wayidx.idx.intersection(way.get_bbox(),
objects=True))
def __init__(self, config_filename):
self.index = Rtree()
# get children from yaml file
config_str = open(config_filename).read()
print config_str
self.children = yaml.load( config_str )
# index children according to their bounding box
for i, child in enumerate(self.children):
self.index.add(i, child['bounds'])
def load_pickle(self, file_path):
timer.start("loading shapes.p")
if os.path.exists(file_path):
self._shapes = pickle.load(open(file_path, "rb"))
if len(self._shapes):
def generator_function(points):
for i, obj in enumerate(points):
if obj == None: continue
yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("loading shapes.p")
def __init__(self, all_trips):
# trips
self.all_trips = all_trips
# cluster seeds
self.cluster_seeds = {}
self.cluster_seed_id = 0
self.cluster_seed_index = Rtree()
# graph edges
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_id = 0
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
class SampleSpace(object):
def __init__(self):
self._index = Rtree()
self._locations = []
self._values = []
def __setitem__(self, location, value):
i = len(self._locations)
self._locations.append(location)
self._values.append(value)
self._index.add(i, self._locations[i])
def __getitem__(self, location):
js = list(self._index.nearest(location, 3))
if len(js) == 0:
return 0
if len(js) == 1:
return self._values[js[0]]
ds = [sqrt(sum([(self._locations[j][i]-location[i])**2 for i in range(2)]))
for j in js]
for i in range(len(js)):
if ds[i] == 0:
return self._values[js[i]]
R = max(ds)
nums = [((R - d)/(R * d))**2 for d in ds]
den = sum(nums)
ws = [num/den for num in nums]
return sum([self._values[js[i]] * ws[i] for i in range(len(js))])
def __iter__(self):
for i in range(len(self._values)):
yield self._locations[i], self._values[i]
def _build_trip_edge_index(self):
sys.stdout.write("\nBuilding trip edge index for clarification algorithm... ")
sys.stdout.flush()
# storage for trip edge index
trip_edge_index = Rtree()
# iterate through all trip edges
for trip_edge in self.trip_edges.values():
# determine trip edge minx, miny, maxx, maxy values
trip_edge_minx = min(trip_edge.in_node.longitude, trip_edge.out_node.longitude)
trip_edge_miny = min(trip_edge.in_node.latitude, trip_edge.out_node.latitude)
trip_edge_maxx = max(trip_edge.in_node.longitude, trip_edge.out_node.longitude)
trip_edge_maxy = max(trip_edge.in_node.latitude, trip_edge.out_node.latitude)
# insert trip edge into spatial index
trip_edge_index.insert(trip_edge.id, (trip_edge_minx, trip_edge_miny, trip_edge_maxx, trip_edge_maxy))
print "done."
# return the trip edge index
return trip_edge_index
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
def clear(self):
self.backward.clear()
props = self.rtree.properties
if props.storage == RT_Disk:
self.rtree.close()
fname = props.filename
try:
os.unlink('%s.%s' % (fname, props.dat_extension))
except OSError:
pass
try:
os.unlink('%s.%s' % (fname, props.idx_extension))
except OSError:
pass
self.rtree = Rtree(*self.rtree_args)
def __init__(self, dbname,overwrite=False,rtree_index=True):
if overwrite:
try:
os.remove( dbname )
except OSError:
pass
self.conn = sqlite3.connect(dbname)
if rtree_index:
self.index = Rtree( dbname )
else:
self.index = None
if overwrite:
self.setup()
def __init__(self, srs, geom_type, fields, values, shapes):
""" Use load() to call this constructor.
"""
self.srs = srs
self.fields = fields
self.geom_type = geom_type
self.values = values
self.shapes = shapes
# this will be changed later
self.tolerance = 0
# guid, src1_id, src2_id, line_id, x1, y1, x2, y2
db = connect(':memory:').cursor()
db.execute("""CREATE table segments (
-- global identifier for this segment
guid INTEGER PRIMARY KEY AUTOINCREMENT,
-- identifiers for source shape or shapes for shared borders
src1_id INTEGER,
src2_id INTEGER,
-- global identifier for this line
line_id INTEGER,
-- start and end coordinates for this segment
x1 REAL,
y1 REAL,
x2 REAL,
y2 REAL,
-- flag
removed INTEGER
)""")
db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)')
db.execute('CREATE INDEX shape1_parts ON segments (src1_id)')
db.execute('CREATE INDEX shape2_parts ON segments (src2_id)')
self.db = db
self.rtree = Rtree()
self.memo_line = make_memo_line()
def __init__(self):
"""Initialise the class by creating an instance of each object."""
self._ratio_index = (0.05, 0.1, 0.2, 0.5, 1, 2, 4, 8, 16, 32, 64) # pagal sita dydi turi kisti tasko atstumas nuo zoominimo centro
self._zoom_level = self._ratio_index.index(4) # 0-dirbame su realiomis koordinatemis - kitur koordinates yra gaunamos atliekant dalyba... todel nera labai tikslios
self._device_area = None # gtk.gdk.Rectangle()
self._shapes = [] # shapes data
self._index_rtree = Rtree() # shapes indexes in self._shapes
self._offset = Point(0, 0) #- naudojamas ctx.translate() #device koordinates
self._prj = GisProjection()
self._styler = Styler(self)
self._painter = Painter(self._styler)
self._drag_to_center = True # jeigu norime kad resize metu (bus iskviestas set_device_area) butu iskvietsas center()
def r_tree(self):
if not hasattr(self, '_r_tree'):
# define properties
p = r_treeIndex.Property()
p.dimension = self.dim()
p.variant = r_treeIndex.RT_Star
index = np.concatenate((range(self.dim()), range(self.dim())))
# Generator provides required point format
def gen():
for id, coord in self:
yield (id, coord[index], id)
self._r_tree = Rtree(gen(), properties=p)
return self._r_tree
def __init__(self, settings_filename):
settings = yaml.load( open( settings_filename ) )
self.home_point = settings['center']
# create cache of osm-node positions
self.osmdb = OSMDB( settings['osmdb_filename'] )
self.gtfsdb = GTFSDatabase( settings['gtfsdb_filename'] )
self.port = settings['port']
self.node_positions = {}
self.index = Rtree()
for node_id, tags, lat, lon in self.osmdb.nodes():
self.node_positions[node_id] = (lon,lat)
self.index.add( int(node_id), (lon,lat,lon,lat) )
# incarnate graph from graphdb
graphdb = GraphDatabase( settings['graphdb_filename'] )
self.graph = graphdb.incarnate()
def __init__(self):
self.nodes = {} # indexed by node id
self.edges = {} # indexed by edge id
self.intersections = {} # indexed by node id
self.node_spatial_index = Rtree()
self.edge_spatial_index = Rtree()
self.intersection_spatial_index = Rtree()
self.edge_lookup_table = {} # indexed by (in_node,out_node)
self.edge_coords_lookup_table = {
} # indexed by (in_node.coords, out_node.coords)
self.segments = {} # indexed by segment id
self.segment_lookup_table = {
} # indexed by (head_edge.in_node, tail_edge.out_node)
class SpatialIndex(Persistent):
def __init__(self, *args):
self.rtree_args = args
self.rtree = Rtree(*args)
self.backward = IOBTree()
def index_doc(self, docid, value):
if docid in self.backward:
self.unindex_doc(docid)
self.backward[docid] = value
self.rtree.add(docid, value, obj=docid)
def unindex_doc(self, docid):
value = self.backward.get(docid)
if value is None:
return
self.rtree.delete(docid, value)
del self.backward[docid]
def apply(self, value):
return [x.object for x in self.rtree.intersection(value, objects=True)]
def clear(self):
self.backward.clear()
props = self.rtree.properties
if props.storage == RT_Disk:
self.rtree.close()
fname = props.filename
try:
os.unlink('%s.%s' % (fname, props.dat_extension))
except OSError:
pass
try:
os.unlink('%s.%s' % (fname, props.idx_extension))
except OSError:
pass
self.rtree = Rtree(*self.rtree_args)
def count(self, value):
return self.rtree.count(value)
def read_dag_to_tree(all_hits):
"""create an rtree, using query as x, subject as y
do this for all htis, then for each diag, do an intersection
(+bbuffer) to find nearby
"""
gdxs = {}
for i, sline in enumerate(open(all_hits)):
if sline[0] == '#': continue
line = sline[:-1].split("\t")
chrs = tuple(sorted([line[0], line[4]]))
# so save the index, which will return i when queried
# and associate i with the text line vie the dict.
if not chrs in gdxs: gdxs[chrs] = ({}, Rtree())
q0, q1 = sorted(map(int, line[2:4]))
s0, s1 = sorted(map(int, line[6:8]))
assert q0 < q1 and s0 < s1
gdxs[chrs][1].add(i, (q0, s0, q1, s1))
gdxs[chrs][0][i] = sline
return gdxs
def __init__(self, psql_conn_string, overwrite=False, rtree_index=True):
self.conn = psycopg2.connect(psql_conn_string)
c = self.conn.cursor()
c.execute("select tablename from pg_tables where schemaname='public'" )
tables = c.fetchall()
for t in ( 'osm_nodes', 'osm_ways' ):
if (t,) not in tables:
overwrite = True
c.close()
if overwrite:
self.setup()
if rtree_index:
self.index = Rtree()
self.index_endnodes()
else:
self.index = None
def add_random_points(self, number, area, generator=True):
"""Kai generator=False - sunaudoja maziau atminties ikrovimo metu, bet trunka gerokai leciau
"""
self._shapes = []
timer.start("Adding random data")
from random import randint
for x in range(0, number):
color = 65536 * randint(0,255) + 256 * randint(0,255) + randint(0,255) # RGBint
x, y = randint(2, area.width), randint(2, area.height)
if not generator: # darysime rtree.add kiekvienam taskui atskirai
self.add(fshape.Shape(1, Point(x, y), color=color))
else:
self._shapes.append((1, color, x, y))
if generator:
def generator_function(points):
for i, obj in enumerate(points):
yield (i, (obj[2], obj[3], obj[2], obj[3]), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("Adding random data")
def to_directed(self, as_view=False):
"""
Convert undirected road network to directed road network
new edge will have new eid, and each original edge will have two edge with reversed coords
:return:
"""
assert as_view is False, "as_view is not supported"
avail_eid = max([eid for u, v, eid in self.edges.data(data='eid')]) + 1
g = nx.DiGraph()
edge_spatial_idx = Rtree()
edge_idx = {}
# add nodes
for n, data in self.nodes(data=True):
# when data=True, it means will data=node's attributes
new_data = copy.deepcopy(data)
g.add_node(n, **new_data)
# add edges
for u, v, data in self.edges(data=True):
mbr = MBR.cal_mbr(data['coords'])
# add forward edge
forward_data = copy.deepcopy(data)
g.add_edge(u, v, **forward_data)
edge_spatial_idx.insert(
forward_data['eid'],
(mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat))
edge_idx[forward_data['eid']] = (u, v)
# add backward edge
backward_data = copy.deepcopy(data)
backward_data['eid'] = avail_eid
avail_eid += 1
backward_data['coords'].reverse()
g.add_edge(v, u, **backward_data)
edge_spatial_idx.insert(
backward_data['eid'],
(mbr.min_lng, mbr.min_lat, mbr.max_lng, mbr.max_lat))
edge_idx[backward_data['eid']] = (v, u)
print('# of nodes:{}'.format(g.number_of_nodes()))
print('# of edges:{}'.format(g.number_of_edges()))
return RoadNetwork(g, edge_spatial_idx, edge_idx)
from flask import redirect, request, current_app, jsonify
import psycopg2
import psycopg2.extras
from collections import defaultdict
import os
from itertools import groupby
from shapely.ops import cascaded_union
from shapely.geometry import mapping, asShape
from shapely import speedups
import geo_utils
import vote_utils
from rtree import Rtree
from shapely.geometry import asShape
import geojson
index = Rtree()
hoodIndex = Rtree()
conn = psycopg2.connect("dbname='gis' user='******' host='localhost' password='******'")
cur = conn.cursor()
def processArea(areaid):
hoodDict = {}
(hoods, id_to_label) = geo_utils.getNeighborhoodsByArea(conn, areaid, None)
for (id, shape) in hoods.iteritems():
hoodIndex.add(int(id), shape.bounds)
hoodDict[int(id)] = shape
(rows, voteDict) = vote_utils.getVotes(conn, areaid, None)
bestVoteDict = {}
def mod_kruskal(G, subgraphs=None, rtree=None):
"""
algorithm to compute the euclidean minimum spanning forest of nodes in
GeoGraph G with 'budget' based restrictions on edges
Uses a modified version of Kruskal's algorithm
NOTE: subgraphs is modified as a side-effect...may remove in future
(useful for testing right now)
Args:
G: GeoGraph of nodes to be connected if appropriate
Nodes should have 'budget' attribute
subgraphs: UnionFind data structure representing existing network's
connected components AND the 'fake' nodes projected onto it. This
is the basis for the agglomerative nearest neighbor approach in
this algorithm.
NOTE: ONLY the existing networks components are represented in
the subgraphs argument. The nodes in G will be added within
this function
rtree: RTree based index of existing network
Returns:
GeoGraph: representing minimum spanning forest of G subject to the
budget based restrictions
"""
# special case (already MST)
if G.number_of_nodes() < 2:
return G
def is_fake(node):
"""
Tests whether the node is a projection on the existing grid,
using its MV
"""
return subgraphs.budget[node] == np.inf
# handy to have coords array
coords = np.row_stack(G.coords.values())
if subgraphs is None:
assert rtree is not None, \
"subgraphs (disjoint set) required when rtree is passed"
rtree = Rtree()
# modified to handle queues, children, mv
subgraphs = UnionFind()
# add nodes and budgets from G to subgraphs as components
for node in G.nodes():
subgraphs.add_component(node, budget=G.node[node]['budget'])
# get fully connected graph
g = G.get_connected_weighted_graph()
# edges in MSF
Et = []
# connect the shortest safe edge until all edges have been tested
# at which point, we have made all possible connections
for u, v, w in sorted(g.edges(data=True), key=lambda x: x[2]['weight']):
# if doesn't create cycle
# and subgraphs have enough MV
# and we're not connecting 2 fake nodes
# then allow the connection
w = w['weight']
if subgraphs[u] != subgraphs[v] and \
(subgraphs.budget[subgraphs[u]] >= w or is_fake(u)) and \
(subgraphs.budget[subgraphs[v]] >= w or is_fake(v)) and \
not (is_fake(u) and is_fake(v)):
# doesn't create cycles from line segment intersection
invalid_edge, intersections = \
line_subgraph_intersection(subgraphs, rtree,
coords[u], coords[v])
if not invalid_edge:
# edges should not intersect a subgraph more than once
assert(filter(lambda n: n > 1,
intersections.values()) == [])
# merge the subgraphs
subgraphs.union(u, v, w)
# For all intersected subgraphs update the mv to that
# created by the edge intersecting them
map(lambda (n, _): subgraphs.union(u, n, 0),
filter(lambda (n, i): i == 1 and
subgraphs[n] != subgraphs[u],
intersections.iteritems()))
# index the newly added edge
box = make_bounding_box(coords[u], coords[v])
# Object is (u.label, v.label), (u.coord, v.coord)
rtree.insert(hash((u, v)), box,
obj=((u, v), (coords[u], coords[v])))
Et += [(u, v, {'weight': w})]
# create new GeoGraph with results
result = G.copy()
result.coords = G.coords
result.remove_edges_from(result.edges())
result.add_edges_from(Et)
return result
''' @author Ben @file rtree_area_example.py In this example we will compare Shapely AGAINST RTREE for performance NOTE: to get this to work I had to install libspatialindex 1.5 from source and then: export LD_LIBRARY_PATH=/usr/local/lib ''' import random, os, sys, time, numpy from rtree import Rtree # instantiate the Rtree class RtreeIndex = Rtree() import random, os, sys, time, numpy from shapely import * from shapely.geometry import Point filename = 'rtree_radius_search.png' numberOfPoints = 50000 gridWidth = 10000 gridHeight = 10000 shapePoints = [] circleRadius = random.randint(500, 1500) circleX = random.randint(0, gridWidth) circleY = random.randint(0, gridHeight)
def api_root():
''' This is the root of the webservice, upon successful authentication a text will be displayed in the browser '''
try:
projectid = request.args.get('projectid')
diagramid = request.args.get('diagramid')
apitoken = request.args.get('apitoken')
except KeyError as ke:
msg = json.dumps({
"message":
"Could not parse Projectid, Diagram ID or API Token ID. One or more of these were not found in your JSON request."
})
return Response(msg, status=400, mimetype='application/json')
if projectid and diagramid and apitoken:
myAPIHelper = GeodesignHub.GeodesignHubClient(
url=config.apisettings['serviceurl'],
project_id=projectid,
token=apitoken)
myGridGenerator = GridGenerator()
myDiagramDecomposer = DiagramDecomposer()
r = myAPIHelper.get_diagram(int(diagramid))
try:
assert r.status_code == 200
except AssertionError as ae:
print("Invalid reponse %s" % ae)
else:
op = json.loads(r.text)
diaggeoms = op['geojson']
sysid = op['sysid']
desc = op['description']
projectorpolicy = op['type']
fundingtype = 'o'
geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms)
grid, allbounds = myGridGenerator.generateGrid(bounds)
gridRTree = Rtree()
for boundsid, bounds in allbounds.items():
gridRTree.insert(boundsid, bounds)
choppedgeomsandareas = []
choppedgeoms = []
totalarea = 0
for curgeom in geoms:
curbounds = curgeom.bounds
igridids = list(gridRTree.intersection(curbounds))
for curintersectid in igridids:
gridfeat = grid[curintersectid]
ifeat = curgeom.intersection(gridfeat)
ifeatarea = ifeat.area
totalarea += ifeatarea
ele = {'area': ifeatarea, 'feature': ifeat}
choppedgeoms.append(ele)
sortedgeomsandareas = sorted(choppedgeoms, key=lambda k: k['area'])
tenpercent = ((totalarea * 10) / 100)
thirtypercent = ((totalarea * 30) / 100)
seventypercent = ((totalarea * 70) / 100)
# print totalarea, tenpercent, thirtypercent, seventypercent
a = 0
tenpercentfeats = {"type": "FeatureCollection", "features": []}
twentypercentfeats = {"type": "FeatureCollection", "features": []}
seventypercentfeats = {"type": "FeatureCollection", "features": []}
for cursortedgeom in sortedgeomsandareas:
cf = {}
j = json.loads(
shapelyHelper.export_to_JSON(cursortedgeom['feature']))
cf['type'] = 'Feature'
cf['properties'] = {}
cf['geometry'] = j
if a < tenpercent:
tenpercentfeats['features'].append(cf)
elif a > tenpercent and a < thirtypercent:
twentypercentfeats['features'].append(cf)
else:
seventypercentfeats['features'].append(cf)
a += float(cursortedgeom['area'])
opdata = [{
'gj': tenpercentfeats,
'desc': "10% " + desc
}, {
'gj': twentypercentfeats,
'desc': "20% " + desc
}, {
'gj': seventypercentfeats,
'desc': "70% " + desc,
"fundingtype": fundingtype
}]
alluploadmessages = []
for curopdata in opdata:
print(json.dumps(curopdata['gj']))
# upload = myAPIHelper.post_as_diagram(geoms = json.dumps(curopdata['gj']), projectorpolicy= projectorpolicy,featuretype = 'polygon', description= curopdata['desc'], sysid = sysid)
# alluploadmessages.append(json.loads(upload.text))
msg = json.dumps({
"message": "Diagrams have been uploaded",
"uploadstatus": alluploadmessages
})
return Response(msg, status=400, mimetype='application/json')
else:
msg = json.dumps({
"message":
"Could not parse Project ID, Diagram ID or API Token ID. One or more of these were not found in your JSON request."
})
return Response(msg, status=400, mimetype='application/json')
''' RTREE is a spatial index that answers the question -> What points are within this 2-dimensional box. RTREE helps us speed up other calculations by only considering legitimate objects for more complex comparisons. ''' from rtree import Rtree # instantiate the Rtree class idx = Rtree() # create a selection boundary (2d bounding box) minx, miny, maxx, maxy = (0.0, 0.0, 1.0, 1.0) # add an item to the index idx.add(0, (minx, miny, maxx, maxy)) # Intersect another bounding box with the original bounding box. list(idx.intersection((1.0, 1.0, 2.0, 2.0))) #[0L] # Point out the accuracy of the spatial calculation list(idx.intersection((1.0000001, 1.0000001, 2.0, 2.0))) #[] #add another item to the index. - show EXACT dimensiion matching index.add(id=id, (left, bottom, right, top)) object = [n for n in index.intersection((left, bottom, right, top))] #[id] # Find nearest object / bounding box idx.add(1, (minx, miny, maxx, maxy)) list(idx.nearest((1.0000001, 1.0000001, 2.0, 2.0), 1)) #[0L, 1L]
def delete(self, id_, coords):
with threading.RLock():
Rtree.delete(self, id_, coords)
class StreetMap:
def __init__(self):
self.nodes = {} # indexed by node id
self.edges = {} # indexed by edge id
self.intersections = {} # indexed by node id
self.node_spatial_index = Rtree()
self.edge_spatial_index = Rtree()
self.intersection_spatial_index = Rtree()
self.edge_lookup_table = {} # indexed by (in_node,out_node)
self.edge_coords_lookup_table = {
} # indexed by (in_node.coords, out_node.coords)
self.segments = {} # indexed by segment id
self.segment_lookup_table = {
} # indexed by (head_edge.in_node, tail_edge.out_node)
def load_osmdb(self, osmdb_filename):
# connect to OSMDB
conn = sqlite3.connect(osmdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
query_iter = cur.execute("select id, lat, lon from nodes")
# query_result = cur.fetchall()
# iterate through all query results
for id, lat, lon in query_iter:
# create and store node in nodes dictionary
self.nodes[int(id)] = Node(float(lat), float(lon), int(id))
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
query_iter = cur.execute("select id, tags, nds from ways")
# query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, tags, nodes in query_iter:
# grab tags associated with current way
way_tags_dict = eval(tags)
# if current way is a valid highway
if ('highway' in way_tags_dict.keys()
and self._valid_highway_edge(way_tags_dict['highway'])):
# grab all nodes that compose this way
way_nodes_list = eval(nodes)
# iterate through list of way nodes
for i in range(1, len(way_nodes_list)):
# grab in_node from nodes dictionary
in_node = self.nodes[int(way_nodes_list[i - 1])]
# grab out_node from nodes dictionary
out_node = self.nodes[int(way_nodes_list[i])]
# create edge_id based on way id
edge_id = int(str(id) + str(i - 1) + "000000")
# if either node on the edge is valid
if (
True
): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[int(edge_id)] = Edge(
in_node, out_node, int(edge_id))
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# if edge is bidirectional
if ('oneway' not in way_tags_dict.keys()):
# create new symmetric edge id
symmetric_edge_id = int(str(edge_id / 10) + "1")
# create and store symmetric edge in edges dictionary
self.edges[int(symmetric_edge_id)] = Edge(
out_node, in_node, int(symmetric_edge_id))
# store in_node in out_node's out_nodes list
if (in_node not in out_node.out_nodes):
out_node.out_nodes.append(in_node)
# store out_node in in_node's in_nodes list
if (out_node not in in_node.in_nodes):
in_node.in_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
print "done."
# close connection to OSMDB
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges and " + str(len(
self.intersections)) + " intersections."
def load_graphdb(self, grapdb_filename):
# connect to graph database
conn = sqlite3.connect(grapdb_filename)
# grab cursor
cur = conn.cursor()
# output that we are loading nodes
sys.stdout.write("\nLoading nodes... ")
sys.stdout.flush()
# execute query on nodes table
cur.execute("select id, latitude, longitude, weight from nodes")
query_result = cur.fetchall()
# iterate through all query results
for id, latitude, longitude, weight in query_result:
# create and store node in nodes dictionary
self.nodes[id] = Node(latitude, longitude, id, weight)
print "done."
# output that we are loading edges
sys.stdout.write("Loading edges... ")
sys.stdout.flush()
# execute query on ways table
cur.execute("select id, in_node, out_node, weight from edges")
query_result = cur.fetchall()
# storage for nodes used in valid edges
valid_edge_nodes = {} # indexed by node id
# iterate through all query results
for id, in_node_id, out_node_id, weight in query_result:
# grab in_node from nodes dictionary
in_node = self.nodes[in_node_id]
# grab out_node from nodes dictionary
out_node = self.nodes[out_node_id]
# if either node on the edge is valid
if (True
): #self._valid_node(in_node) or self._valid_node(out_node)):
# create and store edge in edges dictionary
self.edges[id] = Edge(in_node, out_node, id, weight)
# store in_node in out_node's in_nodes list
if (in_node not in out_node.in_nodes):
out_node.in_nodes.append(in_node)
# store out_node in in_node's out_nodes list
if (out_node not in in_node.out_nodes):
in_node.out_nodes.append(out_node)
# store in_node in valid_edge_nodes dictionary
if (in_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[in_node.id] = in_node
# store out_node in valid_edge_nodes dictionary
if (out_node.id not in valid_edge_nodes.keys()):
valid_edge_nodes[out_node.id] = out_node
# execute query on segments table
cur.execute("select id, edge_ids from segments")
query_result = cur.fetchall()
for id, edge_ids in query_result:
segment_edges = map(lambda edge_id: self.edges[edge_id],
eval(edge_ids))
self.segments[id] = Segment(id, segment_edges)
self.segment_lookup_table[(
self.segments[id].head_edge.in_node,
self.segments[id].tail_edge.out_node)] = self.segments[id]
for segment_edge in segment_edges:
segment_edge.segment = self.segments[id]
# self.segment_lookup_table[segment_edge.id] = self.segments[id]
# execute query on intersections table
cur.execute("select node_id from intersections")
query_result = cur.fetchall()
for node_id in query_result:
self.intersections[node_id[0]] = self.nodes[node_id[0]]
try:
cur.execute(
"select transition_segment, from_segment, to_segment from transitions"
)
query_result = cur.fetchall()
self.transitions = {}
for transition_segment, from_segment, to_segment in query_result:
self.transitions[transition_segment] = (from_segment,
to_segment)
except:
print "Got an error reading "
print "done."
# close connection to graph db
conn.close()
# replace all nodes with valid edge nodes
self.nodes = valid_edge_nodes
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
#self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges, " + str(len(
self.segments)) + " segments and " + str(
len(self.intersections)) + " intersections."
def load_shapedb(self, shapedb_filename):
# connect to graph database
conn = sqlite3.connect(shapedb_filename)
# grab cursor
cur = conn.cursor()
# execute query to find all shape ids
cur.execute("select distinct shape_id from shapes")
# output that we are loading nodes and edges
sys.stdout.write("\nLoading nodes and edges... ")
sys.stdout.flush()
# storage for shape specific edges
self.shape_edges = {} # indexed by shape_id
# storage for node id
node_id = 0
# iterate through all shape ids
for shape_id in cur.fetchall():
# grab shape id
shape_id = shape_id[0]
# if route is a bus route
if (shape_id == "0" or shape_id == "11" or shape_id == "15"
or shape_id == "41" or shape_id == "65"
or shape_id == "22"):
# execute query to find all shape points
cur.execute(
"select shape_pt_lat, shape_pt_lon from shapes where shape_id='"
+ str(shape_id) + "' order by shape_pt_sequence asc")
# amend shape id
if (shape_id == "0"):
shape_id = "10000000"
elif (shape_id == "11"):
shape_id = "10000011"
elif (shape_id == "41"):
shape_id = "10000041"
elif (shape_id == "15"):
shape_id = "10000015"
elif (shape_id == "65"):
shape_id = "10000065"
elif (shape_id == "22"):
shape_id = "10000022"
# storage for first node
first_node = None
# storage for previous node
prev_node = None
# create list for this shape's edges
self.shape_edges[shape_id] = []
# iterate through all shape points
for shape_pt_lat, shape_pt_lon in cur.fetchall():
# create new node
curr_node = Node(shape_pt_lat, shape_pt_lon, node_id)
# store first node
if (first_node is None):
first_node = curr_node
# increment node id
node_id += 1
# add shape id to node
curr_node.shape_id = shape_id
# store new node in nodes dictionary
self.nodes[node_id] = curr_node
# if there exists a previous node
if (prev_node is not None):
# create edge id
edge_id = int(
str(shape_id) + str(prev_node.id) +
str(curr_node.id))
# create new edge
curr_edge = Edge(prev_node, curr_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in current node's in_nodes list
curr_node.in_nodes.append(prev_node)
# store current node in previous node's out_nodes list
prev_node.out_nodes.append(curr_node)
# update previous node
prev_node = curr_node
# create edge id for last edge
edge_id = int(
str(shape_id) + str(prev_node.id) + str(first_node.id))
# create new edge
curr_edge = Edge(prev_node, first_node, edge_id)
# add shape id to edge
curr_edge.shape_id = shape_id
# store new edge in edges dictionary
self.edges[edge_id] = curr_edge
# store new edge in shape edges dictionary
self.shape_edges[shape_id].append(curr_edge)
# store previous node in first node's in_nodes list
first_node.in_nodes.append(prev_node)
# store first node in previous node's out_nodes list
prev_node.out_nodes.append(first_node)
print "done."
# close connection to gtfs db
conn.close()
# index nodes
self._index_nodes()
# index edges
self._index_edges()
# find and index intersections
self._find_and_index_intersections()
# output map statistics
print "Map has " + str(len(self.nodes)) + " nodes, " + str(
len(self.edges)) + " edges and " + str(len(
self.intersections)) + " intersections."
def _index_nodes(self):
# output that we are indexing nodes
sys.stdout.write("Indexing nodes... ")
sys.stdout.flush()
# iterate through all nodes
for curr_node in self.nodes.values():
# insert node into spatial index
self.node_spatial_index.insert(
curr_node.id, (curr_node.longitude, curr_node.latitude))
print "done."
def _index_edges(self):
# output that we are indexing edges
sys.stdout.write("Indexing edges... ")
sys.stdout.flush()
# iterate through all edges
for curr_edge in self.edges.values():
# determine current edge minx, miny, maxx, maxy values
curr_edge_minx = min(curr_edge.in_node.longitude,
curr_edge.out_node.longitude)
curr_edge_miny = min(curr_edge.in_node.latitude,
curr_edge.out_node.latitude)
curr_edge_maxx = max(curr_edge.in_node.longitude,
curr_edge.out_node.longitude)
curr_edge_maxy = max(curr_edge.in_node.latitude,
curr_edge.out_node.latitude)
# insert current edge into spatial index
self.edge_spatial_index.insert(curr_edge.id,
(curr_edge_minx, curr_edge_miny,
curr_edge_maxx, curr_edge_maxy))
# insert current edge into lookup table
self.edge_lookup_table[(curr_edge.in_node,
curr_edge.out_node)] = curr_edge
self.edge_coords_lookup_table[(
curr_edge.in_node.coords(),
curr_edge.out_node.coords())] = curr_edge
# iterate through all edges
for edge in self.edges.values():
# iterate through all out edges
for out_node_neighbor in edge.out_node.out_nodes:
# add out edge to out edges list
edge.out_edges.append(
self.edge_lookup_table[(edge.out_node, out_node_neighbor)])
# iterate through all in edges
for in_node_neighbor in edge.in_node.in_nodes:
# add in edge to in edges list
edge.in_edges.append(self.edge_lookup_table[(in_node_neighbor,
edge.in_node)])
print "done."
def _find_and_index_intersections(self):
# output that we are finding and indexing intersections
sys.stdout.write("Finding and indexing intersections... ")
sys.stdout.flush()
# find intersection nodes and index
(intersection_nodes,
intersection_nodes_index) = self._find_intersection_nodes()
# storage for intersection nodes already placed in intersections
placed_intersection_nodes = set()
# define longitude/latitude offset for bounding box
lon_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LONGITUDE)
lat_offset = ((intersection_size / 2.0) / METERS_PER_DEGREE_LATITUDE)
# storage for intersection id
intersection_id = 0
# iterate through intersection nodes
for intersection_node in intersection_nodes:
# if the intersection node has not yet been placed
if (intersection_node not in placed_intersection_nodes):
# create bounding box
bounding_box = (intersection_node.longitude - lon_offset,
intersection_node.latitude - lat_offset,
intersection_node.longitude + lon_offset,
intersection_node.latitude + lat_offset)
# find intersection node ids within bounding box
intersection_node_ids = intersection_nodes_index.intersection(
bounding_box)
# get intersection nodes
intersection_nodes = map(self._get_node, intersection_node_ids)
# add intersection nodes to placed set
placed_intersection_nodes.update(intersection_nodes)
# create new intersection
new_intersection = Intersection(intersection_id,
intersection_nodes)
# increment intersection id
intersection_id += 1
# add new intersection to intersections list
self.intersections[new_intersection.id] = new_intersection
# insert new intersection into spatial index
self.intersection_spatial_index.insert(
new_intersection.id,
(new_intersection.longitude, new_intersection.latitude))
print "done."
def _get_node(self, node_id):
# return node from dictionary
return self.nodes[node_id]
def _find_intersection_nodes(self):
# storage for intersection nodes
intersection_nodes = []
# spatial index for intersection nodes
intersection_nodes_index = Rtree()
# iterate through all nodes in map
for curr_node in self.nodes.values():
# set storage for current node's unique neighbors
neighbors = set()
# iterate through all in_nodes
for in_node in curr_node.in_nodes:
# add in_node to neighbors set
neighbors.add(in_node)
# iterate through all out_nodes
for out_node in curr_node.out_nodes:
# add out_node to neighbors set
neighbors.add(out_node)
# if current node has more than 2 neighbors
if (len(neighbors) > 2):
# add current node to intersection nodes list
intersection_nodes.append(curr_node)
# add current node to intersection nodes index
intersection_nodes_index.insert(
curr_node.id, (curr_node.longitude, curr_node.latitude))
# return intersection nodes and index
return (intersection_nodes, intersection_nodes_index)
def _valid_node(self, node):
# if node falls inside the designated bounding box
if ((node.latitude >= 41.8619 and node.latitude <= 41.8842) and
(node.longitude >= -87.6874 and node.longitude <= -87.6398)):
return True
else:
return False
def _valid_highway_edge(self, highway_tag_value):
if ((highway_tag_value == 'primary')
or (highway_tag_value == 'secondary')
or (highway_tag_value == 'tertiary')
or (highway_tag_value == 'residential')):
return True
else:
return False
def reset_node_visited_flags(self):
# iterate through all nodes
for node in self.nodes.values():
# set node visited flag to False
node.visited = False
def reset_edge_visited_flags(self):
# iterate through all edges
for edge in self.edges.values():
# set edge visited flag to False
edge.visited = False
def write_map_to_file(self, map_filename="map.txt"):
# output that we are starting the writing process
sys.stdout.write("\nWriting map to file... ")
sys.stdout.flush()
# open map file
map_file = open(map_filename, 'w')
# iterate through all map edges
for curr_edge in self.edges.values():
# output current edge to file
map_file.write(
str(curr_edge.in_node.latitude) + "," +
str(curr_edge.in_node.longitude) + "\n")
map_file.write(
str(curr_edge.out_node.latitude) + "," +
str(curr_edge.out_node.longitude) + "\n\n")
# close map file
map_file.close()
print "done."
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
class Geometry(object):
def __init__(self, wkb):
self._wkb = wkb
self._geom = None
def geom(self):
if not self._geom:
self._geom = wkb.loads(self._wkb)
return self._geom
def __repr__(self):
return self.geom().wkt
data_filename = sys.argv[1] + '.data'
index_filename = sys.argv[1] + '.data.idx'
tree = Rtree(sys.argv[1])
data_file = open(data_filename, 'r+')
index_file = open(index_filename, 'r+')
index_file.seek(0, os.SEEK_END)
index_length = index_file.tell()
index_file.seek(0)
data_file.seek(0, os.SEEK_END)
data_length = data_file.tell()
data_file.seek(0)
query = tuple([float(v) for v in sys.argv[2:6]])
print query
print >> sys.stderr, "Caching points, blocks, and names ..."
pickle.dump((names, blocks, places), file(areaid + ".cache", "w"), -1)
blocks = map(asShape, blocks)
points = []
place_list = set()
count = 0
for place_id, pts in places.items():
count += 1
print >> sys.stderr, "\rPreparing %d of %d places..." % (count,
len(places)),
for pt in pts:
place_list.add((len(points), pt + pt, None))
points.append((place_id, Point(pt)))
print >> sys.stderr, "Indexing...",
index = Rtree(place_list)
print >> sys.stderr, "Done."
def score_block(polygon):
centroid = polygon.centroid
#prepared = prep(polygon)
score = {}
outside_samples = 0
for item in index.nearest((centroid.x, centroid.y),
num_results=SAMPLE_SIZE):
place_id, point = points[item]
score.setdefault(place_id, 0.0)
#if prepared.contains(point):
# score[place_id] += 1.0
#else:
class Graph:
def __init__(self, bus_trips):
self.bus_trips = bus_trips
self.graph_nodes = {} # indexed by "location_id"
self.graph_edge_id = 0
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
self.graph_edge_index = Rtree()
def generate_graph(self):
print "Running graph generation algorithm..."
# initialize trip counter
trip_count = 1
for trip in self.bus_trips:
# initialize location counter
location_count = 1
# storage for previous node
prev_node = None
for location in trip.locations:
sys.stdout.write("\rAnalyzing location " +
str(location_count) + "/" +
str(len(trip.locations)) + " for trip " +
str(trip_count) + "/" +
str(len(self.bus_trips)) + "... ")
sys.stdout.flush()
# find closest edges in graph to location
closest_edges = self._find_closest_edges_in_graph(
location, 100)
# flag variable for whether we merged location
did_merge_location = False
# iterate through candidate edge ids
for candidate_edge_id in closest_edges:
# grab candidate edge from graph edge dictionary
candidate_edge = self.graph_edges[candidate_edge_id]
# determine whether we should merge with candidate edge
if (self._should_merge_location_with_edge(
location, candidate_edge) is True):
# merge location with edge, update previous node
prev_node = self._merge_location_with_edge(
location, candidate_edge, prev_node)
# update merge flag variable
did_merge_location = True
# no need to look at further edges, break out of candidate edges loop
break
# if we did not merge the location with any edge
if (did_merge_location is False):
# add location to graph
self._add_location_to_graph(location, prev_node)
# update previous node with current location
prev_node = location
# increment location counter
location_count += 1
# done with current trip locations
print "done."
# increment trip counter
trip_count += 1
# write graph edges to file
self._write_graph_edges_to_file()
# create graph database
self._output_graph_to_db()
def _merge_location_with_edge(self, location, edge, prev_node):
# get the edge node closest to the location
edge_node = self._get_closest_edge_node(location, edge)
# if prev_node is None
if (prev_node is None):
# increase volume of just this edge
edge.volume += 1
# if prev_node is not None
else:
# find path from prev_node to edge_node
path = self._find_path(prev_node, edge_node, max_path_length)
# if there was a path from prev_node to edge_node
if (path is not None):
# iterate through nodes in path
for i in range(1, len(path)):
# grab in_node
in_node = path[i - 1]
# grab out_node
out_node = path[i]
# find corresponding graph edge
graph_edge = self._find_graph_edge(in_node, out_node)
# increment volume on edge
graph_edge.volume += 1
# if there is no path from prev_node to edge_node
else:
# create a new graph edge between prev_node and edge_node
self._create_graph_edge(prev_node, edge_node)
# return the edge_node
return edge_node
def _get_closest_edge_node(self, location, edge):
# if in_node distance is less than out_node distance
if (self._distance(location, edge.in_node) < self._distance(
location, edge.out_node)):
# return the edge in_node
return edge.in_node
# otherwise, return the edge out_node
return edge.out_node
def _find_path(self, source, destination, max_length):
# reset all node visited flags
self._reset_node_visited_flags()
# get a breath-first search path from source to destination
path = self._bfs_path(source, destination)
# if there is a path from source to destination
if (path is not None):
# and if the path length is less than or equal to the maximum length
if (len(path) <= max_length):
# return the path
return path
# otherwise, return None
return None
def _bfs_path(self, source, destination):
# storage for breadth-first search parents
bfs_parent = {} # key is current node, value is parent node
# source node has no breadth-first search parent
bfs_parent[source] = None
# node queue for breadth-first search
bfs_queue = []
# enqueue source node
bfs_queue.append(source)
# mark source node as visited
source.visited = True
# while the queue is not empty
while (len(bfs_queue) > 0):
# dequeue the first node in the queue
curr_node = bfs_queue.pop(0)
# if the current node is the destination
if (curr_node is destination):
# create storage for breadth-first search path
bfs_path = []
# add the current node to the breadth-first search path
bfs_path.insert(0, curr_node)
# grab the parent of the current node
parent = bfs_parent[curr_node]
# iterate through breadth-first search parents
while (parent is not None):
# add the parent to the breadth-first search path
bfs_path.insert(0, parent)
# grab the next parent
parent = bfs_parent[parent]
# return the breadth-first search path
return bfs_path
# if the current node is not the destination
else:
# iterate through the current node's out_nodes
for out_node in curr_node.out_nodes:
# if the out_node has not been visited
if (out_node.visited is False):
# mark the out_node as visited
out_node.visited = True
# enqueue the out_node
bfs_queue.append(out_node)
# store curr_node as out_node's breadth-first search parent
bfs_parent[out_node] = curr_node
# if we reached here, no path was found
return None
def _should_merge_location_with_edge(self, location, edge):
# project location onto edge
(location_projection, location_projection_fraction,
location_projection_distance) = self._projection_onto_line(
edge.in_node, edge.out_node, location)
# if projection is not onto edge
if (location_projection_fraction < 0.0
or location_projection_fraction > 1.0):
# we cannot merge location with edge
return False
# determine bearing difference between edge and location
bearing_difference = math.cos(
math.radians(
self._path_bearing(edge.in_node, edge.out_node) -
self._location_bearing(location)))
# if location projection distance is less than 20 meters
if (location_projection_distance < location_projection_distance_limit):
# if bearing difference is less than 45 degrees
if (bearing_difference > location_bearing_difference_limit):
# merge location with edge
return True
# otherwise, do not merge location with edge
return False
def _add_location_to_graph(self, location, prev_node):
# add an out_nodes list to location
location.out_nodes = []
# add an in_nodes list to location
location.in_nodes = []
# add a visited flag to location
location.visited = False
# add location to graph nodes list
self.graph_nodes[location.id] = location
# if prev_node is not None
if (prev_node is not None):
# create a new graph edge between prev_node and location
self._create_graph_edge(prev_node, location)
def _create_graph_edge(self, in_node, out_node):
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if (existing_graph_edge is None):
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," +
str(out_node.id)] = new_graph_edge
# add new graph edge to graph edges spatial index
self._add_graph_edge_to_index(new_graph_edge)
# increment graph edge id
self.graph_edge_id += 1
# store out_node in in_nodes's out_nodes list
in_node.out_nodes.append(out_node)
# store in_node in out_node's in_nodes list
out_node.in_nodes.append(in_node)
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if (edge_lookup_key in self.graph_edge_lookup.keys()):
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _add_graph_edge_to_index(self, graph_edge):
# determine graph edge minx, miny, maxx, maxy values
graph_edge_minx = min(graph_edge.in_node.longitude,
graph_edge.out_node.longitude)
graph_edge_miny = min(graph_edge.in_node.latitude,
graph_edge.out_node.latitude)
graph_edge_maxx = max(graph_edge.in_node.longitude,
graph_edge.out_node.longitude)
graph_edge_maxy = max(graph_edge.in_node.latitude,
graph_edge.out_node.latitude)
# insert graph edge into spatial index
self.graph_edge_index.insert(graph_edge.id,
(graph_edge_minx, graph_edge_miny,
graph_edge_maxx, graph_edge_maxy))
def _location_bearing(self, location):
# if location has a previous neighbor and a next neighbor
if ((location.prev_location is not None)
and (location.next_location is not None)):
# determine bearing using previous and next neighbors
return self._path_bearing(location.prev_location,
location.next_location)
# if location has no previous neighbor, but has a next neighbor
elif ((location.prev_location is None)
and (location.next_location is not None)):
# determine bearing using current location and next neighbor
return self._path_bearing(location, location.next_location)
# if location has a previous neighbor, but not a next neighbor
elif ((location.prev_location is not None)
and (location.next_location is None)):
# determine bearing using previous neighbor and current location
return self._path_bearing(location.prev_location, location)
# if we reach here, there is an error
return None
def _find_closest_edges_in_graph(self, location, number_of_edges):
return self.graph_edge_index.nearest(
(location.longitude, location.latitude), number_of_edges)
def _projection_onto_line(self, location1, location2, location3):
return spatialfunclib.projection_onto_line(
location1.latitude, location1.longitude, location2.latitude,
location2.longitude, location3.latitude, location3.longitude)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("cao_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute(
"CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute(
"CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)"
)
# remove values from nodes table
#cur.execute("DELETE FROM nodes")
# remove values from edges table
#cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# insert graph node into nodes table
cur.execute("INSERT INTO nodes VALUES (" + str(graph_node.id) +
"," + str(graph_node.latitude) + "," +
str(graph_node.longitude) + ")")
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# insert graph edge into edges table
cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) +
"," + str(graph_edge.in_node.id) + "," +
str(graph_edge.out_node.id) + ")")
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
def _write_graph_edges_to_file(self):
# output that we are starting the writing process
sys.stdout.write("\nWriting graph edges to file... ")
sys.stdout.flush()
# open graph file
graph_file = open('cao_edges.txt', 'w')
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# if the graph edge has volume greater than or equal to 3
if (graph_edge.volume >= min_graph_edge_volume):
# output edge to file
graph_file.write(
str(graph_edge.in_node.latitude) + "," +
str(graph_edge.in_node.longitude) + "\n")
graph_file.write(
str(graph_edge.out_node.latitude) + "," +
str(graph_edge.out_node.longitude) + "," +
str(graph_edge.volume) + "\n\n")
# close graph file
graph_file.close()
print "done."
def _reset_node_visited_flags(self):
# iterate through all graph nodes
for graph_node in self.graph_nodes.values():
# set visited flag to False
graph_node.visited = False
def main():
# You can edit these vars:
# -----------------------
filename = 'matrix_plot3.png'
numberOfPoints = 200
gridWidth = 10000
gridHeight = 10000
edgesMinimum = 2
edgesMaximum = 4 # The maximum number of edges (connecting lines) a vertex can have
# -----------------------
edgesMaxDistance = (gridWidth / numberOfPoints) * 2
# instantiate the Rtree class
RtreeIndex = Rtree()
# CREATE THE RANDOM POINTS:
shapelyPointIndex = [
] # Stores index = shapely object - We will reference this point from its Index from this point on.
for idx in range(
numberOfPoints): # Each point gets its own index defined by i
x = random.randint(0, gridWidth)
y = random.randint(0, gridHeight)
iPoint = Point(x, y) # Create a new shapely POINT object
shapelyPointIndex.append(iPoint)
RtreeIndex.add(idx, (x, y, x, y)) # Add the point to the RTREE Index
del idx #unset temp variable
# --------------------------------------------------------------------------
#MultiPoint([(0, 0), (1, 1)])
#point.coords
#point.y point.x
lineMatrices = []
ComplexLines = []
graphIndex = {
} # This is a dictionary we will build into a graph/matrix - Each key will be an integer representing an index
# and each value will be a dictionary of dictionaries representing edges.
# iterate over all points to calculate each points nearest points
# build line segments for each point to near points.
# add all edges to a master complex line to compute intersection.
for idx, point in enumerate(shapelyPointIndex):
# Find nearby points quickly
rtreeMatches = list()
sRadiusX = gridWidth / 10
sRadiusY = gridHeight / 10
tries = 0
while (len(rtreeMatches) < edgesMinimum +
1): # Add 1 because we can assume it will always find itself
# Coords is a bounding-box to search within.
coords = (point.x - sRadiusX, point.y - sRadiusY,
point.x + sRadiusX, point.y + sRadiusY)
print "searching coords"
print coords
rtreeMatches = list(RtreeIndex.intersection(coords))
print "found: ", rtreeMatches, " ", len(rtreeMatches)
tries += 1
sRadiusX *= 1.5
sRadiusY *= 1.5
if (tries > 6): #Don't run forever if there is a logic problem.
break
del tries, sRadiusX, sRadiusY
# FIND POSSIBLE EDGES TO ADD BY SEARCHING REGION RECURSIVELY -----------
# AND CALCULATE DISTANCES TO POSSIBLE POINTS ---------------------------
# This dictionary will store our distances to each other point from this point.
edgeDistances = {}
# Calculate distances to matched points
for pointIdx in rtreeMatches:
if (pointIdx == idx):
continue # Don't record a distance to itself
mPoint = shapelyPointIndex[pointIdx]
edgeDistances[pointIdx] = round(
mPoint.distance(point),
2) # keep the distance accuracy reasonable
del mPoint, pointIdx, rtreeMatches
# create a list that references the distances index for the closest indexes and is sorted by close to far.
edgeDistancesIdx = sorted(edgeDistances, key=edgeDistances.get)
#-----------------------------------------------------------------------
# DECIDE WHICH EDGES TO ADD --------------------------------------------
# Now add a minimum number of points, and try to keep them within a logic distance
edgesAdded = 0
edgesIdx = [] # this will contain indexes to our added edges
for pointIdx in edgeDistancesIdx:
distance = edgeDistances[pointIdx]
# Stop adding edges if we've meant the minimum number of edges needed and the rest of the edges are too far away, or we have reach the maximum number of allowed edges
if ((edgesAdded >= edgesMinimum and distance > edgesMaxDistance)
or edgesAdded >= edgesMaximum):
break
edgesIdx.append(pointIdx)
del pointIdx
#-----------------------------------------------------------------------
# Initialize the graphIndex as a dict() for the current vertex if it hasn't been.
if (idx not in graphIndex):
graphIndex[idx] = {}
#ADD THE EDGES TO THE GRID STRUCTURE -----------------------------------
for pointIdx in edgesIdx:
# Add the edge
graphIndex[idx][pointIdx] = edgeDistances[pointIdx]
# Add the reverse edge (so both points share an edge)
if pointIdx not in graphIndex:
graphIndex[pointIdx] = {}
graphIndex[pointIdx][idx] = edgeDistances[pointIdx]
#-----------------------------------------------------------------------
# Print out the graph of vertices
pprint.pprint(graphIndex)
# Randomely select a Start Vertex, and an End Vertex for Shortest Path
# calculation
startIdx = random.randint(0, len(graphIndex) - 1)
endIdx = startIdx
# Calculate a random end index - and make sure its not the same as the start
while (endIdx == startIdx):
endIdx = random.randint(0, len(graphIndex) - 1)
# Setup shortest path calculation!
print "Calculating Shortest Path:\n"
# (8894.6959143222557, [11, 5, 8L, 7, 9])
# GraphIndex is a dictionary of dictionaries, startIdx is the INDEX of the startpoint, endIdx is the index of the endpoint vertex.
shortestPath = dijkstra.shortestpath(graphIndex, startIdx, endIdx)
shortestPathVertices = shortestPath[1]
shortestPathDistance = shortestPath[0]
print "Shortest Path Vertices:"
print shortestPathVertices, "\n"
del shortestPath
#_-------------------------------------------------------------------------------
# DRAW A PLOT FOR THE LINES CALCULATED ABOVE:
import matplotlib
matplotlib.use('Agg') # Do NOT attempt to open X11 instance
from pylab import *
from matplotlib.patches import Circle
import matplotlib.pyplot as pyplot
from matplotlib.text import *
mylist = lineMatrices
matplotlib.rcParams['lines.linewidth'] = 2
pyplot.axis([0, gridWidth, 0, gridHeight])
pyplot.grid(True)
# Setting the axis labels.
pyplot.xlabel('X Space')
pyplot.ylabel('Y Space')
figure = pyplot.figure()
labels = figure.add_subplot(111, projection='rectilinear')
#Give the plot a title
pyplot.title('Shapely Shortest Path Simulation')
del idx
if not isinstance(mylist, list):
print "The matrix is not a list!"
print type(mylist).__name__
for idx, edges in graphIndex.iteritems():
vertex = shapelyPointIndex[idx]
for edgeIdx, edgeDistance in edges.iteritems():
# This shouldn't happen, but lets just make sure.
# DONT PLOT A PATH TO OURSELF
if (edgeIdx == idx):
continue
# Get the edges point-coordinates
edgePoint = shapelyPointIndex[edgeIdx]
# Delete the reverse edge so we don't plot the line twice.
if (edgeIdx in graphIndex):
del graphIndex[edgeIdx][idx]
# Print Debuggin Information
print "PLOTTING EDGE: [%s] to [%s] X:[%s] Y:[%s] DIS:[%s] " % (
idx, edgeIdx, edgePoint.x, edgePoint.y, edgeDistance)
# Plot the edge!
pyplot.plot([vertex.x, edgePoint.x], [vertex.y, edgePoint.y],
'b-',
alpha=0.3,
linewidth=.2,
markersize=1)
# FIND THE MID-POINT OF THE LINE:
xMidpoint = (vertex.x + edgePoint.x) / 2
yMidpoint = (vertex.y + edgePoint.y) / 2
#print "Plotting Text [%s] [%s]" % (xMidpoint, yMidpoint)
# Figure out arrows later... arrowprops=dict(facecolor='black', linewidth=2, alpha=0.4),
# ANNOTATE THE LENGTH OF THE LINE AT ITS MIDPOINT
labels.annotate(edgeDistance,
xy=(xMidpoint, yMidpoint),
xytext=(xMidpoint + 80, yMidpoint),
alpha=0.4,
size='4',
color='green')
# end for ----------------------------------------------------------
# Plot the VERTEX as a Red Circle
pyplot.plot(vertex.x, vertex.y, 'ro', linewidth=2, markersize=2)
# Annotate the VERTEX by its INDEX
labels.annotate(idx,
xy=(vertex.x, vertex.y),
xytext=(vertex.x + 50, vertex.y),
alpha=0.6,
size='6',
color='red')
print "Graphing Shortest Path..."
print "START: %s END: %s" % (startIdx, endIdx)
#shortestPathVertices
for idx, vertexIdx in enumerate(shortestPathVertices):
if (vertexIdx == endIdx):
break # We are done
xy1 = shapelyPointIndex[vertexIdx] # this is the start of the line
vertex2 = shortestPathVertices[
idx +
1] # The end of the line is the next item in the list shortest path list of indexes
xy2 = shapelyPointIndex[vertex2] #this is the end of the line
pyplot.plot([xy1.x, xy2.x], [xy1.y, xy2.y],
'r--',
alpha=0.7,
linewidth=2,
markersize=1)
print "PLOT -> SHORTEST PATH: [%s] to [%s]" % (vertexIdx, vertex2)
print "Saving Plot Image %s" % (filename)
pyplot.savefig(os.getcwd() + '/' + str(filename), dpi=240)
print "\n\n"
"""
class Datasource:
""" Store an exploded representation of a data source, so it can be simplified.
"""
def __init__(self, srs, geom_type, fields, values, shapes):
""" Use load() to call this constructor.
"""
self.srs = srs
self.fields = fields
self.geom_type = geom_type
self.values = values
self.shapes = shapes
# this will be changed later
self.tolerance = 0
# guid, src1_id, src2_id, line_id, x1, y1, x2, y2
db = connect(':memory:').cursor()
db.execute("""CREATE table segments (
-- global identifier for this segment
guid INTEGER PRIMARY KEY AUTOINCREMENT,
-- identifiers for source shape or shapes for shared borders
src1_id INTEGER,
src2_id INTEGER,
-- global identifier for this line
line_id INTEGER,
-- start and end coordinates for this segment
x1 REAL,
y1 REAL,
x2 REAL,
y2 REAL,
-- flag
removed INTEGER
)""")
db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)')
db.execute('CREATE INDEX shape1_parts ON segments (src1_id)')
db.execute('CREATE INDEX shape2_parts ON segments (src2_id)')
self.db = db
self.rtree = Rtree()
self.memo_line = make_memo_line()
def _indexes(self):
return range(len(self.values))
def simplify(self, tolerance, verbose=False):
""" Simplify the polygonal linework.
This method can be called multiple times, but the process is
destructive so it must be called with progressively increasing
tolerance values.
"""
if tolerance < self.tolerance:
raise Exception(
'Repeat calls to simplify must have increasing tolerances.')
self.tolerance = tolerance
q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC'
line_ids = [line_id for (line_id, count) in self.db.execute(q)]
stable_lines = set()
while True:
was = self.db.execute(
'SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0]
preserved, popped = set(), False
for line_id in line_ids:
if line_id in stable_lines:
continue
# For each coordinate that forms the apex of a two-segment
# triangle, find the area of that triangle and put it into a list
# along with the segment identifier and the resulting line if the
# triangle were flattened, ordered from smallest to largest.
rows = self.db.execute(
"""SELECT guid, x1, y1, x2, y2
FROM segments
WHERE line_id = ?
AND removed = 0
ORDER BY guid""", (line_id, ))
segs = [(guid, (x1, y1), (x2, y2))
for (guid, x1, y1, x2, y2) in rows]
triples = [(segs[i][0], segs[i + 1][0], segs[i][1], segs[i][2],
segs[i + 1][2]) for i in range(len(segs) - 1)]
triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3)
for (guid1, guid2, c1, c2, c3) in triples]
areas = sorted([(triangle.area, guid1, guid2, c1, c3)
for (guid1, guid2, triangle, c1,
c3) in triangles])
min_area = self.tolerance**2
if not areas or areas[0][0] > min_area:
# there's nothing to be done
stable_lines.add(line_id)
if verbose:
stderr.write('-')
continue
# Reduce any segments that makes a triangle whose area is below
# the minimum threshold, starting with the smallest and working up.
# Mark segments to be preserved until the next iteration.
for (area, guid1, guid2, ca, cb) in areas:
if area > min_area:
# there won't be any more points to remove.
break
if guid1 in preserved or guid2 in preserved:
# the current segment is too close to a previously-preserved one.
continue
# Check the resulting flattened line against the rest
# any of the original shapefile, to determine if it would
# cross any existing line segment.
(x1, y1), (x2, y2) = ca, cb
new_line = self.memo_line(x1, y1, x2, y2)
old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2))
old_rows = self.db.execute(
'SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0'
% ','.join(map(str, old_guids)))
old_lines = [
self.memo_line(x1, y1, x2, y2)
for (x1, y1, x2, y2) in old_rows
]
if True in [
new_line.crosses(old_line)
for old_line in old_lines
]:
if verbose:
stderr.write('x%d' % line_id)
continue
preserved.add(guid1)
preserved.add(guid2)
popped = True
x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1]
self.db.execute(
'UPDATE segments SET removed=1 WHERE guid=%d' % guid2)
self.db.execute(
'UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?',
(x1, y1, x2, y2, guid1))
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
stderr.write('.')
if verbose:
print >> stderr, ' reduced from', was, 'to',
print >> stderr, self.db.execute(
'SELECT COUNT(guid) FROM segments WHERE removed=0'
).fetchone()[0],
self.rtree = Rtree()
for (guid, x1, y1, x2, y2) in self.db.execute(
'SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0'
):
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
print >> stderr, '.'
if not popped:
break
def populate_shared_segments_by_rtree(datasource, verbose=False):
"""
"""
rtree = Rtree()
indexes = datasource._indexes()
for i in indexes:
xmin, ymin, xmax, ymax = datasource.shapes[i].bounds
xbuf = (xmax - xmin) * .001
ybuf = (ymax - ymin) * .001
bounds = (xmin - xbuf, ymin - ybuf, xmax + xbuf, ymax + ybuf)
rtree.add(i, bounds)
shared = [[] for i in indexes]
for i in indexes:
for j in rtree.intersection(datasource.shapes[i].bounds):
if i >= j:
continue
shape1 = datasource.shapes[i]
shape2 = datasource.shapes[j]
if not shape1.intersects(shape2):
continue
if verbose:
print >> stderr, 'Features %d and %d:' % (i, j), 'of', len(
indexes),
border = linemerge(shape1.intersection(shape2))
geoms = hasattr(border, 'geoms') and border.geoms or [border]
for geom in geoms:
try:
line_id = datasource.rtree.count(
datasource.rtree.get_bounds())
except RTreeError:
line_id = 0
coords = list(geom.coords)
segments = [coords[k:k + 2] for k in range(len(coords) - 1)]
for ((x1, y1), (x2, y2)) in segments:
datasource.db.execute(
"""INSERT INTO segments
(src1_id, src2_id, line_id, x1, y1, x2, y2, removed)
VALUES (?, ?, ?, ?, ?, ?, ?, 0)""",
(i, j, line_id, x1, y1, x2, y2))
bbox = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
datasource.rtree.add(datasource.db.lastrowid, bbox)
if verbose:
print >> stderr, len(coords), '-',
shared[i].append(border)
shared[j].append(border)
if verbose:
print >> stderr, border.type
return shared
def simplify(self, tolerance, verbose=False):
""" Simplify the polygonal linework.
This method can be called multiple times, but the process is
destructive so it must be called with progressively increasing
tolerance values.
"""
if tolerance < self.tolerance:
raise Exception(
'Repeat calls to simplify must have increasing tolerances.')
self.tolerance = tolerance
q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC'
line_ids = [line_id for (line_id, count) in self.db.execute(q)]
stable_lines = set()
while True:
was = self.db.execute(
'SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0]
preserved, popped = set(), False
for line_id in line_ids:
if line_id in stable_lines:
continue
# For each coordinate that forms the apex of a two-segment
# triangle, find the area of that triangle and put it into a list
# along with the segment identifier and the resulting line if the
# triangle were flattened, ordered from smallest to largest.
rows = self.db.execute(
"""SELECT guid, x1, y1, x2, y2
FROM segments
WHERE line_id = ?
AND removed = 0
ORDER BY guid""", (line_id, ))
segs = [(guid, (x1, y1), (x2, y2))
for (guid, x1, y1, x2, y2) in rows]
triples = [(segs[i][0], segs[i + 1][0], segs[i][1], segs[i][2],
segs[i + 1][2]) for i in range(len(segs) - 1)]
triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3)
for (guid1, guid2, c1, c2, c3) in triples]
areas = sorted([(triangle.area, guid1, guid2, c1, c3)
for (guid1, guid2, triangle, c1,
c3) in triangles])
min_area = self.tolerance**2
if not areas or areas[0][0] > min_area:
# there's nothing to be done
stable_lines.add(line_id)
if verbose:
stderr.write('-')
continue
# Reduce any segments that makes a triangle whose area is below
# the minimum threshold, starting with the smallest and working up.
# Mark segments to be preserved until the next iteration.
for (area, guid1, guid2, ca, cb) in areas:
if area > min_area:
# there won't be any more points to remove.
break
if guid1 in preserved or guid2 in preserved:
# the current segment is too close to a previously-preserved one.
continue
# Check the resulting flattened line against the rest
# any of the original shapefile, to determine if it would
# cross any existing line segment.
(x1, y1), (x2, y2) = ca, cb
new_line = self.memo_line(x1, y1, x2, y2)
old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2))
old_rows = self.db.execute(
'SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0'
% ','.join(map(str, old_guids)))
old_lines = [
self.memo_line(x1, y1, x2, y2)
for (x1, y1, x2, y2) in old_rows
]
if True in [
new_line.crosses(old_line)
for old_line in old_lines
]:
if verbose:
stderr.write('x%d' % line_id)
continue
preserved.add(guid1)
preserved.add(guid2)
popped = True
x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1]
self.db.execute(
'UPDATE segments SET removed=1 WHERE guid=%d' % guid2)
self.db.execute(
'UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?',
(x1, y1, x2, y2, guid1))
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
stderr.write('.')
if verbose:
print >> stderr, ' reduced from', was, 'to',
print >> stderr, self.db.execute(
'SELECT COUNT(guid) FROM segments WHERE removed=0'
).fetchone()[0],
self.rtree = Rtree()
for (guid, x1, y1, x2, y2) in self.db.execute(
'SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0'
):
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
print >> stderr, '.'
if not popped:
break
from shapely.geometry import Point, Polygon
from shapely import wkt
from rtree import Rtree
import sys
import struct
output = sys.argv[1]
data_filename = output + '.data'
index_filename = output + '.data.idx'
data = open(data_filename, 'wb')
index = open(index_filename, 'wb')
tree = Rtree(output, overwrite=1)
id = 0
offset = 0
for l in sys.stdin:
x = wkt.loads(l)
tree.add(id, x.bounds)
s = len(x.wkb)
data.write(x.wkb)
index.write(struct.pack('L', offset))
offset += s
id += 1
data.close()
index.close()
places[place_id] = keep
print >> sys.stderr, "%d points discarded." % discarded
if not os.path.exists(point_file + '.cache'):
print >> sys.stderr, "Caching points..."
pickle.dump((names, places), file(point_file + ".cache", "w"), -1)
print >> sys.stderr, "Indexing..."
points = []
place_list = set()
for place_id, pts in places.items():
for pt in pts:
place_list.add((len(points), pt + pt, None))
points.append((place_id, Point(pt)))
index = Rtree(place_list)
"""
REASSIGNMENT_PASSES = 10
iterations = 0
count = 0
queue = places.keys() + [None]
while len(queue) > 1:
place_id = queue.pop(0)
if place_id is None:
count = 0
iterations += 1
queue.append(None)
place_id = queue.pop(0)
if not places[place_id]:
del places[place_id]
r = myAPIHelper.get_diagram(firstDiagID)
try:
assert r.status_code == 200
except AssertionError as ae:
print("Invalid reponse %s" % ae)
else:
op = json.loads(r.text)
diaggeoms = op['geojson']
sysid = op['sysid']
desc = op['description']
projectorpolicy = op['type']
geoms, bounds = myDiagramDecomposer.processGeoms(diaggeoms)
grid, allbounds = myGridGenerator.generateGrid(bounds)
gridRTree = Rtree()
for boundsid, bounds in allbounds.iteritems():
gridRTree.insert(boundsid, bounds)
choppedgeomsandareas = []
choppedgeoms = []
totalarea = 0
for curgeom in geoms:
curbounds = curgeom.bounds
igridids = list(gridRTree.intersection(curbounds))
for curintersectid in igridids:
gridfeat = grid[curintersectid]
ifeat = curgeom.intersection(gridfeat)
ifeatarea = ifeat.area
totalarea += ifeatarea
# http://pypi.python.org/pypi/Rtree
# TODO: path hackery.
if __name__ == "__main__":
import sys, os
mypath = os.path.dirname(sys.argv[0])
sys.path.append(os.path.abspath(os.path.join(mypath, "../../")))
from pyrtree.bench.bench_rtree import ITER, INTERVAL
from tests import RectangleGen
import time
from rtree import Rtree
if __name__ == "__main__":
G = RectangleGen()
idx = Rtree() # this is a libspatialindex one.
start = time.clock()
interval_start = time.clock()
for v in range(ITER):
if 0 == (v % INTERVAL):
# interval time taken, total time taken, # rects, cur max depth
t = time.clock()
dt = t - interval_start
print("%d,%s,%f" % (v, "itime_t", dt))
print("%d,%s,%f" % (v, "avg_insert_t", (dt / float(INTERVAL))))
#print("%d,%s,%d" % (v, "max_depth", rt.node.max_depth()))
#print("%d,%s,%d" % (v, "mean_depth", rt.node.mean_depth()))
interval_start = time.clock()
rect = G.rect(0.000001)
def add(self, id_, coords):
with threading.RLock():
Rtree.add(self, id_, coords)
def snap_points_to_links(points, links):
"""Place points onto closest link in a set of links (arc/edges).
Parameters
----------
points : dict
Point ID as key and (x,y) coordinate as value.
links : list
Elements are of type ``libpysal.cg.shapes.Chain``
** Note ** each element is a link represented as a chain with
*one head and one tail vertex* in other words one link only.
Returns
-------
point2link : dict
Key [point ID (see points in arguments)]; value [a 2-tuple
((head, tail), point) where (head, tail) is the target link,
and point is the snapped location on the link.
Examples
--------
>>> import spaghetti
>>> from libpysal.cg.shapes import Point, Chain
>>> points = {0: Point((1,1))}
>>> link = [Chain([Point((0,0)), Point((2,0))])]
>>> spaghetti.util.snap_points_to_links(points, link)
{0: ([(0.0, 0.0), (2.0, 0.0)], array([1., 0.]))}
"""
# instantiate an rtree
rtree = Rtree()
# set the smallest possible float epsilon on machine
SMALL = numpy.finfo(float).eps
# initialize network vertex to link lookup
vertex_2_link = {}
# iterate over network links
for i, link in enumerate(links):
# extract network link (x,y) vertex coordinates
head, tail = link.vertices
x0, y0 = head
x1, y1 = tail
if (x0, y0) not in vertex_2_link:
vertex_2_link[(x0, y0)] = []
if (x1, y1) not in vertex_2_link:
vertex_2_link[(x1, y1)] = []
vertex_2_link[(x0, y0)].append(link)
vertex_2_link[(x1, y1)].append(link)
# minimally increase the bounding box exterior
bx0, by0, bx1, by1 = link.bounding_box
bx0 -= SMALL
by0 -= SMALL
bx1 += SMALL
by1 += SMALL
# insert the network link and its associated
# rectangle into the rtree
rtree.insert(i, (bx0, by0, bx1, by1), obj=link)
# build a KDtree on link vertices
kdtree = cg.KDTree(list(vertex_2_link.keys()))
point2link = {}
for pt_idx, point in points.items():
# first, find nearest neighbor link vertices for the point
dmin, vertex = kdtree.query(point, k=1)
vertex = tuple(kdtree.data[vertex])
closest = vertex_2_link[vertex][0].vertices
# Use this link as the candidate closest link: closest
# Use the distance as the distance to beat: dmin
point2link[pt_idx] = (closest, numpy.array(vertex))
x0 = point[0] - dmin
y0 = point[1] - dmin
x1 = point[0] + dmin
y1 = point[1] + dmin
# Find all links with bounding boxes that intersect
# a query rectangle centered on the point with sides
# of length dmin * dmin
rtree_lookup = rtree.intersection([x0, y0, x1, y1], objects=True)
candidates = [cand.object for cand in rtree_lookup]
dmin += SMALL
dmin2 = dmin * dmin
# of the candidate arcs, find the nearest to the query point
for candidate in candidates:
dist2cand, nearp = squared_distance_point_link(point, candidate.vertices)
if dist2cand <= dmin2:
closest = candidate.vertices
dmin2 = dist2cand
point2link[pt_idx] = (closest, nearp)
return point2link
allEvalSortedFeatures = []
# iterate over the evaluations
# iterate over the evaluations
opfiles = []
for cureval in evalspriority:
print(
colored(
"Loading Evaluation data for %s system.." % cureval["name"],
"yellow"))
# a dictionary to hold features, we will ignore the red and red2 since allocation should not happen here.
evalfeatcollection = {'green3': [], 'green2': [], 'green': []}
# A dictionary to store the index of the features.
evalfeatRtree = {
'green3': Rtree(),
'green2': Rtree(),
'green': Rtree()
}
# open evaluation file
filename = os.path.join(curPath, cureval['evalfilename'])
try:
assert os.path.isfile(filename)
except AssertionError as e:
print(colored("Input file %s does not exist" % filename, "red"))
sys.exit(0)
with open(filename) as data_file:
try:
geoms = json.load(data_file)
except Exception as e:
print(
def detect_rn_component(self, status_matrix):
node_pixels = np.where(status_matrix == GraphExtractor.NODE)
nb_node_pixels = len(node_pixels[0])
print('\n# of node pixels:{}'.format(nb_node_pixels))
neighbor_deltas = [
dxdy for dxdy in itertools.product([-1, 0, 1], [-1, 0, 1])
if dxdy[0] != 0 or dxdy[1] != 0
]
# node pixel -> center node
nodes = {}
node_pixel_spatial_index = Rtree()
# [node pixel sequence (start and end must be node pixel)]
connected_segments = []
cnt = 1
center_nodes = []
node_pixel_id = 0
for i, j in zip(node_pixels[0], node_pixels[1]):
if (cnt % 100 == 0) or (cnt == nb_node_pixels):
sys.stdout.write("\r" + str(cnt) + "/" + str(nb_node_pixels) +
"... ")
sys.stdout.flush()
cnt += 1
if status_matrix[i][j] == GraphExtractor.VISITED_NODE:
continue
# region merge neighbor node pixels
status_matrix[i][j] = GraphExtractor.VISITED_NODE
candidates = [(i, j)]
node_pixels = []
while len(candidates) > 0:
node_pixel = candidates.pop()
node_pixels.append(node_pixel)
m, n = node_pixel
for dm, dn in neighbor_deltas:
if status_matrix[m + dm][n + dn] == GraphExtractor.NODE:
status_matrix[m + dm][n +
dn] = GraphExtractor.VISITED_NODE
candidates.append((m + dm, n + dn))
center_node = CenterNodePixel(node_pixels)
center_nodes.append(center_node)
for node_pixel in node_pixels:
nodes[node_pixel] = center_node
node_pixel_spatial_index.insert(node_pixel_id,
node_pixel,
obj=node_pixel)
node_pixel_id += 1
# endregion
# region find neighbor segments
# mask current nodes, make sure the edge doesn't return to itself
for m, n in node_pixels:
status_matrix[m][n] = GraphExtractor.INVALID
# find new road segment of the current node in each possible direction
for node_pixel in node_pixels:
connected_segment = self.detect_connected_segment(
status_matrix, node_pixel)
if connected_segment is not None:
connected_segments.append(connected_segment)
# restore masked nodes
for m, n in node_pixels:
status_matrix[m][n] = GraphExtractor.VISITED_NODE
# endregion
print('\n# of directly connected segments:{}'.format(
len(connected_segments)))
# there might be few edge pixels left, that should be fine
nb_unprocessed_edge_pixels = np.sum(
status_matrix[status_matrix == GraphExtractor.EDGE])
print('unprocessed edge pixels:{}'.format(nb_unprocessed_edge_pixels))
print('# of nodes:{}'.format(len(center_nodes)))
print('# of segments:{}'.format(len(connected_segments)))
return nodes, connected_segments
''' @author Ben In this example we will compare Shapely AGAINST RTREE for performance NOTE: to get this to work I had to install libspatialindex 1.5 from source and then: export LD_LIBRARY_PATH=/usr/local/lib ''' import random, os, sys, time, numpy import rtree from rtree import Rtree # instantiate the Rtree class p = rtree.index.Property(variant=rtree.index.RT_Star) RtreeIndex = Rtree(properties=p) import random, os, sys, time, numpy from shapely import * from shapely.geometry import Point filename = 'rtree_radius_search.png' numberOfPoints = 500 gridWidth = 10000 gridHeight = 10000 shapePoints = [] circleRadius = random.randint(500, 1500) circleX = random.randint(0, gridWidth) circleY = random.randint(0, gridHeight)
class OSMDB:
def __init__(self, dbname, overwrite=False, rtree_index=True):
self.dbname = dbname
if overwrite:
try:
os.remove(dbname)
except OSError:
pass
self.conn = sqlite3.connect(dbname)
if rtree_index:
self.index = Rtree(dbname)
else:
self.index = None
if overwrite:
self.setup()
def get_cursor(self):
# Attempts to get a cursor using the current connection to the db. If we've found ourselves in a different thread
# than that which the connection was made in, re-make the connection.
try:
ret = self.conn.cursor()
except sqlite3.ProgrammingError:
self.conn = sqlite3.connect(self.dbname)
ret = self.conn.cursor()
return ret
def setup(self):
c = self.get_cursor()
c.execute(
"CREATE TABLE nodes (id TEXT UNIQUE, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)"
)
c.execute("CREATE TABLE ways (id TEXT UNIQUE, tags TEXT, nds TEXT)")
self.conn.commit()
c.close()
def create_indexes(self):
c = self.get_cursor()
c.execute("CREATE INDEX nodes_id ON nodes (id)")
c.execute("CREATE INDEX nodes_lon ON nodes (lon)")
c.execute("CREATE INDEX nodes_lat ON nodes (lat)")
c.execute("CREATE INDEX ways_id ON ways (id)")
self.conn.commit()
c.close()
def populate(self,
osm_filename,
dryrun=False,
accept=lambda tags: True,
reporter=None,
create_indexes=True):
print "importing %s osm from XML to sqlite database" % osm_filename
c = self.get_cursor()
self.n_nodes = 0
self.n_ways = 0
superself = self
class OSMHandler(xml.sax.ContentHandler):
@classmethod
def setDocumentLocator(self, loc):
pass
@classmethod
def startDocument(self):
pass
@classmethod
def endDocument(self):
pass
@classmethod
def startElement(self, name, attrs):
if name == 'node':
self.currElem = Node(attrs['id'], float(attrs['lon']),
float(attrs['lat']))
elif name == 'way':
self.currElem = Way(attrs['id'])
elif name == 'tag':
self.currElem.tags[attrs['k']] = attrs['v']
elif name == 'nd':
self.currElem.nd_ids.append(attrs['ref'])
@classmethod
def endElement(self, name):
if name == 'node':
if superself.n_nodes % 5000 == 0:
print "node %d" % superself.n_nodes
superself.n_nodes += 1
if not dryrun: superself.add_node(self.currElem, c)
elif name == 'way':
if superself.n_ways % 5000 == 0:
print "way %d" % superself.n_ways
superself.n_ways += 1
if not dryrun and accept(self.currElem.tags):
superself.add_way(self.currElem, c)
@classmethod
def characters(self, chars):
pass
xml.sax.parse(osm_filename, OSMHandler)
self.conn.commit()
c.close()
if not dryrun and create_indexes:
print "indexing primary tables...",
self.create_indexes()
print "done"
def set_endnode_ref_counts(self):
"""Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways"""
print "counting end-node references to find way split-points"
c = self.get_cursor()
endnode_ref_counts = {}
c.execute("SELECT nds from ways")
print "...counting"
for i, (nds_str, ) in enumerate(c):
if i % 5000 == 0:
print i
nds = json.loads(nds_str)
for nd in nds:
endnode_ref_counts[nd] = endnode_ref_counts.get(nd, 0) + 1
print "...updating nodes table"
for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()):
if i % 5000 == 0:
print i
if ref_count > 1:
c.execute("UPDATE nodes SET endnode_refs = ? WHERE id=?",
(ref_count, node_id))
self.conn.commit()
c.close()
def index_endnodes(self):
print "indexing endpoint nodes into rtree"
c = self.get_cursor()
#TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes
c.execute("SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1")
for id, lat, lon in c:
self.index.add(int(id), (lon, lat, lon, lat))
c.close()
def create_and_populate_edges_table(self, tolerant=False):
self.set_endnode_ref_counts()
self.index_endnodes()
print "splitting ways and inserting into edge table"
c = self.get_cursor()
c.execute(
"CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)"
)
for i, way in enumerate(self.ways()):
try:
if i % 5000 == 0:
print i
subways = []
curr_subway = [way.nds[0]
] # add first node to the current subway
for nd in way.nds[1:-1]: # for every internal node of the way
curr_subway.append(nd)
if self.node(
nd
)[4] > 1: # node reference count is greater than one, node is shared by two ways
subways.append(curr_subway)
curr_subway = [nd]
curr_subway.append(
way.nds[-1]
) # add the last node to the current subway, and store the subway
subways.append(curr_subway)
#insert into edge table
for i, subway in enumerate(subways):
coords = [(lambda x: (x[3], x[2]))(self.node(nd))
for nd in subway]
packt = pack_coords(coords)
dist = sum([
vincenty(lat1, lng1, lat2, lng2)
for (lng1, lat1), (lng2, lat2) in cons(coords)
])
c.execute("INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)",
("%s-%s" % (way.id, i), way.id, subway[0],
subway[-1], dist, packt))
except IndexError:
if tolerant:
continue
else:
raise
print "indexing edges...",
c.execute("CREATE INDEX edges_id ON edges (id)")
c.execute("CREATE INDEX edges_parent_id ON edges (parent_id)")
print "done"
self.conn.commit()
c.close()
def edge(self, id):
c = self.get_cursor()
c.execute(
"SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?",
(id, ))
try:
ret = c.next()
way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret
return (way_id, parent_id, from_nd, to_nd, dist,
unpack_coords(geom), json.loads(tags))
except StopIteration:
c.close()
raise IndexError("Database does not have an edge with id '%s'" %
id)
c.close()
return ret
def edges(self):
c = self.get_cursor()
c.execute(
"SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id"
)
for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c:
yield (way_id, parent_id, from_nd, to_nd, dist,
unpack_coords(geom), json.loads(tags))
c.close()
def add_way(self, way, curs=None):
if curs is None:
curs = self.get_cursor()
close_cursor = True
else:
close_cursor = False
curs.execute(
"INSERT OR IGNORE INTO ways (id, tags, nds) VALUES (?, ?, ?)",
(way.id, json.dumps(way.tags), json.dumps(way.nd_ids)))
if close_cursor:
self.conn.commit()
curs.close()
def add_node(self, node, curs=None):
if curs is None:
curs = self.get_cursor()
close_cursor = True
else:
close_cursor = False
curs.execute(
"INSERT OR IGNORE INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)",
(node.id, json.dumps(node.tags), node.lat, node.lon))
if close_cursor:
self.conn.commit()
curs.close()
def nodes(self):
c = self.get_cursor()
c.execute("SELECT * FROM nodes")
for node_row in c:
yield node_row
c.close()
def node(self, id):
c = self.get_cursor()
c.execute("SELECT * FROM nodes WHERE id = ?", (id, ))
try:
ret = c.next()
except StopIteration:
c.close()
raise IndexError("Database does not have node with id '%s'" % id)
c.close()
return ret
def nearest_node(self, lat, lon, range=0.005):
c = self.get_cursor()
if self.index:
#print "YOUR'RE USING THE INDEX"
id = list(self.index.nearest((lon, lat), 1))[0]
#print "THE ID IS %d"%id
c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id, ))
else:
c.execute(
"SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?",
(lat - range, lat + range, lon - range, lon + range))
dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5)
for nid, nlat, nlon in c]
if len(dists) == 0:
return (None, None, None, None)
return min(dists, key=lambda x: x[3])
def nearest_of(self, lat, lon, nodes):
c = self.get_cursor()
c.execute("SELECT id, lat, lon FROM nodes WHERE id IN (%s)" %
",".join([str(x) for x in nodes]))
dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5)
for nid, nlat, nlon in c]
if len(dists) == 0:
return (None, None, None, None)
return min(dists, key=lambda x: x[3])
def way(self, id):
c = self.get_cursor()
c.execute("SELECT id, tags, nds FROM ways WHERE id = ?", (id, ))
try:
id, tags_str, nds_str = c.next()
ret = WayRecord(id, tags_str, nds_str)
except StopIteration:
raise Exception("OSMDB has no way with id '%s'" % id)
finally:
c.close()
return ret
def way_nds(self, id):
c = self.get_cursor()
c.execute("SELECT nds FROM ways WHERE id = ?", (id, ))
(nds_str, ) = c.next()
c.close()
return json.loads(nds_str)
def ways(self):
c = self.get_cursor()
c.execute("SELECT id, tags, nds FROM ways")
for id, tags_str, nds_str in c:
yield WayRecord(id, tags_str, nds_str)
c.close()
def count_ways(self):
c = self.get_cursor()
c.execute("SELECT count(*) FROM ways")
ret = c.next()[0]
c.close()
return ret
def count_edges(self):
c = self.get_cursor()
c.execute("SELECT count(*) FROM edges")
ret = c.next()[0]
c.close()
return ret
def delete_way(self, id):
c = self.get_cursor()
c.execute("DELETE FROM ways WHERE id = ?", (id, ))
c.close()
def bounds(self):
c = self.get_cursor()
c.execute("SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes")
ret = c.next()
c.close()
return ret
def execute(self, sql, args=None):
c = self.get_cursor()
if args:
for row in c.execute(sql, args):
yield row
else:
for row in c.execute(sql):
yield row
c.close()
def cursor(self):
return self.get_cursor()
def enclosure_tree(polygons):
"""
Given a list of shapely polygons with only exteriors,
find which curves represent the exterior shell or root curve
and which represent holes which penetrate the exterior.
This is done with an R-tree for rough overlap detection,
and then exact polygon queries for a final result.
Parameters
-----------
polygons : (n,) shapely.geometry.Polygon
Polygons which only have exteriors and may overlap
Returns
-----------
roots : (m,) int
Index of polygons which are root
contains : networkx.DiGraph
Edges indicate a polygon is
contained by another polygon
"""
tree = Rtree()
# nodes are indexes in polygons
contains = nx.DiGraph()
for i, polygon in enumerate(polygons):
# if a polygon is None it means creation
# failed due to weird geometry so ignore it
if polygon is None or len(polygon.bounds) != 4:
continue
# insert polygon bounds into rtree
tree.insert(i, polygon.bounds)
# make sure every valid polygon has a node
contains.add_node(i)
# loop through every polygon
for i in contains.nodes():
polygon = polygons[i]
# we first query for bounding box intersections from the R-tree
for j in tree.intersection(polygon.bounds):
# if we are checking a polygon against itself continue
if (i == j):
continue
# do a more accurate polygon in polygon test
# for the enclosure tree information
if polygons[i].contains(polygons[j]):
contains.add_edge(i, j)
elif polygons[j].contains(polygons[i]):
contains.add_edge(j, i)
# a root or exterior curve has an even number of parents
# wrap in dict call to avoid networkx view
degree = dict(contains.in_degree())
# convert keys and values to numpy arrays
indexes = np.array(list(degree.keys()))
degrees = np.array(list(degree.values()))
# roots are curves with an even inward degree (parent count)
roots = indexes[(degrees % 2) == 0]
# if there are multiple nested polygons split the graph
# so the contains logic returns the individual polygons
if len(degrees) > 0 and degrees.max() > 1:
# collect new edges for graph
edges = []
# find edges of subgraph for each root and children
for root in roots:
children = indexes[degrees == degree[root] + 1]
edges.extend(contains.subgraph(np.append(children, root)).edges())
# stack edges into new directed graph
contains = nx.from_edgelist(edges, nx.DiGraph())
# if roots have no children add them anyway
contains.add_nodes_from(roots)
return roots, contains
class Graph:
def __init__(self, all_trips):
# trips
self.all_trips = all_trips
# cluster seeds
self.cluster_seeds = {}
self.cluster_seed_id = 0
self.cluster_seed_index = Rtree()
# graph edges
self.graph_edges = {} # indexed by "edge id"
self.graph_edge_id = 0
self.graph_edge_lookup = {} # indexed by "location1_id,location2_id"
def cluster_traces(self):
self._create_all_trip_edges()
self._generate_cluster_seeds()
self._cluster_seeds_with_traces()
self._generate_graph_edges()
self._output_graph_to_db()
def _generate_graph_edges(self):
sys.stdout.write("Generating graph edges... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# grab trip edges
trip_edges = trip.edges.values()
# put trip edges in order
trip_edges.sort(key=lambda x: x.id)
# storage for previous cluster
prev_cluster = None
# iterate through trip edges
for trip_edge in trip_edges:
# if the current trip edge is clustered
if (trip_edge.cluster is not None):
# create a graph edge between the previous cluster and the current cluster
self._create_graph_edge(prev_cluster, trip_edge.cluster)
# update previous cluster with current cluster
prev_cluster = trip_edge.cluster
# output graph edges
self._write_graph_edges_to_file()
print "done."
def _create_graph_edge(self, in_node, out_node):
# if in_node or out_node is None
if ((in_node is None) or (out_node is None)):
# return without doing anything
return
# see if we can find an existing graph edge with the same nodes
existing_graph_edge = self._find_graph_edge(in_node, out_node)
# if there is no existing graph edge with the same nodes
if (existing_graph_edge is None):
# create new graph edge object
new_graph_edge = Edge(self.graph_edge_id, in_node, out_node)
# add new graph edge to graph edge dictionary
self.graph_edges[new_graph_edge.id] = new_graph_edge
# add new graph edge to graph edge lookup dictionary
self.graph_edge_lookup[str(in_node.id) + "," +
str(out_node.id)] = new_graph_edge
# increment graph edge id
self.graph_edge_id += 1
def _find_graph_edge(self, node1, node2):
# generate edge lookup key
edge_lookup_key = str(node1.id) + "," + str(node2.id)
# if edge is in lookup table
if (edge_lookup_key in self.graph_edge_lookup.keys()):
# return the matching edge
return self.graph_edge_lookup[edge_lookup_key]
# if the edge wasn't in the lookup table
return None
def _cluster_seeds_with_traces(self):
# storage for total cluster distance moved
total_cluster_distance_moved = float('infinity')
# iterate until total cluster distance moved below threshold
while (total_cluster_distance_moved >=
cluster_distance_moved_threshold):
# find all points on traces and move clusters
total_cluster_distance_moved = self._find_points_on_traces()
# write cluster seeds to file
self._write_cluster_seeds_to_file(
"edelkamp_cluster_seeds_clustered.txt")
def _find_points_on_traces(self):
# counter for cluster seeds
seed_counter = 1
# storage for total cluster distance moved
total_cluster_distance_moved = 0.0
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# clear current trace points from cluster
cluster_seed.clear_trace_points()
sys.stdout.write("\rFinding intersecting points with cluster " +
str(seed_counter) + "/" +
str(len(self.cluster_seeds)) + "... ")
sys.stdout.flush()
# increment seed counter
seed_counter += 1
# determine leftward cluster bearing
leftward_bearing = math.fmod((cluster_seed.bearing - 90.0) + 360.0,
360.0)
# determine rightward cluster bearing
rightward_bearing = math.fmod(
(cluster_seed.bearing + 90.0) + 360.0, 360.0)
# storage for candidate trace points
candidate_trace_points = []
# iterate through all trips
for trip in self.all_trips:
# find leftward intersection points with trip
candidate_trace_points.extend(
self._find_intersection_points(trip, cluster_seed,
leftward_bearing))
# find rightward intersection points with trip
candidate_trace_points.extend(
self._find_intersection_points(trip, cluster_seed,
rightward_bearing))
# add candidate trace points to cluster
cluster_seed.add_trace_points(candidate_trace_points)
# recompute cluster centroid
total_cluster_distance_moved += cluster_seed.recompute_cluster_centroid(
)
# clear current trace points from cluster
cluster_seed.clear_trace_points()
# add candidate trace points to cluster, again
cluster_seed.add_trace_points(candidate_trace_points)
# normalize total cluster distance moved by number of seeds
total_cluster_distance_moved = (total_cluster_distance_moved /
len(self.cluster_seeds.values()))
# and we're done!
print "done (clusters moved an average of " + str(
total_cluster_distance_moved) + " meters)."
# return total cluster distance moved
return total_cluster_distance_moved
def _find_intersection_points(self, trip, cluster, cluster_bearing):
# find all nearby trip edge id's
nearby_trip_edge_ids = self._find_nearby_trip_edge_ids(
cluster, edge_bounding_box_size, trip.edge_index)
# storage for intersection points
intersection_points = []
# iterate through all nearby edge id's
for edge_id in nearby_trip_edge_ids:
# grab current edge
edge = trip.edges[edge_id]
# determine intersection point between edge and cluster
intersection_point = self._intersection_point(
edge.in_node, edge.bearing, cluster, cluster_bearing)
# if there is an intersection point
if (intersection_point is not None):
# determine distance from edge in_node to intersection point
intersection_distance = self._distance_coords(
edge.in_node.latitude, edge.in_node.longitude,
intersection_point[0], intersection_point[1])
# if intersection distance is less than edge length
if (intersection_distance <= edge.length):
# this edge has a valid intersection point
intersection_points.append(
TracePoint(intersection_point[0],
intersection_point[1], edge.bearing, edge))
# return all intersection points for this trip
return intersection_points
def _generate_cluster_seeds(self):
# iterate through all trips
for i in range(0, len(self.all_trips)):
sys.stdout.write("\rCluster seeding trip " + str(i + 1) + "/" +
str(len(self.all_trips)) + "... ")
sys.stdout.flush()
# grab current trip
trip = self.all_trips[i]
# set last cluster seed distance to zero for first trip location
trip.locations[0].last_cluster_seed_distance = 0.0
# iterate through all trip locations
for j in range(1, len(trip.locations)):
# drop cluster seeds along current edge every 50 meters
self._drop_cluster_seeds_along_edge(trip.locations[j - 1],
trip.locations[j])
print "done (generated " + str(len(
self.cluster_seeds)) + " cluster seeds)."
# write cluster seeds to file
self._write_cluster_seeds_to_file("edelkamp_cluster_seeds_initial.txt")
def _drop_cluster_seeds_along_edge(self, in_node, out_node):
# determine edge length
edge_length = self._distance(in_node, out_node)
# determine distance along edge for first cluster seed
first_cluster_seed_distance = (cluster_seed_interval -
in_node.last_cluster_seed_distance)
# storage for relative cluster seed intervals
rel_cluster_seed_intervals = []
# storage for current cluster seed distance along this edge
curr_cluster_seed_distance = first_cluster_seed_distance
# determine the relative cluster seed intervals needed for this edge
while (curr_cluster_seed_distance <= edge_length):
# append current cluster seed distance to relative cluster seed interval list
rel_cluster_seed_intervals.append(curr_cluster_seed_distance)
# increment current cluster seed distance
curr_cluster_seed_distance += cluster_seed_interval
# determine bearing of current edge
edge_bearing = self._path_bearing(in_node, out_node)
# create cluster seeds for edge
for i in range(0, len(rel_cluster_seed_intervals)):
# determine fraction along current edge to drop cluster seed
fraction_along = (rel_cluster_seed_intervals[i] / edge_length)
# determine point along line to drop cluster seed
(new_cluster_seed_latitude,
new_cluster_seed_longitude) = self._point_along_line(
in_node, out_node, fraction_along)
# locate nearest existing cluster seeds
closest_cluster_seeds = list(
self.cluster_seed_index.nearest(
(new_cluster_seed_longitude, new_cluster_seed_latitude),
25))
# if there does not exist a closest existing cluster seed
if (len(closest_cluster_seeds) == 0):
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude,
edge_bearing)
# else, if there exists a closest existing cluster seed
elif (len(closest_cluster_seeds) > 0):
# storage for matched cluster seed
matched_cluster_seed = None
# iterate through closest existing cluster seeds
for curr_cluster_seed_id in closest_cluster_seeds:
# grab current cluster seed
curr_cluster_seed = self.cluster_seeds[
curr_cluster_seed_id]
# compute distance to current cluster seed
distance = self._distance_coords(
new_cluster_seed_latitude, new_cluster_seed_longitude,
curr_cluster_seed.latitude,
curr_cluster_seed.longitude)
# determine bearing difference between edge and current cluster seed
bearing_difference = math.cos(
math.radians(edge_bearing - curr_cluster_seed.bearing))
# if current cluster is less than 50 meters away and bearing difference is less than or equal to 45 degrees
if ((distance <= cluster_seed_interval)
and (bearing_difference >=
cluster_bearing_difference_limit)):
# store current cluster seed as matched cluster seed
matched_cluster_seed = curr_cluster_seed
# stop searching
break
# if there was not a matched cluster seed
if (matched_cluster_seed is None):
# create a new cluster seed
new_cluster_seed = self._create_new_cluster_seed(
new_cluster_seed_latitude, new_cluster_seed_longitude,
edge_bearing)
# update last cluster seed distance
out_node.last_cluster_seed_distance = self._distance_coords(
new_cluster_seed_latitude, new_cluster_seed_longitude,
out_node.latitude, out_node.longitude)
# if no cluster seeds were generated along this edge
if (len(rel_cluster_seed_intervals) == 0):
# update last cluster seed distance
out_node.last_cluster_seed_distance = (
in_node.last_cluster_seed_distance + edge_length)
def _create_new_cluster_seed(self, latitude, longitude, bearing):
# create a new cluster seed
new_cluster_seed = ClusterSeed(self.cluster_seed_id, latitude,
longitude, bearing)
# add new cluster seed to the cluster seeds dictionary
self.cluster_seeds[new_cluster_seed.id] = new_cluster_seed
# insert new cluster seed into spatial index
self.cluster_seed_index.insert(
new_cluster_seed.id,
(new_cluster_seed.longitude, new_cluster_seed.latitude))
# increment cluster seed id
self.cluster_seed_id += 1
# return new cluster seed
return new_cluster_seed
def _create_all_trip_edges(self):
sys.stdout.write("Creating and indexing edges for all trips... ")
sys.stdout.flush()
# iterate through all trips
for trip in self.all_trips:
# add edge storage to trip
trip.edges = {}
# add edge index to trip
trip.edge_index = Rtree()
# storage for edge id
trip_edge_id = 0
# iterate through all trip locations
for i in range(1, len(trip.locations)):
# create new edge
new_edge = Edge(trip_edge_id, trip.locations[i - 1],
trip.locations[i])
# insert edge into dictionary
trip.edges[trip_edge_id] = new_edge
# insert edge into index
self._index_trip_edge(new_edge, trip.edge_index)
# increment trip edge id
trip_edge_id += 1
# done
print "done."
def _index_trip_edge(self, edge, edge_index):
# determine edge minx, miny, maxx, maxy values
edge_minx = min(edge.in_node.longitude, edge.out_node.longitude)
edge_miny = min(edge.in_node.latitude, edge.out_node.latitude)
edge_maxx = max(edge.in_node.longitude, edge.out_node.longitude)
edge_maxy = max(edge.in_node.latitude, edge.out_node.latitude)
# insert edge into spatial index
edge_index.insert(edge.id,
(edge_minx, edge_miny, edge_maxx, edge_maxy))
def _find_nearby_trip_edge_ids(self, location, distance, edge_index):
# define longitude/latitude offset
lon_offset = ((distance / 2.0) /
spatialfunclib.METERS_PER_DEGREE_LONGITUDE)
lat_offset = ((distance / 2.0) /
spatialfunclib.METERS_PER_DEGREE_LATITUDE)
# create bounding box
bounding_box = (location.longitude - lon_offset,
location.latitude - lat_offset,
location.longitude + lon_offset,
location.latitude + lat_offset)
# return nearby edge id's inside bounding box
return list(edge_index.intersection(bounding_box))
def _intersection_point(self, location1, location1_bearing, location2,
location2_bearing):
return spatialfunclib.intersection_point(
location1.latitude, location1.longitude, location1_bearing,
location2.latitude, location2.longitude, location2_bearing)
def _point_along_line(self, location1, location2, fraction_along):
return spatialfunclib.point_along_line(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude,
fraction_along)
def _path_bearing(self, location1, location2):
return spatialfunclib.path_bearing(location1.latitude,
location1.longitude,
location2.latitude,
location2.longitude)
def _distance(self, location1, location2):
return spatialfunclib.distance(location1.latitude, location1.longitude,
location2.latitude, location2.longitude)
def _distance_coords(self, location1_latitude, location1_longitude,
location2_latitude, location2_longitude):
return spatialfunclib.distance(location1_latitude, location1_longitude,
location2_latitude, location2_longitude)
def _write_cluster_seeds_to_file(self,
filename="edelkamp_cluster_seeds.txt"):
# open graph file
graph_file = open(filename, 'w')
# iterate through all cluster_seeds
for cluster_seed in self.cluster_seeds.values():
# output cluster seed to file
graph_file.write(
str(cluster_seed.latitude) + "," +
str(cluster_seed.longitude) + "," + str(cluster_seed.bearing) +
"\n")
# close graph file
graph_file.close()
def _write_graph_edges_to_file(self):
# open graph file
graph_file = open('edelkamp_cluster_edges.txt', 'w')
# iterate through all graph_edges
for graph_edge in self.graph_edges.values():
# output edge to file
graph_file.write(
str(graph_edge.in_node.latitude) + "," +
str(graph_edge.in_node.longitude) + "\n")
graph_file.write(
str(graph_edge.out_node.latitude) + "," +
str(graph_edge.out_node.longitude) + "\n\n")
# close graph file
graph_file.close()
def _output_graph_to_db(self):
# output that we are starting the database writing process...
sys.stdout.write("\nOutputting graph to database... ")
sys.stdout.flush()
# connect to database
conn = sqlite3.connect("edelkamp_graph.db")
# grab cursor
cur = conn.cursor()
# create nodes table
cur.execute(
"CREATE TABLE nodes (id INTEGER, latitude FLOAT, longitude FLOAT)")
# create edges table
cur.execute(
"CREATE TABLE edges (id INTEGER, in_node INTEGER, out_node INTEGER)"
)
# remove values from nodes table
#cur.execute("DELETE FROM nodes")
# remove values from edges table
#cur.execute("DELETE FROM edges")
# commit creates
conn.commit()
# iterate through all cluster seeds
for cluster_seed in self.cluster_seeds.values():
# insert cluster seed into nodes table
cur.execute("INSERT INTO nodes VALUES (" + str(cluster_seed.id) +
"," + str(cluster_seed.latitude) + "," +
str(cluster_seed.longitude) + ")")
# iterate through all graph edges
for graph_edge in self.graph_edges.values():
# insert graph edge into edges table
cur.execute("INSERT INTO edges VALUES (" + str(graph_edge.id) +
"," + str(graph_edge.in_node.id) + "," +
str(graph_edge.out_node.id) + ")")
# commit inserts
conn.commit()
# close database connection
conn.close()
print "done."
def getFilesForBox(request):
l = float( request.GET['x1'] )
b = float( request.GET['y1'] )
r = float( request.GET['x2'] )
t = float( request.GET['y2'] )
ser = request.GET['ser']
begin = int(request.GET['begin'])
num = int(request.GET['num'])
index = Rtree('/home/renci/geoanalytics/geoanalytics-lib/fl')
ids = list(index.intersection((l,b,r,t), objects=True))[begin:begin+num]
if len(ids)>0:
files = [{
'name' : f.name,
'filepath' : f.filepath,
'bbox4326' : f.bbox4326,
'metadata' : f.metadata,
'type' : "raster",
'band_metadata' : f.band_metadata,
'projection' : f.projection
} for f in [RasterFile.objects(id=i.object[0]).first() for i in filter(lambda x:x.object[1]==ser and x.object[2] == 'r' , ids)]] + [{
'name' : f.name,
'filepath' : f.filepath,
'bbox4326' : f.bbox4326,
'type' : "features",
'attributes' : f.attribute_names,
'layers' : [{'attributes' : zip(l.attribute_names, l.attribute_types),
'geometry_type' : l.geometry_type,
'feature_count' : l.feature_count} for l in f.layers]
} for f in [VectorFile.objects(id=i.object[0]).first() for i in filter(lambda x:x.object[1]==ser and x.object[2] == 'v' , ids)]]
return HttpResponse(json.dumps(files), mimetype='application/json')
else:
return HttpResponse("[]", mimetype='application/json')
