def segment_spatial_index(linestring):
coords = list(linestring.coords)
spatial_index = Rtree()
for i in range(len(coords)-1):
x = map(lambda xy: xy[0],coords[i:i+2])
y = map(lambda xy: xy[1],coords[i:i+2])
spatial_index.add(i,(min(x),min(y),max(x),max(y)))
return spatial_index
class Dispatch:
def __init__(self, config_filename):
self.index = Rtree()
# get children from yaml file
config_str = open(config_filename).read()
print config_str
self.children = yaml.load( config_str )
# index children according to their bounding box
for i, child in enumerate(self.children):
self.index.add(i, child['bounds'])
def register(self, baseurl, left, bottom, right, top):
if baseurl in self.children.values():
raise Exception( "Child with baseurl '%s' already registered"%baseurl )
childindex = len(self.children)
self.children[childindex] = (baseurl, (left,bottom,right,top))
self.index.add( childindex, (left, bottom, right, top) )
return json.dumps(self.children)
def children(self):
return json.dumps( self.children )
def _over(self, lat, lon):
return [self.children[x] for x in self.index.intersection( (lon, lat, lon, lat) )]
def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85):
child_servers = self._over( lat, lon )
if len(child_servers) == 0:
return "NO SHEDS HERE"
child_server = child_servers[0]
args = {'lat':lat,
'lon':lon,
'year':year,
'month':month,
'day':day,
'hour':hour,
'minute':minute,
'second':second,
'cutoff':cutoff,
'step':step,
'speed':speed}
contoururl = "http://%s/contour?%s&encoded=true"%(child_server['url'], urlencode(args))
return urlopen(contoururl).read()
class WayIndex:
def __init__(self, ways):
self.max_way_id = 0
self.way_idx_mapping = {}
self.idx = Rtree()
for way in ways:
self.add(way)
def add(self, way):
self.idx.add(id=self.max_way_id, coordinates=way.get_bbox(),
obj=way.id)
self.way_idx_mapping[way.id] = self.max_way_id
self.max_way_id += 1
def delete(self, way):
self.idx.delete(id=self.way_idx_mapping[way.id],
coordinates=way.get_bbox())
way_ids = map(lambda way: way.object,
wayidx.idx.intersection(way.get_bbox(),
objects=True))
class SpatialIndex(Persistent):
def __init__(self, *args):
self.rtree_args = args
self.rtree = Rtree(*args)
self.backward = IOBTree()
def index_doc(self, docid, value):
if docid in self.backward:
self.unindex_doc(docid)
self.backward[docid] = value
self.rtree.add(docid, value, obj=docid)
def unindex_doc(self, docid):
value = self.backward.get(docid)
if value is None:
return
self.rtree.delete(docid, value)
del self.backward[docid]
def apply(self, value):
return [x.object for x in self.rtree.intersection(value, objects=True)]
def clear(self):
self.backward.clear()
props = self.rtree.properties
if props.storage == RT_Disk:
self.rtree.close()
fname = props.filename
try:
os.unlink('%s.%s' % (fname, props.dat_extension))
except OSError:
pass
try:
os.unlink('%s.%s' % (fname, props.idx_extension))
except OSError:
pass
self.rtree = Rtree(*self.rtree_args)
def count(self, value):
return self.rtree.count(value)
class SampleSpace(object):
def __init__(self):
self._index = Rtree()
self._locations = []
self._values = []
def __setitem__(self, location, value):
i = len(self._locations)
self._locations.append(location)
self._values.append(value)
self._index.add(i, self._locations[i])
def __getitem__(self, location):
js = list(self._index.nearest(location, 3))
if len(js) == 0:
return 0
if len(js) == 1:
return self._values[js[0]]
ds = [sqrt(sum([(self._locations[j][i]-location[i])**2 for i in range(2)]))
for j in js]
for i in range(len(js)):
if ds[i] == 0:
return self._values[js[i]]
R = max(ds)
nums = [((R - d)/(R * d))**2 for d in ds]
den = sum(nums)
ws = [num/den for num in nums]
return sum([self._values[js[i]] * ws[i] for i in range(len(js))])
def __iter__(self):
for i in range(len(self._values)):
yield self._locations[i], self._values[i]
class Datasource:
""" Store an exploded representation of a data source, so it can be simplified.
"""
def __init__(self, srs, geom_type, fields, values, shapes):
""" Use load() to call this constructor.
"""
self.srs = srs
self.fields = fields
self.geom_type = geom_type
self.values = values
self.shapes = shapes
# this will be changed later
self.tolerance = 0
# guid, src1_id, src2_id, line_id, x1, y1, x2, y2
db = connect(':memory:').cursor()
db.execute("""CREATE table segments (
-- global identifier for this segment
guid INTEGER PRIMARY KEY AUTOINCREMENT,
-- identifiers for source shape or shapes for shared borders
src1_id INTEGER,
src2_id INTEGER,
-- global identifier for this line
line_id INTEGER,
-- start and end coordinates for this segment
x1 REAL,
y1 REAL,
x2 REAL,
y2 REAL,
-- flag
removed INTEGER
)""")
db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)')
db.execute('CREATE INDEX shape1_parts ON segments (src1_id)')
db.execute('CREATE INDEX shape2_parts ON segments (src2_id)')
self.db = db
self.rtree = Rtree()
self.memo_line = make_memo_line()
def _indexes(self):
return range(len(self.values))
def simplify(self, tolerance, verbose=False):
""" Simplify the polygonal linework.
This method can be called multiple times, but the process is
destructive so it must be called with progressively increasing
tolerance values.
"""
if tolerance < self.tolerance:
raise Exception(
'Repeat calls to simplify must have increasing tolerances.')
self.tolerance = tolerance
q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC'
line_ids = [line_id for (line_id, count) in self.db.execute(q)]
stable_lines = set()
while True:
was = self.db.execute(
'SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0]
preserved, popped = set(), False
for line_id in line_ids:
if line_id in stable_lines:
continue
# For each coordinate that forms the apex of a two-segment
# triangle, find the area of that triangle and put it into a list
# along with the segment identifier and the resulting line if the
# triangle were flattened, ordered from smallest to largest.
rows = self.db.execute(
"""SELECT guid, x1, y1, x2, y2
FROM segments
WHERE line_id = ?
AND removed = 0
ORDER BY guid""", (line_id, ))
segs = [(guid, (x1, y1), (x2, y2))
for (guid, x1, y1, x2, y2) in rows]
triples = [(segs[i][0], segs[i + 1][0], segs[i][1], segs[i][2],
segs[i + 1][2]) for i in range(len(segs) - 1)]
triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3)
for (guid1, guid2, c1, c2, c3) in triples]
areas = sorted([(triangle.area, guid1, guid2, c1, c3)
for (guid1, guid2, triangle, c1,
c3) in triangles])
min_area = self.tolerance**2
if not areas or areas[0][0] > min_area:
# there's nothing to be done
stable_lines.add(line_id)
if verbose:
stderr.write('-')
continue
# Reduce any segments that makes a triangle whose area is below
# the minimum threshold, starting with the smallest and working up.
# Mark segments to be preserved until the next iteration.
for (area, guid1, guid2, ca, cb) in areas:
if area > min_area:
# there won't be any more points to remove.
break
if guid1 in preserved or guid2 in preserved:
# the current segment is too close to a previously-preserved one.
continue
# Check the resulting flattened line against the rest
# any of the original shapefile, to determine if it would
# cross any existing line segment.
(x1, y1), (x2, y2) = ca, cb
new_line = self.memo_line(x1, y1, x2, y2)
old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2))
old_rows = self.db.execute(
'SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0'
% ','.join(map(str, old_guids)))
old_lines = [
self.memo_line(x1, y1, x2, y2)
for (x1, y1, x2, y2) in old_rows
]
if True in [
new_line.crosses(old_line)
for old_line in old_lines
]:
if verbose:
stderr.write('x%d' % line_id)
continue
preserved.add(guid1)
preserved.add(guid2)
popped = True
x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1]
self.db.execute(
'UPDATE segments SET removed=1 WHERE guid=%d' % guid2)
self.db.execute(
'UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?',
(x1, y1, x2, y2, guid1))
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
stderr.write('.')
if verbose:
print >> stderr, ' reduced from', was, 'to',
print >> stderr, self.db.execute(
'SELECT COUNT(guid) FROM segments WHERE removed=0'
).fetchone()[0],
self.rtree = Rtree()
for (guid, x1, y1, x2, y2) in self.db.execute(
'SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0'
):
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
print >> stderr, '.'
if not popped:
break
class MapMatcher():
"""A very nice MapMatching class
"""
def __init__(self):
self.GPS = GPS()
self.idx = Rtree()
self.nodeidx = Rtree()
self.G = None
self.edgeindex_edge = {}
self.edgecounter = 0
self.nodecounter = 0
self.gps_points = [];
self.edge_id__count = {}
self.node_counter__node = {}
self.result_counter = 0
def saveGraph(self, filename):
"""Saves the graph as a YAML file
"""
nx.write_yaml(self.G,filename)
def readGraphFromYAMLFile(self, filename):
"""Loads a graph from an YAML file.
"""
self.G = nx.read_yaml(filename)
# TODO: buiild up the indexes !!!
def addNodeToIndex(self, node):
"""Adds a node to the node index (RTree)
"""
# self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node)
self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1]))
self.node_counter__node[self.nodecounter] = node
def addEdgeToIndex(self, edge):
"""Add an edge to the edhe index.
"""
self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge)
# print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY())
self.edgeindex_edge[self.edgecounter] = edge
self.edgecounter = self.edgecounter + 1
def openShape(self, inFile, index=0):
self.shapeFile = ogr.Open(inFile)
if self.shapeFile is None:
print "Failed to open " + inFile + ".\n"
sys.exit( 1 )
else:
print "SHP file successfully read"
def getfieldinfo(self, lyr, feature, flds):
f = feature
return [f.GetField(f.GetFieldIndex(x)) for x in flds]
def addlyr(self, G,lyr, fields):
point_coords__nodes = {}
for findex in xrange(lyr.GetFeatureCount()):
f = lyr.GetFeature(findex)
flddata = self.getfieldinfo(lyr, f, fields)
g = f.geometry()
attributes = dict(zip(fields, flddata))
attributes["ShpName"] = lyr.GetName()
if g.GetGeometryType() == 2: #linestring
last = g.GetPointCount() - 1
p_from = g.GetPoint_2D(0)
p_to = g.GetPoint_2D(last)
# check whether we have a node in the index
intersection_mask = (p_from[0]-INTERSECTION_MASK/2,
p_from[1]-INTERSECTION_MASK/2,
p_from[0]+INTERSECTION_MASK/2,
p_from[1]+INTERSECTION_MASK/2)
results = list(self.nodeidx.intersection(intersection_mask))
if len(results)==0:
print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "."
pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
self.node_counter__node[self.nodecounter] = pfrom
self.nodeidx.add(self.nodecounter, (p_from[0],
p_from[1],
p_from[0],
p_from[1]))
# print p_from
self.nodecounter = self.nodecounter + 1
else:
print len(results)
print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "."
pfrom = self.node_counter__node[results[0]]
intersection_mask = (p_to[0]-INTERSECTION_MASK/2,
p_to[1]-INTERSECTION_MASK/2,
p_to[0]+INTERSECTION_MASK/2,
p_to[1]+INTERSECTION_MASK/2)
# print intersection_mask
results = list(self.nodeidx.intersection(intersection_mask))
if len(results)==0:
print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "."
pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
self.node_counter__node[self.nodecounter] = pto
self.nodeidx.add(self.nodecounter, (p_to[0],
p_to[1],
p_to[0],
p_to[1]))
self.nodecounter = self.nodecounter + 1
else:
print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "."
pto = self.node_counter__node[results[0]]
shly_geom = shapely.wkt.loads(g.ExportToWkt())
e = Edge(pfrom, pto, attributes, geometry = shly_geom)
# G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter})
G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter)
self.addEdgeToIndex(e)
#if g.GetGeometryType() == 1: #point
# G.add_node((g.GetPoint_2D(0)), attributes)
# if g.GetGeometryType() == 2: #linestring
# last = g.GetPointCount() - 1
#
# p_from = g.GetPoint_2D(0)
# p_to = g.GetPoint_2D(last)
#
# if point_coords__nodes.get(p_from):
#
# pfrom = point_coords__nodes.get(p_from)
# print "node " + str(pfrom.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR'))
#
# else:
#
# pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
# self.nodecounter = self.nodecounter + 1
# point_coords__nodes[p_from] = pfrom
#
# if point_coords__nodes.get(p_to):
#
# pto = point_coords__nodes.get(p_to)
# print "node " + str(pto.getAttributes().get("nodecounter")) + " edge " + str(attributes.get('ID_NR'))
#
# else:
#
# pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
# self.nodecounter = self.nodecounter + 1
# point_coords__nodes[p_to] = pto
#
# shly_geom = shapely.wkt.loads(g.ExportToWkt())
# e = Edge(pfrom, pto, attributes, geometry = shly_geom)
#
# G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter})
#
# # we pull the nodes out of the graph again to index them
# edges_dict = nx.get_edge_attributes(G,"edgecounter")
#
# # import pdb;pdb.set_trace()
#
# edges_keys = edges_dict.keys()
# for k in edges_keys:
# if self.edgecounter == edges_dict[k]:
# self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0]
# self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1]
# self.addNodeToIndex(k[0])
# self.addNodeToIndex(k[1])
#
# # let us throw the Edge into the index
# self.addEdgeToIndex(e)
#
## # add an edge in the other direction
## e2 = Edge(pto, pfrom, attributes, geometry = shly_geom)
## G.add_edge(pto, pfrom, {"edge": e2, "edgecounter" : self.edgecounter})
##
## edges_dict = nx.get_edge_attributes(G,"edgecounter")
##
## # import pdb;pdb.set_trace()
##
## edges_keys = edges_dict.keys()
## for k in edges_keys:
## if self.edgecounter == edges_dict[k]:
## self.node_counter__node[k[0].getAttributes()['nodecounter']] = k[0]
## self.node_counter__node[k[1].getAttributes()['nodecounter']] = k[1]
## self.addNodeToIndex(k[0])
## self.addNodeToIndex(k[1])
##
## # let us throw the Edge into the index
## self.addEdgeToIndex(e2)
return G
def shapeToGraph(self, inFile, uniqueId="FID"):
"""Loads a shapefile and builds the graph.
uniqueId is the name of a unique field in the shape file.
"""
# self.G = nx.readwrite.nx_shp.read_shp(inFile)
self.G = nx.MultiGraph()
self.shapeFileUniqueId = uniqueId
lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory
for lyrindex in xrange(lyrcount):
lyr = self.shapeFile.GetLayerByIndex(lyrindex)
flds = [x.GetName() for x in lyr.schema]
self.G=self.addlyr(self.G, lyr, flds)
self.routefinder = RouteFinder(self.G)
def readGPS(self, inFile):
"""Parses a shapefile and build the GPS object
"""
self.GPS.readFromShapeFile(inFile)
self.gps_points = self.GPS.getGPSPoints()
def maxGPSDistance(self):
"""Calculate the maximum distance of two consecutive GPS Points
"""
# TODO check whether GPS points are already there
# TODO: move into sl.gps.GPS()
maxDistance = 0
gps_point = self.gps_points[0]
for gpspoint in self.gps_points:
distance = gpspoint.getGeometry().distance(gps_point.getGeometry())
gps_point = gps_point
if distance > maxDistance:
maxDistance = distance
return maxDistance
def nearPoints(self):
"""Sums up the gps point per edge segment. Stores in self.edge_id__count
"""
# initialize the edge counter
for edge in self.G.edges():
self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0
for point in self.gps_points:
nearest_edge = self.getNearestEdge(point)
# print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id'))
self.addPointCountToEdge(nearest_edge)
def addPointCountToEdge(self, edge):
"""Increments the point counter for the given edge by one.
"""
attributes = edge.getAttributes()
if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)):
self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1
else:
self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1
edge.setAttributes(attributes)
def getNearestEdge(self, point):
"""Returns the edge closes to a Shapely entity given (point)
"""
edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True)
edges = [e.object for e in edge]
if len(edges) == 1:
result = edges[0]
else:
dist = 99999999999999999999999999999999999999999
for edge in edges:
distance = point.getPoint().distance(edge.getGeometry())
if distance < dist:
dist = distance
result = edge
return result
def getNearestNode(self, point):
"""Returns the closest node to a GPS point.
"""
nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y)))
return self.node_counter__node.get(nodes[0])
def findRoute(self, returnNonSelection=False):
"""Finds a route from the node closest to the first GPS point to
the node closest to the latest GPS point.
"""
# pick the start and end GPS points # TODO: sort GPS Points first
start_point = self.gps_points[0]
end_point = self.gps_points[-1]
start_node = self.getNearestNode(start_point)
end_node = self.getNearestNode(end_point)
# the start and endnodes returnes by the index are not in the graph,
# therefore we need to look them up ....
start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter"))
end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter"))
self.routfinder = RouteFinder(self.G)
label_list = self.routefinder.findroutes(start_node, end_node)
import pdb;pdb.set_trace()
label_scores = []
# let us loop through the label list
for label in label_list:
number_of_points = 0
# we sum up the number of points and relate them to the length of the route
for edge in label.getEdges():
edge_id = edge.getAttributes().get(self.shapeFileUniqueId)
number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0)
#we add the scores to a dict
label_scores.append((label, number_of_points/label.getLength()))
# print label_scores
# and extract the maximum score
score = 0
selected = None
for ls in label_scores:
if ls[1] > score:
selected = ls[0]
score = ls[1]
if returnNonSelection:
pass
else:
return selected
def eliminiateEmptyEdges(self, distance = 100):
"""Loops through the GPS pointset and selects edges within a boundary
of <distance> meters
"""
print "Edge elimination started"
selected_edge_ids = []
# let us
for point in self.gps_points:
results = self.idx.nearest(((point.getPoint().x-distance/2),
(point.getPoint().y-distance/2),
(point.getPoint().x+distance/2),
(point.getPoint().y+distance/2)), objects=True)
for result in results:
from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter"))
to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter"))
edge_counter = self.G.edge[from_node][to_node].get("edgecounter")
if edge_counter not in selected_edge_ids:
selected_edge_ids.append(edge_counter)
print str(len(selected_edge_ids)) + " edges found to keep."
elimination_counter = 0
for edge in self.G.edges():
edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter")
if edgecounter not in selected_edge_ids:
edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node)
self.G.remove_edge(*edge_tuple)
elimination_counter = elimination_counter + 1
print str(elimination_counter) + " edges eliminated."
class OSMDB:
def __init__(self, dbname, overwrite=False, rtree_index=True):
self.dbname = dbname
if overwrite:
try:
os.remove(dbname)
except OSError:
pass
self.conn = sqlite3.connect(dbname)
if rtree_index:
self.index = Rtree(dbname)
else:
self.index = None
if overwrite:
self.setup()
def get_cursor(self):
# Attempts to get a cursor using the current connection to the db. If we've found ourselves in a different thread
# than that which the connection was made in, re-make the connection.
try:
ret = self.conn.cursor()
except sqlite3.ProgrammingError:
self.conn = sqlite3.connect(self.dbname)
ret = self.conn.cursor()
return ret
def setup(self):
c = self.get_cursor()
c.execute(
"CREATE TABLE nodes (id TEXT UNIQUE, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)"
)
c.execute("CREATE TABLE ways (id TEXT UNIQUE, tags TEXT, nds TEXT)")
self.conn.commit()
c.close()
def create_indexes(self):
c = self.get_cursor()
c.execute("CREATE INDEX nodes_id ON nodes (id)")
c.execute("CREATE INDEX nodes_lon ON nodes (lon)")
c.execute("CREATE INDEX nodes_lat ON nodes (lat)")
c.execute("CREATE INDEX ways_id ON ways (id)")
self.conn.commit()
c.close()
def populate(self,
osm_filename,
dryrun=False,
accept=lambda tags: True,
reporter=None,
create_indexes=True):
print "importing %s osm from XML to sqlite database" % osm_filename
c = self.get_cursor()
self.n_nodes = 0
self.n_ways = 0
superself = self
class OSMHandler(xml.sax.ContentHandler):
@classmethod
def setDocumentLocator(self, loc):
pass
@classmethod
def startDocument(self):
pass
@classmethod
def endDocument(self):
pass
@classmethod
def startElement(self, name, attrs):
if name == 'node':
self.currElem = Node(attrs['id'], float(attrs['lon']),
float(attrs['lat']))
elif name == 'way':
self.currElem = Way(attrs['id'])
elif name == 'tag':
self.currElem.tags[attrs['k']] = attrs['v']
elif name == 'nd':
self.currElem.nd_ids.append(attrs['ref'])
@classmethod
def endElement(self, name):
if name == 'node':
if superself.n_nodes % 5000 == 0:
print "node %d" % superself.n_nodes
superself.n_nodes += 1
if not dryrun: superself.add_node(self.currElem, c)
elif name == 'way':
if superself.n_ways % 5000 == 0:
print "way %d" % superself.n_ways
superself.n_ways += 1
if not dryrun and accept(self.currElem.tags):
superself.add_way(self.currElem, c)
@classmethod
def characters(self, chars):
pass
xml.sax.parse(osm_filename, OSMHandler)
self.conn.commit()
c.close()
if not dryrun and create_indexes:
print "indexing primary tables...",
self.create_indexes()
print "done"
def set_endnode_ref_counts(self):
"""Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways"""
print "counting end-node references to find way split-points"
c = self.get_cursor()
endnode_ref_counts = {}
c.execute("SELECT nds from ways")
print "...counting"
for i, (nds_str, ) in enumerate(c):
if i % 5000 == 0:
print i
nds = json.loads(nds_str)
for nd in nds:
endnode_ref_counts[nd] = endnode_ref_counts.get(nd, 0) + 1
print "...updating nodes table"
for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()):
if i % 5000 == 0:
print i
if ref_count > 1:
c.execute("UPDATE nodes SET endnode_refs = ? WHERE id=?",
(ref_count, node_id))
self.conn.commit()
c.close()
def index_endnodes(self):
print "indexing endpoint nodes into rtree"
c = self.get_cursor()
#TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes
c.execute("SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1")
for id, lat, lon in c:
self.index.add(int(id), (lon, lat, lon, lat))
c.close()
def create_and_populate_edges_table(self, tolerant=False):
self.set_endnode_ref_counts()
self.index_endnodes()
print "splitting ways and inserting into edge table"
c = self.get_cursor()
c.execute(
"CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)"
)
for i, way in enumerate(self.ways()):
try:
if i % 5000 == 0:
print i
subways = []
curr_subway = [way.nds[0]
] # add first node to the current subway
for nd in way.nds[1:-1]: # for every internal node of the way
curr_subway.append(nd)
if self.node(
nd
)[4] > 1: # node reference count is greater than one, node is shared by two ways
subways.append(curr_subway)
curr_subway = [nd]
curr_subway.append(
way.nds[-1]
) # add the last node to the current subway, and store the subway
subways.append(curr_subway)
#insert into edge table
for i, subway in enumerate(subways):
coords = [(lambda x: (x[3], x[2]))(self.node(nd))
for nd in subway]
packt = pack_coords(coords)
dist = sum([
vincenty(lat1, lng1, lat2, lng2)
for (lng1, lat1), (lng2, lat2) in cons(coords)
])
c.execute("INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)",
("%s-%s" % (way.id, i), way.id, subway[0],
subway[-1], dist, packt))
except IndexError:
if tolerant:
continue
else:
raise
print "indexing edges...",
c.execute("CREATE INDEX edges_id ON edges (id)")
c.execute("CREATE INDEX edges_parent_id ON edges (parent_id)")
print "done"
self.conn.commit()
c.close()
def edge(self, id):
c = self.get_cursor()
c.execute(
"SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?",
(id, ))
try:
ret = c.next()
way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret
return (way_id, parent_id, from_nd, to_nd, dist,
unpack_coords(geom), json.loads(tags))
except StopIteration:
c.close()
raise IndexError("Database does not have an edge with id '%s'" %
id)
c.close()
return ret
def edges(self):
c = self.get_cursor()
c.execute(
"SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id"
)
for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c:
yield (way_id, parent_id, from_nd, to_nd, dist,
unpack_coords(geom), json.loads(tags))
c.close()
def add_way(self, way, curs=None):
if curs is None:
curs = self.get_cursor()
close_cursor = True
else:
close_cursor = False
curs.execute(
"INSERT OR IGNORE INTO ways (id, tags, nds) VALUES (?, ?, ?)",
(way.id, json.dumps(way.tags), json.dumps(way.nd_ids)))
if close_cursor:
self.conn.commit()
curs.close()
def add_node(self, node, curs=None):
if curs is None:
curs = self.get_cursor()
close_cursor = True
else:
close_cursor = False
curs.execute(
"INSERT OR IGNORE INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)",
(node.id, json.dumps(node.tags), node.lat, node.lon))
if close_cursor:
self.conn.commit()
curs.close()
def nodes(self):
c = self.get_cursor()
c.execute("SELECT * FROM nodes")
for node_row in c:
yield node_row
c.close()
def node(self, id):
c = self.get_cursor()
c.execute("SELECT * FROM nodes WHERE id = ?", (id, ))
try:
ret = c.next()
except StopIteration:
c.close()
raise IndexError("Database does not have node with id '%s'" % id)
c.close()
return ret
def nearest_node(self, lat, lon, range=0.005):
c = self.get_cursor()
if self.index:
#print "YOUR'RE USING THE INDEX"
id = list(self.index.nearest((lon, lat), 1))[0]
#print "THE ID IS %d"%id
c.execute("SELECT id, lat, lon FROM nodes WHERE id = ?", (id, ))
else:
c.execute(
"SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?",
(lat - range, lat + range, lon - range, lon + range))
dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5)
for nid, nlat, nlon in c]
if len(dists) == 0:
return (None, None, None, None)
return min(dists, key=lambda x: x[3])
def nearest_of(self, lat, lon, nodes):
c = self.get_cursor()
c.execute("SELECT id, lat, lon FROM nodes WHERE id IN (%s)" %
",".join([str(x) for x in nodes]))
dists = [(nid, nlat, nlon, ((nlat - lat)**2 + (nlon - lon)**2)**0.5)
for nid, nlat, nlon in c]
if len(dists) == 0:
return (None, None, None, None)
return min(dists, key=lambda x: x[3])
def way(self, id):
c = self.get_cursor()
c.execute("SELECT id, tags, nds FROM ways WHERE id = ?", (id, ))
try:
id, tags_str, nds_str = c.next()
ret = WayRecord(id, tags_str, nds_str)
except StopIteration:
raise Exception("OSMDB has no way with id '%s'" % id)
finally:
c.close()
return ret
def way_nds(self, id):
c = self.get_cursor()
c.execute("SELECT nds FROM ways WHERE id = ?", (id, ))
(nds_str, ) = c.next()
c.close()
return json.loads(nds_str)
def ways(self):
c = self.get_cursor()
c.execute("SELECT id, tags, nds FROM ways")
for id, tags_str, nds_str in c:
yield WayRecord(id, tags_str, nds_str)
c.close()
def count_ways(self):
c = self.get_cursor()
c.execute("SELECT count(*) FROM ways")
ret = c.next()[0]
c.close()
return ret
def count_edges(self):
c = self.get_cursor()
c.execute("SELECT count(*) FROM edges")
ret = c.next()[0]
c.close()
return ret
def delete_way(self, id):
c = self.get_cursor()
c.execute("DELETE FROM ways WHERE id = ?", (id, ))
c.close()
def bounds(self):
c = self.get_cursor()
c.execute("SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes")
ret = c.next()
c.close()
return ret
def execute(self, sql, args=None):
c = self.get_cursor()
if args:
for row in c.execute(sql, args):
yield row
else:
for row in c.execute(sql):
yield row
c.close()
def cursor(self):
return self.get_cursor()
def populate_shared_segments_by_rtree(datasource, verbose=False):
"""
"""
rtree = Rtree()
indexes = datasource._indexes()
for i in indexes:
xmin, ymin, xmax, ymax = datasource.shapes[i].bounds
xbuf = (xmax - xmin) * .001
ybuf = (ymax - ymin) * .001
bounds = (xmin - xbuf, ymin - ybuf, xmax + xbuf, ymax + ybuf)
rtree.add(i, bounds)
shared = [[] for i in indexes]
for i in indexes:
for j in rtree.intersection(datasource.shapes[i].bounds):
if i >= j:
continue
shape1 = datasource.shapes[i]
shape2 = datasource.shapes[j]
if not shape1.intersects(shape2):
continue
if verbose:
print >> stderr, 'Features %d and %d:' % (i, j), 'of', len(
indexes),
border = linemerge(shape1.intersection(shape2))
geoms = hasattr(border, 'geoms') and border.geoms or [border]
for geom in geoms:
try:
line_id = datasource.rtree.count(
datasource.rtree.get_bounds())
except RTreeError:
line_id = 0
coords = list(geom.coords)
segments = [coords[k:k + 2] for k in range(len(coords) - 1)]
for ((x1, y1), (x2, y2)) in segments:
datasource.db.execute(
"""INSERT INTO segments
(src1_id, src2_id, line_id, x1, y1, x2, y2, removed)
VALUES (?, ?, ?, ?, ?, ?, ?, 0)""",
(i, j, line_id, x1, y1, x2, y2))
bbox = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
datasource.rtree.add(datasource.db.lastrowid, bbox)
if verbose:
print >> stderr, len(coords), '-',
shared[i].append(border)
shared[j].append(border)
if verbose:
print >> stderr, border.type
return shared
class ContourServer(Servable):
def __init__(self, settings_filename):
settings = yaml.load( open( settings_filename ) )
self.home_point = settings['center']
# create cache of osm-node positions
self.osmdb = OSMDB( settings['osmdb_filename'] )
self.gtfsdb = GTFSDatabase( settings['gtfsdb_filename'] )
self.port = settings['port']
self.node_positions = {}
self.index = Rtree()
for node_id, tags, lat, lon in self.osmdb.nodes():
self.node_positions[node_id] = (lon,lat)
self.index.add( int(node_id), (lon,lat,lon,lat) )
# incarnate graph from graphdb
graphdb = GraphDatabase( settings['graphdb_filename'] )
self.graph = graphdb.incarnate()
def strgraph(self):
return str(self.graph)
def vertices(self):
return "\n".join( [vv.label for vv in self.graph.vertices] )
def _get_important_routes(self, spt):
# set edge thicknesses
t0 = time.time()
print "setting thicknesses"
spt.set_thicknesses( vertex_label )
t1 = time.time()
print "took %s"%(t1-t0)
t0 = time.time()
print "finding gateway boardings"
# use thicknesses to determine important boardings
origin = spt.get_vertex( vertex_label )
sum_thickness = sum( [edge.thickness for edge in origin.outgoing] )
important_boardings = sorted( filter(lambda x: x.payload.__class__==core.TripBoard and \
x.thickness/float(sum_thickness) > 0.01, spt.edges),
key = lambda x:x.thickness )
for edge in important_boardings:
print "gateway to %f%% vertices"%(100*edge.thickness/float(sum_thickness))
print "hop onto trip '%s' at stop '%s', time '%s'"%(edge.to_v.payload.trip_id, edge.from_v.label, edge.to_v.payload.time)
print "took %ss"%(time.time()-t0)
def _points(self, vertex_label, starttime, cutoff, speed):
starttime = starttime or time.time()
#=== find shortest path tree ===
print "Finding shortest path tree"
t0 = time.time()
wo = WalkOptions()
wo.walking_speed = speed
spt = self.graph.shortest_path_tree( vertex_label, None, State(1,starttime), wo, maxtime=starttime+int(cutoff*1.25) )
wo.destroy()
t1 = time.time()
print "took %s s"%(t1-t0)
#=== cobble together ETA surface ===
print "Creating ETA surface from OSM points..."
points = []
t0 = time.time()
for vertex in spt.vertices:
if "osm" in vertex.label:
x, y = self.node_positions[vertex.label[3:]]
points.append( (x, y, vertex.payload.time-starttime) )
t1 = time.time()
print "Took %s s"%(t1-t0)
spt.destroy()
return points
def _contour(self, vertex_label, starttime, cutoff, step=None, speed=0.85):
points = self._points( vertex_label, starttime, cutoff, speed )
#=== create contour ===
print "creating contour...",
t0 = time.time()
contours = travel_time_contour( points, cutoff=cutoff, cellsize=0.004, fudge=1.7, step=step )
print "%s sec"%(time.time()-t0)
print "done. here you go..."
return contours
def label_contour(self, vertex_label, starttime=None, cutoff=1800):
starttime = starttime or time.time()
return json.dumps( self._contour( vertex_label, starttime, cutoff ) )
def contour(self, lat, lon, year, month, day, hour, minute, second, cutoff, step=60*15, encoded=False, speed=0.85):
if step is not None and step < 600:
raise Exception( "Step cannot be less than 600 seconds" )
starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" )
#=== get osm vertex ==
print( "getting nearest vertex" )
#find osmid of origin intersection
t0 = time.time()
range = 0.001
bbox = (lon-range, lat-range, lon+range, lat+range)
candidates = self.index.intersection( bbox )
vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates )
t1 = time.time()
print( "done, took %s seconds"%(t1-t0) )
#vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon )
if vlabel is None:
return json.dumps( "NO NEARBY INTERSECTION" )
print( "found - %s"%vlabel )
contours = self._contour( "osm"+vlabel, starttime, cutoff, step, speed )
if encoded:
encoded_contours = []
for contour in contours:
encoded_contour = []
for ring in contour:
encoded_contour.append( encode_pairs( [(lat,lon) for lon,lat in ring] ) )
encoded_contours.append( encoded_contour )
contours = encoded_contours
return json.dumps( contours )
def _surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed, cellsize=0.004):
starttime = TimeHelpers.localtime_to_unix( year, month, day, hour, minute, second, "America/Los_Angeles" )
#=== get osm vertex ==
print( "getting nearest vertex" )
#find osmid of origin intersection
t0 = time.time()
range = 0.001
bbox = (lon-range, lat-range, lon+range, lat+range)
candidates = self.index.intersection( bbox )
vlabel, vlat, vlon, vdist = self.osmdb.nearest_of( lat, lon, candidates )
t1 = time.time()
print( "done, took %s seconds"%(t1-t0) )
#vlabel, vlat, vlon, vdist = self.osmdb.nearest_node( lat, lon )
if vlabel is None:
return json.dumps( "NO NEARBY INTERSECTION" )
print( "found - %s"%vlabel )
#=== get points which comprise ETA surface ===
points = self._points( "osm"+vlabel, starttime, cutoff, speed )
#=== create regular grid from ETA surface ===
print "creating surface...",
t0 = time.time()
ret = points_to_surface_grid( points, cutoff=cutoff, fudge=1.1, margin=2, closure_tolerance=0.05, cellsize=cellsize )
#ret = travel_time_surface( points, cutoff=cutoff, cellsize=0.004, fudge=1.7 )
print "%s sec"%(time.time()-t0)
print "done. here you go..."
return ret
def surface(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85):
return json.dumps( self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed).to_matrix() )
def transitability(self, lat, lon, year, month, day, hour, minute, second, cutoff, speed=0.85):
grid = self._surface(lat,lon,year,month,day,hour,minute,second,cutoff,speed)
if type(grid) == str:
return None
ret = 0
for i in range(10):
contourslice=sum( [len(filter(lambda x:x[2]<=(cutoff/10.0)*(i+1),col)) for col in grid] )
print contourslice
ret += contourslice
return ret
def transitability_surface(self, left, bottom, right, top, res, year, month, day, hour, minute, second, cutoff, speed=0.85):
step = (right-left)/res
return json.dumps([[(lon,lat,self.transitability(lat,lon,year,month,day,hour,minute,second,cutoff,speed)) for lat in frange(bottom,top,step)] for lon in frange(left,right,step)])
def nodes(self):
return "\n".join( ["%s-%s"%(k,v) for k,v in self.node_positions.items()] )
def nearest_node(self, lat, lon):
range = 0.005
bbox = (lon-range, lat-range, lon+range, lat+range)
print bbox
print self.index.intersection( bbox )
return json.dumps(self.osmdb.nearest_node( lat, lon ))
def bounds(self):
return json.dumps(self.osmdb.bounds())
def run_test_server(self):
Servable.run_test_server(self, self.port)
class GisCanvas:
"""
Top-level drawing class that contains a list of all the objects to draw.
"""
def __init__(self):
"""Initialise the class by creating an instance of each object."""
self._ratio_index = (0.05, 0.1, 0.2, 0.5, 1, 2, 4, 8, 16, 32, 64) # pagal sita dydi turi kisti tasko atstumas nuo zoominimo centro
self._zoom_level = self._ratio_index.index(4) # 0-dirbame su realiomis koordinatemis - kitur koordinates yra gaunamos atliekant dalyba... todel nera labai tikslios
self._device_area = None # gtk.gdk.Rectangle()
self._shapes = [] # shapes data
self._index_rtree = Rtree() # shapes indexes in self._shapes
self._offset = Point(0, 0) #- naudojamas ctx.translate() #device koordinates
self._prj = GisProjection()
self._styler = Styler(self)
self._painter = Painter(self._styler)
self._drag_to_center = True # jeigu norime kad resize metu (bus iskviestas set_device_area) butu iskvietsas center()
def load(self, name):
self._styler.load_css_file("%s.css" % name)
self.load_fcad_file("%s.fcad" % name)
def save(self, file_name):
#self.create_fcad_file("%s.fcad" % file_name)
ShapeFile.create_fcad_file("%s.fcad" % file_name, self._shapes)
StyleFile.create_css_file("%s.css" % file_name, self._styler.get_shapes_style(), self._styler.get_symbols_style(), prettyprint=True)
#self._styler.create_css_file("%s.css" % file_name, prettyprint=True)
def clear(self):
print "clear canvas!"
self._zoom_level = self._ratio_index.index(1)
self._offset = Point(0, 0)
self._shapes = []
#self._cairo_paths = {}
self._index_rtree = Rtree()
self._styler.load_default_style()
def load_pickle(self, file_path):
timer.start("loading shapes.p")
if os.path.exists(file_path):
self._shapes = pickle.load(open(file_path, "rb"))
if len(self._shapes):
def generator_function(points):
for i, obj in enumerate(points):
if obj == None: continue
yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("loading shapes.p")
def save_pickle(self, file_path):
pickle.dump(self._shapes, open(file_path, "wb"))
def load_fcad_file(self, file_path):
timer.start("loading shapes.fcad")
self._shapes = ShapeFile.read_fcad_file(file_path)
if len(self._shapes):
def generator_function(points):
for i, obj in enumerate(points):
if obj == None: continue
yield (i, self._styler.get_bbox(fshape.Shape.decompress(obj)), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("loading shapes.fcad")
def set_device_area(self, area): # atnaujina screen.resize()
self._device_area = area
if self._drag_to_center:
self.center()
self._drag_to_center = False
def get_shape_by_id(self, id):
compressed_shape = self._shapes[id]
if compressed_shape == None: return None
return fshape.Shape.decompress(compressed_shape)
def get_object_by_id(self, id):
return self._shapes[id]
def draw_100(self, ctx, area):
"""100% draw for printing, no scale, area in user coordinates"""
page_area = Area(0, 0, area.width, area.height)
#ctx.rectangle(0, 0, area.width, area.height)
#ctx.clip() # tam kad nepaisytu uzh sito staciakampio ribu (tada gaunasi dubliuotos linijos)
#self.background(ctx, page_area) # paint white background
self._painter.background(ctx, page_area, color=(1,1,1), clip=True) # paint white background
self._painter.setup(ctx, transform={"translate":(-area.x, -area.y)})
radius = 0 #self.pixel_radius + self.line_width # ne cia reikia prideti radiusa, o ten kur dedame i cavas'a shape'us
elements = self._index_rtree.intersection((area.x-radius, area.y-radius, area.x+area.width+radius, area.y+area.height+radius))
elements_zindex = self._styler.create_zindex(elements) # jis yra pilnas visu galimu zindex'u sarasas - pradzioje dalis ju gali buti ir tusti []
timer.start("draw100")
for zindex in sorted(elements_zindex.keys()):
for element in elements_zindex[zindex]:
self._painter.draw(element, update=elements_zindex) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u
# tie elementai bus nupaisomi veliau.
timer.end("draw100")
def draw_object(self, ctx, id, fill_or_stroke=True):
#self.apply_transforms(ctx) # drag and scale
self._painter.setup(ctx, transform={"translate":self.get_offset(), "scale": self.get_ratio()})
elements_zindex = self._styler.create_zindex([id])
timer.start("draw single object")
for zindex in sorted(elements_zindex.keys()):
for element in elements_zindex[zindex]:
self._painter.draw(element, update=elements_zindex, fill_or_stroke=fill_or_stroke) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u
timer.end("draw single object")
def draw(self, ctx, area, fill_or_stroke=True):
"""Draw the complete drawing by drawing each object in turn."""
self._painter.background(ctx, area, color=(1,1,1), clip=True) # paint white background
self._painter.setup(ctx, transform={"translate":self.get_offset(), "scale": self.get_ratio()})
# paishysime tik tuos tashkus kurie pakliuna i vartotojo langa
# reikia device_area konvertuoti i user koordinates
x, y = self.device_to_user(Point(area.x, area.y))
x2, y2 = self.device_to_user(Point(area.x + area.width, area.y + area.height))
radius = 0 #self.pixel_radius + self.line_width # ne cia reikia prideti radiusa, o ten kur dedame i cavas'a shape'us
# geriau cia, nes paprasciau yra iskviesti nupaisyti didesni gabala nei paskaiciuoti tikslu pvz linijo simboliu dydi...
elements = self._index_rtree.intersection((x-radius, y-radius, x2+radius, y2+radius))
elements_zindex = self._styler.create_zindex(elements) # jis yra pilnas visu galimu zindex'u sarasas - pradzioje dalis ju gali buti ir tusti []
timer.start("draw")
for zindex in sorted(elements_zindex.keys()):
for element in elements_zindex[zindex]:
#print "element: ", element[0][0]
self._painter.draw(element, update=elements_zindex, fill_or_stroke=fill_or_stroke) # kazka visada nupaiso - bet dar papildomai gali iterpti elementu su didesniu zindex'u
# tie elementai bus nupaisomi veliau.
timer.end("draw")
def get_ratio(self, level=None):
if level == None: level = self._zoom_level
return self._ratio_index[level]
def drag2(self, drag_offset):
self._offset = Point(self._offset.x + drag_offset.x, self._offset.y + drag_offset.y)
def get_offset(self):
return self._offset
def find_objects_at_position(self, point):
#ctx = self.get_context(transform=False) # naujas kontekstas kur paisysime, transformuoti nereikia - nes tai padarys .draw()
ctx = cairo.Context(cairo.ImageSurface(cairo.FORMAT_ARGB32, 0, 0)) # naujas kontekstas kur paisysime, transformuoti nereikia - nes tai padarys .draw()
area = Area(point.x, point.y, 1, 1)
listener = ContextObjectsListener(point)
listener_id = self._painter.addContextListener(listener)
try:
self.draw(ctx, area, fill_or_stroke=False) # nepaisysime tik sukursime path'us context'e ir su jais kazka atliks ContextListeneris
finally:
self._painter.removeContextListener(listener_id)
return listener.get_objects() # rastu elementu indeksai
def get_shape_redraw_area(self, id):
#ctx = self.get_context(transform=False)
ctx = cairo.Context(cairo.ImageSurface(cairo.FORMAT_ARGB32, 0, 0))
shape = self.get_object_by_id(id)
if shape == None: return None
listener = ContextBoundsListener()
listener_id = self._painter.addContextListener(listener)
try:
self.draw_object(ctx, id, fill_or_stroke=False)
finally:
self._painter.removeContextListener(listener_id)
area = listener.get_area()
#print "Area: ", area
return area
def device_to_user(self, point):
#x, y = self.ctx.device_to_user(point.x, point.y)
#x, y = self.get_context().device_to_user(point.x, point.y)
ratio = self.get_ratio()
offset = self.get_offset()
x, y = (point.x - offset.x)/ratio, (point.y - offset.y)/ratio
return Point(x, y)
def user_to_device(self, point, offset=(0, 0)):
#x, y = self.ctx.user_to_device(point.x, point.y)
#x, y = self.get_context().user_to_device(point.x, point.y)
ratio = self.get_ratio()
drag_offset = self.get_offset()
x, y = (point.x * ratio) + drag_offset.x, (point.y * ratio) + drag_offset.y
return Point(x + offset[0], y + offset[1])
def add(self, shape):
id = len(self._shapes) # top element index
self._shapes.append(shape.compress())
self._index_rtree.add(id, self._styler.get_bbox(shape))
return id
def remove(self, id):
shape = self.get_shape_by_id(id)
self._index_rtree.delete(id, shape.bbox()) # po sito as jau niekada tokio id negausiu (nes viskas eina per rtree)
self._shapes[id] = None
def replace(self, id, shape):
old = self.get_shape_by_id(id)
#print "before :", list(self._index_rtree.intersection(old.bbox()))
if old != None:
self._index_rtree.delete(id, old.bbox())
#print "after :", list(self._index_rtree.intersection(old.bbox()))
self._shapes[id] = shape.compress()
self._index_rtree.add(id, self._styler.get_bbox(shape))
def zoom(self, direction, center=None):
"""center cia yra device koordinatemis - jeigu butu paspausta su zoom irankiu peles pagalba"""
new_zoom_level = self._zoom_level+direction
if new_zoom_level in range(0, len(self._ratio_index)):
#esamas centro taskas atlikus zooma turi islikti toje pacioje vietoje
center = center or Point(self._device_area.width/2, self._device_area.height/2) # vartotojo parinktas tashkas arba tiesiog centras
center_user = self.device_to_user(center)
# gauname priesh tai buvusio centro koordinates naujame zoom lygyje
new_ratio = self.get_ratio(new_zoom_level)
new_center = Point(center_user.x * new_ratio, center_user.y * new_ratio)
# gauname centro poslinki (per tiek reikia perstumti visa vaizdeli)
self._offset = Point(center.x - new_center.x, center.y - new_center.y) # naujas poslinkis
#print "zoom: ", self._offset
self._zoom_level = new_zoom_level
return True
return False
def center(self, point=None):
"""center to user point"""
if not point:
bounds = self._index_rtree.bounds
point = Point((bounds[0]+bounds[2])/2.0, (bounds[1]+bounds[3])/2.0)
hand = self.user_to_device(point)
to = Point(self._device_area.width/2, self._device_area.height/2)
self.drag2(Point(to.x-hand.x, to.y-hand.y))
def get_projection(self):
return self._prj
def get_styler(self):
return self._styler
def load_ocad_file(self, file_path, generator=True):
#import ocadfile
#self.add(Shape(1, Point(0, 0), symbol="graphics")) # koordinaciu centras
self._prj = GisProjection()
self._zoom_level = self._ratio_index.index(4)
of = OcadFile(file_path, self._prj)
#self._styler.load_ocad_symbols(of, prj)
timer.start("Adding ocad symbols")
self._styler.set_symbols_style(of.get_symbols_style())
timer.end("Adding ocad symbols")
timer.start("Adding ocad elements")
shapes = of.get_shapes()
print "Shapes: ", len(shapes)
for shape in shapes:
self.add(shape)
timer.end("Adding ocad elements")
self.center()
def load_shape_file(self, file_path, generator=True):
import shapefile
from random import randint
sf = shapefile.Reader(file_path)
print "Number of shapes: ", sf.numRecords
self.add(fshape.Shape(1, Point(0, 0), style={"z-index":99})) # koordinaciu centras
if self.prj.scale == 1: # pirmas kartas
prj = GisProjection(self, sf.bbox)
#po sito ciklo jau turesime zemelapio ribas
center = prj.map_to_user(prj.get_center())
self.center(center)
self.prj = prj
timer.start("Adding shapes")
symbol = sf.shapeName.split("/")[-1]
self._styler.set_symbol_style(symbol, {"color": (randint(0,255),randint(0,255),randint(0,255))})
for shape in sf.ogis_shapes(self.prj):
self.add(fshape.Shape(shape.shapeType, shape.points, symbol=symbol))
timer.end("Adding shapes")
def add_random_points(self, number, area, generator=True):
"""Kai generator=False - sunaudoja maziau atminties ikrovimo metu, bet trunka gerokai leciau
"""
self._shapes = []
timer.start("Adding random data")
from random import randint
for x in range(0, number):
color = 65536 * randint(0,255) + 256 * randint(0,255) + randint(0,255) # RGBint
x, y = randint(2, area.width), randint(2, area.height)
if not generator: # darysime rtree.add kiekvienam taskui atskirai
self.add(fshape.Shape(1, Point(x, y), color=color))
else:
self._shapes.append((1, color, x, y))
if generator:
def generator_function(points):
for i, obj in enumerate(points):
yield (i, (obj[2], obj[3], obj[2], obj[3]), obj)
self._index_rtree = Rtree(generator_function(self._shapes))
timer.end("Adding random data")
def add(self, id_, coords):
with threading.RLock():
Rtree.add(self, id_, coords)
from shapely.geometry import Point, Polygon
from shapely import wkt
from rtree import Rtree
import sys
import struct
output = sys.argv[1]
data_filename = output + '.data'
index_filename = output + '.data.idx'
data = open(data_filename, 'wb')
index = open(index_filename, 'wb')
tree = Rtree(output, overwrite=1)
id = 0
offset = 0
for l in sys.stdin:
x = wkt.loads(l)
tree.add(id, x.bounds)
s = len(x.wkb)
data.write(x.wkb)
index.write(struct.pack('L', offset))
offset += s
id += 1
data.close()
index.close()
def geometry_spatial_index(geometries):
spatial_index = Rtree()
for g,geom in enumerate(geometries):
spatial_index.add(g,geom.bounds)
return spatial_index
# TODO: path hackery.
if __name__ == "__main__":
import sys, os
mypath = os.path.dirname(sys.argv[0])
sys.path.append(os.path.abspath(os.path.join(mypath, "../../")))
from pyrtree.bench.bench_rtree import ITER, INTERVAL
from tests import RectangleGen
import time
from rtree import Rtree
if __name__ == "__main__":
G = RectangleGen()
idx = Rtree() # this is a libspatialindex one.
start = time.clock()
interval_start = time.clock()
for v in range(ITER):
if 0 == (v % INTERVAL):
# interval time taken, total time taken, # rects, cur max depth
t = time.clock()
dt = t - interval_start
print("%d,%s,%f" % (v, "itime_t", dt))
print("%d,%s,%f" % (v, "avg_insert_t", (dt / float(INTERVAL))))
#print("%d,%s,%d" % (v, "max_depth", rt.node.max_depth()))
#print("%d,%s,%d" % (v, "mean_depth", rt.node.mean_depth()))
interval_start = time.clock()
rect = G.rect(0.000001)
idx.add(v, rect.coords())
from rtree import Rtree idx = Rtree() idx.add(5, (1,2,1,2)) idx.add(1, (2,0,2,0)) idx.add(2, (3,0,3,0)) idx.add(3, (4,0,4,0)) idx.add(4, (5,0,5,0)) idx2 = Rtree() idx2.add(10, (0,0,0,0)) idx2.add(11, (1,1,1,1)) print 'Dimensions: %d' % (idx.properties.dimension,) h = idx.hausdorff(idx2) id1 = h[1].contents.value id2 = h[2].contents.value print 'Hausdorff distance: %.2f, %d, %d' % (h[0], id1, id2)
def main():
# You can edit these vars:
# -----------------------
filename = 'matrix_plot3.png'
numberOfPoints = 200
gridWidth = 10000
gridHeight = 10000
edgesMinimum = 2
edgesMaximum = 4 # The maximum number of edges (connecting lines) a vertex can have
# -----------------------
edgesMaxDistance = (gridWidth / numberOfPoints) * 2
# instantiate the Rtree class
RtreeIndex = Rtree()
# CREATE THE RANDOM POINTS:
shapelyPointIndex = [
] # Stores index = shapely object - We will reference this point from its Index from this point on.
for idx in range(
numberOfPoints): # Each point gets its own index defined by i
x = random.randint(0, gridWidth)
y = random.randint(0, gridHeight)
iPoint = Point(x, y) # Create a new shapely POINT object
shapelyPointIndex.append(iPoint)
RtreeIndex.add(idx, (x, y, x, y)) # Add the point to the RTREE Index
del idx #unset temp variable
# --------------------------------------------------------------------------
#MultiPoint([(0, 0), (1, 1)])
#point.coords
#point.y point.x
lineMatrices = []
ComplexLines = []
graphIndex = {
} # This is a dictionary we will build into a graph/matrix - Each key will be an integer representing an index
# and each value will be a dictionary of dictionaries representing edges.
# iterate over all points to calculate each points nearest points
# build line segments for each point to near points.
# add all edges to a master complex line to compute intersection.
for idx, point in enumerate(shapelyPointIndex):
# Find nearby points quickly
rtreeMatches = list()
sRadiusX = gridWidth / 10
sRadiusY = gridHeight / 10
tries = 0
while (len(rtreeMatches) < edgesMinimum +
1): # Add 1 because we can assume it will always find itself
# Coords is a bounding-box to search within.
coords = (point.x - sRadiusX, point.y - sRadiusY,
point.x + sRadiusX, point.y + sRadiusY)
print "searching coords"
print coords
rtreeMatches = list(RtreeIndex.intersection(coords))
print "found: ", rtreeMatches, " ", len(rtreeMatches)
tries += 1
sRadiusX *= 1.5
sRadiusY *= 1.5
if (tries > 6): #Don't run forever if there is a logic problem.
break
del tries, sRadiusX, sRadiusY
# FIND POSSIBLE EDGES TO ADD BY SEARCHING REGION RECURSIVELY -----------
# AND CALCULATE DISTANCES TO POSSIBLE POINTS ---------------------------
# This dictionary will store our distances to each other point from this point.
edgeDistances = {}
# Calculate distances to matched points
for pointIdx in rtreeMatches:
if (pointIdx == idx):
continue # Don't record a distance to itself
mPoint = shapelyPointIndex[pointIdx]
edgeDistances[pointIdx] = round(
mPoint.distance(point),
2) # keep the distance accuracy reasonable
del mPoint, pointIdx, rtreeMatches
# create a list that references the distances index for the closest indexes and is sorted by close to far.
edgeDistancesIdx = sorted(edgeDistances, key=edgeDistances.get)
#-----------------------------------------------------------------------
# DECIDE WHICH EDGES TO ADD --------------------------------------------
# Now add a minimum number of points, and try to keep them within a logic distance
edgesAdded = 0
edgesIdx = [] # this will contain indexes to our added edges
for pointIdx in edgeDistancesIdx:
distance = edgeDistances[pointIdx]
# Stop adding edges if we've meant the minimum number of edges needed and the rest of the edges are too far away, or we have reach the maximum number of allowed edges
if ((edgesAdded >= edgesMinimum and distance > edgesMaxDistance)
or edgesAdded >= edgesMaximum):
break
edgesIdx.append(pointIdx)
del pointIdx
#-----------------------------------------------------------------------
# Initialize the graphIndex as a dict() for the current vertex if it hasn't been.
if (idx not in graphIndex):
graphIndex[idx] = {}
#ADD THE EDGES TO THE GRID STRUCTURE -----------------------------------
for pointIdx in edgesIdx:
# Add the edge
graphIndex[idx][pointIdx] = edgeDistances[pointIdx]
# Add the reverse edge (so both points share an edge)
if pointIdx not in graphIndex:
graphIndex[pointIdx] = {}
graphIndex[pointIdx][idx] = edgeDistances[pointIdx]
#-----------------------------------------------------------------------
# Print out the graph of vertices
pprint.pprint(graphIndex)
# Randomely select a Start Vertex, and an End Vertex for Shortest Path
# calculation
startIdx = random.randint(0, len(graphIndex) - 1)
endIdx = startIdx
# Calculate a random end index - and make sure its not the same as the start
while (endIdx == startIdx):
endIdx = random.randint(0, len(graphIndex) - 1)
# Setup shortest path calculation!
print "Calculating Shortest Path:\n"
# (8894.6959143222557, [11, 5, 8L, 7, 9])
# GraphIndex is a dictionary of dictionaries, startIdx is the INDEX of the startpoint, endIdx is the index of the endpoint vertex.
shortestPath = dijkstra.shortestpath(graphIndex, startIdx, endIdx)
shortestPathVertices = shortestPath[1]
shortestPathDistance = shortestPath[0]
print "Shortest Path Vertices:"
print shortestPathVertices, "\n"
del shortestPath
#_-------------------------------------------------------------------------------
# DRAW A PLOT FOR THE LINES CALCULATED ABOVE:
import matplotlib
matplotlib.use('Agg') # Do NOT attempt to open X11 instance
from pylab import *
from matplotlib.patches import Circle
import matplotlib.pyplot as pyplot
from matplotlib.text import *
mylist = lineMatrices
matplotlib.rcParams['lines.linewidth'] = 2
pyplot.axis([0, gridWidth, 0, gridHeight])
pyplot.grid(True)
# Setting the axis labels.
pyplot.xlabel('X Space')
pyplot.ylabel('Y Space')
figure = pyplot.figure()
labels = figure.add_subplot(111, projection='rectilinear')
#Give the plot a title
pyplot.title('Shapely Shortest Path Simulation')
del idx
if not isinstance(mylist, list):
print "The matrix is not a list!"
print type(mylist).__name__
for idx, edges in graphIndex.iteritems():
vertex = shapelyPointIndex[idx]
for edgeIdx, edgeDistance in edges.iteritems():
# This shouldn't happen, but lets just make sure.
# DONT PLOT A PATH TO OURSELF
if (edgeIdx == idx):
continue
# Get the edges point-coordinates
edgePoint = shapelyPointIndex[edgeIdx]
# Delete the reverse edge so we don't plot the line twice.
if (edgeIdx in graphIndex):
del graphIndex[edgeIdx][idx]
# Print Debuggin Information
print "PLOTTING EDGE: [%s] to [%s] X:[%s] Y:[%s] DIS:[%s] " % (
idx, edgeIdx, edgePoint.x, edgePoint.y, edgeDistance)
# Plot the edge!
pyplot.plot([vertex.x, edgePoint.x], [vertex.y, edgePoint.y],
'b-',
alpha=0.3,
linewidth=.2,
markersize=1)
# FIND THE MID-POINT OF THE LINE:
xMidpoint = (vertex.x + edgePoint.x) / 2
yMidpoint = (vertex.y + edgePoint.y) / 2
#print "Plotting Text [%s] [%s]" % (xMidpoint, yMidpoint)
# Figure out arrows later... arrowprops=dict(facecolor='black', linewidth=2, alpha=0.4),
# ANNOTATE THE LENGTH OF THE LINE AT ITS MIDPOINT
labels.annotate(edgeDistance,
xy=(xMidpoint, yMidpoint),
xytext=(xMidpoint + 80, yMidpoint),
alpha=0.4,
size='4',
color='green')
# end for ----------------------------------------------------------
# Plot the VERTEX as a Red Circle
pyplot.plot(vertex.x, vertex.y, 'ro', linewidth=2, markersize=2)
# Annotate the VERTEX by its INDEX
labels.annotate(idx,
xy=(vertex.x, vertex.y),
xytext=(vertex.x + 50, vertex.y),
alpha=0.6,
size='6',
color='red')
print "Graphing Shortest Path..."
print "START: %s END: %s" % (startIdx, endIdx)
#shortestPathVertices
for idx, vertexIdx in enumerate(shortestPathVertices):
if (vertexIdx == endIdx):
break # We are done
xy1 = shapelyPointIndex[vertexIdx] # this is the start of the line
vertex2 = shortestPathVertices[
idx +
1] # The end of the line is the next item in the list shortest path list of indexes
xy2 = shapelyPointIndex[vertex2] #this is the end of the line
pyplot.plot([xy1.x, xy2.x], [xy1.y, xy2.y],
'r--',
alpha=0.7,
linewidth=2,
markersize=1)
print "PLOT -> SHORTEST PATH: [%s] to [%s]" % (vertexIdx, vertex2)
print "Saving Plot Image %s" % (filename)
pyplot.savefig(os.getcwd() + '/' + str(filename), dpi=240)
print "\n\n"
"""
if __name__ == "__main__":
import sys, os
mypath = os.path.dirname(sys.argv[0])
sys.path.append(os.path.abspath(os.path.join(mypath, "../../")))
from pyrtree.bench.bench_rtree import ITER,INTERVAL
from pyrtree.tests.test_rtree import RectangleGen,TstO
import time
from rtree import Rtree
if __name__ == "__main__":
G = RectangleGen()
idx = Rtree() # this is a libspatialindex one.
start = time.clock()
interval_start = time.clock()
for v in range(ITER):
if 0 == (v % INTERVAL):
# interval time taken, total time taken, # rects, cur max depth
t = time.clock()
dt = t - interval_start
print("%d,%s,%f" % (v, "itime_t", dt))
print("%d,%s,%f" % (v, "avg_insert_t", (dt/float(INTERVAL))))
#print("%d,%s,%d" % (v, "max_depth", rt.node.max_depth()))
#print("%d,%s,%d" % (v, "mean_depth", rt.node.mean_depth()))
interval_start = time.clock()
rect = G.rect(0.000001)
idx.add(v,rect.coords())
def populate_shared_segments_by_rtree(datasource, verbose=False):
"""
"""
rtree = Rtree()
indexes = datasource._indexes()
for i in indexes:
xmin, ymin, xmax, ymax = datasource.shapes[i].bounds
xbuf = (xmax - xmin) * .001
ybuf = (ymax - ymin) * .001
bounds = (xmin-xbuf, ymin-ybuf, xmax+xbuf, ymax+ybuf)
rtree.add(i, bounds)
shared = [[] for i in indexes]
for i in indexes:
for j in rtree.intersection(datasource.shapes[i].bounds):
if i >= j:
continue
shape1 = datasource.shapes[i]
shape2 = datasource.shapes[j]
if not shape1.intersects(shape2):
continue
if verbose:
print >> stderr, 'Features %d and %d:' % (i, j), 'of', len(indexes),
border = linemerge(shape1.intersection(shape2))
geoms = hasattr(border, 'geoms') and border.geoms or [border]
for geom in geoms:
try:
line_id = datasource.rtree.count(datasource.rtree.get_bounds())
except RTreeError:
line_id = 0
coords = list(geom.coords)
segments = [coords[k:k+2] for k in range(len(coords) - 1)]
for ((x1, y1), (x2, y2)) in segments:
datasource.db.execute("""INSERT INTO segments
(src1_id, src2_id, line_id, x1, y1, x2, y2, removed)
VALUES (?, ?, ?, ?, ?, ?, ?, 0)""",
(i, j, line_id, x1, y1, x2, y2))
bbox = min(x1, x2), min(y1, y2), max(x1, x2), max(y1, y2)
datasource.rtree.add(datasource.db.lastrowid, bbox)
if verbose:
print >> stderr, len(coords), '-',
shared[i].append(border)
shared[j].append(border)
if verbose:
print >> stderr, border.type
return shared
shapePoints = [] pointList = [] for i in range(numberOfPoints): x = random.randint(0, gridWidth) y = random.randint(0, gridHeight) # Add the points to a python list of lists pointList.append((x, y)) # Create a Shapely point object iPoint = Point(x, y) iPoint.idx = i # set our custom attribute. (if this doesnt work I have other ways) shapePoints.append(iPoint) # Add the point to an RTREE index # index.add(id=id, (left, bottom, right, top)) # Note that this would work if the RtreeIndex.add(i, (x, y, x, y)) matchingPoints = [] searchBench = time.time() for idx, point in enumerate(shapePoints): if sCircle.contains(point): matchingPoints.append(idx) searchBench = time.time() - searchBench #------------------------------------------------------------------------------- # NOW TEST RTREE RtreeBench = time.time() RtreeIndex rtreeMatches = list(RtreeIndex.intersection((boxLeft, boxBottom, boxRight, boxTop)))
class Datasource:
""" Store an exploded representation of a data source, so it can be simplified.
"""
def __init__(self, srs, geom_type, fields, values, shapes):
""" Use load() to call this constructor.
"""
self.srs = srs
self.fields = fields
self.geom_type = geom_type
self.values = values
self.shapes = shapes
# this will be changed later
self.tolerance = 0
# guid, src1_id, src2_id, line_id, x1, y1, x2, y2
db = connect(':memory:').cursor()
db.execute("""CREATE table segments (
-- global identifier for this segment
guid INTEGER PRIMARY KEY AUTOINCREMENT,
-- identifiers for source shape or shapes for shared borders
src1_id INTEGER,
src2_id INTEGER,
-- global identifier for this line
line_id INTEGER,
-- start and end coordinates for this segment
x1 REAL,
y1 REAL,
x2 REAL,
y2 REAL,
-- flag
removed INTEGER
)""")
db.execute('CREATE INDEX segments_lines ON segments (line_id, guid)')
db.execute('CREATE INDEX shape1_parts ON segments (src1_id)')
db.execute('CREATE INDEX shape2_parts ON segments (src2_id)')
self.db = db
self.rtree = Rtree()
self.memo_line = make_memo_line()
def _indexes(self):
return range(len(self.values))
def simplify(self, tolerance, verbose=False):
""" Simplify the polygonal linework.
This method can be called multiple times, but the process is
destructive so it must be called with progressively increasing
tolerance values.
"""
if tolerance < self.tolerance:
raise Exception('Repeat calls to simplify must have increasing tolerances.')
self.tolerance = tolerance
q = 'SELECT line_id, COUNT(guid) AS guids FROM segments WHERE removed=0 GROUP BY line_id order by guids DESC'
line_ids = [line_id for (line_id, count) in self.db.execute(q)]
stable_lines = set()
while True:
was = self.db.execute('SELECT COUNT(*) FROM segments WHERE removed=0').fetchone()[0]
preserved, popped = set(), False
for line_id in line_ids:
if line_id in stable_lines:
continue
# For each coordinate that forms the apex of a two-segment
# triangle, find the area of that triangle and put it into a list
# along with the segment identifier and the resulting line if the
# triangle were flattened, ordered from smallest to largest.
rows = self.db.execute("""SELECT guid, x1, y1, x2, y2
FROM segments
WHERE line_id = ?
AND removed = 0
ORDER BY guid""",
(line_id, ))
segs = [(guid, (x1, y1), (x2, y2)) for (guid, x1, y1, x2, y2) in rows]
triples = [(segs[i][0], segs[i+1][0], segs[i][1], segs[i][2], segs[i+1][2]) for i in range(len(segs) - 1)]
triangles = [(guid1, guid2, Polygon([c1, c2, c3, c1]), c1, c3) for (guid1, guid2, c1, c2, c3) in triples]
areas = sorted( [(triangle.area, guid1, guid2, c1, c3) for (guid1, guid2, triangle, c1, c3) in triangles] )
min_area = self.tolerance ** 2
if not areas or areas[0][0] > min_area:
# there's nothing to be done
stable_lines.add(line_id)
if verbose:
stderr.write('-')
continue
# Reduce any segments that makes a triangle whose area is below
# the minimum threshold, starting with the smallest and working up.
# Mark segments to be preserved until the next iteration.
for (area, guid1, guid2, ca, cb) in areas:
if area > min_area:
# there won't be any more points to remove.
break
if guid1 in preserved or guid2 in preserved:
# the current segment is too close to a previously-preserved one.
continue
# Check the resulting flattened line against the rest
# any of the original shapefile, to determine if it would
# cross any existing line segment.
(x1, y1), (x2, y2) = ca, cb
new_line = self.memo_line(x1, y1, x2, y2)
old_guids = self.rtree.intersection(bbox(x1, y1, x2, y2))
old_rows = self.db.execute('SELECT x1, y1, x2, y2 FROM segments WHERE guid IN (%s) AND removed=0' % ','.join(map(str, old_guids)))
old_lines = [self.memo_line(x1, y1, x2, y2) for (x1, y1, x2, y2) in old_rows]
if True in [new_line.crosses(old_line) for old_line in old_lines]:
if verbose:
stderr.write('x%d' % line_id)
continue
preserved.add(guid1)
preserved.add(guid2)
popped = True
x1, y1, x2, y2 = ca[0], ca[1], cb[0], cb[1]
self.db.execute('UPDATE segments SET removed=1 WHERE guid=%d' % guid2)
self.db.execute('UPDATE segments SET x1=?, y1=?, x2=?, y2=? WHERE guid=?',
(x1, y1, x2, y2, guid1))
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
stderr.write('.')
if verbose:
print >> stderr, ' reduced from', was, 'to',
print >> stderr, self.db.execute('SELECT COUNT(guid) FROM segments WHERE removed=0').fetchone()[0],
self.rtree = Rtree()
for (guid, x1, y1, x2, y2) in self.db.execute('SELECT guid, x1, y1, x2, y2 FROM segments WHERE removed=0'):
self.rtree.add(guid1, bbox(x1, y1, x2, y2))
if verbose:
print >> stderr, '.'
if not popped:
break
class GraphicScene(object):
ITEM_SELECTION_RADIUS = 2 # px
_logger = logging.getLogger(__name__)
##############################################
def __init__(self, glortho2d):
self._glortho2d = glortho2d # Fixme: used for window_to_scene_coordinate window_to_scene_distance
self._items = {}
self._rtree = Rtree()
self._selected_item = None
##############################################
def _window_to_scene_coordinate(self, event):
location = np.array((event.x(), event.y()), dtype=np.float)
return list(self._glortho2d.window_to_gl_coordinate(location))
##############################################
def _window_to_scene_distance(self, x):
return self._glortho2d.window_to_gl_distance(x)
##############################################
def add_item(self, item):
item_id = hash(item)
self._items[item_id] = item
self._rtree.add(item_id, item.bounding_box)
##############################################
def remove_item(self, item):
if self._selected_item is item:
self.deselect_item(item)
item_id = hash(item)
del self._items[item_id]
self._rtree.delete(item_id, item.bounding_box)
##############################################
def select_item(self, item):
item.is_selected = True
self._selected_item = item
##############################################
def deselect_item(self, item):
item.is_selected = False
self._selected_item = None
##############################################
def items_in(self, interval):
self._logger.debug(str( interval))
items = [self._items[x] for x in self._rtree.intersection(interval.bounding_box())]
items.sort() # accordind to z value
return items
##############################################
def items_at(self, x, y):
return self.items_in(point_interval(x, y))
##############################################
def items_around(self, x, y, radius):
return self.items_in(centred_interval(x, y, radius))
##############################################
def item_under_mouse(self, event):
x, y = self._window_to_scene_coordinate(event)
radius = self._window_to_scene_distance(GraphicScene.ITEM_SELECTION_RADIUS)
items = self.items_around(x, y, radius)
if items:
# return the highest z value
# Fixme: ok ?
return items[-1]
else:
return None
##############################################
def keyPressEvent(self, event):
return False
##############################################
def keyReleaseEvent(self, event):
return False
##############################################
def mousePressEvent(self, event):
item = self.item_under_mouse(event)
if item is not None:
self.select_item(item)
item.mousePressEvent(event)
return True
else:
return False
##############################################
def mouseReleaseEvent(self, event):
if self._selected_item is not None:
self._selected_item.mouseReleaseEvent(event)
self.deselect_item(self._selected_item)
return True
else:
return False
##############################################
def mouseDoubleClickEvent(self, event):
return False
##############################################
def wheelEvent(self, event):
return False
##############################################
def mouseMoveEvent(self, event):
if self._selected_item is not None:
self._selected_item.mouseMoveEvent(event)
return True
else:
return False
class IntRtreeIndex(BaseIndex):
"""Avoids the slower Rtree query object=True interface
"""
_v_nextuid = None
family = BTrees.family32
def clear(self):
self.fwd = Rtree()
self.bwd = self.family.OO.BTree()
self.keys = self.family.IO.BTree()
self.intids = self.family.OI.BTree()
self.ids = self.family.OO.BTree()
def __init__(self):
self.clear()
def key(self, item):
try:
return item['id'], tuple(self.bbox(item))
except:
return tuple(item.items())
def fid(self, item):
return item['id']
def intid(self, item):
# Get and track next available key using zope.intid algorithm
# Item might be already registered
uid = self.intids.get(self.key(item))
if uid is not None:
return uid
# But if not registered
nextuid = getattr(self, '_v_nextuid', None)
while True:
if nextuid is None:
nextuid = random.randrange(0, self.family.maxint)
uid = nextuid
if uid not in self.keys:
nextuid += 1
if nextuid > self.family.maxint:
nextuid = None
self._v_nextuid = nextuid
return uid
nextuid = None
def intersection(self, bbox):
"""Return an iterator over Items that intersect with the bbox"""
for hit in self.fwd.intersection(bbox, objects=False):
yield self.bwd[int(hit)]
def nearest(self, bbox, limit=1):
"""Return an iterator over the nearest N=limit Items to the bbox"""
for hit in self.fwd.nearest(bbox, num_results=limit, objects=False):
yield self.bwd[int(hit)]
def item(self, fid, bbox):
return self.bwd[self.intids[(fid, bbox)]]
def items(self, fid):
return [self.bwd[intid] for intid in self.ids[fid]]
def index_item(self, itemid, bbox, item):
"""Add an Item to the index"""
if itemid in self.bwd:
self.unindex_item(itemid, bbox)
# Store an id for the item if it has None
try:
item.update(id=item.get('id') or str(uuid.uuid4()))
key = self.key(item)
sid = self.fid(item)
# Map keys <-> intids
intid = self.intid(item)
self.keys[intid] = key
self.intids[key] = intid
if sid not in self.ids:
self.ids[sid] = IISet([])
self.ids[sid].add(intid)
self.bwd[intid] = item
self.fwd.add(intid, bbox)
except:
import pdb; pdb.set_trace()
raise
def unindex_item(self, itemid, bbox):
"""Remove an Item from the index"""
intid = int(itemid)
key = self.keys.get(intid)
if key is None:
return
self.ids[key[0]].remove(intid)
del self.keys[intid]
del self.intids[key]
del self.bwd[intid]
self.fwd.delete(intid, bbox)
def batch(self, changeset):
BaseIndex.batch(self, changeset)
def commit(self):
transaction.commit()
rtree_storage = self.fwd.properties.filename
self.fwd.close()
self.fwd = Rtree(rtree_storage)
def close(self):
self.fwd.close()
class OSMDB:
def __init__(self, dbname,overwrite=False,rtree_index=True):
if overwrite:
try:
os.remove( dbname )
except OSError:
pass
self.conn = sqlite3.connect(dbname)
if rtree_index:
self.index = Rtree( dbname )
else:
self.index = None
if overwrite:
self.setup()
def setup(self):
c = self.conn.cursor()
c.execute( "CREATE TABLE nodes (id TEXT, tags TEXT, lat FLOAT, lon FLOAT, endnode_refs INTEGER DEFAULT 1)" )
c.execute( "CREATE TABLE ways (id TEXT, tags TEXT, nds TEXT)" )
self.conn.commit()
c.close()
def create_indexes(self):
c = self.conn.cursor()
c.execute( "CREATE INDEX nodes_id ON nodes (id)" )
c.execute( "CREATE INDEX nodes_lon ON nodes (lon)" )
c.execute( "CREATE INDEX nodes_lat ON nodes (lat)" )
c.execute( "CREATE INDEX ways_id ON ways (id)" )
self.conn.commit()
c.close()
def populate(self, osm_filename, accept=lambda tags: True, reporter=None):
print "importing osm from XML to sqlite database"
c = self.conn.cursor()
self.n_nodes = 0
self.n_ways = 0
superself = self
class OSMHandler(xml.sax.ContentHandler):
@classmethod
def setDocumentLocator(self,loc):
pass
@classmethod
def startDocument(self):
pass
@classmethod
def endDocument(self):
pass
@classmethod
def startElement(self, name, attrs):
if name=='node':
self.currElem = Node(attrs['id'], float(attrs['lon']), float(attrs['lat']))
elif name=='way':
self.currElem = Way(attrs['id'])
elif name=='tag':
self.currElem.tags[attrs['k']] = attrs['v']
elif name=='nd':
self.currElem.nd_ids.append( attrs['ref'] )
@classmethod
def endElement(self,name):
if name=='node':
if superself.n_nodes%5000==0:
print "node %d"%superself.n_nodes
superself.n_nodes += 1
superself.add_node( self.currElem, c )
elif name=='way':
if superself.n_ways%5000==0:
print "way %d"%superself.n_ways
superself.n_ways += 1
superself.add_way( self.currElem, c )
@classmethod
def characters(self, chars):
pass
xml.sax.parse(osm_filename, OSMHandler)
self.conn.commit()
c.close()
print "indexing primary tables...",
self.create_indexes()
print "done"
def set_endnode_ref_counts( self ):
"""Populate ways.endnode_refs. Necessary for splitting ways into single-edge sub-ways"""
print "counting end-node references to find way split-points"
c = self.conn.cursor()
endnode_ref_counts = {}
c.execute( "SELECT nds from ways" )
print "...counting"
for i, (nds_str,) in enumerate(c):
if i%5000==0:
print i
nds = json.loads( nds_str )
for nd in nds:
endnode_ref_counts[ nd ] = endnode_ref_counts.get( nd, 0 )+1
print "...updating nodes table"
for i, (node_id, ref_count) in enumerate(endnode_ref_counts.items()):
if i%5000==0:
print i
if ref_count > 1:
c.execute( "UPDATE nodes SET endnode_refs = ? WHERE id=?", (ref_count, node_id) )
self.conn.commit()
c.close()
def index_endnodes( self ):
print "indexing endpoint nodes into rtree"
c = self.conn.cursor()
#TODO index endnodes if they're at the end of oneways - which only have one way ref, but are still endnodes
c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1" )
for id, lat, lon in c:
self.index.add( int(id), (lon, lat, lon, lat) )
c.close()
def create_and_populate_edges_table( self, tolerant=False ):
self.set_endnode_ref_counts()
self.index_endnodes()
print "splitting ways and inserting into edge table"
c = self.conn.cursor()
c.execute( "CREATE TABLE edges (id TEXT, parent_id TEXT, start_nd TEXT, end_nd TEXT, dist FLOAT, geom TEXT)" )
for i, way in enumerate(self.ways()):
try:
if i%5000==0:
print i
subways = []
curr_subway = [ way.nds[0] ] # add first node to the current subway
for nd in way.nds[1:-1]: # for every internal node of the way
curr_subway.append( nd )
if self.node(nd)[4] > 1: # node reference count is greater than one, node is shared by two ways
subways.append( curr_subway )
curr_subway = [ nd ]
curr_subway.append( way.nds[-1] ) # add the last node to the current subway, and store the subway
subways.append( curr_subway );
#insert into edge table
for i, subway in enumerate(subways):
coords = [(lambda x:(x[3],x[2]))(self.node(nd)) for nd in subway]
packt = pack_coords( coords )
dist = sum([vincenty(lat1, lng1, lat2, lng2) for (lng1, lat1), (lng2, lat2) in cons(coords)])
c.execute( "INSERT INTO edges VALUES (?, ?, ?, ?, ?, ?)", ("%s-%s"%(way.id, i),
way.id,
subway[0],
subway[-1],
dist,
packt) )
except IndexError:
if tolerant:
continue
else:
raise
print "indexing edges...",
c.execute( "CREATE INDEX edges_id ON edges (id)" )
c.execute( "CREATE INDEX edges_parent_id ON edges (parent_id)" )
print "done"
self.conn.commit()
c.close()
def edge(self, id):
c = self.conn.cursor()
c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id AND edges.id = ?", (id,) )
try:
ret = c.next()
way_id, parent_id, from_nd, to_nd, dist, geom, tags = ret
return (way_id, parent_id, from_nd, to_nd, dist, unpack_coords( geom ), json.loads(tags))
except StopIteration:
c.close()
raise IndexError( "Database does not have an edge with id '%s'"%id )
c.close()
return ret
def edges(self):
c = self.conn.cursor()
c.execute( "SELECT edges.*, ways.tags FROM edges, ways WHERE ways.id = edges.parent_id" )
for way_id, parent_id, from_nd, to_nd, dist, geom, tags in c:
yield (way_id, parent_id, from_nd, to_nd, dist, unpack_coords(geom), json.loads(tags))
c.close()
def add_way( self, way, curs=None ):
if curs is None:
curs = self.conn.cursor()
close_cursor = True
else:
close_cursor = False
curs.execute("INSERT INTO ways (id, tags, nds) VALUES (?, ?, ?)", (way.id, json.dumps(way.tags), json.dumps(way.nd_ids) ))
if close_cursor:
self.conn.commit()
curs.close()
def add_node( self, node, curs=None ):
if curs is None:
curs = self.conn.cursor()
close_cursor = True
else:
close_cursor = False
curs.execute("INSERT INTO nodes (id, tags, lat, lon) VALUES (?, ?, ?, ?)", ( node.id, json.dumps(node.tags), node.lat, node.lon ) )
if close_cursor:
self.conn.commit()
curs.close()
def nodes(self):
c = self.conn.cursor()
c.execute( "SELECT * FROM nodes" )
for node_row in c:
yield node_row
c.close()
def node(self, id):
c = self.conn.cursor()
c.execute( "SELECT * FROM nodes WHERE id = ?", (id,) )
try:
ret = c.next()
except StopIteration:
c.close()
raise IndexError( "Database does not have node with id '%s'"%id )
c.close()
return ret
def nearest_node(self, lat, lon, range=0.005):
c = self.conn.cursor()
if self.index:
print "YOU'RE USING THE INDEX"
id = list(self.index.nearest( (lon, lat), 1 ))[0]
print "THE ID IS %d"%id
c.execute( "SELECT id, lat, lon FROM nodes WHERE id = ?", (id,) )
else:
c.execute( "SELECT id, lat, lon FROM nodes WHERE endnode_refs > 1 AND lat > ? AND lat < ? AND lon > ? AND lon < ?", (lat-range, lat+range, lon-range, lon+range) )
dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]
if len(dists)==0:
return (None, None, None, None)
return min( dists, key = lambda x:x[3] )
def nearest_of( self, lat, lon, nodes ):
c = self.conn.cursor()
c.execute( "SELECT id, lat, lon FROM nodes WHERE id IN (%s)"%",".join([str(x) for x in nodes]) )
dists = [(nid, nlat, nlon, ((nlat-lat)**2+(nlon-lon)**2)**0.5) for nid, nlat, nlon in c]
if len(dists)==0:
return (None, None, None, None)
return min( dists, key = lambda x:x[3] )
def way(self, id):
c = self.conn.cursor()
c.execute( "SELECT id, tags, nds FROM ways WHERE id = ?", (id,) )
try:
id, tags_str, nds_str = c.next()
ret = WayRecord(id, tags_str, nds_str)
except StopIteration:
raise Exception( "OSMDB has no way with id '%s'"%id )
finally:
c.close()
return ret
def way_nds(self, id):
c = self.conn.cursor()
c.execute( "SELECT nds FROM ways WHERE id = ?", (id,) )
(nds_str,) = c.next()
c.close()
return json.loads( nds_str )
def ways(self):
c = self.conn.cursor()
c.execute( "SELECT id, tags, nds FROM ways" )
for id, tags_str, nds_str in c:
yield WayRecord( id, tags_str, nds_str )
c.close()
def count_ways(self):
c = self.conn.cursor()
c.execute( "SELECT count(*) FROM ways" )
ret = c.next()[0]
c.close()
return ret
def count_edges(self):
c = self.conn.cursor()
c.execute( "SELECT count(*) FROM edges" )
ret = c.next()[0]
c.close()
return ret
def delete_way(self, id):
c = self.conn.cursor()
c.execute("DELETE FROM ways WHERE id = ?", (id,))
c.close()
def bounds(self):
c = self.conn.cursor()
c.execute( "SELECT min(lon), min(lat), max(lon), max(lat) FROM nodes" )
ret = c.next()
c.close()
return ret
def execute(self,sql,args=None):
c = self.conn.cursor()
if args:
for row in c.execute(sql,args):
yield row
else:
for row in c.execute(sql):
yield row
c.close()
def cursor(self):
return self.conn.cursor()
class MapMatcher():
"""A very nice MapMatching class
"""
def __init__(self):
self.GPS = GPS()
self.idx = Rtree()
self.nodeidx = Rtree()
self.G = None
self.edgeindex_edge = {}
self.edgecounter = 0
self.nodecounter = 0
self.gps_points = [];
self.edge_id__count = {}
self.node_counter__node = {}
self.distance_matrix = {}
def saveGraph(self, filename):
"""Saves the graph as a YAML file
"""
nx.write_yaml(self.G,filename)
def readGraphFromYAMLFile(self, filename):
"""Loads a graph from an YAML file.
"""
self.G = nx.read_yaml(filename)
# TODO: buiild up the indexes !!!
def addNodeToIndex(self, node):
"""Adds a node to the node index (RTree)
"""
# self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1]), obj=node)
self.nodeidx.add(self.nodecounter, (node.getPoint()[0], node.getPoint()[1], node.getPoint()[0], node.getPoint()[1]))
self.node_counter__node[self.nodecounter] = node
def addEdgeToIndex(self, edge):
"""Add an edge to the edhe index.
"""
self.idx.add(self.edgecounter, (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY()),obj=edge)
# print "%d/%d -> %d/%d" % (edge.getMinX(), edge.getMinY(), edge.getMaxX(), edge.getMaxY())
self.edgeindex_edge[self.edgecounter] = edge
self.edgecounter = self.edgecounter + 1
def openShape(self, inFile, index=0):
self.shapeFile = ogr.Open(inFile)
if self.shapeFile is None:
print "Failed to open " + inFile + ".\n"
sys.exit( 1 )
else:
print "SHP file successfully read"
def getfieldinfo(self, lyr, feature, flds):
f = feature
return [f.GetField(f.GetFieldIndex(x)) for x in flds]
def addlyr(self, G,lyr, fields):
point_coords__nodes = {}
for findex in xrange(lyr.GetFeatureCount()):
f = lyr.GetFeature(findex)
flddata = self.getfieldinfo(lyr, f, fields)
g = f.geometry()
attributes = dict(zip(fields, flddata))
attributes["ShpName"] = lyr.GetName()
if g.GetGeometryType() == 2: #linestring
last = g.GetPointCount() - 1
p_from = g.GetPoint_2D(0)
p_to = g.GetPoint_2D(last)
# check whether we have a node in the index
intersection_mask = (p_from[0]-INTERSECTION_MASK/2,
p_from[1]-INTERSECTION_MASK/2,
p_from[0]+INTERSECTION_MASK/2,
p_from[1]+INTERSECTION_MASK/2)
results = list(self.nodeidx.intersection(intersection_mask))
if len(results)==0:
#print "New from-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "."
pfrom = Node(p_from, attributes={'from_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
self.node_counter__node[self.nodecounter] = pfrom
self.nodeidx.add(self.nodecounter, (p_from[0],
p_from[1],
p_from[0],
p_from[1]))
# print p_from
self.nodecounter = self.nodecounter + 1
else:
#print len(results)
#print "From-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "."
pfrom = self.node_counter__node[results[0]]
intersection_mask = (p_to[0]-INTERSECTION_MASK/2,
p_to[1]-INTERSECTION_MASK/2,
p_to[0]+INTERSECTION_MASK/2,
p_to[1]+INTERSECTION_MASK/2)
# print intersection_mask
results = list(self.nodeidx.intersection(intersection_mask))
if len(results)==0:
#print "New to-node " + str(self.nodecounter) + " for edge " + str(attributes.get("ID_NR")) + "."
pto = Node(p_to, attributes={'to_edge':attributes.get(self.shapeFileUniqueId), "nodecounter":self.nodecounter})
self.node_counter__node[self.nodecounter] = pto
self.nodeidx.add(self.nodecounter, (p_to[0],
p_to[1],
p_to[0],
p_to[1]))
self.nodecounter = self.nodecounter + 1
else:
#print "To-node " + str(results[0]) + " recycled for edge " + str(attributes.get("ID_NR")) + "."
pto = self.node_counter__node[results[0]]
shly_geom = shapely.wkt.loads(g.ExportToWkt())
e = Edge(pfrom, pto, attributes, geometry = shly_geom)
# G.add_edge(pfrom, pto, {"edge": e, "edgecounter" : self.edgecounter})
G.add_edge(pfrom, pto, edge=e, edgecounter=self.edgecounter)
self.addEdgeToIndex(e)
return G
def shapeToGraph(self, inFile, uniqueId="FID"):
"""Loads a shapefile and builds the graph.
uniqueId is the name of a unique field in the shape file.
"""
# self.G = nx.readwrite.nx_shp.read_shp(inFile)
self.G = nx.MultiGraph()
self.shapeFileUniqueId = uniqueId
lyrcount = self.shapeFile.GetLayerCount() # multiple layers indicate a directory
for lyrindex in xrange(lyrcount):
lyr = self.shapeFile.GetLayerByIndex(lyrindex)
flds = [x.GetName() for x in lyr.schema]
self.G=self.addlyr(self.G, lyr, flds)
self.routefinder = RouteFinder(self.G, euclideanOD=self.distance_matrix)
def readGPS(self, inFile):
"""Parses a shapefile and build the GPS object
"""
self.GPS.readFromShapeFile(inFile)
self.gps_points = self.GPS.getGPSPoints()
def maxGPSDistance(self):
"""Calculate the maximum distance of two consecutive GPS Points
"""
# TODO check whether GPS points are already there
# TODO: move into sl.gps.GPS()
maxDistance = 0
gps_point = self.gps_points[0]
for gpspoint in self.gps_points:
distance = gpspoint.getGeometry().distance(gps_point.getGeometry())
gps_point = gps_point
if distance > maxDistance:
maxDistance = distance
return maxDistance
def nearPoints(self):
"""Sums up the gps point per edge segment. Stores in self.edge_id__count
"""
# initialize the edge counter
for edge in self.G.edges():
self.edge_id__count[self.G[edge[0]][edge[1]].get("edgecounter")] = 0
for point in self.gps_points:
nearest_edge = self.getNearestEdge(point)
# print str(point.getAttributes().get("ID")) + "->" + str(nearest_edge.getAttributes().get('Id'))
self.addPointCountToEdge(nearest_edge)
def addPointCountToEdge(self, edge):
"""Increments the point counter for the given edge by one.
"""
attributes = edge.getAttributes()
if self.edge_id__count.has_key(attributes.get(self.shapeFileUniqueId)):
self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = self.edge_id__count[attributes.get(self.shapeFileUniqueId)] + 1
else:
self.edge_id__count[attributes.get(self.shapeFileUniqueId)] = 1
edge.setAttributes(attributes)
def getNearestEdge(self, point):
"""Returns the edge closes to a Shapely entity given (point)
"""
edge = mm.idx.nearest((point.getPoint().x, point.getPoint().y), objects=True)
edges = [e.object for e in edge]
if len(edges) == 1:
result = edges[0]
else:
dist = 99999999999999999999999999999999999999999
for edge in edges:
distance = point.getPoint().distance(edge.getGeometry())
if distance < dist:
dist = distance
result = edge
return result
def getNearestNode(self, point):
"""Returns the closest node to a GPS point.
"""
nodes = list(mm.nodeidx.nearest((point.getPoint().x, point.getPoint().y)))
return self.node_counter__node.get(nodes[0])
def find_all_paths(self, graph, start, end, path=[]):
path = path + [start]
if start == end:
return [path]
if not graph.has_key(start):
return []
paths = []
for node in graph[start]:
if node not in path:
newpaths = self.find_all_paths(graph, node, end, path)
for newpath in newpaths:
paths.append(newpath)
return paths
def findRoutes2(self):
start_point = self.gps_points[0]
end_point = self.gps_points[-1]
start_node = self.getNearestNode(start_point)
end_node = self.getNearestNode(end_point)
graph = {}
print "preparing python graph"
for node in self.node_counter__node.values():
graph[node.getNodeID()] = [n.getOutNode2(node).getNodeID() for n in node.getOutEdges()]
import pprint
pprint.pprint( graph )
print "From ", start_node.getNodeID(), " to ", end_node.getNodeID(), "."
results = self.find_all_paths(graph, start_node.getNodeID(), end_node.getNodeID())
# let us find the edges
route_list = []
for result in results:
i=1
edges = []
while i<len(result):
node_from = self.node_counter__node.get(result[i-1] )
node_to = self.node_counter__node.get(result[i] )
i = i +1
edge_dict = self.G[node_from][node_to]
if edge_dict.keys()>1:
lngth = 9E99999
for k in edge_dict.keys():
if edge_dict[k]['edge'].getLength() < lngth:
lngth = edge_dict[k]['edge'].getLength()
edge = edge_dict[k]['edge']
else:
edge = edge_dict[0]['edge']
edges.append(edge)
route_list.append(Route(edges))
factor__selected_route = {}
for route in route_list:
number_of_points = 0
length = 0
for edge in route.getEdges():
# import pdb;pdb.set_trace()
length = length + edge.getLength()
edge_id = edge.getAttributes().get(self.shapeFileUniqueId)
number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0)
if number_of_points > 1:
factor__selected_route[number_of_points/length] = route
keys = factor__selected_route.keys()
keys.sort()
return factor__selected_route.get(keys[0])
def findRoute(self, returnNonSelection=False):
"""Finds a route from the node closest to the first GPS point to
the node closest to the latest GPS point.
"""
# pick the start and end GPS points # TODO: sort GPS Points first
start_point = self.gps_points[0]
end_point = self.gps_points[-1]
start_node = self.getNearestNode(start_point)
end_node = self.getNearestNode(end_point)
# the start and endnodes returns by the index are not in the graph,
# therefore we need to look them up ....
start_node = self.node_counter__node.get(start_node.getAttributes().get("nodecounter"))
end_node = self.node_counter__node.get(end_node.getAttributes().get("nodecounter"))
self.routfinder = RouteFinder(self.G, euclideanOD=self.distance_matrix)
label_list = self.routefinder.findroutes(start_node, end_node)
label_scores = []
# import pdb;pdb.set_trace()
# let us loop through the label list
for label in label_list:
number_of_points = 0
# we sum up the number of points and relate them to the length of the route
print label
for edge in label.getEdges():
edge_id = edge.getAttributes().get(self.shapeFileUniqueId)
number_of_points = number_of_points + self.edge_id__count.get(edge_id, 0)
print " ", number_of_points
#we add the scores to a dict
if number_of_points > 1:
label_scores.append((label, number_of_points/label.getLength()))
# print label_scores
# and extract the maximum score
score = 0
selected = None
for ls in label_scores:
if ls[1] > score:
selected = ls[0]
score = ls[1]
if returnNonSelection:
pass
else:
return selected
def eliminiateEmptyEdges(self, distance = 100):
"""Loops through the GPS pointset and selects edges within a boundary
of <distance> meters
"""
print "Edge elimination started"
selected_edge_ids = []
# let us
for point in self.gps_points:
results = self.idx.nearest(((point.getPoint().x-distance/2),
(point.getPoint().y-distance/2),
(point.getPoint().x+distance/2),
(point.getPoint().y+distance/2)), objects=True)
for result in results:
from_node = self.node_counter__node.get(result.object.from_node.getAttributes().get("nodecounter"))
to_node = self.node_counter__node.get(result.object.to_node.getAttributes().get("nodecounter"))
edge_counter = self.G.edge[from_node][to_node].get("edgecounter")
if edge_counter not in selected_edge_ids:
selected_edge_ids.append(edge_counter)
print str(len(selected_edge_ids)) + " edges found to keep."
elimination_counter = 0
for edge in self.G.edges():
edgecounter = self.G.edge[edge[0]][edge[1]].get("edgecounter")
if edgecounter not in selected_edge_ids:
edge_tuple = (self.G.edge[edge[0]][edge[1]].get("edge").from_node, self.G.edge[edge[0]][edge[1]].get("edge").to_node)
self.G.remove_edge(*edge_tuple)
elimination_counter = elimination_counter + 1
print str(elimination_counter) + " edges eliminated."
def dumpPointShape(self, filename, original_coverage=None):
if filename:
driverName = "ESRI Shapefile"
drv = ogr.GetDriverByName( driverName )
if drv:
drv.DeleteDataSource(filename)
if drv is None:
print "%s driver not available.\n" % driverName
ds = drv.CreateDataSource( filename)
lyr = ds.CreateLayer( "blabla", None, ogr.wkbPoint )
if lyr is None:
print "Layer creation failed.\n"
field_defn = ogr.FieldDefn( "node_count", ogr.OFTInteger )
if lyr.CreateField ( field_defn ) != 0:
print "Creating Name field failed.\n"
sys.exit( 1 )
field_defn = ogr.FieldDefn( "edge_list", ogr.OFTString )
field_defn.SetWidth( 1024)
if lyr.CreateField ( field_defn ) != 0:
print "Creating Name field failed.\n"
sys.exit( 1 )
for node in self.node_counter__node.values():
feat = ogr.Feature( lyr.GetLayerDefn() )
nc = node.getAttributes().get("nodecounter")
print nc
# import pdb;pdb.set_trace()
feat.SetField( "node_count", nc )
s = ""
for edge in node.getOutEdges():
s = s + str(int(edge.getAttributes().get("ID_NR"))) + ", "
feat.SetField( "edge_list", s )
node_entity = ogr.Geometry(ogr.wkbPoint)
# wkb = edge.getGeometry().to_wkb()
node_entity.SetPoint_2D(0,node.getGeometry().x, node.getGeometry().y)
feat.SetGeometry(node_entity)
lyr.CreateFeature(feat)
feat.Destroy()
print "Shapefile (%s) written." % filename
shapePoints = []
pointList = []
for i in range(numberOfPoints):
x = random.randint(0, gridWidth)
y = random.randint(0, gridHeight)
# Add the points to a python list of lists
pointList.append((x, y))
# Create a Shapely point object
iPoint = Point(x, y)
iPoint.idx = i # set our custom attribute. (if this doesnt work I have other ways)
shapePoints.append(iPoint)
# Add the point to an RTREE index
# index.add(id=id, (left, bottom, right, top))
# Note that this would work if the
RtreeIndex.add(i, (x, y, x, y))
matchingPoints = []
searchBench = time.time()
for idx, point in enumerate(shapePoints):
if sCircle.contains(point):
matchingPoints.append(idx)
searchBench = time.time() - searchBench
#-------------------------------------------------------------------------------
# NOW TEST RTREE
RtreeBench = time.time()
RtreeIndex
rtreeMatches = list(
RtreeIndex.intersection((boxLeft, boxBottom, boxRight, boxTop)))
''' RTREE is a spatial index that answers the question -> What points are within this 2-dimensional box. RTREE helps us speed up other calculations by only considering legitimate objects for more complex comparisons. ''' from rtree import Rtree # instantiate the Rtree class idx = Rtree() # create a selection boundary (2d bounding box) minx, miny, maxx, maxy = (0.0, 0.0, 1.0, 1.0) # add an item to the index idx.add(0, (minx, miny, maxx, maxy)) # Intersect another bounding box with the original bounding box. list(idx.intersection((1.0, 1.0, 2.0, 2.0))) #[0L] # Point out the accuracy of the spatial calculation list(idx.intersection((1.0000001, 1.0000001, 2.0, 2.0))) #[] #add another item to the index. - show EXACT dimensiion matching index.add(id=id, (left, bottom, right, top)) object = [n for n in index.intersection((left, bottom, right, top))] #[id] # Find nearest object / bounding box idx.add(1, (minx, miny, maxx, maxy)) list(idx.nearest((1.0000001, 1.0000001, 2.0, 2.0), 1)) #[0L, 1L]
from shapely.geometry import Point, Polygon
from shapely import wkt
from rtree import Rtree
import sys
import struct
output = sys.argv[1]
data_filename = output + '.data'
index_filename = output + '.data.idx'
data = open(data_filename, 'wb')
index = open(index_filename, 'wb')
tree = Rtree(output, overwrite = 1)
id = 0
offset = 0
for l in sys.stdin:
x = wkt.loads(l)
tree.add(id, x.bounds)
s = len(x.wkb)
data.write(x.wkb)
index.write(struct.pack('L', offset))
offset += s
id += 1
data.close()
index.close()
class GraphicScene(object):
ITEM_SELECTION_RADIUS = 2 # px
_logger = _module_logger.getChild('GraphicScene')
##############################################
def __init__(self):
self._items = OrderedDict()
self._rtree = Rtree()
self._selected_items = OrderedDict()
##############################################
def add_item(self, item):
item_id = item._id
self._items[item_id] = item
self._rtree.add(item_id, item.bounding_box)
item._scene = self
##############################################
def remove_item(self, item):
if item in self._selected_items:
item.deselect()
item_id = item._id
del self._items[item_id]
self._rtree.delete(item_id, item.bounding_box)
item._scene = None
##############################################
def add_items(self, items):
for item in items:
self.add_item(item)
##############################################
def remove_items(self, items):
for item in items:
self.remove_item(item)
##############################################
def items_in(self, interval):
items = [self._items[x] for x in self._rtree.intersection(interval.bounding_box())]
items.sort() # accordind to z value
return items
##############################################
def items_at(self, x, y):
# Fixme: vector or x, y
return self.items_in(point_interval(x, y))
##############################################
def items_around(self, x, y, radius):
return self.items_in(centred_interval(x, y, radius))
##############################################
def __iter__(self):
return iter(self._items.values())
##############################################
def _select_item(self, item):
self._selected_items[item._id] = self
##############################################
def _deselect_item(self, item):
del self._selected_items[item._id]
##############################################
def iter_on_selected_items(self):
return self._selected_items.values()
##############################################
def erase(self, position, radius):
removed_items = []
new_items = []
for item in self:
# for path in self.items_around(position.x, position.y, radius):
# erase return None, the item or an iterable of new items
items = item.erase(position, radius)
if items is not item:
self._logger.info("Erase item {}".format(item.id))
removed_items.append(item)
if items is not None:
new_items.extend(items)
self.remove_items(removed_items)
self.add_items(new_items)
return removed_items, new_items
def main():
# You can edit these vars:
# -----------------------
filename='matrix_plot3.png'
numberOfPoints = 200
gridWidth = 10000
gridHeight = 10000
edgesMinimum = 2
edgesMaximum = 4 # The maximum number of edges (connecting lines) a vertex can have
# -----------------------
edgesMaxDistance = (gridWidth / numberOfPoints) * 2
# instantiate the Rtree class
RtreeIndex = Rtree()
# CREATE THE RANDOM POINTS:
shapelyPointIndex = [] # Stores index = shapely object - We will reference this point from its Index from this point on.
for idx in range(numberOfPoints): # Each point gets its own index defined by i
x = random.randint(0, gridWidth)
y = random.randint(0, gridHeight)
iPoint = Point(x, y) # Create a new shapely POINT object
shapelyPointIndex.append(iPoint)
RtreeIndex.add(idx, (x, y, x, y)) # Add the point to the RTREE Index
del idx #unset temp variable
# --------------------------------------------------------------------------
#MultiPoint([(0, 0), (1, 1)])
#point.coords
#point.y point.x
lineMatrices = []
ComplexLines = []
graphIndex = {} # This is a dictionary we will build into a graph/matrix - Each key will be an integer representing an index
# and each value will be a dictionary of dictionaries representing edges.
# iterate over all points to calculate each points nearest points
# build line segments for each point to near points.
# add all edges to a master complex line to compute intersection.
for idx, point in enumerate(shapelyPointIndex):
# Find nearby points quickly
rtreeMatches = list()
sRadiusX = gridWidth / 10
sRadiusY = gridHeight / 10
tries = 0
while(len(rtreeMatches) < edgesMinimum+1): # Add 1 because we can assume it will always find itself
# Coords is a bounding-box to search within.
coords = (point.x-sRadiusX, point.y-sRadiusY, point.x+sRadiusX, point.y+sRadiusY)
print "searching coords"
print coords
rtreeMatches = list(RtreeIndex.intersection(coords))
print "found: ", rtreeMatches, " ", len(rtreeMatches)
tries +=1
sRadiusX *= 1.5
sRadiusY *= 1.5
if(tries > 6): #Don't run forever if there is a logic problem.
break
del tries, sRadiusX, sRadiusY
# FIND POSSIBLE EDGES TO ADD BY SEARCHING REGION RECURSIVELY -----------
# AND CALCULATE DISTANCES TO POSSIBLE POINTS ---------------------------
# This dictionary will store our distances to each other point from this point.
edgeDistances={}
# Calculate distances to matched points
for pointIdx in rtreeMatches:
if(pointIdx == idx):
continue # Don't record a distance to itself
mPoint = shapelyPointIndex[pointIdx]
edgeDistances[pointIdx] = round(mPoint.distance(point), 2) # keep the distance accuracy reasonable
del mPoint, pointIdx, rtreeMatches
# create a list that references the distances index for the closest indexes and is sorted by close to far.
edgeDistancesIdx = sorted(edgeDistances, key=edgeDistances.get)
#-----------------------------------------------------------------------
# DECIDE WHICH EDGES TO ADD --------------------------------------------
# Now add a minimum number of points, and try to keep them within a logic distance
edgesAdded = 0
edgesIdx=[] # this will contain indexes to our added edges
for pointIdx in edgeDistancesIdx:
distance = edgeDistances[pointIdx]
# Stop adding edges if we've meant the minimum number of edges needed and the rest of the edges are too far away, or we have reach the maximum number of allowed edges
if((edgesAdded >= edgesMinimum and distance > edgesMaxDistance) or edgesAdded >= edgesMaximum):
break
edgesIdx.append(pointIdx)
del pointIdx
#-----------------------------------------------------------------------
# Initialize the graphIndex as a dict() for the current vertex if it hasn't been.
if(idx not in graphIndex):
graphIndex[idx] = {}
#ADD THE EDGES TO THE GRID STRUCTURE -----------------------------------
for pointIdx in edgesIdx:
# Add the edge
graphIndex[idx][pointIdx] = edgeDistances[pointIdx]
# Add the reverse edge (so both points share an edge)
if pointIdx not in graphIndex:
graphIndex[pointIdx] = {}
graphIndex[pointIdx][idx] = edgeDistances[pointIdx]
#-----------------------------------------------------------------------
# Print out the graph of vertices
pprint.pprint(graphIndex)
# Randomely select a Start Vertex, and an End Vertex for Shortest Path
# calculation
startIdx = random.randint(0, len(graphIndex)-1)
endIdx = startIdx
# Calculate a random end index - and make sure its not the same as the start
while(endIdx == startIdx):
endIdx = random.randint(0, len(graphIndex)-1)
# Setup shortest path calculation!
print "Calculating Shortest Path:\n"
# (8894.6959143222557, [11, 5, 8L, 7, 9])
# GraphIndex is a dictionary of dictionaries, startIdx is the INDEX of the startpoint, endIdx is the index of the endpoint vertex.
shortestPath = dijkstra.shortestpath(graphIndex, startIdx, endIdx)
shortestPathVertices = shortestPath[1]
shortestPathDistance = shortestPath[0]
print "Shortest Path Vertices:"
print shortestPathVertices, "\n"
del shortestPath
#_-------------------------------------------------------------------------------
# DRAW A PLOT FOR THE LINES CALCULATED ABOVE:
import matplotlib
matplotlib.use('Agg') # Do NOT attempt to open X11 instance
from pylab import *
from matplotlib.patches import Circle
import matplotlib.pyplot as pyplot
from matplotlib.text import *
mylist = lineMatrices
matplotlib.rcParams['lines.linewidth'] = 2
pyplot.axis([0, gridWidth, 0, gridHeight])
pyplot.grid(True)
# Setting the axis labels.
pyplot.xlabel('X Space')
pyplot.ylabel('Y Space')
figure = pyplot.figure()
labels = figure.add_subplot(111, projection='rectilinear')
#Give the plot a title
pyplot.title('Shapely Shortest Path Simulation')
del idx
if not isinstance(mylist, list):
print "The matrix is not a list!"
print type(mylist).__name__
for idx, edges in graphIndex.iteritems():
vertex = shapelyPointIndex[idx]
for edgeIdx, edgeDistance in edges.iteritems():
# This shouldn't happen, but lets just make sure.
# DONT PLOT A PATH TO OURSELF
if(edgeIdx == idx):
continue
# Get the edges point-coordinates
edgePoint = shapelyPointIndex[edgeIdx]
# Delete the reverse edge so we don't plot the line twice.
if(edgeIdx in graphIndex):
del graphIndex[edgeIdx][idx]
# Print Debuggin Information
print "PLOTTING EDGE: [%s] to [%s] X:[%s] Y:[%s] DIS:[%s] " % (idx, edgeIdx, edgePoint.x, edgePoint.y, edgeDistance)
# Plot the edge!
pyplot.plot([vertex.x, edgePoint.x], [vertex.y, edgePoint.y], 'b-', alpha=0.3, linewidth=.2, markersize=1)
# FIND THE MID-POINT OF THE LINE:
xMidpoint = (vertex.x + edgePoint.x) / 2
yMidpoint = (vertex.y + edgePoint.y) / 2
#print "Plotting Text [%s] [%s]" % (xMidpoint, yMidpoint)
# Figure out arrows later... arrowprops=dict(facecolor='black', linewidth=2, alpha=0.4),
# ANNOTATE THE LENGTH OF THE LINE AT ITS MIDPOINT
labels.annotate(edgeDistance, xy=(xMidpoint, yMidpoint), xytext=(xMidpoint+80, yMidpoint), alpha=0.4, size='4', color='green')
# end for ----------------------------------------------------------
# Plot the VERTEX as a Red Circle
pyplot.plot(vertex.x, vertex.y, 'ro', linewidth=2, markersize=2)
# Annotate the VERTEX by its INDEX
labels.annotate(idx, xy=(vertex.x, vertex.y), xytext=(vertex.x+50, vertex.y), alpha=0.6, size='6', color='red')
print "Graphing Shortest Path..."
print "START: %s END: %s" % (startIdx, endIdx)
#shortestPathVertices
for idx, vertexIdx in enumerate(shortestPathVertices):
if(vertexIdx == endIdx):
break # We are done
xy1 = shapelyPointIndex[vertexIdx] # this is the start of the line
vertex2 = shortestPathVertices[idx+1] # The end of the line is the next item in the list shortest path list of indexes
xy2 = shapelyPointIndex[vertex2] #this is the end of the line
pyplot.plot([xy1.x, xy2.x], [xy1.y, xy2.y], 'r--', alpha=0.7, linewidth=2, markersize=1)
print "PLOT -> SHORTEST PATH: [%s] to [%s]" % (vertexIdx, vertex2)
print "Saving Plot Image %s" % (filename)
pyplot.savefig(os.getcwd()+'/'+str(filename), dpi=240)
print "\n\n"
"""
