def isRingValid(self, points):
"""
Check if ring (ordered list of points) is well formed.
Return False if self-intersecting or self-touching.
"""
assert points[0] == points[-1], ("Ring not closed %r <-> %r" %
(points[0], points[-1]))
# LinearRing should be simple
# - point (except first/last) appearing once
# - no intersecting line
uniqpoints = set(points)
if len(uniqpoints) < len(points) - 1:
for coord in points[:-1]:
try:
uniqpoints.remove(coord)
except KeyError:
logo.DEBUG("Duplicate point in ring at %s" % (coord, ))
return False
crossing = findLineIntersection(points)
if crossing:
for coord in crossing.values():
logo.DEBUG("Ring self-intersect at %s" % (coord, ))
return False
return True
def main():
logo.init(filename=caop_config.logfile,
verbose=caop_config.verbose,
progress=caop_config.progress)
if len(sys.argv) < 2:
raise logo.ERROR("Missing input Shapefile")
logo.DEBUG("Connect to DB(%s)" % caop_config.dbname)
db = psycopg2.connect(caop_config.dbname)
if not check_db_caop(db):
logo.INFO("Creating PostgreSQL tables")
create_caop_table(db)
create_temp_table(db)
shapeu = ShapeUtil(caop_config.cachesize)
for i in xrange(1, len(sys.argv)):
logo.INFO("Reading geometries '%s'" % sys.argv[i])
read_CAOP(sys.argv[i], shapeu)
logo.INFO("Simplify geometries")
shapeu.buildSimplifiedLines()
logo.INFO("Building administrative area")
admins = {}
for i in xrange(1, len(sys.argv)):
admin_CAOP(sys.argv[i], shapeu, admins)
logo.INFO("Verifying administrative area")
verify_admin(shapeu, admins)
logo.INFO("Importing into database")
import_caop(db, shapeu, admins)
vacuum_analyze_db(db)
logo.close()
def vacuum_analyze_db(db):
""" Update DB statistics. """
logo.DEBUG("Vacuum Analyze")
isolation_level = db.isolation_level
db.set_isolation_level(0)
cursor = db.cursor()
cursor.execute("VACUUM ANALYZE")
db.set_isolation_level(isolation_level)
def link_caop_osm(self, adminlvl):
"""
Search the OSM relation fitting the CAOP admin area.
Link each CAOP relation in 'adminlvl' with the best
existing OSM relation.
"""
self.adminlvl = adminlvl
cursor = self.db.cursor()
# Search for each caop relation the unlinked osm relation
# fitting the area
cursor.execute("""SELECT A.caop_id
FROM caop_relations A, caop_relation_tags B
WHERE A.caop_id = B.caop_id
AND B.k = 'admin_level' AND B.v = %s
AND NOT EXISTS (SELECT 1 FROM matching_relation
WHERE associationid = A.caop_id)
""", (str(adminlvl),) )
data = cursor.fetchall()
cursor.close()
self.db.commit()
logo.starting("Searching for admin level %d" % adminlvl, len(data))
for (caop_id,) in data:
logo.progress()
self.best_match = None
self.do_search_admin(caop_id) # Delegate
if self.best_match:
logo.DEBUG("Found for caop %s osm=%s)" % (caop_id, self.best_match))
if self.best_match.adminlvl != self.adminlvl:
logo.WARN("Relation found with unmatched admin level caop=%s (%s), osm=%s (%s)"
% (caop_id, self.adminlvl,
self.best_match.osm_id,
self.best_match.adminlvl))
self.set_association(caop_id, self.best_match.osm_id)
else:
logo.DEBUG("No relation found for caop=%s" % caop_id)
logo.ending()
def create_temp_table(db):
"""
Create temporary table to assign caop_id to line, point, admin.
"""
cursor = db.cursor()
# Table converting id into unique id
logo.DEBUG("Create Temporary tables")
cursor.execute("""CREATE TEMPORARY TABLE caop_points (
point_id int NOT NULL,
caop_id bigint NOT NULL
DEFAULT nextval('seq_caop_id'),
PRIMARY KEY (point_id)
)""")
cursor.execute("""SELECT AddGeometryColumn('caop_points', 'geom',
4326, 'POINT', 2)
""")
cursor.execute("""CREATE TEMPORARY TABLE caop_lines (
line_id int NOT NULL,
caop_id bigint NOT NULL
DEFAULT nextval('seq_caop_id'),
PRIMARY KEY (line_id)
)""")
cursor.execute("""CREATE TEMPORARY TABLE caop_admins (
admin_id int NOT NULL,
caop_id bigint NOT NULL
DEFAULT nextval('seq_caop_id'),
name text NOT NULL,
level int NOT NULL,
PRIMARY KEY (admin_id)
)""")
cursor.execute("""SELECT AddGeometryColumn('caop_admins', 'bbox',
4326, 'GEOMETRY', 2)
""")
# Table for bulk copy content in lines/admins
cursor.execute("""CREATE TEMPORARY TABLE caop_linepts (
line_id int NOT NULL,
sequence_id int NOT NULL,
point_id int NOT NULL,
PRIMARY KEY (line_id, sequence_id)
)""")
cursor.execute("""CREATE TEMPORARY TABLE caop_adminlines (
admin_id int NOT NULL,
line_id int NOT NULL,
role text NOT NULL,
sequence_id int NOT NULL,
PRIMARY KEY (admin_id, sequence_id)
)""")
db.commit()
def check_db_caop(db):
""" Check for special caop tables. """
logo.DEBUG("Checking for CAOP tables ...")
cursor = db.cursor()
try:
cursor.execute("""SELECT max(caop_id) FROM caop_nodes
UNION
SELECT max(caop_id) FROM caop_ways
UNION
SELECT max(caop_id) FROM caop_relations
UNION
SELECT last_value FROM seq_caop_id
""")
cursor.fetchall() # ignore result, just check if table exists
except psycopg2.ProgrammingError:
db.rollback()
logo.DEBUG("... no CAOP tables")
return False
db.commit()
logo.DEBUG("... CAOP tables exists")
return True
def read_CAOP(filename, shapeu):
"""
Read the shapefile and build the geometry.
We expect only 1 layer of type polygon, coordinates are reprojected
to WGS84.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Verify field and geometry type
for field in ("DICOFRE", "MUNICIPIO", "FREGUESIA"):
if layerDef.GetFieldIndex(field) == -1:
raise logo.ERROR("Field '%s' not found" % field)
if (layerDef.GetFieldIndex("DISTRITO") == -1
and layerDef.GetFieldIndex("ILHA") == -1):
raise logo.ERROR("Field 'DISTRITO' or 'ILHA' not found")
if layerDef.GetGeomType() != ogr.wkbPolygon:
raise logo.ERROR("Not a POLYGON file")
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Read each polygon and build the connection arrays (point, segment, line)
logo.starting("Geometry read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
# Outer Ring (1) followed by Inner Rings (n-1)
# we create all segments for each ring to find the topology ...
logo.DEBUG("Feature %d with %d rings" %
(featnum, newgeometry.GetGeometryCount()))
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
lon1, lat1 = ring.GetPoint_2D(0)
for pnt in xrange(1, ring.GetPointCount()):
lon2, lat2 = ring.GetPoint_2D(pnt)
shapeu.makeSegment(lon1, lat1, lon2, lat2)
lon1, lat1 = lon2, lat2
logo.ending()
def main():
logo.init(filename = caop_config.logfile,
verbose = caop_config.verbose,
progress = caop_config.progress)
logo.DEBUG("Connect to DB(%s)" % caop_config.dbname)
db = psycopg2.connect(caop_config.dbname)
create_table(db)
prepare_table(db)
# Search OSM admin area using bounding box
logo.INFO("Search existing admin area by bounding box")
matching = MatchRelationBbox(db)
for adminlevel in (4, 6, 7, 8):
matching.link_caop_osm(adminlevel)
# Search OSM admin area using geometry
matching = MatchRelationGeom(db)
logo.INFO("Search existing admin area by geometry")
for adminlevel in (8, 7, 6, 4):
matching.link_caop_osm(adminlevel)
def simplifyShapeZV(points, ptsdeleted):
"""
Simplify some very big angles in line.
Remove 1 point if shapes looking like a Z or V.
"""
angledist = [
angledistance(points[i - 1][0], points[i - 1][1], points[i][0],
points[i][1]) for i in xrange(1, len(points))
]
ptsdiscard = []
i = 0
while i < len(angledist) - 1:
# Search very big angles
diffangle = diffheading(angledist[i][0], angledist[i + 1][0])
if abs(diffangle) <= 90.0:
i += 1
continue
#
# Two big angles in a row = Z shape
# 0---1
# /
# 2---3
#
if abs(diffangle) > 135.0 and i < len(angledist) - 2 and abs(
diffheading(angledist[i + 1][0], angledist[i + 2][0])) > 135.0:
angle_1, dist_1 = angledist[i]
angle_2, dist_2 = angledist[i + 1]
angle_3, dist_3 = angledist[i + 2]
angle_A, dist_A = angledistance(points[i + 1][0], points[i + 1][1],
points[i + 3][0], points[i + 3][1])
# Distance of point 1 to line 2-3
d1 = getdeviation(diffheading(angle_3, angle_A), dist_3, dist_A,
dist_2)
angle_B, dist_B = angledistance(points[i][0], points[i][1],
points[i + 2][0], points[i + 2][1])
# Distance of point 2 to line 0-1
d2 = getdeviation(diffheading(angle_1, angle_B), dist_1, dist_B,
dist_2)
if min(d1, d2) >= 7.0:
i += 1
continue
# Keep closest point to line
if d2 < d1:
# Remove point 1
logo.DEBUG("Discard %s from Z shape %s %s %s %s" %
(points[i + 1], points[i], points[i + 1],
points[i + 2], points[i + 3]))
angledist[i:i + 2] = [(angle_B, dist_B)]
ptsdiscard.append(points[i + 1])
points = points[:i + 1] + points[i + 2:]
if i > 0:
i -= 1 # recheck with previous point
else:
# Remove point 2
logo.DEBUG("Discard %s from Z shape %s %s %s %s" %
(points[i + 2], points[i], points[i + 1],
points[i + 2], points[i + 3]))
angledist[i + 1:i + 3] = [(angle_A, dist_A)]
ptsdiscard.append(points[i + 2])
points = points[:i + 2] + points[i + 3:]
continue # Retry current position (don't increment i)
#
# One big angles = V shape
# 0 2
# \ /
# 1
#
angle_0, dist_0 = angledistance(points[i][0], points[i][1],
points[i + 2][0], points[i + 2][1])
angle_1, dist_1 = angledist[i]
angle_2, dist_2 = angledist[i + 1]
# Distance of point 0 to line 1-2
d1 = getdeviation(diffheading(angle_2, angle_0), dist_2, dist_0,
dist_1)
# Distance of point 2 to line 0-1
d2 = getdeviation(diffheading(angle_1, angle_0), dist_1, dist_0,
dist_2)
dist_1 = dist_1 * 6371000.0
dist_2 = dist_2 * 6371000.0
# Distance for simplification given by length of line 0-1 or 1-2
# if very big angle
# length (smallest) simplification
# < 9m 2.25m
# 9 to 22m length/4
# > 22m 5.5m
# if big angle
# angle length (longest) simplification
# 105-135 both < 9m 1.25m
# " < 9m 1.9m
# 90-105 18 to 28m 1.25m
# " > 28m 1.9m
if abs(diffangle) >= 135.0:
distmax = min(dist_1, dist_2) * 0.25
distmax = min(distmax, 5.5)
distmax = max(distmax, 2.25)
elif abs(diffangle) >= 105.0:
if max(dist_1, dist_2) < 9.0:
distmax = 1.25
else:
distmax = 1.9
else:
if max(dist_1, dist_2) < 18.0:
i += 1
continue
elif max(dist_1, dist_2) < 28.0:
distmax = 1.25
else:
distmax = 1.9
if min(d1, d2) < distmax:
# Remove point 1
logo.DEBUG(
"Discard %s from V shape %s %s %s" %
(points[i + 1], points[i], points[i + 1], points[i + 2]))
angledist[i:i + 2] = [(angle_0, dist_0)]
ptsdiscard.append(points[i + 1])
points = points[:i + 1] + points[i + 2:]
if i > 0:
i -= 1 # recheck with previous point
continue
i += 1
if ptsdiscard:
# Some point removed, redo Douglas-Peucker
points, ptsresimplify = simplifyPoints(points)
ptsdeleted = ptsdeleted + ptsdiscard + ptsresimplify
return (points, ptsdeleted)
def buildSimplifiedLines(self):
"""
Grab each segment and build polylines (OSM way compatible).
Attach a way to its successor/predecessor if there's only 1
connection, remove useless point (simplify geometry) and make
sure there's not too much point in a line (limit of 2000 OSM
nodes per way).
"""
logo.DEBUG("Before simplification %d points, %d segments" %
(len(self.point_pos), self.segment_count / 2))
logo.starting("Line simplification", self.segment_count)
if self.line_count > 0:
# Restart build if polylines have already been made
self.line_seg = array.array('i', [0] * (self.cachemem / 2))
self.line_ends = array.array('i')
self.line_count = 0
specialjoinset = set()
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum / 2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
# Orphaned segment, happens when a point is simplified
# and the segment is dropped
continue
self._simplifyLineSegment(coordpts, specialjoinset)
logo.ending()
# Special case for merged segment (duplicate segment removed)
# Redo search+simplify on the longest possible line
nbpass = 1
while specialjoinset:
nbpass += 1
logo.starting("Line simplification (pass %d)" % nbpass,
len(specialjoinset))
newjoinset = set()
for lineid in specialjoinset:
logo.progress()
try:
segmentnum = self.line_seg.index(lineid) * 2
except ValueError:
continue
coordpts = self._buildLineFromSegment(segmentnum, lineid)
self._simplifyLineSegment(coordpts, newjoinset)
logo.ending()
specialjoinset = newjoinset
# Renumbering line id, no gap and less than 2000 nodes per line
logo.starting("Build way with 2000 nodes limit",
self.segment_count / 2)
self.line_seg = array.array('i', [0] * (self.segment_count / 2))
self.line_count = 0
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum / 2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
continue
# Split if we are too close to the limit of 2000 nodes
# and ensure that a new line have more than a few points
# we also record both extremity of a line for later use
segmentdir1 = self.point_pos[coordpts[0]]
segmentdir2 = self.point_pos[coordpts[1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
while len(coordpts) > 1980:
# End of previous line and start a new one
self.line_count += 1
lineid = self.line_count
segmentdir1 = self.point_pos[coordpts[1949]]
coordpts = coordpts[1950:]
segmentdir2 = self.point_pos[coordpts[0]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
segmentnum = self.segment_connect[segmentnum]
self.line_ends.append(segmentnum)
for i in xrange(1, min(1980, len(coordpts))):
self.line_seg[int(segmentnum / 2)] = lineid
segmentnum = self.segment_connect[segmentnum ^ 1]
segmentdir1 = self.point_pos[coordpts[-2]]
segmentdir2 = self.point_pos[coordpts[-1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[-1]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
logo.ending()
logo.DEBUG("After simplification %d points, %d lines" %
(len(self.point_pos), self.line_count))
def import_caop(db, shapeu, admins):
"""
Import with an unique id all nodes, ways, relations.
"""
cursor = db.cursor()
logo.starting("Saving nodes, ways, relations",
shapeu.nbrPoints() + shapeu.nbrLines() + len(admins))
# Points -> Nodes
# - bulk copy to a temp table to get a new unique id
# - do only one big insert with new ids to the finale table
logo.DEBUG("Write nodes to database")
buffcopy = StringIO()
for pointid, coord in shapeu.iterPoints():
logo.progress()
pointEwkt = "SRID=4326;POINT(%.7f %.7f)" % (coord[0], coord[1])
buffcopy.write("%d\t%s\n" % (pointid, pointEwkt))
buffcopy.seek(0)
cursor.copy_from(buffcopy, 'caop_points', columns=('point_id', 'geom'))
cursor.execute("""INSERT INTO caop_nodes (caop_id, geom)
SELECT caop_id, geom FROM caop_points
""")
db.commit()
buffcopy.close()
# Lines -> Ways
# - bulk copy to a temp table to get a new unique id
# - bulk copy points in lines in a temp table
# - insert all ways with new ids as administrative level 8
logo.DEBUG("Write ways to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for lineid, pntids in shapeu.iterLines():
logo.progress()
buffcopy1.write("%d\n" % lineid)
for orderpntid in enumerate(pntids):
buffcopy2.write("%d\t" % lineid)
buffcopy2.write("%d\t%d\n" % orderpntid)
buffcopy1.seek(0)
cursor.copy_from(buffcopy1, 'caop_lines', columns=('line_id', ))
cursor.execute("""INSERT INTO caop_ways (caop_id)
SELECT caop_id FROM caop_lines
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_linepts')
cursor.execute("""INSERT INTO caop_way_nodes
SELECT A.caop_id, B.caop_id, C.sequence_id
FROM caop_lines A, caop_points B, caop_linepts C
WHERE A.line_id = C.line_id
AND C.point_id = B.point_id
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_lines
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'admin_level', 8
FROM caop_lines
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
# Admins -> Relations
# - bulk copy to a temp table to get a new unique id
# - bulk copy lines in admins in a temp table
# - correct outer ways administrative level
# - insert all tags for administrative area
logo.DEBUG("Write relations to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for (num, dicofre) in enumerate(admins):
logo.progress()
buffcopy1.write("%d\t" % num)
buffcopy1.write("%(name)s\t%(level)d\t" % admins[dicofre])
buffcopy1.write(
"SRID=4326;POLYGON((%(x1).7f %(y1).7f,%(x1).7f %(y2).7f,%(x2).7f %(y2).7f,%(x2).7f %(y1).7f,%(x1).7f %(y1).7f))\n"
% dict(zip(['x1', 'x2', 'y1', 'y2'], admins[dicofre]['bbox'])))
sequenceid = 0
for role in ("outer", "inner"):
for lineid in admins[dicofre][role]:
buffcopy2.write("%d\t%d\t%s\t%d\n" %
(num, lineid, role, sequenceid))
sequenceid += 1
if admins[dicofre]['level'] < 8:
cursor.execute(
"""UPDATE caop_way_tags SET v = %(level)s
FROM caop_lines A
WHERE caop_way_tags.caop_id = A.caop_id
AND A.line_id IN %(outer)s
AND k = 'admin_level'
AND v::int > %(level)s
""", admins[dicofre])
db.commit()
buffcopy1.seek(0)
cursor.copy_from(buffcopy1,
'caop_admins',
columns=('admin_id', 'name', 'level', 'bbox'))
cursor.execute("""INSERT INTO caop_relations (caop_id, bbox)
SELECT caop_id, bbox FROM caop_admins
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_adminlines')
cursor.execute("""INSERT INTO caop_relation_members
SELECT A.caop_id, B.caop_id, 'W', C.role, C.sequence_id
FROM caop_admins A, caop_lines B, caop_adminlines C
WHERE A.admin_id = C.admin_id
AND C.line_id = B.line_id
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'type', 'boundary'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'admin_level', level::text
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'name', name
FROM caop_admins
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
logo.ending()
def create_caop_table(db):
""" Recreate caop tables. """
cursor = db.cursor()
# Create node tables
logo.DEBUG("Create Node tables")
cursor.execute("""DROP TABLE IF EXISTS caop_nodes""")
cursor.execute("""CREATE TABLE caop_nodes (
caop_id bigint NOT NULL,
osmid bigint,
version int,
action character(1)
)""")
cursor.execute("""SELECT AddGeometryColumn('caop_nodes', 'geom',
4326, 'POINT', 2)
""")
cursor.execute("""DROP TABLE IF EXISTS caop_node_tags""")
cursor.execute("""CREATE TABLE caop_node_tags (
caop_id bigint NOT NULL,
k text NOT NULL,
v text NOT NULL
)""")
# Create way tables
logo.DEBUG("Create Way tables")
cursor.execute("""DROP TABLE IF EXISTS caop_ways""")
cursor.execute("""CREATE TABLE caop_ways (
caop_id bigint NOT NULL,
osmid bigint,
version int,
action character(1)
)""")
cursor.execute("""DROP TABLE IF EXISTS caop_way_nodes""")
cursor.execute("""CREATE TABLE caop_way_nodes (
caop_id bigint NOT NULL,
node_id bigint NOT NULL,
sequence_id int NOT NULL
)""")
cursor.execute("""DROP TABLE IF EXISTS caop_way_tags""")
cursor.execute("""CREATE TABLE caop_way_tags (
caop_id bigint NOT NULL,
k text NOT NULL,
v text NOT NULL
)""")
# Create relation tables
logo.DEBUG("Create Relation tables")
cursor.execute("""DROP TABLE IF EXISTS caop_relations""")
cursor.execute("""CREATE TABLE caop_relations (
caop_id bigint NOT NULL,
osmid bigint,
version int,
action character(1)
)""")
cursor.execute("""SELECT AddGeometryColumn('caop_relations', 'bbox',
4326, 'GEOMETRY', 2)
""")
cursor.execute("""DROP TABLE IF EXISTS caop_relation_members""")
cursor.execute("""CREATE TABLE caop_relation_members (
caop_id bigint NOT NULL,
member_id bigint NOT NULL,
member_type character(1) NOT NULL,
member_role text NOT NULL,
sequence_id int NOT NULL
)""")
cursor.execute("""DROP TABLE IF EXISTS caop_relation_tags""")
cursor.execute("""CREATE TABLE caop_relation_tags (
caop_id bigint NOT NULL,
k text NOT NULL,
v text NOT NULL
)""")
# Primary key for node, way, relation
logo.DEBUG("Create primary key")
cursor.execute("""ALTER TABLE caop_nodes
ADD CONSTRAINT pk_caop_nodes
PRIMARY KEY (caop_id)
""")
cursor.execute("""ALTER TABLE caop_ways
ADD CONSTRAINT pk_caop_ways
PRIMARY KEY (caop_id)
""")
cursor.execute("""ALTER TABLE caop_relations
ADD CONSTRAINT pk_caop_relations
PRIMARY KEY (caop_id)
""")
# Primary key for nodes in way, members in relation
cursor.execute("""ALTER TABLE caop_way_nodes
ADD CONSTRAINT pk_caop_way_nodes
PRIMARY KEY (caop_id, sequence_id)
""")
cursor.execute("""ALTER TABLE caop_relation_members
ADD CONSTRAINT pk_caop_relation_members
PRIMARY KEY (caop_id, sequence_id)
""")
# Create spatial index
logo.DEBUG("Create index")
cursor.execute("""CREATE INDEX idx_caop_node_geom
ON caop_nodes USING gist (geom)
""")
cursor.execute("""CREATE INDEX idx_caop_relation_bbox
ON caop_relations USING gist (bbox)
""")
# Create index for tags
cursor.execute("""CREATE INDEX idx_caop_node_tags
ON caop_node_tags USING btree (caop_id)
""")
cursor.execute("""CREATE INDEX idx_caop_way_tags
ON caop_way_tags USING btree (caop_id)
""")
cursor.execute("""CREATE INDEX idx_caop_relation_tags
ON caop_relation_tags USING btree (caop_id)
""")
# Auto-incrementing sequence for caop_id
logo.DEBUG("Create sequence")
cursor.execute("""DROP SEQUENCE IF EXISTS seq_caop_id""")
cursor.execute("""CREATE SEQUENCE seq_caop_id INCREMENT BY -1""")
db.commit()
def verify_admin(shapeu, admins):
"""
Check that all administrative area are closed.
Also search for inner ring and update 'admins'.
"""
logo.starting("Verify admin area", len(admins))
verifyinner = {}
for dicofre in admins:
logo.progress()
logo.DEBUG("Area level=%(level)d '%(name)s'" % admins[dicofre])
# Administrative areas read from the shapefile are also checked
# and dispatched into outer/inner ring, even if technically only
# the upper and reconstructed admin level need it (the shapefile
# already knows what's outer and inner, but we avoid a special
# case and it cannot fail unless something was really wrong).
closedrings = FindClosedRings(shapeu, admins[dicofre]["outer"])
if not closedrings.isValid():
logo.ERROR("Area '%s' (DICOFRE=%s) not a valid closed ring\n" %
(admins[dicofre]["name"], dicofre))
for ring, pntid1, pntid2 in closedrings.iterRingDiscarded():
lineids = closedrings.getLineDiscarded(ring)
if pntid1 == pntid2:
logo.WARN(
"Ring with %d lines is self-intersecting, still building admin area with this defect"
% len(lineids))
else:
points = closedrings.getGeometryDiscarded(ring)
logo.WARN(
"Ring with %d lines is open at %s -> %s, still building admin area with this defect"
% (len(lineids), points[0], points[-1]))
xmin, xmax, ymin, ymax = closedrings.getExtentLineDiscarded()
admins[dicofre]["bbox"] = [xmin, xmax, ymin, ymax]
# Moving lineids from outer to inner and compute envelope
for outer, inner in closedrings.iterPolygons():
for ring in inner:
lineids = closedrings.getLineRing(ring)
admins[dicofre]["outer"].difference_update(lineids)
admins[dicofre]["inner"].update(lineids)
for line in lineids:
# Remember lines used in inner ring for later verification
key = (line, admins[dicofre]["level"])
verifyinner[key] = [dicofre]
# Bounding box on outer rings
xmin, xmax, ymin, ymax = closedrings.getExtentRing(outer)
if not admins[dicofre]["bbox"]:
admins[dicofre]["bbox"] = [xmin, xmax, ymin, ymax]
else:
if xmin < admins[dicofre]["bbox"][0]:
admins[dicofre]["bbox"][0] = xmin
if xmax > admins[dicofre]["bbox"][1]:
admins[dicofre]["bbox"][1] = xmax
if ymin < admins[dicofre]["bbox"][2]:
admins[dicofre]["bbox"][2] = ymin
if ymax > admins[dicofre]["bbox"][3]:
admins[dicofre]["bbox"][3] = ymax
logo.ending()
# Each inner line on each admin level should be used as outer line
# in one and only one admin area with the same level
for dicofre in admins:
for line in admins[dicofre]["outer"]:
key = (line, admins[dicofre]["level"])
if key in verifyinner:
verifyinner[key].append(dicofre)
for key in verifyinner:
if len(verifyinner[key]) != 2:
dicofre = verifyinner[key][0]
if len(verifyinner[key]) == 1:
logo.ERROR(
"Inner line in area '%s' (DICOFRE=%s) not present as outer in any admin area with level=%d\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"]))
else:
logo.ERROR(
"Inner line in area '%s' (DICOFRE=%s) exist as multiple outer in level=%d : %s\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"], ', '.join([
"%s (DICOFRE=%s)" % (admins[i]["name"], i)
for i in verifyinner[key][1:]
])))
def admin_CAOP(filename, shapeu, admins):
"""
Reread the shapefile and build each administrative entity.
Geometry described by a set of lines, attributes converted to UTF8.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Detect if we are dealing with Portugal or the autonomous regions
if layerDef.GetFieldIndex("DISTRITO") != -1:
logo.DEBUG("Found DISTRITO using admin level 6, 7, 8")
isregion = False
toplevel = "DISTRITO"
elif layerDef.GetFieldIndex("ILHA") != -1:
logo.DEBUG("Found ILHA using admin level 4, 7, 8")
isregion = True
toplevel = "ILHA"
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Reread each polygon and create the right administrative area
logo.starting("Attributes read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
dicofre = feature.GetField("DICOFRE")
distrito = convertname(feature.GetField(toplevel))
municipio = convertname(feature.GetField("MUNICIPIO"))
freguesia = convertname(feature.GetField("FREGUESIA"))
logo.DEBUG("Feature %d %s='%s' MUNICIPIO='%s' FREGUESIA='%s'" %
(featnum, toplevel, distrito, municipio, freguesia))
# Distrito or Region
if isregion:
dicofre1 = dicofre[0:1]
if not admins.has_key(dicofre1):
# Extract archipelago name from island name
m = re.search("\(([^)]+)\)", distrito)
if m:
distrito = m.group(1)
admins[dicofre1] = {
"name": distrito,
"level": 4,
"inner": set(),
"outer": set(),
"bbox": None
}
else:
dicofre1 = dicofre[0:2]
if not admins.has_key(dicofre1):
admins[dicofre1] = {
"name": distrito,
"level": 6,
"inner": set(),
"outer": set(),
"bbox": None
}
# Municipio
dicofre2 = dicofre[0:4]
if not admins.has_key(dicofre2):
admins[dicofre2] = {
"name": municipio,
"level": 7,
"inner": set(),
"outer": set(),
"bbox": None
}
# Freguesia
if not admins.has_key(dicofre):
admins[dicofre] = {
"name": freguesia,
"level": 8,
"inner": set(),
"outer": set(),
"bbox": None
}
# Build sets of lineid, don't distinguish outer and inner rings
# we deal it later when verifying and grouping rings
lineset = set()
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
pntinring = []
for pnt in xrange(ring.GetPointCount()):
lon, lat = ring.GetPoint_2D(pnt)
pointid = shapeu.getPoint(lon, lat)
if pointid is not None:
pntinring.append(pointid)
if pntinring[0] != pntinring[-1]:
# Simplification have broken the ring,
# starting point was in the middle of a simplified line
pntinring.append(pntinring[0])
for pnt in xrange(1, len(pntinring)):
if pntinring[pnt - 1] == pntinring[pnt]:
# If 2 coordinates after rounding give the same point id
# (safety measure, normaly doesn't happen)
continue
segment = shapeu.getSegment(pntinring[pnt - 1], pntinring[pnt])
lineset.add(shapeu.getLine(segment))
# Update each administrative level
admins[dicofre]["outer"].update(lineset)
admins[dicofre2]["outer"].symmetric_difference_update(lineset)
admins[dicofre1]["outer"].symmetric_difference_update(lineset)
logo.ending()
