def verify_admin(shapeu, admins):
"""
Check that all administrative area are closed.
Also search for inner ring and update 'admins'.
"""
logo.starting("Verify admin area", len(admins))
for adm in admins:
logo.progress()
logo.DEBUG("Area level=%(level)d '%(name)s'" % admins[adm])
# Administrative areas read from the shapefile are also checked
# and dispatched into outer/inner ring, even if technically only
# the upper and reconstructed admin level need it (the shapefile
# already knows what's outer and inner, but we avoid a special
# case and it cannot fail unless something was really wrong).
closedrings = FindClosedRings(shapeu, admins[adm]["outer"])
if closedrings.getLineDiscarded():
logo.ERROR("Area '%s' ring not closed\n"
% (admins[adm]["name"]) )
for line in closedrings.getLineDiscarded():
coords = shapeu.getLineCoords(line)
logo.DEBUG("Line in ring with %d points still open %s -> %s"
% (len(coords), coords[0], coords[-1]) )
# Moving lineids from outer to inner and compute envelope
for outer, inner in closedrings.iterPolygons():
for ring in inner:
lineids = closedrings.getLineRing(ring)
admins[adm]["outer"].difference_update(lineids)
admins[adm]["inner"].update(lineids)
logo.ending()
def admin_UGANDA_OSM(filename, shapeu, admins):
logo.starting("Attributes read", parseosm.getNbRelation())
for relationid in parseosm.getIterRelation():
logo.progress()
relation = parseosm.getRelation(relationid)
admins[relationid] = { "name" : relation['tags']['name'],
"level" : int(relation['tags']['admin_level']),
"inner" : set(),
"outer" : set(),
}
if 'old_name' in relation['tags']:
admins[relationid]["old_name"] = relation['tags']['old_name']
wayids = [ data[1] for data in relation['members']
if data[0] == 'way' and data[2] in ('outer', 'inner') ]
lineset = set()
for wayid in wayids:
pntinring = []
linegeometry = parseosm.getGeometryWay(wayid)
for lon, lat in linegeometry:
pointid = shapeu.getPoint(lon, lat)
if pointid is not None:
pntinring.append(pointid)
for pnt in xrange(1, len(pntinring)):
if pntinring[pnt-1] == pntinring[pnt]:
# If 2 coordinates after rounding give the same point id
continue
segment = shapeu.getSegment(pntinring[pnt-1], pntinring[pnt])
lineset.add(shapeu.getLine(segment))
# Update each administrative level
admins[relationid]["outer"].update(lineset)
logo.ending()
def read_UGANDA(filename, shapeu):
"""
Read the shapefile and build the geometry.
We expect only 1 layer of type polygon, coordinates are reprojected
to WGS84.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Verify field and geometry type
for possiblefields in [ ("DNAME_2010", "SUBREGION"), ("region", "place") ]:
for field in possiblefields:
if layerDef.GetFieldIndex(field) == -1:
break
else:
break
else:
raise logo.ERROR("Important field missing, where is the admin area name ?")
if layerDef.GetGeomType() != ogr.wkbPolygon:
raise logo.ERROR("Not a POLYGON file")
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Read each polygon and build the connection arrays (point, segment, line)
logo.starting("Geometry read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
# MultiPolygon: only deal with first polygon
# Polygon: Outer Ring (1) followed by Inner Rings (n-1)
# we create all segments for outer ring only, drop
# inner rings (very exotic ...)
if newgeometry.GetGeometryType() == ogr.wkbMultiPolygon:
logo.DEBUG("Feature %d with %d polygons" % (featnum,
newgeometry.GetGeometryCount()))
ring = newgeometry.GetGeometryRef(0).GetGeometryRef(0)
else:
logo.DEBUG("Feature %d with %d rings" % (featnum,
newgeometry.GetGeometryCount()))
ring = newgeometry.GetGeometryRef(0)
lon1, lat1 = ring.GetPoint_2D(0)
for pnt in xrange(1, ring.GetPointCount()):
lon2, lat2 = ring.GetPoint_2D(pnt)
shapeu.makeSegment(lon1, lat1, lon2, lat2)
lon1, lat1 = lon2, lat2
logo.ending()
def write_uganda(fileout, shapeu, admins):
"""
Import with an unique id all nodes, ways, relations.
"""
logo.starting("Saving nodes, ways, relations",
shapeu.nbrPoints() + shapeu.nbrLines() + len(admins))
out = open(fileout+"_out.osm", "w")
tmstamp = datetime.datetime.utcnow().strftime("%Y-%m-%dT%H:%M:%SZ")
out.write('<?xml version="1.0" encoding="UTF-8"?>\n')
out.write('<osm version="0.6" generator="test">\n')
# Points -> Nodes
logo.DEBUG("Write nodes")
for pointid, coord in shapeu.iterPoints():
logo.progress()
out.write(' <node id="-%d" lat="%.7f" lon="%.7f" version="0" timestamp="%s"/>\n'
% (pointid+1, coord[1], coord[0], tmstamp))
# Lines -> Ways
logo.DEBUG("Write ways")
waylevel = {}
for adm in admins:
for lineid in admins[adm]["outer"]:
level = min(waylevel.get(lineid, 8), admins[adm]["level"])
waylevel[lineid] = level
for lineid, pntids in shapeu.iterLines():
logo.progress()
out.write(' <way id="-%d" version="0" timestamp="%s">\n' % (lineid, tmstamp))
for pointid in pntids:
out.write(' <nd ref="-%d"/>\n' % (pointid+1))
out.write(' <tag k="boundary" v="administrative"/>\n')
try:
out.write(' <tag k="admin_level" v="%s"/>\n' % waylevel[lineid])
except KeyError:
pass # because of inner ring in middle of river and not in any admin area outer
out.write(' </way>\n')
# Admins -> Relations
logo.DEBUG("Write relations")
for (num,adm) in enumerate(admins):
logo.progress()
out.write(' <relation id="-%d" version="0" timestamp="%s">\n' % (num+1, tmstamp))
for role in ("outer", "inner"):
for lineid in admins[adm][role]:
out.write(' <member type="way" ref="-%d" role="%s"/>\n' % (lineid, role))
out.write(' <tag k="type" v="boundary"/>\n')
out.write(' <tag k="boundary" v="administrative"/>\n')
out.write(' <tag k="admin_level" v="%d"/>\n' % admins[adm]["level"])
out.write(' <tag k="name" v="%s"/>\n' % admins[adm]["name"])
if "old_name" in admins[adm]:
out.write(' <tag k="old_name" v="%s"/>\n' % admins[adm]["old_name"])
out.write(' </relation>\n')
out.write(' </osm>\n')
logo.ending()
def read_CAOP(filename, shapeu):
"""
Read the shapefile and build the geometry.
We expect only 1 layer of type polygon, coordinates are reprojected
to WGS84.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Verify field and geometry type
for field in ( "DICOFRE", "MUNICIPIO", "FREGUESIA" ):
if layerDef.GetFieldIndex(field) == -1:
raise logo.ERROR("Field '%s' not found" % field)
if (layerDef.GetFieldIndex("DISTRITO") == -1
and layerDef.GetFieldIndex("ILHA") == -1):
raise logo.ERROR("Field 'DISTRITO' or 'ILHA' not found")
if layerDef.GetGeomType() != ogr.wkbPolygon:
raise logo.ERROR("Not a POLYGON file")
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Read each polygon and build the connection arrays (point, segment, line)
logo.starting("Geometry read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
# Outer Ring (1) followed by Inner Rings (n-1)
# we create all segments for each ring to find the topology ...
logo.DEBUG("Feature %d with %d rings" % (featnum,
newgeometry.GetGeometryCount()))
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
lon1, lat1 = ring.GetPoint_2D(0)
for pnt in xrange(1, ring.GetPointCount()):
lon2, lat2 = ring.GetPoint_2D(pnt)
shapeu.makeSegment(lon1, lat1, lon2, lat2)
lon1, lat1 = lon2, lat2
logo.ending()
def read_CAOP(filename, shapeu):
"""
Read the shapefile and build the geometry.
We expect only 1 layer of type polygon, coordinates are reprojected
to WGS84.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Verify field and geometry type
for field in ("DICOFRE", "MUNICIPIO", "FREGUESIA"):
if layerDef.GetFieldIndex(field) == -1:
raise logo.ERROR("Field '%s' not found" % field)
if (layerDef.GetFieldIndex("DISTRITO") == -1
and layerDef.GetFieldIndex("ILHA") == -1):
raise logo.ERROR("Field 'DISTRITO' or 'ILHA' not found")
if layerDef.GetGeomType() != ogr.wkbPolygon:
raise logo.ERROR("Not a POLYGON file")
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Read each polygon and build the connection arrays (point, segment, line)
logo.starting("Geometry read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
# Outer Ring (1) followed by Inner Rings (n-1)
# we create all segments for each ring to find the topology ...
logo.DEBUG("Feature %d with %d rings" %
(featnum, newgeometry.GetGeometryCount()))
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
lon1, lat1 = ring.GetPoint_2D(0)
for pnt in xrange(1, ring.GetPointCount()):
lon2, lat2 = ring.GetPoint_2D(pnt)
shapeu.makeSegment(lon1, lat1, lon2, lat2)
lon1, lat1 = lon2, lat2
logo.ending()
def verify_admin(shapeu, admins):
"""
Check that all administrative area are closed.
Also search for inner ring and update 'admins'.
"""
logo.starting("Verify admin area", len(admins))
for dicofre in admins:
logo.progress()
logo.DEBUG("Area level=%(level)d '%(name)s'" % admins[dicofre])
# Administrative areas read from the shapefile are also checked
# and dispatched into outer/inner ring, even if technically only
# the upper and reconstructed admin level need it (the shapefile
# already knows what's outer and inner, but we avoid a special
# case and it cannot fail unless something was really wrong).
closedrings = FindClosedRings(shapeu, admins[dicofre]["outer"])
if closedrings.getLineDiscarded():
logo.ERROR("Area '%s' (DICOFRE=%s) ring not closed\n"
% (admins[dicofre]["name"], dicofre) )
# Moving lineids from outer to inner and compute envelope
for outer, inner in closedrings.iterPolygons():
for ring in inner:
lineids = closedrings.getLineRing(ring)
admins[dicofre]["outer"].difference_update(lineids)
admins[dicofre]["inner"].update(lineids)
# Bounding box on outer rings
xmin, xmax, ymin, ymax = closedrings.getExtentRing(outer)
if not admins[dicofre]["bbox"]:
admins[dicofre]["bbox"] = [ xmin, xmax, ymin, ymax ]
else:
if xmin < admins[dicofre]["bbox"][0]:
admins[dicofre]["bbox"][0] = xmin
if xmax > admins[dicofre]["bbox"][1]:
admins[dicofre]["bbox"][1] = xmax
if ymin < admins[dicofre]["bbox"][2]:
admins[dicofre]["bbox"][2] = ymin
if ymax > admins[dicofre]["bbox"][3]:
admins[dicofre]["bbox"][3] = ymax
logo.ending()
def link_caop_osm(self, adminlvl):
"""
Search the OSM relation fitting the CAOP admin area.
Link each CAOP relation in 'adminlvl' with the best
existing OSM relation.
"""
self.adminlvl = adminlvl
cursor = self.db.cursor()
# Search for each caop relation the unlinked osm relation
# fitting the area
cursor.execute(
"""SELECT A.caop_id
FROM caop_relations A, caop_relation_tags B
WHERE A.caop_id = B.caop_id
AND B.k = 'admin_level' AND B.v = %s
AND NOT EXISTS (SELECT 1 FROM matching_relation
WHERE associationid = A.caop_id)
""",
(str(adminlvl),),
)
data = cursor.fetchall()
cursor.close()
self.db.commit()
logo.starting("Searching for admin level %d" % adminlvl, len(data))
for (caop_id,) in data:
logo.progress()
self.best_match = None
self.do_search_admin(caop_id) # Delegate
if self.best_match:
logo.DEBUG("Found for caop %s osm=%s)" % (caop_id, self.best_match))
if self.best_match.adminlvl != self.adminlvl:
logo.WARN(
"Relation found with unmatched admin level caop=%s (%s), osm=%s (%s)"
% (caop_id, self.adminlvl, self.best_match.osm_id, self.best_match.adminlvl)
)
self.set_association(caop_id, self.best_match.osm_id)
else:
logo.DEBUG("No relation found for caop=%s" % caop_id)
logo.ending()
def read_UGANDA_OSM(filename, shapeu):
parseosm.parse_xml(open(filename).read())
shapeutil.precision = 14 # don't do aggressive rounding
shapeutil.testnearest = [] # nor neighbour hack
# Read each polygon and build the connection arrays (point, segment, line)
logo.starting("Geometry read", parseosm.getNbRelation())
for relationid in parseosm.getIterRelation():
logo.progress()
relation = parseosm.getRelation(relationid)
wayids = [ data[1] for data in relation['members']
if data[0] == 'way' and data[2] in ('outer', 'inner') ]
for wayid in wayids:
linegeometry = parseosm.getGeometryWay(wayid)
lon1, lat1 = linegeometry[0]
for lon2, lat2 in linegeometry[1:]:
shapeu.makeSegment(lon1, lat1, lon2, lat2)
lon1, lat1 = lon2, lat2
logo.ending()
def link_caop_osm(self, adminlvl):
"""
Search the OSM relation fitting the CAOP admin area.
Link each CAOP relation in 'adminlvl' with the best
existing OSM relation.
"""
self.adminlvl = adminlvl
cursor = self.db.cursor()
# Search for each caop relation the unlinked osm relation
# fitting the area
cursor.execute("""SELECT A.caop_id
FROM caop_relations A, caop_relation_tags B
WHERE A.caop_id = B.caop_id
AND B.k = 'admin_level' AND B.v = %s
AND NOT EXISTS (SELECT 1 FROM matching_relation
WHERE associationid = A.caop_id)
""", (str(adminlvl),) )
data = cursor.fetchall()
cursor.close()
self.db.commit()
logo.starting("Searching for admin level %d" % adminlvl, len(data))
for (caop_id,) in data:
logo.progress()
self.best_match = None
self.do_search_admin(caop_id) # Delegate
if self.best_match:
logo.DEBUG("Found for caop %s osm=%s)" % (caop_id, self.best_match))
if self.best_match.adminlvl != self.adminlvl:
logo.WARN("Relation found with unmatched admin level caop=%s (%s), osm=%s (%s)"
% (caop_id, self.adminlvl,
self.best_match.osm_id,
self.best_match.adminlvl))
self.set_association(caop_id, self.best_match.osm_id)
else:
logo.DEBUG("No relation found for caop=%s" % caop_id)
logo.ending()
def buildSimplifiedLines(self):
"""
Grab each segment and build polylines (OSM way compatible).
Attach a way to its successor/predecessor if there's only 1
connection, remove useless point (simplify geometry) and make
sure there's not too much point in a line (limit of 2000 OSM
nodes per way).
"""
logo.DEBUG("Before simplification %d points, %d segments" % (
len(self.point_pos), self.segment_count/2))
logo.starting("Line simplification", self.segment_count)
self.line_seg = array.array('i', [0] * (self.cachemem/2))
self.line_ends = array.array('i')
self.line_count = 0
specialjoinset = set()
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum/2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
# Orphaned segment, happens when a point is simplified
continue
self._simplifyLineSegment(coordpts, specialjoinset)
logo.ending()
# Special case for merged segment (duplicate segment removed)
# Redo search+simplify on the longest possible line
nbpass = 1
while specialjoinset:
nbpass += 1
logo.starting("Line simplification (pass %d)" % nbpass,
len(specialjoinset))
newjoinset = set()
for lineid in specialjoinset:
logo.progress()
try:
segmentnum = self.line_seg.index(lineid) * 2
except ValueError:
continue
coordpts = self._buildLineFromSegment(segmentnum, lineid)
self._simplifyLineSegment(coordpts, newjoinset)
logo.ending()
specialjoinset = newjoinset
logo.starting("Build way with 2000 nodes limit", self.segment_count)
self.line_seg = array.array('i', [0] * (self.cachemem/2))
self.line_ends = array.array('i')
self.line_count = 0
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum/2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
continue
# Split if we are too close to the limit of 2000 nodes
# and ensure that a new line have more than a few points
# we also record both extremity of a line for later use
segmentdir1 = self.point_pos[coordpts[0]]
segmentdir2 = self.point_pos[coordpts[1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
while len(coordpts) > 1980:
# End of previous line and start a new one
self.line_count += 1
lineid = self.line_count
segmentdir1 = self.point_pos[coordpts[1949]]
coordpts = coordpts[1950:]
segmentdir2 = self.point_pos[coordpts[0]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
segmentnum = self.segment_connect[segmentnum]
self.line_ends.append(segmentnum)
for i in xrange(1, min(1980, len(coordpts))):
self.line_seg[int(segmentnum/2)] = lineid
segmentnum = self.segment_connect[segmentnum^1]
segmentdir1 = self.point_pos[coordpts[-2]]
segmentdir2 = self.point_pos[coordpts[-1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[-1]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
logo.ending()
logo.DEBUG("After simplification %d points, %d lines" % (
len(self.point_pos), self.line_count))
def admin_CAOP(filename, shapeu, admins):
"""
Reread the shapefile and build each administrative entity.
Geometry described by a set of lines, attributes converted to UTF8.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Detect if we are dealing with Portugal or the autonomous regions
if layerDef.GetFieldIndex("DISTRITO") != -1:
logo.DEBUG("Found DISTRITO using admin level 6, 7, 8")
isregion = False
toplevel = "DISTRITO"
elif layerDef.GetFieldIndex("ILHA") != -1:
logo.DEBUG("Found ILHA using admin level 4, 7, 8")
isregion = True
toplevel = "ILHA"
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Reread each polygon and create the right administrative area
logo.starting("Attributes read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
dicofre = feature.GetField("DICOFRE")
distrito = convertname(feature.GetField(toplevel))
municipio = convertname(feature.GetField("MUNICIPIO"))
freguesia = convertname(feature.GetField("FREGUESIA"))
logo.DEBUG("Feature %d %s='%s' MUNICIPIO='%s' FREGUESIA='%s'" %
(featnum, toplevel, distrito, municipio, freguesia))
# Distrito or Region
if isregion:
dicofre1 = dicofre[0:1]
if not admins.has_key(dicofre1):
# Extract archipelago name from island name
m = re.search("\(([^)]+)\)", distrito)
if m:
distrito = m.group(1)
admins[dicofre1] = {
"name": distrito,
"level": 4,
"inner": set(),
"outer": set(),
"bbox": None
}
else:
dicofre1 = dicofre[0:2]
if not admins.has_key(dicofre1):
admins[dicofre1] = {
"name": distrito,
"level": 6,
"inner": set(),
"outer": set(),
"bbox": None
}
# Municipio
dicofre2 = dicofre[0:4]
if not admins.has_key(dicofre2):
admins[dicofre2] = {
"name": municipio,
"level": 7,
"inner": set(),
"outer": set(),
"bbox": None
}
# Freguesia
if not admins.has_key(dicofre):
admins[dicofre] = {
"name": freguesia,
"level": 8,
"inner": set(),
"outer": set(),
"bbox": None
}
# Build sets of lineid, don't distinguish outer and inner rings
# we deal it later when verifying and grouping rings
lineset = set()
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
pntinring = []
for pnt in xrange(ring.GetPointCount()):
lon, lat = ring.GetPoint_2D(pnt)
pointid = shapeu.getPoint(lon, lat)
if pointid is not None:
pntinring.append(pointid)
if pntinring[0] != pntinring[-1]:
# Simplification have broken the ring,
# starting point was in the middle of a simplified line
pntinring.append(pntinring[0])
for pnt in xrange(1, len(pntinring)):
if pntinring[pnt - 1] == pntinring[pnt]:
# If 2 coordinates after rounding give the same point id
# (safety measure, normaly doesn't happen)
continue
segment = shapeu.getSegment(pntinring[pnt - 1], pntinring[pnt])
lineset.add(shapeu.getLine(segment))
# Update each administrative level
admins[dicofre]["outer"].update(lineset)
admins[dicofre2]["outer"].symmetric_difference_update(lineset)
admins[dicofre1]["outer"].symmetric_difference_update(lineset)
logo.ending()
def verify_admin(shapeu, admins):
"""
Check that all administrative area are closed.
Also search for inner ring and update 'admins'.
"""
logo.starting("Verify admin area", len(admins))
verifyinner = {}
for dicofre in admins:
logo.progress()
logo.DEBUG("Area level=%(level)d '%(name)s'" % admins[dicofre])
# Administrative areas read from the shapefile are also checked
# and dispatched into outer/inner ring, even if technically only
# the upper and reconstructed admin level need it (the shapefile
# already knows what's outer and inner, but we avoid a special
# case and it cannot fail unless something was really wrong).
closedrings = FindClosedRings(shapeu, admins[dicofre]["outer"])
if not closedrings.isValid():
logo.ERROR("Area '%s' (DICOFRE=%s) not a valid closed ring\n" %
(admins[dicofre]["name"], dicofre))
for ring, pntid1, pntid2 in closedrings.iterRingDiscarded():
lineids = closedrings.getLineDiscarded(ring)
if pntid1 == pntid2:
logo.WARN(
"Ring with %d lines is self-intersecting, still building admin area with this defect"
% len(lineids))
else:
points = closedrings.getGeometryDiscarded(ring)
logo.WARN(
"Ring with %d lines is open at %s -> %s, still building admin area with this defect"
% (len(lineids), points[0], points[-1]))
xmin, xmax, ymin, ymax = closedrings.getExtentLineDiscarded()
admins[dicofre]["bbox"] = [xmin, xmax, ymin, ymax]
# Moving lineids from outer to inner and compute envelope
for outer, inner in closedrings.iterPolygons():
for ring in inner:
lineids = closedrings.getLineRing(ring)
admins[dicofre]["outer"].difference_update(lineids)
admins[dicofre]["inner"].update(lineids)
for line in lineids:
# Remember lines used in inner ring for later verification
key = (line, admins[dicofre]["level"])
verifyinner[key] = [dicofre]
# Bounding box on outer rings
xmin, xmax, ymin, ymax = closedrings.getExtentRing(outer)
if not admins[dicofre]["bbox"]:
admins[dicofre]["bbox"] = [xmin, xmax, ymin, ymax]
else:
if xmin < admins[dicofre]["bbox"][0]:
admins[dicofre]["bbox"][0] = xmin
if xmax > admins[dicofre]["bbox"][1]:
admins[dicofre]["bbox"][1] = xmax
if ymin < admins[dicofre]["bbox"][2]:
admins[dicofre]["bbox"][2] = ymin
if ymax > admins[dicofre]["bbox"][3]:
admins[dicofre]["bbox"][3] = ymax
logo.ending()
# Each inner line on each admin level should be used as outer line
# in one and only one admin area with the same level
for dicofre in admins:
for line in admins[dicofre]["outer"]:
key = (line, admins[dicofre]["level"])
if key in verifyinner:
verifyinner[key].append(dicofre)
for key in verifyinner:
if len(verifyinner[key]) != 2:
dicofre = verifyinner[key][0]
if len(verifyinner[key]) == 1:
logo.ERROR(
"Inner line in area '%s' (DICOFRE=%s) not present as outer in any admin area with level=%d\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"]))
else:
logo.ERROR(
"Inner line in area '%s' (DICOFRE=%s) exist as multiple outer in level=%d : %s\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"], ', '.join([
"%s (DICOFRE=%s)" % (admins[i]["name"], i)
for i in verifyinner[key][1:]
])))
def import_caop(db, shapeu, admins):
"""
Import with an unique id all nodes, ways, relations.
"""
cursor = db.cursor()
logo.starting("Saving nodes, ways, relations",
shapeu.nbrPoints() + shapeu.nbrLines() + len(admins))
# Points -> Nodes
# - bulk copy to a temp table to get a new unique id
# - do only one big insert with new ids to the finale table
logo.DEBUG("Write nodes to database")
buffcopy = StringIO()
for pointid, coord in shapeu.iterPoints():
logo.progress()
pointEwkt = "SRID=4326;POINT(%.7f %.7f)" % (coord[0], coord[1])
buffcopy.write("%d\t%s\n" % (pointid, pointEwkt))
buffcopy.seek(0)
cursor.copy_from(buffcopy, 'caop_points', columns=('point_id', 'geom'))
cursor.execute("""INSERT INTO caop_nodes (caop_id, geom)
SELECT caop_id, geom FROM caop_points
""")
db.commit()
buffcopy.close()
# Lines -> Ways
# - bulk copy to a temp table to get a new unique id
# - bulk copy points in lines in a temp table
# - insert all ways with new ids as administrative level 8
logo.DEBUG("Write ways to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for lineid, pntids in shapeu.iterLines():
logo.progress()
buffcopy1.write("%d\n" % lineid)
for orderpntid in enumerate(pntids):
buffcopy2.write("%d\t" % lineid)
buffcopy2.write("%d\t%d\n" % orderpntid)
buffcopy1.seek(0)
cursor.copy_from(buffcopy1, 'caop_lines', columns=('line_id', ))
cursor.execute("""INSERT INTO caop_ways (caop_id)
SELECT caop_id FROM caop_lines
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_linepts')
cursor.execute("""INSERT INTO caop_way_nodes
SELECT A.caop_id, B.caop_id, C.sequence_id
FROM caop_lines A, caop_points B, caop_linepts C
WHERE A.line_id = C.line_id
AND C.point_id = B.point_id
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_lines
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'admin_level', 8
FROM caop_lines
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
# Admins -> Relations
# - bulk copy to a temp table to get a new unique id
# - bulk copy lines in admins in a temp table
# - correct outer ways administrative level
# - insert all tags for administrative area
logo.DEBUG("Write relations to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for (num, dicofre) in enumerate(admins):
logo.progress()
buffcopy1.write("%d\t" % num)
buffcopy1.write("%(name)s\t%(level)d\t" % admins[dicofre])
buffcopy1.write(
"SRID=4326;POLYGON((%(x1).7f %(y1).7f,%(x1).7f %(y2).7f,%(x2).7f %(y2).7f,%(x2).7f %(y1).7f,%(x1).7f %(y1).7f))\n"
% dict(zip(['x1', 'x2', 'y1', 'y2'], admins[dicofre]['bbox'])))
sequenceid = 0
for role in ("outer", "inner"):
for lineid in admins[dicofre][role]:
buffcopy2.write("%d\t%d\t%s\t%d\n" %
(num, lineid, role, sequenceid))
sequenceid += 1
if admins[dicofre]['level'] < 8:
cursor.execute(
"""UPDATE caop_way_tags SET v = %(level)s
FROM caop_lines A
WHERE caop_way_tags.caop_id = A.caop_id
AND A.line_id IN %(outer)s
AND k = 'admin_level'
AND v::int > %(level)s
""", admins[dicofre])
db.commit()
buffcopy1.seek(0)
cursor.copy_from(buffcopy1,
'caop_admins',
columns=('admin_id', 'name', 'level', 'bbox'))
cursor.execute("""INSERT INTO caop_relations (caop_id, bbox)
SELECT caop_id, bbox FROM caop_admins
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_adminlines')
cursor.execute("""INSERT INTO caop_relation_members
SELECT A.caop_id, B.caop_id, 'W', C.role, C.sequence_id
FROM caop_admins A, caop_lines B, caop_adminlines C
WHERE A.admin_id = C.admin_id
AND C.line_id = B.line_id
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'type', 'boundary'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'admin_level', level::text
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'name', name
FROM caop_admins
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
logo.ending()
def admin_UGANDA(filename, shapeu, admins):
"""
Reread the shapefile and build each administrative entity.
Geometry described by a set of lines, attributes converted to UTF8.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Extract attributes from district or merged file
if layerDef.GetFieldIndex("SUBREGION") == -1:
fieldregion = "region"
fielddistrict = "place"
else:
fieldregion = "SUBREGION"
fielddistrict = "DNAME_2010"
# Change here the admin area level !!!
LevelSubRegion = 6
LevelDistrict = 7
# Reread each polygon and create the right administrative area
logo.starting("Attributes read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
subregion = feature.GetField(fieldregion)
district = feature.GetField(fielddistrict)
logo.DEBUG("Feature %d SUBREGION='%s' DISTRICT='%s'" % (
featnum, subregion, district))
# Subregion / District
if district is None:
# Merged file and polygon is region
key_admin1 = None
key_admin2 = "SR_" + subregion.upper()
if key_admin2 not in admins:
admins[key_admin2] = { "name" : convertname(subregion),
"level" : LevelSubRegion,
"inner" : set(),
"outer" : set(),
}
elif subregion is None:
# Merged file and polygon is district
key_admin1 = None
key_admin2 = "DI_" + district.upper()
if key_admin2 not in admins:
admins[key_admin2] = { "name" : convertname(district),
"level" : LevelDistrict,
"inner" : set(),
"outer" : set(),
}
else:
# District only file, automagicaly build region from district
key_admin1 = "SR_" + subregion.upper()
key_admin2 = "DI_" + district.upper()
if key_admin1 not in admins:
admins[key_admin1] = { "name" : convertname(subregion),
"level" : LevelSubRegion,
"inner" : set(),
"outer" : set(),
}
if key_admin2 not in admins:
admins[key_admin2] = { "name" : convertname(district),
"level" : LevelDistrict,
"inner" : set(),
"outer" : set(),
}
# Build sets of lineid, deal only outer, inner rings
# are useless and wrong
lineset = set()
if newgeometry.GetGeometryType() == ogr.wkbMultiPolygon:
ring = newgeometry.GetGeometryRef(0).GetGeometryRef(0)
else:
ring = newgeometry.GetGeometryRef(0)
pntinring = []
for pnt in xrange(ring.GetPointCount()):
lon, lat = ring.GetPoint_2D(pnt)
pointid = shapeu.getPoint(lon, lat)
if pointid is not None:
pntinring.append(pointid)
if pntinring[0] != pntinring[-1]:
# Simplification have broken the ring,
# starting point was in the middle of a simplified line
pntinring.append(pntinring[0])
for pnt in xrange(1, len(pntinring)):
if pntinring[pnt-1] == pntinring[pnt]:
# If 2 coordinates after rounding give the same point id
# (safety measure, normaly doesn't happen)
continue
segment = shapeu.getSegment(pntinring[pnt-1], pntinring[pnt])
lineset.add(shapeu.getLine(segment))
# Update each administrative level
admins[key_admin2]["outer"].update(lineset)
if key_admin1 is not None:
admins[key_admin1]["outer"].symmetric_difference_update(lineset)
logo.ending()
def buildSimplifiedLines(self):
"""
Grab each segment and build polylines (OSM way compatible).
Attach a way to its successor/predecessor if there's only 1
connection, remove useless point (simplify geometry) and make
sure there's not too much point in a line (limit of 2000 OSM
nodes per way).
"""
logo.DEBUG("Before simplification %d points, %d segments" %
(len(self.point_pos), self.segment_count / 2))
logo.starting("Line simplification", self.segment_count)
if self.line_count > 0:
# Restart build if polylines have already been made
self.line_seg = array.array('i', [0] * (self.cachemem / 2))
self.line_ends = array.array('i')
self.line_count = 0
specialjoinset = set()
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum / 2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
# Orphaned segment, happens when a point is simplified
# and the segment is dropped
continue
self._simplifyLineSegment(coordpts, specialjoinset)
logo.ending()
# Special case for merged segment (duplicate segment removed)
# Redo search+simplify on the longest possible line
nbpass = 1
while specialjoinset:
nbpass += 1
logo.starting("Line simplification (pass %d)" % nbpass,
len(specialjoinset))
newjoinset = set()
for lineid in specialjoinset:
logo.progress()
try:
segmentnum = self.line_seg.index(lineid) * 2
except ValueError:
continue
coordpts = self._buildLineFromSegment(segmentnum, lineid)
self._simplifyLineSegment(coordpts, newjoinset)
logo.ending()
specialjoinset = newjoinset
# Renumbering line id, no gap and less than 2000 nodes per line
logo.starting("Build way with 2000 nodes limit",
self.segment_count / 2)
self.line_seg = array.array('i', [0] * (self.segment_count / 2))
self.line_count = 0
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum / 2]:
# Already attached
continue
coordpts = self._buildLineFromSegment(segmentnum)
if coordpts is None:
continue
# Split if we are too close to the limit of 2000 nodes
# and ensure that a new line have more than a few points
# we also record both extremity of a line for later use
segmentdir1 = self.point_pos[coordpts[0]]
segmentdir2 = self.point_pos[coordpts[1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
while len(coordpts) > 1980:
# End of previous line and start a new one
self.line_count += 1
lineid = self.line_count
segmentdir1 = self.point_pos[coordpts[1949]]
coordpts = coordpts[1950:]
segmentdir2 = self.point_pos[coordpts[0]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
segmentnum = self.segment_connect[segmentnum]
self.line_ends.append(segmentnum)
for i in xrange(1, min(1980, len(coordpts))):
self.line_seg[int(segmentnum / 2)] = lineid
segmentnum = self.segment_connect[segmentnum ^ 1]
segmentdir1 = self.point_pos[coordpts[-2]]
segmentdir2 = self.point_pos[coordpts[-1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[-1]:
segmentnum = segmentnum ^ 1
self.line_ends.append(segmentnum)
logo.ending()
logo.DEBUG("After simplification %d points, %d lines" %
(len(self.point_pos), self.line_count))
def buildSimplifiedLines(self):
"""
Grab each segment and build polylines (OSM way compatible).
Attach a way to its successor/predecessor if there's only 1
connection, remove useless point (simplify geometry) and make
sure there's not too much point in a line (limit of 2000 OSM
nodes per way).
"""
logo.DEBUG("Before simplification %d points, %d segments" % (
len(self.point_pos), self.segment_count/2))
logo.starting("Line simplification", self.segment_count)
for segmentnum in xrange(0, self.segment_count, 2):
logo.progress(segmentnum)
if self.line_seg[segmentnum/2]: # Already attached
continue
segmentdir1 = segmentnum
segmentdir2 = segmentnum + 1
# Count predecessors/successors
nbprev = self.nbrConnection(segmentdir1)
nbnext = self.nbrConnection(segmentdir2)
if nbprev == 0 and nbnext == 0:
# Orphaned segment, happens when a point is simplified
continue
# Affect a lineid to the current segment
self.line_count += 1
lineid = self.line_count
self.line_seg[segmentdir1/2] = lineid
coordpts = [ self.coord_pnt[segmentdir1],
self.coord_pnt[segmentdir2] ]
# Join previous segments if it's the only connection
while nbprev == 1:
if self.line_seg[self.segment_connect[segmentdir1]/2]:
break # loop on closed ring
segmentdir1 = self.segment_connect[segmentdir1] ^ 1
self.line_seg[segmentdir1/2] = lineid
coordpts.insert(0, self.coord_pnt[segmentdir1])
nbprev = self.nbrConnection(segmentdir1)
# Join next segments if it's the only connection
while nbnext == 1:
if self.line_seg[self.segment_connect[segmentdir2]/2]:
break # loop on closed ring
segmentdir2 = self.segment_connect[segmentdir2] ^ 1
self.line_seg[segmentdir2/2] = lineid
coordpts.append(self.coord_pnt[segmentdir2])
nbnext = self.nbrConnection(segmentdir2)
# Find useless points
coordpts, purgepts = simplifyPoints(coordpts)
coordpts, purgepts = simplifyShapeZV(coordpts, purgepts)
coordpts, purgepts = fixSelfIntersect(coordpts, purgepts)
# Now the *not so* fun part, we change and delete some segments.
# The ids will change so we work with point coordinates and we
# keep track of all the dependencies.
# A simplified point have only 2 segments, the first segment will
# adopt a new location for its end, the second segment will be
# entirely dereferenced.
for coord in purgepts:
segmentnum = self.point_pos[coord]
segmentdir1 = self.segment_connect[segmentnum]
segmentdir2 = segmentdir1^1
self.segment_connect[segmentnum] = self.segment_connect[segmentdir2]
seg = self.segment_connect[segmentdir2]
while self.segment_connect[seg] != segmentdir2:
seg = self.segment_connect[seg]
self.segment_connect[seg] = segmentnum
self.segment_connect[segmentdir1] = segmentdir1
self.segment_connect[segmentdir2] = segmentdir2
# Update new end point location
coord2 = self.coord_pnt[segmentdir2]
self.coord_pnt[segmentnum] = coord2
if self.point_pos[coord2] == segmentdir2:
self.point_pos[coord2] = segmentnum
del self.point_pos[coord]
self.coord_pnt[segmentdir1] = None
self.coord_pnt[segmentdir2] = None
self.line_seg[segmentdir2/2] = 0
# Split if we are too close to the limit of 2000 nodes
# and ensure that a new line have more than a few points
# we also record both extremity of a line for later use
segmentdir1 = self.point_pos[coordpts[0]]
segmentdir2 = self.point_pos[coordpts[1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
while len(coordpts) > 1980:
# End of previous line and start a new one
self.line_count += 1
lineid = self.line_count
segmentdir1 = self.point_pos[coordpts[1949]]
coordpts = coordpts[1950:]
segmentdir2 = self.point_pos[coordpts[0]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[0]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
segmentnum = self.segment_connect[segmentnum]
self.line_ends.append(segmentnum)
for i in xrange(1, min(1980, len(coordpts))):
self.line_seg[segmentnum/2] = lineid
segmentnum = self.segment_connect[segmentnum^1]
segmentdir1 = self.point_pos[coordpts[-2]]
segmentdir2 = self.point_pos[coordpts[-1]]
segmentnum = self.getSegment(segmentdir1, segmentdir2)
if self.coord_pnt[segmentnum] != coordpts[-1]:
segmentnum = segmentnum^1
self.line_ends.append(segmentnum)
logo.ending()
logo.DEBUG("After simplification %d points, %d lines" % (
len(self.point_pos), self.line_count))
def verify_admin(shapeu, admins):
"""
Check that all administrative area are closed.
Also search for inner ring and update 'admins'.
"""
logo.starting("Verify admin area", len(admins))
verifyinner = {}
for dicofre in admins:
logo.progress()
logo.DEBUG("Area level=%(level)d '%(name)s'" % admins[dicofre])
# Administrative areas read from the shapefile are also checked
# and dispatched into outer/inner ring, even if technically only
# the upper and reconstructed admin level need it (the shapefile
# already knows what's outer and inner, but we avoid a special
# case and it cannot fail unless something was really wrong).
closedrings = FindClosedRings(shapeu, admins[dicofre]["outer"])
if not closedrings.isValid():
logo.ERROR("Area '%s' (DICOFRE=%s) not a valid closed ring\n"
% (admins[dicofre]["name"], dicofre) )
for ring, pntid1, pntid2 in closedrings.iterRingDiscarded():
lineids = closedrings.getLineDiscarded(ring)
if pntid1 == pntid2:
logo.WARN("Ring with %d lines is self-intersecting, still building admin area with this defect"
% len(lineids))
else:
points = closedrings.getGeometryDiscarded(ring)
logo.WARN("Ring with %d lines is open at %s -> %s, still building admin area with this defect"
% (len(lineids), points[0], points[-1]))
xmin, xmax, ymin, ymax = closedrings.getExtentLineDiscarded()
admins[dicofre]["bbox"] = [ xmin, xmax, ymin, ymax ]
# Moving lineids from outer to inner and compute envelope
for outer, inner in closedrings.iterPolygons():
for ring in inner:
lineids = closedrings.getLineRing(ring)
admins[dicofre]["outer"].difference_update(lineids)
admins[dicofre]["inner"].update(lineids)
for line in lineids:
# Remember lines used in inner ring for later verification
key = (line, admins[dicofre]["level"])
verifyinner[key] = [dicofre]
# Bounding box on outer rings
xmin, xmax, ymin, ymax = closedrings.getExtentRing(outer)
if not admins[dicofre]["bbox"]:
admins[dicofre]["bbox"] = [ xmin, xmax, ymin, ymax ]
else:
if xmin < admins[dicofre]["bbox"][0]:
admins[dicofre]["bbox"][0] = xmin
if xmax > admins[dicofre]["bbox"][1]:
admins[dicofre]["bbox"][1] = xmax
if ymin < admins[dicofre]["bbox"][2]:
admins[dicofre]["bbox"][2] = ymin
if ymax > admins[dicofre]["bbox"][3]:
admins[dicofre]["bbox"][3] = ymax
logo.ending()
# Each inner line on each admin level should be used as outer line
# in one and only one admin area with the same level
for dicofre in admins:
for line in admins[dicofre]["outer"]:
key = (line, admins[dicofre]["level"])
if key in verifyinner:
verifyinner[key].append(dicofre)
for key in verifyinner:
if len(verifyinner[key]) != 2:
dicofre = verifyinner[key][0]
if len(verifyinner[key]) == 1:
logo.ERROR("Inner line in area '%s' (DICOFRE=%s) not present as outer in any admin area with level=%d\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"])
)
else:
logo.ERROR("Inner line in area '%s' (DICOFRE=%s) exist as multiple outer in level=%d : %s\n"
% (admins[dicofre]["name"], dicofre,
admins[dicofre]["level"],
', '.join([ "%s (DICOFRE=%s)" % (
admins[i]["name"], i)
for i in verifyinner[key][1:] ]))
)
def admin_CAOP(filename, shapeu, admins):
"""
Reread the shapefile and build each administrative entity.
Geometry described by a set of lines, attributes converted to UTF8.
"""
shapefile = ogr.Open(filename)
layer = shapefile.GetLayer(0)
layerDef = layer.GetLayerDefn()
# Detect if we are dealing with Portugal or the autonomous regions
if layerDef.GetFieldIndex("DISTRITO") != -1:
logo.DEBUG("Found DISTRITO using admin level 6, 7, 8")
isregion = False
toplevel = "DISTRITO"
elif layerDef.GetFieldIndex("ILHA") != -1:
logo.DEBUG("Found ILHA using admin level 4, 7, 8")
isregion = True
toplevel = "ILHA"
# Reproject on the fly
srcSpatialRef = layer.GetSpatialRef()
dstSpatialRef = osr.SpatialReference()
dstSpatialRef.SetWellKnownGeogCS('WGS84')
transform = osr.CoordinateTransformation(srcSpatialRef, dstSpatialRef)
# Reread each polygon and create the right administrative area
logo.starting("Attributes read", layer.GetFeatureCount())
for featnum in xrange(layer.GetFeatureCount()):
logo.progress(featnum)
feature = layer.GetFeature(featnum)
geometry = feature.GetGeometryRef()
newgeometry = geometry.Clone()
newgeometry.Transform(transform)
dicofre = feature.GetField("DICOFRE")
distrito = convertname(feature.GetField(toplevel))
municipio = convertname(feature.GetField("MUNICIPIO"))
freguesia = convertname(feature.GetField("FREGUESIA"))
logo.DEBUG("Feature %d %s='%s' MUNICIPIO='%s' FREGUESIA='%s'" % (
featnum, toplevel, distrito,
municipio, freguesia))
# Distrito or Region
if isregion:
dicofre1 = dicofre[0:1]
if not admins.has_key(dicofre1):
# Extract archipelago name from island name
m = re.search("\(([^)]+)\)", distrito)
if m:
distrito = m.group(1)
admins[dicofre1] = { "name" : distrito,
"level" : 4,
"inner" : set(),
"outer" : set(),
"bbox" : None
}
else:
dicofre1 = dicofre[0:2]
if not admins.has_key(dicofre1):
admins[dicofre1] = { "name" : distrito,
"level" : 6,
"inner" : set(),
"outer" : set(),
"bbox" : None
}
# Municipio
dicofre2 = dicofre[0:4]
if not admins.has_key(dicofre2):
admins[dicofre2] = { "name" : municipio,
"level" : 7,
"inner" : set(),
"outer" : set(),
"bbox" : None
}
# Freguesia
if not admins.has_key(dicofre):
admins[dicofre] = { "name" : freguesia,
"level" : 8,
"inner" : set(),
"outer" : set(),
"bbox" : None
}
# Build sets of lineid, don't distinguish outer and inner rings
# we deal it later when verifying and grouping rings
lineset = set()
for i in xrange(newgeometry.GetGeometryCount()):
ring = newgeometry.GetGeometryRef(i)
pntinring = []
for pnt in xrange(ring.GetPointCount()):
lon, lat = ring.GetPoint_2D(pnt)
pointid = shapeu.getPoint(lon, lat)
if pointid is not None:
pntinring.append(pointid)
if pntinring[0] != pntinring[-1]:
# Simplification have broken the ring,
# starting point was in the middle of a simplified line
pntinring.append(pntinring[0])
for pnt in xrange(1, len(pntinring)):
if pntinring[pnt-1] == pntinring[pnt]:
# If 2 coordinates after rounding give the same point id
# (safety measure, normaly doesn't happen)
continue
segment = shapeu.getSegment(pntinring[pnt-1], pntinring[pnt])
lineset.add(shapeu.getLine(segment))
# Update each administrative level
admins[dicofre]["outer"].update(lineset)
admins[dicofre2]["outer"].symmetric_difference_update(lineset)
admins[dicofre1]["outer"].symmetric_difference_update(lineset)
logo.ending()
def import_caop(db, shapeu, admins):
"""
Import with an unique id all nodes, ways, relations.
"""
cursor = db.cursor()
logo.starting("Saving nodes, ways, relations",
shapeu.nbrPoints() + shapeu.nbrLines() + len(admins))
# Points -> Nodes
# - bulk copy to a temp table to get a new unique id
# - do only one big insert with new ids to the finale table
logo.DEBUG("Write nodes to database")
buffcopy = StringIO()
for pointid, coord in shapeu.iterPoints():
logo.progress()
pointEwkt = "SRID=4326;POINT(%.7f %.7f)" % (coord[0], coord[1])
buffcopy.write("%d\t%s\n" % (pointid, pointEwkt))
buffcopy.seek(0)
cursor.copy_from(buffcopy, 'caop_points', columns=('point_id', 'geom'))
cursor.execute("""INSERT INTO caop_nodes (caop_id, geom)
SELECT caop_id, geom FROM caop_points
""")
db.commit()
buffcopy.close()
# Lines -> Ways
# - bulk copy to a temp table to get a new unique id
# - bulk copy points in lines in a temp table
# - insert all ways with new ids as administrative level 8
logo.DEBUG("Write ways to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for lineid, pntids in shapeu.iterLines():
logo.progress()
buffcopy1.write("%d\n" % lineid)
for orderpntid in enumerate(pntids):
buffcopy2.write("%d\t" % lineid)
buffcopy2.write("%d\t%d\n" % orderpntid)
buffcopy1.seek(0)
cursor.copy_from(buffcopy1, 'caop_lines', columns=('line_id',))
cursor.execute("""INSERT INTO caop_ways (caop_id)
SELECT caop_id FROM caop_lines
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_linepts')
cursor.execute("""INSERT INTO caop_way_nodes
SELECT A.caop_id, B.caop_id, C.sequence_id
FROM caop_lines A, caop_points B, caop_linepts C
WHERE A.line_id = C.line_id
AND C.point_id = B.point_id
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_lines
""")
cursor.execute("""INSERT INTO caop_way_tags
SELECT caop_id, 'admin_level', 8
FROM caop_lines
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
# Admins -> Relations
# - bulk copy to a temp table to get a new unique id
# - bulk copy lines in admins in a temp table
# - correct outer ways administrative level
# - insert all tags for administrative area
logo.DEBUG("Write relations to database")
buffcopy1 = StringIO()
buffcopy2 = StringIO()
for (num,dicofre) in enumerate(admins):
logo.progress()
buffcopy1.write("%d\t" % num)
buffcopy1.write("%(name)s\t%(level)d\t" % admins[dicofre])
buffcopy1.write("SRID=4326;POLYGON((%(x1).7f %(y1).7f,%(x1).7f %(y2).7f,%(x2).7f %(y2).7f,%(x2).7f %(y1).7f,%(x1).7f %(y1).7f))\n" % dict(zip(['x1', 'x2', 'y1', 'y2'], admins[dicofre]['bbox'])) )
sequenceid = 0
for role in ("outer", "inner"):
for lineid in admins[dicofre][role]:
buffcopy2.write("%d\t%d\t%s\t%d\n" % (
num, lineid, role, sequenceid))
sequenceid += 1
if admins[dicofre]['level'] < 8:
cursor.execute("""UPDATE caop_way_tags SET v = %(level)s
FROM caop_lines A
WHERE caop_way_tags.caop_id = A.caop_id
AND A.line_id IN %(outer)s
AND k = 'admin_level'
AND v::int > %(level)s
""", admins[dicofre])
db.commit()
buffcopy1.seek(0)
cursor.copy_from(buffcopy1, 'caop_admins', columns=('admin_id', 'name',
'level', 'bbox'))
cursor.execute("""INSERT INTO caop_relations (caop_id, bbox)
SELECT caop_id, bbox FROM caop_admins
""")
buffcopy2.seek(0)
cursor.copy_from(buffcopy2, 'caop_adminlines')
cursor.execute("""INSERT INTO caop_relation_members
SELECT A.caop_id, B.caop_id, 'W', C.role, C.sequence_id
FROM caop_admins A, caop_lines B, caop_adminlines C
WHERE A.admin_id = C.admin_id
AND C.line_id = B.line_id
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'type', 'boundary'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'boundary', 'administrative'
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'admin_level', level::text
FROM caop_admins
""")
cursor.execute("""INSERT INTO caop_relation_tags
SELECT caop_id, 'name', name
FROM caop_admins
""")
db.commit()
buffcopy1.close()
buffcopy2.close()
logo.ending()