def addOverlay(self, property, labelfield=""):
'''
Overlay a polygon onto the baselayer and set the label to use for the
polygon.
The feature will be transported in GeoJSON format since it is the most
efficient format for AJAX loading.
'''
if property is None: return
#Set the name of the field to use for labeling
self._style.setLabelField(labelfield)
#Update the style of the property on each overlay operation
self._updateLayerStyle()
#Set label object
labelJSObject = "null"
if hasattr(property, labelfield):
propVal = getattr(property, labelfield)
labelJSObject = "{'%s':'%s'}" % (labelfield, str(propVal))
#Reproject to web mercator
prop_wkb = self.dbSession.scalar(property.geom.ST_Transform(900913))
web_geom = WKBElement(prop_wkb)
prop_geo_json = self.dbSession.scalar(web_geom.ST_AsGeoJSON())
overlayJS = "drawProperty('%s',%s);" % (prop_geo_json, labelJSObject)
zoomLevel, ok = self._setJS(overlayJS).toInt()
#Raise map zoom changed event
self.onZoomLevelChanged(zoomLevel)
def add_overlay(self, sp_unit, geometry_col, labelfield=""):
"""
Overlay a point/line/polygon onto the baselayer and set the label to use for the
feature.
The feature will be transported in GeoJSON format since it is the most
efficient format for AJAX loading.
"""
if sp_unit is None:
return
# Set the name of the field to use for labeling
# self._style.setLabelField(labelfield)
# Update the style of the property on each overlay operation
self._updateLayerStyle()
# Set label object
label_js_object = "null"
if hasattr(sp_unit, labelfield):
lbl_val = getattr(sp_unit, labelfield)
label_js_object = "{'%s':'%s'}" % (labelfield, str(lbl_val))
# Reproject to web mercator
geom = getattr(sp_unit, geometry_col)
sp_unit_wkb = self.dbSession.scalar(geom.ST_Transform(900913))
web_geom = WKBElement(sp_unit_wkb)
sp_unit_geo_json = self.dbSession.scalar(web_geom.ST_AsGeoJSON())
overlay_js = "drawSpatialUnit('%s',%s);" % (sp_unit_geo_json,
label_js_object)
zoom_level = self._setJS(overlay_js)
# Raise map zoom changed event
self.onZoomLevelChanged(zoom_level)
def valuesFromOgrFeat(self, feat, transform=None):
"""Returns a dictionary with loaded values from a feature"""
df = feat.GetDefnRef()
vals = {}
for i in range(feat.GetFieldCount()):
fld = df.GetFieldDefn(i)
name = fld.GetName()
if self.ignoreRegex.search(name):
continue
if name.lower() == 'id':
#skip columns with id (will be automatically filled)
continue
if fld.GetTypeName() == 'String':
# vals[name.lower()]=feat.GetFieldAsBinary(i).decode('iso-8859-1')
vals[name.lower()] = feat.GetFieldAsBinary(i).decode(
self.encoding)
else:
vals[name.lower()] = feat.GetField(i)
#append geometry values
geom = feat.GetGeometryRef()
if self.swapxy:
geom.SwapXY()
if transform:
geom.Transform(transform)
vals['geom'] = WKBElement(geom.ExportToWkb(), srid=self.targetsrid)
return vals
class TestToGeojson:
@pytest.mark.parametrize('element, expected_value', (
(
WKBElement('010600000002000000010300000001000000040000000000000000'
'003e40000000000000344000000000008046400000000000004440'
'000000000000244000000000000044400000000000003e40000000'
'0000003440010300000001000000050000000000000000002e4000'
'000000000014400000000000004440000000000000244000000000'
'000024400000000000003440000000000000144000000000000024'
'400000000000002e400000000000001440'),
{
'type': 'MultiPolygon', 'coordinates': [
(((30.0, 20.0), (45.0, 40.0), (10.0, 40.0), (30.0, 20.0)),),
(((15.0, 5.0), (40.0, 10.0), (10.0, 20.0), (5.0, 10.0),
(15.0, 5.0)),)]
}
),
(
'MULTIPOLYGON (((30 20, 45 40, 10 40, 30 20)), '
'((15 5, 40 10, 10 20, 5 10, 15 5)))',
{
'type': 'MultiPolygon', 'coordinates': [
(((30.0, 20.0), (45.0, 40.0), (10.0, 40.0),
(30.0, 20.0)),),
(((15.0, 5.0), (40.0, 10.0), (10.0, 20.0), (5.0, 10.0),
(15.0, 5.0)),)]
}
),
(
WKBElement('0101000000a18499b67f4947c058207a5226c935c0'),
{
"type": "Point",
"coordinates": (-46.57421, -21.785741)
}
),
(
'POINT (-46.57421 -21.785741)',
{
"type": "Point",
"coordinates": (-46.57421, -21.785741)
}
),
))
def test_must_parse_element(self, element, expected_value):
assert parsers.to_geojson(element) == expected_value
def _read_sector_route(self, s, sector_id):
q = text(f'''
select st_transform(st_setsrid(s.route, sg.srid), {WGS84_SRID})
from sector as s,
sector_group as sg
where s.id = :sector_id''')
return self._wkb_to_latlon(
WKBElement(s.connection().execute(
q, sector_id=sector_id).fetchone()[0]))
def test_deserialize_valid_geojson(self):
from c2corg_api.ext.colander_ext import Geometry
geom_schema = Geometry()
from shapely.geometry.point import Point
expected_wkb = WKBElement(Point(1.0, 2.0).wkb)
wkb = geom_schema.deserialize(
{}, '{"type": "Point", "coordinates": [1.0, 2.0]}')
self.assertEquals(expected_wkb.desc, wkb.desc)
def GRDCmetaExtractor(uri):
encoding = 'iso-8859-1'
meta = {}
headerregex = re.compile(b"^#")
dataregex = re.compile(b"^[0-9]")
header = {}
epoch = []
slurplogger().info("Extracting meta info from: %s" % (uri.url))
with gzip.open(uri.url, 'r') as gzid:
for ln in gzid:
if headerregex.search(ln):
#parse header
lnspl = ln[1:].split(b":", 1)
if len(lnspl) == 2:
ky = lnspl[0].lstrip(b" ").rstrip(b" ").decode(encoding)
val = lnspl[1].lstrip(b" ").rstrip(b"\r\n ").decode(
encoding)
if not val:
#get the nextline as a value
val = gzid.readline()[1:].lstrip(b" ").rstrip(
b"\r\n ").decode(encoding)
header[ky] = val
continue
if dataregex.search(ln):
#parse data line
lnspl = ln.decode(encoding).split(";")
datestr = lnspl[0] + lnspl[1].replace("--:--", ":00:00")
epoch.append(datetime.strptime(datestr, '%Y-%m-%d:%H:%S'))
nextdownstream = header["Next downstream station"]
if "-" in nextdownstream:
nextdownstream = None
else:
nextdownstream = int(nextdownstream)
area = float(header["Catchment area (km²)"])
if area == -999.0:
area = None
alt = float(header["Altitude (m ASL)"])
if alt == -999.0:
alt = None
location = Point(float(header["Longitude (DD)"]),
float(header["Latitude (DD)"]), 0)
else:
location = Point(float(header["Longitude (DD)"]),
float(header["Latitude (DD)"]), alt)
# import pdb;pdb.set_trace()
try:
meta = {
"grdc_no": int(header["GRDC-No."]),
"tstart": epoch[0],
"tend": epoch[-1],
"lastupdate": datetime.strptime(header["Last update"], "%Y-%m-%d"),
"area": area,
"altitude": alt,
"river": header["River"],
"statname": header["Station"],
"country": header["Country"],
"nextstation": nextdownstream,
"uri": uri.url,
"geom": WKBElement(wkbdump(location), srid=4326, extended=True)
}
except KeyError as e:
import pdb
pdb.set_trace()
raise (e)
return meta
def hull128_obj(self):
return WKBElement(self.hull128) if type(
self.hull128) == type("STR") else self.hull128
def geojson_to_wkbelement(geojson):
return WKBElement(WKBWriter(geojson).hex_wkb, srid=SRID)
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
bucketKey = '%s/%s/%s.terrain' % (tileXYZ[2], tileXYZ[0], tileXYZ[1])
with db.userSession() as session:
bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY),
# 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
geomCoords = []
for q in query:
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
geomCoords.append(coords)
nbGeoms = len(geomCoords)
if nbGeoms > 0:
try:
terrainTile = encode(geomCoords,
bounds=bounds,
autocorrectGeometries=True,
hasLighting=hasLighting,
watermask=watermask)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' % pid
msg += 'encoding terrain tile\n'
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
writeToS3(bucket,
bucketKey,
terrainTile.toBytesIO(gzipped=True),
model.__tablename__,
bucketBasePath,
contentType=terrainTile.getContentType())
tend = time.time()
tilecount.value += 1
tilesCreated = tilecount.value
total = tilesCreated + skipcount.value
if tilesCreated % 1000 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, nbGeoms,
str(datetime.timedelta(seconds=tend - t0)),
tilesCreated, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0
def _read_activity_route_wkb(self, s, activity_journal_id):
q = text(f'''
select st_force2d(aj.route_et::geometry)
from activity_journal as aj
where aj.id = :activity_journal_id''')
return WKBElement(s.connection().execute(q, activity_journal_id=activity_journal_id).fetchone()[0])
def geometry_formatter(self, geometry):
"""
Formats the passed geoalchemy2 geometry WKB element into its valid GeoJSON representation. This is a
list of coordinate tuples.
Args:
geometry (geoalchemy2.WKBElement): A geoalchemy wkb element object which should be converted.
Returns:
list: A list of coordinates formatted string
Raises:
HTTPServerError: If the geometry type is not supported.
"""
shapely_object = to_shape(geometry)
geom_type = shapely_object.geom_type
if geom_type.upper() == 'POINT':
return {
"type": geom_type,
"coordinates": list(shapely_object.coords[0])
}
elif geom_type.upper() == 'LINESTRING':
return {
"type": geom_type,
"coordinates": list(shapely_object.coords)
}
elif geom_type.upper() == 'POLYGON':
return {
"type": geom_type,
"coordinates": [list(shapely_object.exterior.coords)]
}
elif geom_type.upper() == 'MULTIPOINT':
coordinates = list()
for point in shapely_object.geoms:
coordinates.append([list(point.coords[0])])
return {
"type": geom_type,
"coordinates": coordinates
}
elif geom_type.upper() == 'MULTILINESTRING':
coordinates = list()
for point in shapely_object.geoms:
coordinates.append([list(point.coords)])
return {
"type": geom_type,
"coordinates": coordinates
}
elif geom_type.upper() == 'MULTIPOLYGON':
coordinates = list()
for polygon in shapely_object.geoms:
coordinates.append([list(polygon.exterior.coords)])
return {
"type": geom_type,
"coordinates": coordinates
}
elif geom_type.upper() == 'GEOMETRYCOLLECTION':
members = []
for inner_geometry in shapely_object.geoms:
members.append(self.geometry_formatter(WKBElement(inner_geometry.wkb)))
return {
"type": geom_type,
"members": members
}
else:
raise HTTPServerError(
'You try to access a Dataset with a "{type}" geometry type. This is not supported in the '
'moment. Sorry...'.format(type=geom_type)
)
def from_wkb(wkb, srid=-1):
return WKBElement(buffer(wkb), srid=srid)
def createTile(tile):
session = None
pid = os.getpid()
try:
(bounds, tileXYZ, t0, dbConfigFile, bucketBasePath, hasLighting,
hasWatermask) = tile
db = DB(dbConfigFile)
with db.userSession() as session:
# bucket = getBucket()
# Get the model according to the zoom level
model = modelsPyramid.getModelByZoom(tileXYZ[2])
watermask = []
if hasWatermask:
lakeModel = modelsPyramid.getLakeModelByZoom(tileXYZ[2])
query = session.query(
lakeModel.watermaskRasterize(bounds).label('watermask'))
for q in query:
watermask = q.watermask
# Get the interpolated point at the 4 corners
# 0: (minX, minY), 1: (minX, maxY), 2: (maxX, maxY), 3: (maxX, minY)
pts = [(bounds[0], bounds[1], 0), (bounds[0], bounds[3], 0),
(bounds[2], bounds[3], 0), (bounds[2], bounds[1], 0)]
def toSubQuery(x):
return session.query(
model.id, model.interpolateHeightOnPlane(pts[x])).filter(
and_(model.bboxIntersects(createBBox(pts[x], 0.01)),
model.pointIntersects(pts[x]))).subquery('p%s' %
x)
subqueries = [toSubQuery(i) for i in range(0, len(pts))]
# Get the height of the corner points as postgis cannot properly
# clip a polygon
cornerPts = {}
step = 2
j = step
query = session.query(*subqueries)
for q in query:
for i in range(0, len(q), step):
sub = q[i:j]
j += step
cornerPts[sub[0]] = list(
to_shape(WKBElement(sub[1])).coords)
# Clip using the bounds
clippedGeometry = model.bboxClippedGeom(bounds)
query = session.query(model.id,
clippedGeometry.label('clip')).filter(
model.bboxIntersects(bounds))
terrainTopo = TerrainTopology(hasLighting=hasLighting)
for q in query:
if to_shape(q.clip).geom_type == "GeometryCollection":
continue
if to_shape(q.clip).geom_type == "LineString" or to_shape(
q.clip).geom_type == "Point":
continue
coords = list(to_shape(q.clip).exterior.coords)
if q.id in cornerPts:
pt = cornerPts[q.id][0]
for i in range(0, len(coords)):
c = coords[i]
if c[0] == pt[0] and c[1] == pt[1]:
coords[i] = [c[0], c[1], pt[2]]
try:
rings = processRingCoordinates(coords)
except Exception as e:
msg = '[%s] --------- ERROR ------- occured while ' \
'collapsing non triangular shapes\n' % pid
msg += '[%s]: %s' % (pid, e)
logger.error(msg, exc_info=True)
raise Exception(e)
# Redundant coord has been remove already
for vertices in rings:
terrainTopo.addVertices(vertices)
baseDir = '/Users/liberostelios/3d_forge_geodata/output'
terrainDir = '%s/%s/%s' % (baseDir, tileXYZ[2], tileXYZ[0])
bucketKey = '%s/%s.terrain' % (terrainDir, tileXYZ[1])
verticesLength = len(terrainTopo.vertices)
if verticesLength > 0:
terrainTopo.create()
# Prepare terrain tile
terrainFormat = TerrainTile(watermask=watermask)
terrainFormat.fromTerrainTopology(terrainTopo, bounds=bounds)
# STELIOS: We want this in a file, instead
if not os.path.isdir(terrainDir):
os.makedirs(terrainDir)
terrainFormat.toFile(bucketKey)
logger.info('[%s] Just did %s %s' % (pid, bucketKey, bounds))
# Bytes manipulation and compression
# fileObject = terrainFormat.toStringIO()
# compressedFile = gzipFileObject(fileObject)
# writeToS3(
# bucket, bucketKey, compressedFile, model.__tablename__,
# bucketBasePath, contentType=terrainFormat.getContentType()
# )
tend = time.time()
tilecount.value += 1
val = tilecount.value
total = val + skipcount.value
if val % 10 == 0:
logger.info('[%s] Last tile %s (%s rings). '
'%s to write %s tiles. (total processed: %s)' %
(pid, bucketKey, verticesLength,
str(datetime.timedelta(seconds=tend - t0)),
val, total))
else:
skipcount.value += 1
val = skipcount.value
total = val + tilecount.value
# One should write an empyt tile
logger.info('[%s] Skipping %s %s because no features found '
'for this tile (%s skipped from %s total)' %
(pid, bucketKey, bounds, val, total))
except Exception as e:
logger.error(e, exc_info=True)
raise Exception(e)
finally:
db.userEngine.dispose()
return 0