def populate_datamart():
print 'populate datamart'
adminlevel_REG = AdminLevelType.get(u'REG')
from functools import partial
import pyproj
from shapely.ops import transform
project = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
pyproj.Proj(init='epsg:3857'))
shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])])
reprojected = transform(project, shape)
geometry = from_shape(reprojected, 3857)
div = AdministrativeDivision(
**{
'code': 30,
'leveltype_id': adminlevel_REG.id,
'name': u'Administrative division level 3'
})
div.geom = geometry
div.hazardcategories = []
DBSession.add(div)
DBSession.flush()
def populate_datamart():
print 'populate datamart'
adminlevel_REG = AdminLevelType.get(u'REG')
shape = MultiPolygon([
Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
])
geometry = from_shape(shape, 4326)
div = AdministrativeDivision(**{
'code': 30,
'leveltype_id': adminlevel_REG.id,
'name': u'Administrative division level 3'
})
div.geom = geometry
div.hazardcategories = []
DBSession.add(div)
DBSession.flush()
def create_outputs(hazardset, layers, readers):
type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic]
adminlevel_REG = AdminLevelType.get(u'REG')
bbox = None
for reader in readers.itervalues():
polygon = polygon_from_boundingbox(reader.bounds)
if bbox is None:
bbox = polygon
else:
bbox = bbox.intersection(polygon)
admindivs = DBSession.query(AdministrativeDivision) \
.filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \
.filter(func.ST_Intersects(AdministrativeDivision.geom,
func.ST_GeomFromText(bbox.wkt, 4326))) \
.order_by(AdministrativeDivision.id) # Needed for windowed querying
current = 0
last_percent = 0
outputs = []
total = admindivs.count()
logger.info(' Iterating over {} administrative divisions'.format(total))
# Windowed querying to limit memory usage
limit = 1000 # 1000 records <=> 10 Mo
admindivs = admindivs.limit(limit)
for offset in xrange(0, total, limit):
admindivs = admindivs.offset(offset)
for admindiv in admindivs:
current += 1
if admindiv.geom is None:
logger.warning(' {}-{} has null geometry'
.format(admindiv.code, admindiv.name))
continue
shape = to_shape(admindiv.geom)
# Try block to include admindiv.code in exception message
try:
if 'values' in type_settings.keys():
# preprocessed layer
hazardlevel = preprocessed_hazardlevel(
type_settings,
layers[0], readers[0],
shape)
else:
hazardlevel = notpreprocessed_hazardlevel(
hazardset.hazardtype.mnemonic, type_settings, layers,
readers, shape)
except Exception as e:
e.message = ("{}-{} raises an exception :\n{}"
.format(admindiv.code, admindiv.name, e.message))
raise
# Create output record
if hazardlevel is not None:
output = Output()
output.hazardset = hazardset
output.admin_id = admindiv.id
output.hazardlevel = hazardlevel
# TODO: calculate coverage ratio
output.coverage_ratio = 100
outputs.append(output)
# Remove admindiv from memory
DBSession.expunge(admindiv)
percent = int(100.0 * current / total)
if percent % 10 == 0 and percent != last_percent:
logger.info(' ... processed {}%'.format(percent))
last_percent = percent
return outputs
def upscaling_query(level):
return '''
/*
* Upscale hazard categories for each administrative division
*/
INSERT INTO datamart.rel_hazardcategory_administrativedivision (
administrativedivision_id,
hazardcategory_id
)
SELECT DISTINCT
admindiv_parent.id AS administrativedivision_id,
first_value(category.id) OVER w AS hazardcategory_id
FROM
datamart.rel_hazardcategory_administrativedivision AS category_admindiv
JOIN datamart.hazardcategory AS category
ON category.id = category_admindiv.hazardcategory_id
JOIN datamart.enum_hazardlevel AS level
ON level.id = category.hazardlevel_id
JOIN datamart.administrativedivision AS admindiv_child
ON admindiv_child.id = category_admindiv.administrativedivision_id
JOIN datamart.administrativedivision AS admindiv_parent
ON admindiv_parent.code = admindiv_child.parent_code
WHERE admindiv_parent.leveltype_id = {leveltype_id}
WINDOW w AS (
PARTITION BY
admindiv_parent.id,
category.hazardtype_id
ORDER BY
level.order
);
/*
* Upscale relations with hazardsets (sources)
*/
INSERT INTO datamart.rel_hazardcategory_administrativedivision_hazardset (
rel_hazardcategory_administrativedivision_id,
hazardset_id
)
SELECT DISTINCT
hc_ad_parent.id,
hc_ad_hs.hazardset_id
FROM
-- Get hazardset_id from hc_ad_hs child level
datamart.rel_hazardcategory_administrativedivision_hazardset AS hc_ad_hs
-- Get hc_ad child level
JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_child
ON hc_ad_child.id =
hc_ad_hs.rel_hazardcategory_administrativedivision_id
-- Get ad child level
JOIN datamart.administrativedivision AS ad_child
ON ad_child.id = hc_ad_child.administrativedivision_id
-- Get ad parent level
JOIN datamart.administrativedivision AS ad_parent
ON ad_parent.code = ad_child.parent_code
-- Get hc_ad parent level to obtain destination hc_ad identifier
JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_parent
ON hc_ad_parent.administrativedivision_id = ad_parent.id
AND hc_ad_parent.hazardcategory_id = hc_ad_child.hazardcategory_id
WHERE ad_parent.leveltype_id = {leveltype_id};
'''.format(leveltype_id=AdminLevelType.get(level).id)
def process_hazardset(hazardset, force=False):
print hazardset.id
chrono = datetime.datetime.now()
last_percent = 0
level_VLO = HazardLevel.get(u'VLO')
if hazardset is None:
raise ProcessException('HazardSet {} does not exist.'
.format(hazardset.id))
if hazardset.processed:
if force:
hazardset.processed = False
else:
raise ProcessException('HazardSet {} has already been processed.'
.format(hazardset.id))
# clean previous outputs
DBSession.query(Output) \
.filter(Output.hazardset_id == hazardset.id) \
.delete()
DBSession.flush()
hazardtype = hazardset.hazardtype
hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
thresholds = hazardtype_settings['thresholds']
project = partial(
pyproj.transform,
pyproj.Proj(init='epsg:3857'),
pyproj.Proj(init='epsg:4326'))
layers = {}
for level in (u'HIG', u'MED', u'LOW'):
hazardlevel = HazardLevel.get(level)
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset.id) \
.filter(Layer.hazardlevel_id == hazardlevel.id) \
.one()
layers[level] = layer
with rasterio.drivers():
with rasterio.open(layers['HIG'].path()) as src_hig, \
rasterio.open(layers['MED'].path()) as src_med, \
rasterio.open(layers['LOW'].path()) as src_low:
readers = {}
readers['HIG'] = src_hig
readers['MED'] = src_med
readers['LOW'] = src_low
polygon_hig = polygonFromBounds(src_hig.bounds)
polygon_med = polygonFromBounds(src_med.bounds)
polygon_low = polygonFromBounds(src_low.bounds)
polygon = cascaded_union((
polygon_hig,
polygon_med,
polygon_low))
adminlevel_REG = AdminLevelType.get(u'REG')
admindivs = DBSession.query(AdministrativeDivision) \
.filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \
if hazardset.local:
admindivs = admindivs \
.filter(
func.ST_Transform(AdministrativeDivision.geom, 4326)
.intersects(
func.ST_GeomFromText(polygon.wkt, 4326))) \
.filter(func.ST_Intersects(
func.ST_Transform(AdministrativeDivision.geom, 4326),
func.ST_GeomFromText(polygon.wkt, 4326)))
current = 0
outputs = 0
total = admindivs.count()
for admindiv in admindivs:
# print ' ', admindiv.id, admindiv.code, admindiv.name
current += 1
if admindiv.geom is None:
print ' ', ('{}-{} has null geometry'
.format(admindiv.code, admindiv.name))
continue
reprojected = transform(
project,
to_shape(admindiv.geom))
output = Output()
output.hazardset = hazardset
output.administrativedivision = admindiv
output.hazardlevel = None
# TODO: calculate coverage ratio
output.coverage_ratio = 100
for level in (u'HIG', u'MED', u'LOW'):
layer = layers[level]
src = readers[level]
if not reprojected.intersects(polygon):
continue
window = src.window(*reprojected.bounds)
data = src.read(1, window=window, masked=True)
if data.shape[0] * data.shape[1] == 0:
continue
threshold = thresholds[layer.hazardunit]
positive_data = (data > threshold).astype(rasterio.uint8)
division = features.rasterize(
((g, 1) for g in [reprojected]),
out_shape=data.shape,
transform=src.window_transform(window),
all_touched=True)
masked = numpy.ma.masked_array(positive_data,
mask=~division.astype(bool))
if str(numpy.max(masked)) == str(numpy.ma.masked):
break
else:
if output.hazardlevel is None:
output.hazardlevel = level_VLO
if numpy.max(masked) > 0:
output.hazardlevel = layer.hazardlevel
break
if output.hazardlevel is not None:
# print ' hazardlevel :', output.hazardlevel.mnemonic
DBSession.add(output)
outputs += 1
percent = int(100.0 * current / total)
if percent % 10 == 0 and percent != last_percent:
print ' ... processed {}%'.format(percent)
last_percent = percent
pass
hazardset.processed = True
DBSession.flush()
transaction.commit()
print ('Successfully processed {} divisions, {} outputs generated in {}'
.format(total, outputs, datetime.datetime.now() - chrono))