def populate_notpreprocessed(type, unit):
hazardset_id = u'notpreprocessed'
hazardtype = HazardType.get(type)
hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
print 'Populating hazardset {}'.format(hazardset_id)
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now())
DBSession.add(hazardset)
return_periods = hazardtype_settings['return_periods']
for level in (u'HIG', u'MED', u'LOW'):
hazardlevel = HazardLevel.get(level)
level_return_periods = return_periods[level]
if isinstance(level_return_periods, list):
return_period = level_return_periods[0]
else:
return_period = level_return_periods
layer = Layer(
hazardlevel=hazardlevel,
mask=False,
return_period=return_period,
hazardunit=unit,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(),
download_url='test',
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True
)
hazardset.layers.append(layer)
layer = Layer(
hazardlevel=None,
mask=True,
return_period=hazardtype_settings['mask_return_period'],
hazardunit=unit,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(),
download_url='test',
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True
)
hazardset.layers.append(layer)
hazardset.complete = True
DBSession.flush()
def make_layers():
layers = []
for level in (u'HIG', u'MED', u'LOW'):
layer = Layer()
layer.hazardlevel = HazardLevel.get(level)
layer.return_period = 1
layer.hazardunit = u'm'
layer.data_lastupdated_date = datetime.now()
layer.metadata_lastupdated_date = datetime.now()
layer.geonode_id = new_geonode_id()
layer.download_url = u'http://something'
layer.calculation_method_quality = 5
layer.scientific_quality = 1
layer.local = False
layer.downloaded = True
layers.append(layer)
return layers
def make_layers():
layers = []
for level in (u'HIG', u'MED', u'LOW'):
layer = Layer()
layer.hazardlevel = HazardLevel.get(level)
layer.return_period = 1
layer.hazardunit = u'm'
layer.data_lastupdated_date = datetime.now()
layer.metadata_lastupdated_date = datetime.now()
layer.geonode_id = new_geonode_id()
layer.download_url = u'http://something'
layer.calculation_method_quality = 5
layer.scientific_quality = 1
layer.local = False
layer.downloaded = True
layers.append(layer)
return layers
def populate_processing():
print 'populate processing'
hazardset_id = u'test'
hazardtype = HazardType.get(u'EQ')
hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
print 'Populating hazardset {}'.format(hazardset_id)
hazardset = HazardSet()
hazardset.id = hazardset_id
hazardset.hazardtype = hazardtype
hazardset.local = False
hazardset.data_lastupdated_date = datetime.now()
hazardset.metadata_lastupdated_date = datetime.now()
DBSession.add(hazardset)
return_periods = hazardtype_settings['global']['return_periods']
unit = hazardtype_settings['thresholds'].keys()[0]
for level in (u'HIG', u'MED', u'LOW'):
hazardlevel = HazardLevel.get(level)
return_period = return_periods[level]
layer = Layer()
layer.title = "{}-{}".format(id, return_period)
layer.hazardlevel = hazardlevel
layer.return_period = return_period
layer.hazardunit = unit
layer.data_lastupdated_date = datetime.now()
layer.metadata_lastupdated_date = datetime.now()
layer.geonode_id = new_geonode_id()
layer.download_url = 'test'
layer.calculation_method_quality = 5
layer.scientific_quality = 1
layer.local = False
layer.downloaded = True
hazardset.layers.append(layer)
DBSession.flush()
hazardset.complete = True
DBSession.flush()
def preprocessed_hazardlevel(type_settings, layer, reader, geometry):
hazardlevel = None
for polygon in geometry.geoms:
window = reader.window(*polygon.bounds)
data = reader.read(1, window=window, masked=True)
if data.shape[0] * data.shape[1] == 0:
continue
if data.mask.all():
continue
mask = features.geometry_mask(
[polygon],
out_shape=data.shape,
transform=reader.window_transform(window),
all_touched=True)
data.mask = data.mask | mask
if data.mask.all():
continue
for level in (u'HIG', u'MED', u'LOW', u'VLO'):
level_obj = HazardLevel.get(level)
if level_obj <= hazardlevel:
break
if level in type_settings['values']:
values = type_settings['values'][level]
for value in values:
if value in data:
hazardlevel = level_obj
break
return hazardlevel
def process_hazardset(hazardset_id, force=False):
hazardset = DBSession.query(HazardSet).get(hazardset_id)
if hazardset is None:
raise ProcessException('Hazardset {} does not exist.'
.format(hazardset_id))
chrono = datetime.datetime.now()
if hazardset.processed:
if force:
hazardset.processed = False
else:
raise ProcessException('Hazardset {} has already been processed.'
.format(hazardset.id))
logger.info(" Cleaning previous outputs")
DBSession.query(Output) \
.filter(Output.hazardset_id == hazardset.id) \
.delete()
DBSession.flush()
type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic]
with rasterio.drivers():
try:
logger.info(" Opening raster files")
# Open rasters
layers = {}
readers = {}
if 'values' in type_settings.keys():
# preprocessed layer
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset.id) \
.one()
reader = rasterio.open(layer_path(layer))
layers[0] = layer
readers[0] = reader
else:
for level in (u'HIG', u'MED', u'LOW'):
hazardlevel = HazardLevel.get(level)
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset.id) \
.filter(Layer.hazardlevel_id == hazardlevel.id) \
.one()
reader = rasterio.open(layer_path(layer))
layers[level] = layer
readers[level] = reader
if ('mask_return_period' in type_settings):
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset.id) \
.filter(Layer.mask.is_(True)) \
.one()
reader = rasterio.open(layer_path(layer))
layers['mask'] = layer
readers['mask'] = reader
outputs = create_outputs(hazardset, layers, readers)
if outputs:
DBSession.add_all(outputs)
finally:
logger.info(" Closing raster files")
for key, reader in readers.iteritems():
if reader and not reader.closed:
reader.close()
hazardset.processed = True
DBSession.flush()
logger.info(' Successfully processed {}, {} outputs generated in {}'
.format(hazardset.id,
len(outputs),
datetime.datetime.now() - chrono))
return True
def notpreprocessed_hazardlevel(hazardtype, type_settings, layers, readers,
geometry):
level_VLO = HazardLevel.get(u'VLO')
hazardlevel = None
# Create some optimization caches
polygons = {}
bboxes = {}
masks = {}
inverted_comparison = ('inverted_comparison' in type_settings and
type_settings['inverted_comparison'])
for level in (u'HIG', u'MED', u'LOW'):
layer = layers[level]
reader = readers[level]
threshold = get_threshold(hazardtype,
layer.local,
layer.hazardlevel.mnemonic,
layer.hazardunit)
if threshold is None:
raise ProcessException(
'No threshold found for {} {} {} {}'
.format(hazardtype,
'local' if layer.local else 'global',
layer.hazardlevel.mnemonic,
layer.hazardunit))
for i in xrange(0, len(geometry.geoms)):
if i not in polygons:
polygon = geometry.geoms[i]
bbox = polygon.bounds
polygons[i] = polygon
bboxes[i] = bbox
else:
polygon = polygons[i]
bbox = bboxes[i]
window = reader.window(*bbox)
data = reader.read(1, window=window, masked=True)
if data.shape[0] * data.shape[1] == 0:
continue
if data.mask.all():
continue
if inverted_comparison:
data = data < threshold
else:
data = data > threshold
if ('mask_return_period' in type_settings):
mask = readers['mask'].read(1, window=window, masked=True)
if inverted_comparison:
mask = mask < threshold
else:
mask = mask > threshold
data.mask = ma.getmaskarray(data) | mask.filled(False)
if data.mask.all():
continue
if i in masks:
mask = masks[i]
else:
mask = features.geometry_mask(
[polygon],
out_shape=data.shape,
transform=reader.window_transform(window),
all_touched=True)
masks[i] = mask
data.mask = ma.getmaskarray(data) | mask
if data.any():
hazardlevel = layer.hazardlevel
break # No need to go further
if data.mask.all():
continue
if hazardlevel is None:
hazardlevel = level_VLO
if hazardlevel == layer.hazardlevel:
break # No need to go further
return hazardlevel
from thinkhazard_common.models import (
DBSession,
AdministrativeDivision,
HazardLevel,
HazardCategory,
HazardCategoryAdministrativeDivisionAssociation,
HazardType,
ClimateChangeRecommendation,
TechnicalRecommendation,
FurtherResource,
)
# An object for the "no data" category type.
_hazardlevel_nodata = HazardLevel()
_hazardlevel_nodata.mnemonic = 'no-data'
_hazardlevel_nodata.title = 'No data available'
_hazardlevel_nodata.description = 'No data for this hazard type.'
_hazardlevel_nodata.order = float('inf')
@view_config(route_name='report_overview', renderer='templates/report.jinja2')
@view_config(route_name='report_overview_slash',
renderer='templates/report.jinja2')
@view_config(route_name='report', renderer='templates/report.jinja2')
def report(request):
try:
division_code = request.matchdict.get('divisioncode')
except:
raise HTTPBadRequest(detail='incorrect value for parameter '
def populate_processing():
hazardtypeEQ = HazardType.get(u'EQ')
hazardtypeFL = HazardType.get(u'FL')
hazardset_id = u'hazardset1'
print 'Populating hazardset {}'.format(hazardset_id)
hazardset1 = HazardSet()
hazardset1.id = hazardset_id
hazardset1.hazardtype = hazardtypeEQ
hazardset1.local = False
hazardset1.calculation_method_quality = 0
hazardset1.data_lastupdated_date = datetime.now()
hazardset1.metadata_lastupdated_date = datetime.now()
hazardset1.complete = True
hazardset1.layers.extend(make_layers())
DBSession.add(hazardset1)
hazardset_id = u'hazardset1b'
print 'Populating hazardset {}'.format(hazardset_id)
hazardset1b = HazardSet()
hazardset1b.id = hazardset_id
hazardset1b.hazardtype = hazardtypeFL
hazardset1b.local = True
hazardset1b.calculation_method_quality = 3
hazardset1b.data_lastupdated_date = datetime.now()
hazardset1b.metadata_lastupdated_date = datetime.now()
hazardset1b.complete = True
hazardset1b.layers.extend(make_layers())
DBSession.add(hazardset1b)
# create hazardsets 2, 3 ... 5
hazardset_id = u'hazardset2'
# calculation_method_quality = 1 / scientific_quality = 0
print 'Populating hazardset {}'.format(hazardset_id)
hazardset2 = HazardSet()
hazardset2.id = hazardset_id
hazardset2.hazardtype = hazardtypeEQ
hazardset2.local = True
hazardset2.calculation_method_quality = 1
hazardset2.scientific_quality = 0
hazardset2.data_lastupdated_date = datetime.now()
hazardset2.metadata_lastupdated_date = datetime.now()
hazardset2.complete = True
hazardset2.layers.extend(make_layers())
DBSession.add(hazardset2)
hazardset_id = u'hazardset3'
# date = 2015-01-01 / global /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset3 = HazardSet()
hazardset3.id = hazardset_id
hazardset3.hazardtype = hazardtypeEQ
hazardset3.local = False
hazardset3.calculation_method_quality = 1
hazardset3.scientific_quality = 2
hazardset3.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
hazardset3.metadata_lastupdated_date = datetime.now()
hazardset3.complete = True
hazardset3.layers.extend(make_layers())
DBSession.add(hazardset3)
hazardset_id = u'hazardset4'
# date = 2015-01-01 / local /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset4 = HazardSet()
hazardset4.id = hazardset_id
hazardset4.hazardtype = hazardtypeEQ
hazardset4.local = True
hazardset4.calculation_method_quality = 1
hazardset4.scientific_quality = 2
hazardset4.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
hazardset4.metadata_lastupdated_date = datetime.now()
hazardset4.complete = True
hazardset4.layers.extend(make_layers())
DBSession.add(hazardset4)
hazardset_id = u'hazardset5'
# date = now() / local /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset5 = HazardSet()
hazardset5.id = hazardset_id
hazardset5.hazardtype = hazardtypeEQ
hazardset5.local = True
hazardset5.calculation_method_quality = 1
hazardset5.scientific_quality = 2
hazardset5.data_lastupdated_date = datetime.now()
hazardset5.metadata_lastupdated_date = datetime.now()
hazardset5.complete = True
hazardset5.layers.extend(make_layers())
DBSession.add(hazardset5)
hazardset_id = u'hazardset6'
# date = now() / global /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset6 = HazardSet()
hazardset6.id = hazardset_id
hazardset6.hazardtype = hazardtypeEQ
hazardset6.local = False
hazardset6.calculation_method_quality = 1
hazardset6.scientific_quality = 2
hazardset6.data_lastupdated_date = datetime.now()
hazardset6.metadata_lastupdated_date = datetime.now()
hazardset6.complete = True
hazardset6.layers.extend(make_layers())
DBSession.add(hazardset6)
# populate output table
# admin div (code 30) has only one hazardset for EQ
admin30 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 30).one()
# => test outcome = hazardset1
output1 = Output()
output1.coverage_ratio = 10
output1.hazardset = hazardset1
output1.administrativedivision = admin30
output1.hazardlevel = HazardLevel.get(u'HIG')
DBSession.add(output1)
# admin div (code 30) also has another hazardset
# but this one is for FL
# => test outcome = hazardset1b
output1b = Output()
output1b.coverage_ratio = 11
output1b.hazardset = hazardset1b
output1b.administrativedivision = admin30
output1b.hazardlevel = HazardLevel.get(u'NPR')
DBSession.add(output1b)
# admin div (code 31) has 2 hazardsets,
# one with a higher calculation_method_quality
# => test outcome = hazardset2
admin31 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 31).one()
output2 = Output()
output2.coverage_ratio = 20
output2.hazardset = hazardset1 # calculation_method_quality = 0
output2.administrativedivision = admin31
output2.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output2)
output3 = Output()
output3.coverage_ratio = 30
output3.hazardset = hazardset2 # calculation_method_quality = 1
output3.administrativedivision = admin31
output3.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output3)
# admin div (code 32) has 2 hazardsets,
# both share the same calculation_method_quality,
# one with a higher scientific_quality
# => test outcome = hazardset3
admin32 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 32).one()
output4 = Output()
output4.coverage_ratio = 40
output4.hazardset = hazardset2
# calculation_method_quality = 1 / scientific_quality = 0
output4.administrativedivision = admin32
output4.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output4)
output5 = Output()
output5.coverage_ratio = 50
output5.hazardset = hazardset3
# calculation_method_quality = 1 / scientific_quality = 2
output5.administrativedivision = admin32
output5.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output5)
# admin div (code 33) has 2 hazardsets,
# both share the same ratings, one is global, one local
# => test outcome = hazardset4
admin33 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 33).one()
output6 = Output()
output6.coverage_ratio = 60
output6.hazardset = hazardset3
# global / calculation_method_quality = 1 / scientific_quality = 2
output6.administrativedivision = admin33
output6.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output6)
output7 = Output()
output7.coverage_ratio = 70
output7.hazardset = hazardset4
# local / calculation_method_quality = 1 / scientific_quality = 2
output7.administrativedivision = admin33
output7.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output7)
# admin div (code 34) has 2 hazardsets,
# both share the same ratings, are local, one is more recent
# => test outcome = hazardset5
admin34 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 34).one()
output8 = Output()
output8.coverage_ratio = 80
output8.hazardset = hazardset4
# date = 2015-01-01 / local /
# calculation_method_quality = 1 / scientific_quality = 2
output8.administrativedivision = admin34
output8.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output8)
output9 = Output()
output9.coverage_ratio = 90
output9.hazardset = hazardset5
# date = now() / local /
# calculation_method_quality = 1 / scientific_quality = 2
output9.administrativedivision = admin34
output9.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output9)
# admin div (code 35) has 2 hazardsets,
# both share the same ratings, are global, one is more recent
# => test outcome = hazardset6
admin35 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 35).one()
output10 = Output()
output10.coverage_ratio = 95
output10.hazardset = hazardset3
# date = 2015-01-01 / global /
# calculation_method_quality = 1 / scientific_quality = 2
output10.administrativedivision = admin35
output10.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output10)
output11 = Output()
output11.coverage_ratio = 99
output11.hazardset = hazardset6
# date = now() / global /
# calculation_method_quality = 1 / scientific_quality = 2
output11.administrativedivision = admin35
output11.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output11)
DBSession.flush()
def populate_processing():
hazardtypeEQ = HazardType.get(u'EQ')
hazardtypeFL = HazardType.get(u'FL')
hazardset_id = u'hazardset1'
print 'Populating hazardset {}'.format(hazardset_id)
hazardset1 = HazardSet()
hazardset1.id = hazardset_id
hazardset1.hazardtype = hazardtypeEQ
hazardset1.local = False
hazardset1.calculation_method_quality = 0
hazardset1.data_lastupdated_date = datetime.now()
hazardset1.metadata_lastupdated_date = datetime.now()
hazardset1.complete = True
hazardset1.layers.extend(make_layers())
DBSession.add(hazardset1)
hazardset_id = u'hazardset1b'
print 'Populating hazardset {}'.format(hazardset_id)
hazardset1b = HazardSet()
hazardset1b.id = hazardset_id
hazardset1b.hazardtype = hazardtypeFL
hazardset1b.local = True
hazardset1b.calculation_method_quality = 3
hazardset1b.data_lastupdated_date = datetime.now()
hazardset1b.metadata_lastupdated_date = datetime.now()
hazardset1b.complete = True
hazardset1b.layers.extend(make_layers())
DBSession.add(hazardset1b)
# create hazardsets 2, 3 ... 5
hazardset_id = u'hazardset2'
# calculation_method_quality = 1 / scientific_quality = 0
print 'Populating hazardset {}'.format(hazardset_id)
hazardset2 = HazardSet()
hazardset2.id = hazardset_id
hazardset2.hazardtype = hazardtypeEQ
hazardset2.local = True
hazardset2.calculation_method_quality = 1
hazardset2.scientific_quality = 0
hazardset2.data_lastupdated_date = datetime.now()
hazardset2.metadata_lastupdated_date = datetime.now()
hazardset2.complete = True
hazardset2.layers.extend(make_layers())
DBSession.add(hazardset2)
hazardset_id = u'hazardset3'
# date = 2015-01-01 / global /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset3 = HazardSet()
hazardset3.id = hazardset_id
hazardset3.hazardtype = hazardtypeEQ
hazardset3.local = False
hazardset3.calculation_method_quality = 1
hazardset3.scientific_quality = 2
hazardset3.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
hazardset3.metadata_lastupdated_date = datetime.now()
hazardset3.complete = True
hazardset3.layers.extend(make_layers())
DBSession.add(hazardset3)
hazardset_id = u'hazardset4'
# date = 2015-01-01 / local /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset4 = HazardSet()
hazardset4.id = hazardset_id
hazardset4.hazardtype = hazardtypeEQ
hazardset4.local = True
hazardset4.calculation_method_quality = 1
hazardset4.scientific_quality = 2
hazardset4.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
hazardset4.metadata_lastupdated_date = datetime.now()
hazardset4.complete = True
hazardset4.layers.extend(make_layers())
DBSession.add(hazardset4)
hazardset_id = u'hazardset5'
# date = now() / local /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset5 = HazardSet()
hazardset5.id = hazardset_id
hazardset5.hazardtype = hazardtypeEQ
hazardset5.local = True
hazardset5.calculation_method_quality = 1
hazardset5.scientific_quality = 2
hazardset5.data_lastupdated_date = datetime.now()
hazardset5.metadata_lastupdated_date = datetime.now()
hazardset5.complete = True
hazardset5.layers.extend(make_layers())
DBSession.add(hazardset5)
hazardset_id = u'hazardset6'
# date = now() / global /
# calculation_method_quality = 1 / scientific_quality = 2
print 'Populating hazardset {}'.format(hazardset_id)
hazardset6 = HazardSet()
hazardset6.id = hazardset_id
hazardset6.hazardtype = hazardtypeEQ
hazardset6.local = False
hazardset6.calculation_method_quality = 1
hazardset6.scientific_quality = 2
hazardset6.data_lastupdated_date = datetime.now()
hazardset6.metadata_lastupdated_date = datetime.now()
hazardset6.complete = True
hazardset6.layers.extend(make_layers())
DBSession.add(hazardset6)
# populate output table
# admin div (code 30) has only one hazardset for EQ
admin30 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 30).one()
# => test outcome = hazardset1
output1 = Output()
output1.coverage_ratio = 10
output1.hazardset = hazardset1
output1.administrativedivision = admin30
output1.hazardlevel = HazardLevel.get(u'HIG')
DBSession.add(output1)
# admin div (code 30) also has another hazardset
# but this one is for FL
# => test outcome = hazardset1b
output1b = Output()
output1b.coverage_ratio = 11
output1b.hazardset = hazardset1b
output1b.administrativedivision = admin30
output1b.hazardlevel = HazardLevel.get(u'NPR')
DBSession.add(output1b)
# admin div (code 31) has 2 hazardsets,
# one with a higher calculation_method_quality
# => test outcome = hazardset2
admin31 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 31).one()
output2 = Output()
output2.coverage_ratio = 20
output2.hazardset = hazardset1 # calculation_method_quality = 0
output2.administrativedivision = admin31
output2.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output2)
output3 = Output()
output3.coverage_ratio = 30
output3.hazardset = hazardset2 # calculation_method_quality = 1
output3.administrativedivision = admin31
output3.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output3)
# admin div (code 32) has 2 hazardsets,
# both share the same calculation_method_quality,
# one with a higher scientific_quality
# => test outcome = hazardset3
admin32 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 32).one()
output4 = Output()
output4.coverage_ratio = 40
output4.hazardset = hazardset2
# calculation_method_quality = 1 / scientific_quality = 0
output4.administrativedivision = admin32
output4.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output4)
output5 = Output()
output5.coverage_ratio = 50
output5.hazardset = hazardset3
# calculation_method_quality = 1 / scientific_quality = 2
output5.administrativedivision = admin32
output5.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output5)
# admin div (code 33) has 2 hazardsets,
# both share the same ratings, one is global, one local
# => test outcome = hazardset4
admin33 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 33).one()
output6 = Output()
output6.coverage_ratio = 60
output6.hazardset = hazardset3
# global / calculation_method_quality = 1 / scientific_quality = 2
output6.administrativedivision = admin33
output6.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output6)
output7 = Output()
output7.coverage_ratio = 70
output7.hazardset = hazardset4
# local / calculation_method_quality = 1 / scientific_quality = 2
output7.administrativedivision = admin33
output7.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output7)
# admin div (code 34) has 2 hazardsets,
# both share the same ratings, are local, one is more recent
# => test outcome = hazardset5
admin34 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 34).one()
output8 = Output()
output8.coverage_ratio = 80
output8.hazardset = hazardset4
# date = 2015-01-01 / local /
# calculation_method_quality = 1 / scientific_quality = 2
output8.administrativedivision = admin34
output8.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output8)
output9 = Output()
output9.coverage_ratio = 90
output9.hazardset = hazardset5
# date = now() / local /
# calculation_method_quality = 1 / scientific_quality = 2
output9.administrativedivision = admin34
output9.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output9)
# admin div (code 35) has 2 hazardsets,
# both share the same ratings, are global, one is more recent
# => test outcome = hazardset6
admin35 = DBSession.query(AdministrativeDivision)\
.filter(AdministrativeDivision.code == 35).one()
output10 = Output()
output10.coverage_ratio = 95
output10.hazardset = hazardset3
# date = 2015-01-01 / global /
# calculation_method_quality = 1 / scientific_quality = 2
output10.administrativedivision = admin35
output10.hazardlevel = HazardLevel.get(u'MED')
DBSession.add(output10)
output11 = Output()
output11.coverage_ratio = 99
output11.hazardset = hazardset6
# date = now() / global /
# calculation_method_quality = 1 / scientific_quality = 2
output11.administrativedivision = admin35
output11.hazardlevel = HazardLevel.get(u'LOW')
DBSession.add(output11)
DBSession.flush()
def harvest_layer(object, dry_run=False):
title = object['title']
'''
typename = urllib.unquote(
object['distribution_url'].split('/').pop())
# print title
'''
hazardset_id = object['hazard_set']
if not hazardset_id:
logger.info('{} - hazard_set is empty'.format(title))
return False
hazard_type = object['hazard_type']
if not hazard_type:
logger.warning('{} - hazard_type is empty'.format(title))
return False
hazardtype = hazardtype_from_geonode(hazard_type)
if hazardtype is None:
logger.warning('{} - hazard_type not supported: {}'
.format(title, hazard_type))
return False
type_settings = settings['hazard_types'][hazardtype.mnemonic]
preprocessed = 'values' in type_settings
'''
csw_wkt_geometry = object['csw_wkt_geometry']
bounds = wkt_loads(csw_wkt_geometry).bounds
# print ' bounds :', bounds
# TODO: minimum global bbox should be defined in settings
minimum_global_bounds = (-175, -45, 175, 45)
from shapely.geometry import box
local = not box(bounds).contains(box(minimum_global_bounds))
'''
local = 'GLOBAL' not in hazardset_id
'''
local = (
bounds[0] > -175 or
bounds[1] > -45 or
bounds[2] < 175 or
bounds[3] < 45)
'''
mask = False
if preprocessed is True:
hazardlevel = None
hazard_unit = None
if object['hazard_period']:
logger.info('{} - Has a return period'.format(title))
return False
hazard_period = None
else:
hazard_period = int(object['hazard_period'])
hazardlevel = None
for level in (u'LOW', u'MED', u'HIG'):
return_periods = type_settings['return_periods'][level]
if isinstance(return_periods, list):
if (hazard_period >= return_periods[0] and
hazard_period <= return_periods[1]):
hazardlevel = HazardLevel.get(level)
break
else:
if hazard_period == return_periods:
hazardlevel = HazardLevel.get(level)
if ('mask_return_period' in type_settings and
hazard_period == type_settings['mask_return_period']):
mask = True
if hazardlevel is None and not mask:
logger.info('{} - No corresponding hazard_level'.format(title))
return False
hazard_unit = object['hazard_unit']
if hazard_unit == '':
logger.info('{} - hazard_unit is empty'.format(title))
return False
if object['srid'] != 'EPSG:4326':
logger.info('{} - srid is different from "EPSG:4326"'
.format(title))
return False
data_update_date = object['data_update_date']
if not data_update_date:
logger.warning('{} - data_update_date is empty'.format(title))
# TODO: Restore bypassed constraint to get Volcanic data
# return False
data_update_date = datetime.now()
metadata_update_date = object['metadata_update_date']
if not metadata_update_date:
logger.warning('{} - metadata_update_date is empty'.format(title))
# return False
metadata_update_date = datetime.now()
calculation_method_quality = object['calculation_method_quality']
if not calculation_method_quality:
logger.warning('{} - calculation_method_quality is empty'
.format(title))
return False
calculation_method_quality = int(float(calculation_method_quality))
scientific_quality = object['scientific_quality']
if not scientific_quality:
logger.warning('{} - scientific_quality is empty'.format(title))
return False
scientific_quality = int(float(scientific_quality))
download_url = object['download_url']
if not download_url:
logger.warning('{} - download_url is empty'.format(title))
return False
hazardset = DBSession.query(HazardSet).get(hazardset_id)
if hazardset is None:
logger.info('{} - Create new hazardset {}'
.format(title, hazardset_id))
hazardset = HazardSet()
hazardset.id = hazardset_id
hazardset.hazardtype = hazardtype
hazardset.data_lastupdated_date = data_update_date
hazardset.metadata_lastupdated_date = metadata_update_date
DBSession.add(hazardset)
else:
# print ' Hazardset {} found'.format(hazardset_id)
pass
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset_id)
if hazardlevel is not None:
layer = layer.filter(Layer.hazardlevel_id == hazardlevel.id)
if mask:
layer = layer.filter(Layer.mask.is_(True))
layer = layer.first()
if layer is None:
layer = Layer()
logger.info('{} - Create new Layer {}'.format(title, title))
DBSession.add(layer)
else:
if object['id'] == layer.geonode_id:
# TODO: metadata change
return False
if hazard_period > layer.return_period:
logger.info('{} - Use preferred return period {}'
.format(title, layer.return_period))
return False
logger.info('{} - Replace layer for level {}'
.format(title, hazardlevel.mnemonic))
layer.hazardset = hazardset
layer.hazardlevel = hazardlevel
layer.mask = mask
layer.return_period = hazard_period
layer.hazardunit = hazard_unit
layer.data_lastupdated_date = data_update_date
layer.metadata_lastupdated_date = metadata_update_date
layer.geonode_id = object['id']
layer.download_url = download_url
# TODO: retrieve quality attributes
layer.calculation_method_quality = calculation_method_quality
layer.scientific_quality = scientific_quality
layer.local = local
DBSession.flush()
return True
def layerByLevel(self, level):
hazardlevel = HazardLevel.get(level)
return DBSession.query(Layer) \
.filter(Layer.hazardset_id == self.id) \
.filter(Layer.hazardlevel_id == hazardlevel.id) \
.one_or_none()
def process_hazardset(hazardset, force=False):
print hazardset.id
chrono = datetime.datetime.now()
last_percent = 0
level_VLO = HazardLevel.get(u'VLO')
if hazardset is None:
raise ProcessException('HazardSet {} does not exist.'
.format(hazardset.id))
if hazardset.processed:
if force:
hazardset.processed = False
else:
raise ProcessException('HazardSet {} has already been processed.'
.format(hazardset.id))
# clean previous outputs
DBSession.query(Output) \
.filter(Output.hazardset_id == hazardset.id) \
.delete()
DBSession.flush()
hazardtype = hazardset.hazardtype
hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
thresholds = hazardtype_settings['thresholds']
project = partial(
pyproj.transform,
pyproj.Proj(init='epsg:3857'),
pyproj.Proj(init='epsg:4326'))
layers = {}
for level in (u'HIG', u'MED', u'LOW'):
hazardlevel = HazardLevel.get(level)
layer = DBSession.query(Layer) \
.filter(Layer.hazardset_id == hazardset.id) \
.filter(Layer.hazardlevel_id == hazardlevel.id) \
.one()
layers[level] = layer
with rasterio.drivers():
with rasterio.open(layers['HIG'].path()) as src_hig, \
rasterio.open(layers['MED'].path()) as src_med, \
rasterio.open(layers['LOW'].path()) as src_low:
readers = {}
readers['HIG'] = src_hig
readers['MED'] = src_med
readers['LOW'] = src_low
polygon_hig = polygonFromBounds(src_hig.bounds)
polygon_med = polygonFromBounds(src_med.bounds)
polygon_low = polygonFromBounds(src_low.bounds)
polygon = cascaded_union((
polygon_hig,
polygon_med,
polygon_low))
adminlevel_REG = AdminLevelType.get(u'REG')
admindivs = DBSession.query(AdministrativeDivision) \
.filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \
if hazardset.local:
admindivs = admindivs \
.filter(
func.ST_Transform(AdministrativeDivision.geom, 4326)
.intersects(
func.ST_GeomFromText(polygon.wkt, 4326))) \
.filter(func.ST_Intersects(
func.ST_Transform(AdministrativeDivision.geom, 4326),
func.ST_GeomFromText(polygon.wkt, 4326)))
current = 0
outputs = 0
total = admindivs.count()
for admindiv in admindivs:
# print ' ', admindiv.id, admindiv.code, admindiv.name
current += 1
if admindiv.geom is None:
print ' ', ('{}-{} has null geometry'
.format(admindiv.code, admindiv.name))
continue
reprojected = transform(
project,
to_shape(admindiv.geom))
output = Output()
output.hazardset = hazardset
output.administrativedivision = admindiv
output.hazardlevel = None
# TODO: calculate coverage ratio
output.coverage_ratio = 100
for level in (u'HIG', u'MED', u'LOW'):
layer = layers[level]
src = readers[level]
if not reprojected.intersects(polygon):
continue
window = src.window(*reprojected.bounds)
data = src.read(1, window=window, masked=True)
if data.shape[0] * data.shape[1] == 0:
continue
threshold = thresholds[layer.hazardunit]
positive_data = (data > threshold).astype(rasterio.uint8)
division = features.rasterize(
((g, 1) for g in [reprojected]),
out_shape=data.shape,
transform=src.window_transform(window),
all_touched=True)
masked = numpy.ma.masked_array(positive_data,
mask=~division.astype(bool))
if str(numpy.max(masked)) == str(numpy.ma.masked):
break
else:
if output.hazardlevel is None:
output.hazardlevel = level_VLO
if numpy.max(masked) > 0:
output.hazardlevel = layer.hazardlevel
break
if output.hazardlevel is not None:
# print ' hazardlevel :', output.hazardlevel.mnemonic
DBSession.add(output)
outputs += 1
percent = int(100.0 * current / total)
if percent % 10 == 0 and percent != last_percent:
print ' ... processed {}%'.format(percent)
last_percent = percent
pass
hazardset.processed = True
DBSession.flush()
transaction.commit()
print ('Successfully processed {} divisions, {} outputs generated in {}'
.format(total, outputs, datetime.datetime.now() - chrono))
