def populate_datamart(): print 'populate datamart' adminlevel_REG = AdminLevelType.get(u'REG') from functools import partial import pyproj from shapely.ops import transform project = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'), pyproj.Proj(init='epsg:3857')) shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])]) reprojected = transform(project, shape) geometry = from_shape(reprojected, 3857) div = AdministrativeDivision( **{ 'code': 30, 'leveltype_id': adminlevel_REG.id, 'name': u'Administrative division level 3' }) div.geom = geometry div.hazardcategories = [] DBSession.add(div) DBSession.flush()
def populate_datamart(): print 'populate datamart' adminlevel_REG = AdminLevelType.get(u'REG') shape = MultiPolygon([ Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)]) ]) geometry = from_shape(shape, 4326) div = AdministrativeDivision(**{ 'code': 30, 'leveltype_id': adminlevel_REG.id, 'name': u'Administrative division level 3' }) div.geom = geometry div.hazardcategories = [] DBSession.add(div) DBSession.flush()
def create_outputs(hazardset, layers, readers): type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic] adminlevel_REG = AdminLevelType.get(u'REG') bbox = None for reader in readers.itervalues(): polygon = polygon_from_boundingbox(reader.bounds) if bbox is None: bbox = polygon else: bbox = bbox.intersection(polygon) admindivs = DBSession.query(AdministrativeDivision) \ .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \ .filter(func.ST_Intersects(AdministrativeDivision.geom, func.ST_GeomFromText(bbox.wkt, 4326))) \ .order_by(AdministrativeDivision.id) # Needed for windowed querying current = 0 last_percent = 0 outputs = [] total = admindivs.count() logger.info(' Iterating over {} administrative divisions'.format(total)) # Windowed querying to limit memory usage limit = 1000 # 1000 records <=> 10 Mo admindivs = admindivs.limit(limit) for offset in xrange(0, total, limit): admindivs = admindivs.offset(offset) for admindiv in admindivs: current += 1 if admindiv.geom is None: logger.warning(' {}-{} has null geometry' .format(admindiv.code, admindiv.name)) continue shape = to_shape(admindiv.geom) # Try block to include admindiv.code in exception message try: if 'values' in type_settings.keys(): # preprocessed layer hazardlevel = preprocessed_hazardlevel( type_settings, layers[0], readers[0], shape) else: hazardlevel = notpreprocessed_hazardlevel( hazardset.hazardtype.mnemonic, type_settings, layers, readers, shape) except Exception as e: e.message = ("{}-{} raises an exception :\n{}" .format(admindiv.code, admindiv.name, e.message)) raise # Create output record if hazardlevel is not None: output = Output() output.hazardset = hazardset output.admin_id = admindiv.id output.hazardlevel = hazardlevel # TODO: calculate coverage ratio output.coverage_ratio = 100 outputs.append(output) # Remove admindiv from memory DBSession.expunge(admindiv) percent = int(100.0 * current / total) if percent % 10 == 0 and percent != last_percent: logger.info(' ... processed {}%'.format(percent)) last_percent = percent return outputs
def upscaling_query(level): return ''' /* * Upscale hazard categories for each administrative division */ INSERT INTO datamart.rel_hazardcategory_administrativedivision ( administrativedivision_id, hazardcategory_id ) SELECT DISTINCT admindiv_parent.id AS administrativedivision_id, first_value(category.id) OVER w AS hazardcategory_id FROM datamart.rel_hazardcategory_administrativedivision AS category_admindiv JOIN datamart.hazardcategory AS category ON category.id = category_admindiv.hazardcategory_id JOIN datamart.enum_hazardlevel AS level ON level.id = category.hazardlevel_id JOIN datamart.administrativedivision AS admindiv_child ON admindiv_child.id = category_admindiv.administrativedivision_id JOIN datamart.administrativedivision AS admindiv_parent ON admindiv_parent.code = admindiv_child.parent_code WHERE admindiv_parent.leveltype_id = {leveltype_id} WINDOW w AS ( PARTITION BY admindiv_parent.id, category.hazardtype_id ORDER BY level.order ); /* * Upscale relations with hazardsets (sources) */ INSERT INTO datamart.rel_hazardcategory_administrativedivision_hazardset ( rel_hazardcategory_administrativedivision_id, hazardset_id ) SELECT DISTINCT hc_ad_parent.id, hc_ad_hs.hazardset_id FROM -- Get hazardset_id from hc_ad_hs child level datamart.rel_hazardcategory_administrativedivision_hazardset AS hc_ad_hs -- Get hc_ad child level JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_child ON hc_ad_child.id = hc_ad_hs.rel_hazardcategory_administrativedivision_id -- Get ad child level JOIN datamart.administrativedivision AS ad_child ON ad_child.id = hc_ad_child.administrativedivision_id -- Get ad parent level JOIN datamart.administrativedivision AS ad_parent ON ad_parent.code = ad_child.parent_code -- Get hc_ad parent level to obtain destination hc_ad identifier JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_parent ON hc_ad_parent.administrativedivision_id = ad_parent.id AND hc_ad_parent.hazardcategory_id = hc_ad_child.hazardcategory_id WHERE ad_parent.leveltype_id = {leveltype_id}; '''.format(leveltype_id=AdminLevelType.get(level).id)
def process_hazardset(hazardset, force=False): print hazardset.id chrono = datetime.datetime.now() last_percent = 0 level_VLO = HazardLevel.get(u'VLO') if hazardset is None: raise ProcessException('HazardSet {} does not exist.' .format(hazardset.id)) if hazardset.processed: if force: hazardset.processed = False else: raise ProcessException('HazardSet {} has already been processed.' .format(hazardset.id)) # clean previous outputs DBSession.query(Output) \ .filter(Output.hazardset_id == hazardset.id) \ .delete() DBSession.flush() hazardtype = hazardset.hazardtype hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic] thresholds = hazardtype_settings['thresholds'] project = partial( pyproj.transform, pyproj.Proj(init='epsg:3857'), pyproj.Proj(init='epsg:4326')) layers = {} for level in (u'HIG', u'MED', u'LOW'): hazardlevel = HazardLevel.get(level) layer = DBSession.query(Layer) \ .filter(Layer.hazardset_id == hazardset.id) \ .filter(Layer.hazardlevel_id == hazardlevel.id) \ .one() layers[level] = layer with rasterio.drivers(): with rasterio.open(layers['HIG'].path()) as src_hig, \ rasterio.open(layers['MED'].path()) as src_med, \ rasterio.open(layers['LOW'].path()) as src_low: readers = {} readers['HIG'] = src_hig readers['MED'] = src_med readers['LOW'] = src_low polygon_hig = polygonFromBounds(src_hig.bounds) polygon_med = polygonFromBounds(src_med.bounds) polygon_low = polygonFromBounds(src_low.bounds) polygon = cascaded_union(( polygon_hig, polygon_med, polygon_low)) adminlevel_REG = AdminLevelType.get(u'REG') admindivs = DBSession.query(AdministrativeDivision) \ .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \ if hazardset.local: admindivs = admindivs \ .filter( func.ST_Transform(AdministrativeDivision.geom, 4326) .intersects( func.ST_GeomFromText(polygon.wkt, 4326))) \ .filter(func.ST_Intersects( func.ST_Transform(AdministrativeDivision.geom, 4326), func.ST_GeomFromText(polygon.wkt, 4326))) current = 0 outputs = 0 total = admindivs.count() for admindiv in admindivs: # print ' ', admindiv.id, admindiv.code, admindiv.name current += 1 if admindiv.geom is None: print ' ', ('{}-{} has null geometry' .format(admindiv.code, admindiv.name)) continue reprojected = transform( project, to_shape(admindiv.geom)) output = Output() output.hazardset = hazardset output.administrativedivision = admindiv output.hazardlevel = None # TODO: calculate coverage ratio output.coverage_ratio = 100 for level in (u'HIG', u'MED', u'LOW'): layer = layers[level] src = readers[level] if not reprojected.intersects(polygon): continue window = src.window(*reprojected.bounds) data = src.read(1, window=window, masked=True) if data.shape[0] * data.shape[1] == 0: continue threshold = thresholds[layer.hazardunit] positive_data = (data > threshold).astype(rasterio.uint8) division = features.rasterize( ((g, 1) for g in [reprojected]), out_shape=data.shape, transform=src.window_transform(window), all_touched=True) masked = numpy.ma.masked_array(positive_data, mask=~division.astype(bool)) if str(numpy.max(masked)) == str(numpy.ma.masked): break else: if output.hazardlevel is None: output.hazardlevel = level_VLO if numpy.max(masked) > 0: output.hazardlevel = layer.hazardlevel break if output.hazardlevel is not None: # print ' hazardlevel :', output.hazardlevel.mnemonic DBSession.add(output) outputs += 1 percent = int(100.0 * current / total) if percent % 10 == 0 and percent != last_percent: print ' ... processed {}%'.format(percent) last_percent = percent pass hazardset.processed = True DBSession.flush() transaction.commit() print ('Successfully processed {} divisions, {} outputs generated in {}' .format(total, outputs, datetime.datetime.now() - chrono))