Exemplo n.º 1
0
def populate_datamart():
    print 'populate datamart'
    adminlevel_REG = AdminLevelType.get(u'REG')

    from functools import partial
    import pyproj
    from shapely.ops import transform

    project = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
                      pyproj.Proj(init='epsg:3857'))

    shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])])
    reprojected = transform(project, shape)
    geometry = from_shape(reprojected, 3857)

    div = AdministrativeDivision(
        **{
            'code': 30,
            'leveltype_id': adminlevel_REG.id,
            'name': u'Administrative division level 3'
        })
    div.geom = geometry
    div.hazardcategories = []
    DBSession.add(div)

    DBSession.flush()
Exemplo n.º 2
0
def populate_datamart():
    print 'populate datamart'
    adminlevel_REG = AdminLevelType.get(u'REG')

    shape = MultiPolygon([
        Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
    ])
    geometry = from_shape(shape, 4326)

    div = AdministrativeDivision(**{
        'code': 30,
        'leveltype_id': adminlevel_REG.id,
        'name': u'Administrative division level 3'
    })
    div.geom = geometry
    div.hazardcategories = []
    DBSession.add(div)

    DBSession.flush()
Exemplo n.º 3
0
def create_outputs(hazardset, layers, readers):
    type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic]
    adminlevel_REG = AdminLevelType.get(u'REG')

    bbox = None
    for reader in readers.itervalues():
        polygon = polygon_from_boundingbox(reader.bounds)
        if bbox is None:
            bbox = polygon
        else:
            bbox = bbox.intersection(polygon)

    admindivs = DBSession.query(AdministrativeDivision) \
        .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \
        .filter(func.ST_Intersects(AdministrativeDivision.geom,
                func.ST_GeomFromText(bbox.wkt, 4326))) \
        .order_by(AdministrativeDivision.id)  # Needed for windowed querying

    current = 0
    last_percent = 0
    outputs = []
    total = admindivs.count()
    logger.info('  Iterating over {} administrative divisions'.format(total))

    # Windowed querying to limit memory usage
    limit = 1000  # 1000 records <=> 10 Mo
    admindivs = admindivs.limit(limit)
    for offset in xrange(0, total, limit):
        admindivs = admindivs.offset(offset)

        for admindiv in admindivs:
            current += 1

            if admindiv.geom is None:
                logger.warning('    {}-{} has null geometry'
                               .format(admindiv.code, admindiv.name))
                continue

            shape = to_shape(admindiv.geom)

            # Try block to include admindiv.code in exception message
            try:
                if 'values' in type_settings.keys():
                    # preprocessed layer
                    hazardlevel = preprocessed_hazardlevel(
                        type_settings,
                        layers[0], readers[0],
                        shape)
                else:
                    hazardlevel = notpreprocessed_hazardlevel(
                        hazardset.hazardtype.mnemonic, type_settings, layers,
                        readers, shape)

            except Exception as e:
                e.message = ("{}-{} raises an exception :\n{}"
                             .format(admindiv.code, admindiv.name, e.message))
                raise

            # Create output record
            if hazardlevel is not None:
                output = Output()
                output.hazardset = hazardset
                output.admin_id = admindiv.id
                output.hazardlevel = hazardlevel
                # TODO: calculate coverage ratio
                output.coverage_ratio = 100
                outputs.append(output)

            # Remove admindiv from memory
            DBSession.expunge(admindiv)

            percent = int(100.0 * current / total)
            if percent % 10 == 0 and percent != last_percent:
                logger.info('  ... processed {}%'.format(percent))
                last_percent = percent

    return outputs
Exemplo n.º 4
0
def upscaling_query(level):
    return '''
/*
 * Upscale hazard categories for each administrative division
 */
INSERT INTO datamart.rel_hazardcategory_administrativedivision (
    administrativedivision_id,
    hazardcategory_id
)
SELECT DISTINCT
    admindiv_parent.id AS administrativedivision_id,
    first_value(category.id) OVER w AS hazardcategory_id
FROM
    datamart.rel_hazardcategory_administrativedivision AS category_admindiv
    JOIN datamart.hazardcategory AS category
        ON category.id = category_admindiv.hazardcategory_id
    JOIN datamart.enum_hazardlevel AS level
        ON level.id =  category.hazardlevel_id
    JOIN datamart.administrativedivision AS admindiv_child
        ON admindiv_child.id = category_admindiv.administrativedivision_id
    JOIN datamart.administrativedivision AS admindiv_parent
        ON admindiv_parent.code = admindiv_child.parent_code
WHERE admindiv_parent.leveltype_id = {leveltype_id}
WINDOW w AS (
    PARTITION BY
        admindiv_parent.id,
        category.hazardtype_id
    ORDER BY
        level.order
);


/*
 * Upscale relations with hazardsets (sources)
 */
INSERT INTO datamart.rel_hazardcategory_administrativedivision_hazardset (
    rel_hazardcategory_administrativedivision_id,
    hazardset_id
)
SELECT DISTINCT
    hc_ad_parent.id,
    hc_ad_hs.hazardset_id
FROM
    -- Get hazardset_id from hc_ad_hs child level
    datamart.rel_hazardcategory_administrativedivision_hazardset AS hc_ad_hs

    -- Get hc_ad child level
    JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_child
        ON hc_ad_child.id =
            hc_ad_hs.rel_hazardcategory_administrativedivision_id

    -- Get ad child level
    JOIN datamart.administrativedivision AS ad_child
        ON ad_child.id = hc_ad_child.administrativedivision_id

    -- Get ad parent level
    JOIN datamart.administrativedivision AS ad_parent
        ON ad_parent.code = ad_child.parent_code

    -- Get hc_ad parent level to obtain destination hc_ad identifier
    JOIN datamart.rel_hazardcategory_administrativedivision AS hc_ad_parent
        ON hc_ad_parent.administrativedivision_id = ad_parent.id
        AND hc_ad_parent.hazardcategory_id = hc_ad_child.hazardcategory_id
WHERE ad_parent.leveltype_id = {leveltype_id};
'''.format(leveltype_id=AdminLevelType.get(level).id)
Exemplo n.º 5
0
def process_hazardset(hazardset, force=False):
    print hazardset.id
    chrono = datetime.datetime.now()
    last_percent = 0

    level_VLO = HazardLevel.get(u'VLO')

    if hazardset is None:
        raise ProcessException('HazardSet {} does not exist.'
                               .format(hazardset.id))

    if hazardset.processed:
        if force:
            hazardset.processed = False
        else:
            raise ProcessException('HazardSet {} has already been processed.'
                                   .format(hazardset.id))

    # clean previous outputs
    DBSession.query(Output) \
        .filter(Output.hazardset_id == hazardset.id) \
        .delete()
    DBSession.flush()

    hazardtype = hazardset.hazardtype
    hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
    thresholds = hazardtype_settings['thresholds']

    project = partial(
        pyproj.transform,
        pyproj.Proj(init='epsg:3857'),
        pyproj.Proj(init='epsg:4326'))

    layers = {}
    for level in (u'HIG', u'MED', u'LOW'):
        hazardlevel = HazardLevel.get(level)
        layer = DBSession.query(Layer) \
            .filter(Layer.hazardset_id == hazardset.id) \
            .filter(Layer.hazardlevel_id == hazardlevel.id) \
            .one()
        layers[level] = layer

    with rasterio.drivers():
        with rasterio.open(layers['HIG'].path()) as src_hig, \
                rasterio.open(layers['MED'].path()) as src_med, \
                rasterio.open(layers['LOW'].path()) as src_low:
            readers = {}
            readers['HIG'] = src_hig
            readers['MED'] = src_med
            readers['LOW'] = src_low

            polygon_hig = polygonFromBounds(src_hig.bounds)
            polygon_med = polygonFromBounds(src_med.bounds)
            polygon_low = polygonFromBounds(src_low.bounds)
            polygon = cascaded_union((
                polygon_hig,
                polygon_med,
                polygon_low))

            adminlevel_REG = AdminLevelType.get(u'REG')

            admindivs = DBSession.query(AdministrativeDivision) \
                .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \

            if hazardset.local:
                admindivs = admindivs \
                    .filter(
                        func.ST_Transform(AdministrativeDivision.geom, 4326)
                        .intersects(
                            func.ST_GeomFromText(polygon.wkt, 4326))) \
                    .filter(func.ST_Intersects(
                        func.ST_Transform(AdministrativeDivision.geom, 4326),
                        func.ST_GeomFromText(polygon.wkt, 4326)))

            current = 0
            outputs = 0
            total = admindivs.count()
            for admindiv in admindivs:
                # print ' ', admindiv.id, admindiv.code, admindiv.name

                current += 1
                if admindiv.geom is None:
                    print '   ', ('{}-{} has null geometry'
                                  .format(admindiv.code, admindiv.name))
                    continue

                reprojected = transform(
                    project,
                    to_shape(admindiv.geom))

                output = Output()
                output.hazardset = hazardset
                output.administrativedivision = admindiv
                output.hazardlevel = None

                # TODO: calculate coverage ratio
                output.coverage_ratio = 100

                for level in (u'HIG', u'MED', u'LOW'):
                    layer = layers[level]
                    src = readers[level]

                    if not reprojected.intersects(polygon):
                        continue

                    window = src.window(*reprojected.bounds)
                    data = src.read(1, window=window, masked=True)
                    if data.shape[0] * data.shape[1] == 0:
                        continue

                    threshold = thresholds[layer.hazardunit]
                    positive_data = (data > threshold).astype(rasterio.uint8)

                    division = features.rasterize(
                        ((g, 1) for g in [reprojected]),
                        out_shape=data.shape,
                        transform=src.window_transform(window),
                        all_touched=True)

                    masked = numpy.ma.masked_array(positive_data,
                                                   mask=~division.astype(bool))

                    if str(numpy.max(masked)) == str(numpy.ma.masked):
                        break
                    else:
                        if output.hazardlevel is None:
                            output.hazardlevel = level_VLO

                    if numpy.max(masked) > 0:
                        output.hazardlevel = layer.hazardlevel
                        break

                if output.hazardlevel is not None:
                    # print '    hazardlevel :', output.hazardlevel.mnemonic
                    DBSession.add(output)
                    outputs += 1

                percent = int(100.0 * current / total)
                if percent % 10 == 0 and percent != last_percent:
                    print '  ... processed {}%'.format(percent)
                    last_percent = percent
                    pass

    hazardset.processed = True

    DBSession.flush()
    transaction.commit()

    print ('Successfully processed {} divisions, {} outputs generated in {}'
           .format(total, outputs, datetime.datetime.now() - chrono))