def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        "-f",
        "--force",
        dest="force",
        action="store_const",
        const=True,
        default=False,
        help="Force execution even if hazardset has already been \
              marked as complete",
    )
    parser.add_argument(
        "--dry-run",
        dest="dry_run",
        action="store_const",
        const=True,
        default=False,
        help="Perform a trial run that does not commit changes",
    )
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, "sqlalchemy.")
    DBSession.configure(bind=engine)

    complete(force=args.force, dry_run=args.dry_run)
def complete(hazardset_id=None, force=False, dry_run=False):
    if force:
        try:
            clearall()
            if dry_run:
                transaction.abort()
            else:
                transaction.commit()
        except:
            transaction.abort()
            raise

    ids = DBSession.query(HazardSet.id)
    if not force:
        ids = ids.filter(HazardSet.complete.is_(False))
    if hazardset_id is not None:
        ids = ids.filter(HazardSet.id == hazardset_id)
    for id in ids:
        logger.info('  Working on hazardset {}'.format(id[0]))
        try:
            if not complete_hazardset(id[0]):
                hazardset = DBSession.query(HazardSet).get(id)
                if hazardset.processed:
                    logger.info('Deleting {} previous outputs related \
                                to this hazardset'.format(
                                hazardset.outputs.count()))
                    hazardset.outputs.delete()
            if dry_run:
                transaction.abort()
            else:
                transaction.commit()
        except Exception as e:
            transaction.abort()
            logger.error(e.message)
def populate_preprocessed(type):
    hazardset_id = u'preprocessed'
    hazardtype = HazardType.get(type)

    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset = HazardSet(
        id=hazardset_id,
        hazardtype=hazardtype,
        local=False,
        data_lastupdated_date=datetime.now(),
        metadata_lastupdated_date=datetime.now())
    DBSession.add(hazardset)

    layer = Layer(
        hazardlevel=None,
        mask=False,
        return_period=None,
        data_lastupdated_date=datetime.now(),
        metadata_lastupdated_date=datetime.now(),
        geonode_id=new_geonode_id(),
        download_url='test',
        calculation_method_quality=5,
        scientific_quality=1,
        local=False,
        downloaded=True
    )
    hazardset.layers.append(layer)

    hazardset.complete = True
    DBSession.flush()
Beispiel #4
0
def upscale_hazardcategories(target_adminlevel_mnemonic):
    # identify the admin divisions matching the given level:
    admindivs = DBSession.query(AdministrativeDivision)\
        .join(AdminLevelType) \
        .filter(AdminLevelType.mnemonic == target_adminlevel_mnemonic)
    hazardtypes = DBSession.query(HazardType)
    # for each of these admin divisions and hazard types,
    # identify the highest HazardCategory among their children:
    for admindiv in admindivs:
        # identify max(level) for each hazardtype across children
        for hazardtype in hazardtypes:
            hazardcategory = DBSession.query(HazardCategory) \
                .join((AdministrativeDivision, HazardCategory.administrativedivisions)) \
                .join(HazardType) \
                .join(HazardLevel) \
                .filter(HazardType.id == hazardtype.id) \
                .filter(AdministrativeDivision.parent == admindiv) \
                .order_by(HazardLevel.order.asc()).first()
            if hazardcategory:
                # find the highest hazardlevel for all children admindivs
                print '[upscaling] admindiv {} inherits hazardlevel {} for {}'\
                    .format(admindiv.code, hazardcategory.hazardlevel_id,
                            hazardcategory.hazardtype.mnemonic)
                admindiv.hazardcategories.append(hazardcategory)
                DBSession.add(admindiv)
Beispiel #5
0
def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """

    load_local_settings(settings)

    engine = engine_from_config(settings, "sqlalchemy.")
    DBSession.configure(bind=engine)
    Base.metadata.bind = engine

    config = Configurator(settings=settings)

    config.include("pyramid_jinja2")
    config.include("papyrus")

    config.add_static_view("static", "static", cache_max_age=3600, cachebust=True)
    config.add_static_view("lib", settings.get("node_modules"), cache_max_age=86000, cachebust=True)

    config.add_route("index", "/")
    config.add_route("report", "/report/{divisioncode:\d+}/{hazardtype:([A-Z]{2})}")
    config.add_route("report_json", "/report/{divisioncode:\d+}/{hazardtype:([A-Z]{2})}.json")
    config.add_route("report_overview", "/report/{divisioncode:\d+}")
    config.add_route("report_overview_slash", "/report/{divisioncode:\d+}/")
    config.add_route("report_overview_json", "/report/{divisioncode:\d+}.json")
    config.add_route("administrativedivision", "/administrativedivision")

    config.add_renderer("geojson", GeoJSON())

    config.scan(ignore=["thinkhazard.tests"])
    return config.make_wsgi_app()
Beispiel #6
0
def populate_datamart():
    print 'populate datamart'
    adminlevel_REG = AdminLevelType.get(u'REG')

    from functools import partial
    import pyproj
    from shapely.ops import transform

    project = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'),
                      pyproj.Proj(init='epsg:3857'))

    shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])])
    reprojected = transform(project, shape)
    geometry = from_shape(reprojected, 3857)

    div = AdministrativeDivision(
        **{
            'code': 30,
            'leveltype_id': adminlevel_REG.id,
            'name': u'Administrative division level 3'
        })
    div.geom = geometry
    div.hazardcategories = []
    DBSession.add(div)

    DBSession.flush()
def download_layer(geonode_id):
    layer = DBSession.query(Layer).get(geonode_id)
    if layer is None:
        raise Exception('Layer {} does not exist.'.format(geonode_id))

    logger.info('Downloading layer {}'.format(layer.name()))

    dir_path = os.path.dirname(layer_path(layer))
    if not os.path.exists(dir_path):
        os.makedirs(dir_path)

    if not os.path.isfile(layer_path(layer)):
        h = Http()
        url = urlunsplit((geonode['scheme'],
                          geonode['netloc'],
                          layer.download_url,
                          '',
                          ''))
        logger.info('Retrieving {}'.format(url))
        response, content = h.request(url)

        try:
            with open(layer_path(layer), 'wb') as f:
                f.write(content)
                layer.downloaded = True
        except EnvironmentError:
            logger.error('Writing data from layer {} failed'.format(
                layer.name()))

    DBSession.flush()
def populate_notpreprocessed(type, unit):
    hazardset_id = u'notpreprocessed'
    hazardtype = HazardType.get(type)
    hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]

    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset = HazardSet(
        id=hazardset_id,
        hazardtype=hazardtype,
        local=False,
        data_lastupdated_date=datetime.now(),
        metadata_lastupdated_date=datetime.now())
    DBSession.add(hazardset)

    return_periods = hazardtype_settings['return_periods']

    for level in (u'HIG', u'MED', u'LOW'):
        hazardlevel = HazardLevel.get(level)
        level_return_periods = return_periods[level]
        if isinstance(level_return_periods, list):
            return_period = level_return_periods[0]
        else:
            return_period = level_return_periods

        layer = Layer(
            hazardlevel=hazardlevel,
            mask=False,
            return_period=return_period,
            hazardunit=unit,
            data_lastupdated_date=datetime.now(),
            metadata_lastupdated_date=datetime.now(),
            geonode_id=new_geonode_id(),
            download_url='test',
            calculation_method_quality=5,
            scientific_quality=1,
            local=False,
            downloaded=True
        )
        hazardset.layers.append(layer)

    layer = Layer(
        hazardlevel=None,
        mask=True,
        return_period=hazardtype_settings['mask_return_period'],
        hazardunit=unit,
        data_lastupdated_date=datetime.now(),
        metadata_lastupdated_date=datetime.now(),
        geonode_id=new_geonode_id(),
        download_url='test',
        calculation_method_quality=5,
        scientific_quality=1,
        local=False,
        downloaded=True
    )
    hazardset.layers.append(layer)

    hazardset.complete = True
    DBSession.flush()
def add_resource(hazard_type, resource):
    categories = DBSession.query(HazardCategory) \
        .join(HazardType) \
        .filter(HazardType.mnemonic == hazard_type)
    for category in categories:
        association = HazardCategoryFurtherResourceAssociation(order=1)
        association.hazardcategory = category
        resource.hazardcategory_associations.append(association)
    DBSession.add(resource)
def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--dry-run', dest='dry_run',
        action='store_const', const=True, default=False,
        help='Perform a trial run that does not commit changes')
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)

    apply_decision_tree(dry_run=args.dry_run)
def populate():
    DBSession.query(Output).delete()
    DBSession.query(AdministrativeDivision).delete()
    DBSession.query(Layer).delete()
    DBSession.query(HazardSet).delete()
    populate_datamart()
    populate_processing()
    transaction.commit()
Beispiel #12
0
def populate():
    DBSession.query(Output).delete()
    DBSession.query(Layer).delete()
    DBSession.query(HazardSet).delete()
    DBSession.query(AdministrativeDivision).delete()
    populate_datamart()
    populate_processing()
    transaction.commit()
def download(title=None, force=False, dry_run=False):
    if force:
        try:
            clearall()
            if dry_run:
                transaction.abort()
            else:
                transaction.commit()
        except:
            transaction.abort()
            raise

    geonode_ids = DBSession.query(Layer.geonode_id)

    if not force:
        geonode_ids = geonode_ids.filter(Layer.downloaded.is_(False))

    if title is not None:
        geonode_ids = geonode_ids.filter(Layer.title == title)

    for geonode_id in geonode_ids:
        try:
            download_layer(geonode_id)
            if dry_run:
                transaction.abort()
            else:
                transaction.commit()
        except Exception as e:
            transaction.abort()
            logger.error('{} raise an exception :\n{}'
                         .format(geonode_id, e.message))
def populate():
    DBSession.query(Output).delete()
    DBSession.query(Layer).delete()
    DBSession.query(HazardSet).delete()
    DBSession.query(AdministrativeDivision).delete()
    populate_datamart()
    populate_notpreprocessed(notpreprocessed_type, notpreprocessed_unit)
    populate_preprocessed(preprocessed_type)
    transaction.commit()
Beispiel #15
0
def report_json(request):

    try:
        division_code = request.matchdict.get('divisioncode')
    except:
        raise HTTPBadRequest(detail='incorrect value for parameter '
                                    '"divisioncode"')

    try:
        resolution = float(request.params.get('resolution'))
    except:
        raise HTTPBadRequest(detail='invalid value for parameter "resolution"')

    hazard_type = request.matchdict.get('hazardtype', None)

    _filter = or_(AdministrativeDivision.code == division_code,
                  AdministrativeDivision.parent_code == division_code)

    simplify = func.ST_Simplify(
        func.ST_Transform(AdministrativeDivision.geom, 3857), resolution / 2)

    if hazard_type is not None:
        divisions = DBSession.query(AdministrativeDivision) \
            .add_columns(simplify, HazardLevel.mnemonic, HazardLevel.title) \
            .outerjoin(AdministrativeDivision.hazardcategories) \
            .join(HazardCategory) \
            .outerjoin(HazardType)\
            .outerjoin(HazardLevel) \
            .filter(and_(_filter, HazardType.mnemonic == hazard_type))
    else:
        divisions = DBSession.query(AdministrativeDivision) \
            .add_columns(simplify, literal_column("'None'"),
                         literal_column("'blah'")) \
            .filter(_filter)

    return [{
        'type': 'Feature',
        'geometry': to_shape(geom_simplified),
        'properties': {
            'name': division.name,
            'code': division.code,
            'hazardLevelMnemonic': hazardlevel_mnemonic,
            'hazardLevelTitle': hazardlevel_title
        }
    } for division, geom_simplified, hazardlevel_mnemonic,
          hazardlevel_title in divisions]
 def test_preprocessed_empty(self, open_mock):
     '''Test mask layer'''
     open_mock.side_effect = [
         global_reader(),
     ]
     process(hazardset_id='preprocessed')
     output = DBSession.query(Output).first()
     self.assertEqual(output, None)
Beispiel #17
0
def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-f', '--force', dest='force',
        action='store_const', const=True, default=False,
        help='Force execution even if layer metadata have already \
              been fetched')
    parser.add_argument(
        '--dry-run', dest='dry_run',
        action='store_const', const=True, default=False,
        help='Perform a trial run that does not commit changes')
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)

    harvest(force=args.force,
            dry_run=args.dry_run)
def populate_datamart():
    print 'populate datamart'
    adminlevel_REG = AdminLevelType.get(u'REG')

    shape = MultiPolygon([
        Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
    ])
    geometry = from_shape(shape, 4326)

    div = AdministrativeDivision(**{
        'code': 30,
        'leveltype_id': adminlevel_REG.id,
        'name': u'Administrative division level 3'
    })
    div.geom = geometry
    div.hazardcategories = []
    DBSession.add(div)

    DBSession.flush()
 def test_preprocessed_hig(self, open_mock):
     '''Test mask layer'''
     hazardtype = HazardType.get(preprocessed_type)
     hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
     open_mock.side_effect = [
         global_reader(hazardtype_settings['values']['HIG'][0]),
     ]
     process(hazardset_id='preprocessed')
     output = DBSession.query(Output).first()
     self.assertEqual(output.hazardlevel.mnemonic, u'HIG')
def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument('--hazardset_id',
                        dest='hazardset_id',
                        action='store',
                        help='The hazard set identifier')
    parser.add_argument(
        '--force',
        dest='force',
        action='store_const',
        const=True,
        default=False,
        help='Force execution even if hazardset has already been processed')
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)

    process(hazardset_id=args.hazardset_id, force=args.force)
Beispiel #21
0
 def test_process_medium(self, open_mock):
     '''Test value > threshold in MED layer'''
     open_mock.side_effect = [
         rasterio_open(global_reader()),
         rasterio_open(global_reader(100.0)),
         rasterio_open(global_reader())
     ]
     process(force=True)
     output = DBSession.query(Output).first()
     self.assertEqual(output.hazardlevel.mnemonic, 'MED')
 def test_process_hig(self, open_mock):
     '''Test value > threshold in HIG layer'''
     open_mock.side_effect = [
         global_reader(100.0),
         global_reader(),
         global_reader(),
         global_reader()
     ]
     process(hazardset_id='notpreprocessed')
     output = DBSession.query(Output).first()
     self.assertEqual(output.hazardlevel.mnemonic, u'HIG')
 def test_process_empty(self, open_mock):
     '''Test nodata everywhere'''
     open_mock.side_effect = [
         global_reader(),
         global_reader(),
         global_reader(),
         global_reader()
     ]
     process(hazardset_id='notpreprocessed')
     output = DBSession.query(Output).first()
     self.assertEqual(output, None)
 def test_process_vlo(self, open_mock):
     '''Test value < threshold <=> hazardlevel VLO'''
     open_mock.side_effect = [
         global_reader(0.0),
         global_reader(),
         global_reader(),
         global_reader()
     ]
     process(hazardset_id='notpreprocessed')
     output = DBSession.query(Output).first()
     self.assertEqual(output.hazardlevel.mnemonic, 'VLO')
Beispiel #25
0
 def test_process_nodata(self, open_mock):
     '''Test nodata everywhere'''
     open_mock.side_effect = [
         rasterio_open(global_reader()),
         rasterio_open(global_reader()),
         rasterio_open(global_reader())
     ]
     rasterio_open.return_period = None
     rasterio_open.value = None
     process(force=True)
     output = DBSession.query(Output).first()
     self.assertEqual(output.hazardlevel.mnemonic, 'VLO')
Beispiel #26
0
def process(hazardset_id=None, force=False):
    hazardsets = DBSession.query(HazardSet)
    hazardsets = hazardsets.filter(HazardSet.complete.is_(True))
    if hazardset_id is not None:
        hazardsets = hazardsets.filter(HazardSet.id == hazardset_id)
    if not force:
        hazardsets = hazardsets.filter(HazardSet.processed.is_(False))
    if hazardsets.count() == 0:
        print 'No hazardsets to process'
        return
    for hazardset in hazardsets:
        process_hazardset(hazardset, force=force)
Beispiel #27
0
def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--hazardset_id',  dest='hazardset_id', action='store',
        help='The hazardset identifier')
    parser.add_argument(
        '--force', dest='force',
        action='store_const', const=True, default=False,
        help='Force execution even if hazardset has already been processed')
    parser.add_argument(
        '--dry-run', dest='dry_run',
        action='store_const', const=True, default=False,
        help='Perform a trial run that does not commit changes')
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)

    process(
        hazardset_id=args.hazardset_id,
        force=args.force,
        dry_run=args.dry_run)
def main(argv=sys.argv):
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '--title', dest='title', action='store',
        help='The layer title')
    parser.add_argument(
        '--force', dest='force',
        action='store_const', const=True, default=False,
        help='Force download even if layer has already been downloaded')
    parser.add_argument(
        '--dry-run', dest='dry_run',
        action='store_const', const=True, default=False,
        help='Perform a trial run that does not commit changes')
    args = parser.parse_args(argv[1:])

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)

    download(
        title=args.title,
        force=args.force,
        dry_run=args.dry_run)
Beispiel #29
0
def administrativedivision(request):

    if 'q' not in request.params:
        raise HTTPBadRequest(detail='parameter "q" is missing')

    ilike = '%%%s%%' % request.params['q']

    admin0s = DBSession.query(AdministrativeDivision).filter(
        and_(AdministrativeDivision.leveltype_id == 1,
             AdministrativeDivision.name.ilike(ilike))).limit(10)

    admin1s = DBSession.query(AdministrativeDivision).filter(
        and_(AdministrativeDivision.leveltype_id == 2,
             AdministrativeDivision.name.ilike(ilike))).limit(10)

    admin2s = DBSession.query(AdministrativeDivision).filter(
        and_(AdministrativeDivision.leveltype_id == 3,
             AdministrativeDivision.name.ilike(ilike))).limit(10)

    data = sorted(_roundrobin(admin0s, admin1s, admin2s, limit=10),
                  key=lambda o: o.leveltype_id)

    return {'data': data}
Beispiel #30
0
def main(global_config, **settings):
    """ This function returns a Pyramid WSGI application.
    """

    load_local_settings(settings)

    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)
    Base.metadata.bind = engine

    config = Configurator(settings=settings)

    config.include('pyramid_jinja2')
    config.include('papyrus')

    config.add_static_view('static', 'static', cache_max_age=3600,
                           cachebust=True)
    config.add_static_view('lib', settings.get('node_modules'),
                           cache_max_age=86000, cachebust=True)

    config.add_route('index', '/')
    config.add_route('about', '/about')
    config.add_route('faq', '/faq')
    config.add_route('report',
                     '/report/{divisioncode:\d+}/{hazardtype:([A-Z]{2})}')
    config.add_route('report_json',
                     '/report/{divisioncode:\d+}/{hazardtype:([A-Z]{2})}.json')
    config.add_route('report_overview', '/report/{divisioncode:\d+}')
    config.add_route('report_overview_slash', '/report/{divisioncode:\d+}/')
    config.add_route('report_overview_json', '/report/{divisioncode:\d+}.json')
    config.add_route('administrativedivision', '/administrativedivision')

    config.add_renderer('geojson', GeoJSON())

    config.scan(ignore=['thinkhazard.tests'])
    return config.make_wsgi_app()
Beispiel #31
0
def process_outputs():
    print "Decision Tree running..."
    # first of all, remove all records
    # in the datamart table linking admin divs with hazard categories:
    hazardcategory_administrativedivision_table.delete()
    # identify the admin level for which we run the decision tree:
    # (REG)ion aka admin level 2
    dt_level = DBSession.query(AdminLevelType)\
        .filter(AdminLevelType.mnemonic == u'REG').one()
    # then, identify the unique (admindiv, hazardtype) tuples
    # contained in the Output table:
    admindiv_hazardtype_tuples = (
        DBSession.query(AdministrativeDivision, HazardType).distinct()
        .join(Output)
        .join(HazardSet)
        .join(HazardType)
        # the following should not be necessary in production
        # because only the lowest admin levels should be inserted
        # in the Output table:
        .filter(AdministrativeDivision.leveltype_id == dt_level.id)
        # not necessary, but practical for debugging:
        .order_by(AdministrativeDivision.code)
    )
    # for each tuple, identify the most relevant HazardSet
    # in the light of the criteria that we all agreed on (cf Decision Tree):
    for (admindiv, hazardtype) in admindiv_hazardtype_tuples:
        (hazardset, output) = DBSession.query(HazardSet, Output) \
            .join(Output) \
            .filter(Output.admin_id == admindiv.id) \
            .filter(HazardSet.hazardtype_id == hazardtype.id) \
            .order_by(HazardSet.calculation_method_quality.desc(),
                      HazardSet.scientific_quality.desc(),
                      HazardSet.local.desc(),
                      HazardSet.data_lastupdated_date.desc()).first()
        print "[decision tree] admindiv {} gets hazardlevel {} from {} for {}"\
            .format(admindiv.code, output.hazardlevel_id, hazardset.id,
                    hazardtype.mnemonic)
        # find the relevant HazardCategory for the current hazardtype
        # and the hazardset's hazardlevel
        hazardcategory = DBSession.query(HazardCategory) \
            .filter(HazardCategory.hazardtype_id == hazardtype.id) \
            .filter(HazardCategory.hazardlevel_id == output.hazardlevel_id) \
            .one()
        # append new hazardcategory to current admin div:
        admindiv.hazardcategories.append(hazardcategory)
        DBSession.add(admindiv)

    # UpScaling level2 (REG)ion -> level1 (PRO)vince
    upscale_hazardcategories(u'PRO')
    # UpScaling level1 (PRO)vince -> level0 (COU)ntry
    upscale_hazardcategories(u'COU')

    transaction.commit()
def process(hazardset_id=None, force=False, dry_run=False):
    ids = DBSession.query(HazardSet.id) \
        .filter(HazardSet.complete.is_(True))
    if hazardset_id is not None:
        ids = ids.filter(HazardSet.id == hazardset_id)
    if not force:
        ids = ids.filter(HazardSet.processed.is_(False))
    if ids.count() == 0:
        logger.info('No hazardset to process')
        return
    for id in ids:
        logger.info(id[0])
        try:
            process_hazardset(id[0], force=force)
            if dry_run:
                logger.info('  Abording transaction')
                transaction.abort()
            else:
                logger.info('  Committing transaction')
                transaction.commit()
        except Exception:
            transaction.abort()
            logger.error(traceback.format_exc())
Beispiel #33
0
def populate_processing():
    print 'populate processing'
    hazardset_id = u'test'
    hazardtype = HazardType.get(u'EQ')
    hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]

    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset = HazardSet()
    hazardset.id = hazardset_id
    hazardset.hazardtype = hazardtype
    hazardset.local = False
    hazardset.data_lastupdated_date = datetime.now()
    hazardset.metadata_lastupdated_date = datetime.now()
    DBSession.add(hazardset)

    return_periods = hazardtype_settings['global']['return_periods']
    unit = hazardtype_settings['thresholds'].keys()[0]

    for level in (u'HIG', u'MED', u'LOW'):
        hazardlevel = HazardLevel.get(level)
        return_period = return_periods[level]

        layer = Layer()
        layer.title = "{}-{}".format(id, return_period)
        layer.hazardlevel = hazardlevel
        layer.return_period = return_period
        layer.hazardunit = unit
        layer.data_lastupdated_date = datetime.now()
        layer.metadata_lastupdated_date = datetime.now()
        layer.geonode_id = new_geonode_id()
        layer.download_url = 'test'
        layer.calculation_method_quality = 5
        layer.scientific_quality = 1
        layer.local = False
        layer.downloaded = True
        hazardset.layers.append(layer)
        DBSession.flush()

    hazardset.complete = True
    DBSession.flush()
Beispiel #34
0
def process_hazardset(hazardset, force=False):
    print hazardset.id
    chrono = datetime.datetime.now()
    last_percent = 0

    level_VLO = HazardLevel.get(u'VLO')

    if hazardset is None:
        raise ProcessException('HazardSet {} does not exist.'
                               .format(hazardset.id))

    if hazardset.processed:
        if force:
            hazardset.processed = False
        else:
            raise ProcessException('HazardSet {} has already been processed.'
                                   .format(hazardset.id))

    # clean previous outputs
    DBSession.query(Output) \
        .filter(Output.hazardset_id == hazardset.id) \
        .delete()
    DBSession.flush()

    hazardtype = hazardset.hazardtype
    hazardtype_settings = settings['hazard_types'][hazardtype.mnemonic]
    thresholds = hazardtype_settings['thresholds']

    project = partial(
        pyproj.transform,
        pyproj.Proj(init='epsg:3857'),
        pyproj.Proj(init='epsg:4326'))

    layers = {}
    for level in (u'HIG', u'MED', u'LOW'):
        hazardlevel = HazardLevel.get(level)
        layer = DBSession.query(Layer) \
            .filter(Layer.hazardset_id == hazardset.id) \
            .filter(Layer.hazardlevel_id == hazardlevel.id) \
            .one()
        layers[level] = layer

    with rasterio.drivers():
        with rasterio.open(layers['HIG'].path()) as src_hig, \
                rasterio.open(layers['MED'].path()) as src_med, \
                rasterio.open(layers['LOW'].path()) as src_low:
            readers = {}
            readers['HIG'] = src_hig
            readers['MED'] = src_med
            readers['LOW'] = src_low

            polygon_hig = polygonFromBounds(src_hig.bounds)
            polygon_med = polygonFromBounds(src_med.bounds)
            polygon_low = polygonFromBounds(src_low.bounds)
            polygon = cascaded_union((
                polygon_hig,
                polygon_med,
                polygon_low))

            adminlevel_REG = AdminLevelType.get(u'REG')

            admindivs = DBSession.query(AdministrativeDivision) \
                .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \

            if hazardset.local:
                admindivs = admindivs \
                    .filter(
                        func.ST_Transform(AdministrativeDivision.geom, 4326)
                        .intersects(
                            func.ST_GeomFromText(polygon.wkt, 4326))) \
                    .filter(func.ST_Intersects(
                        func.ST_Transform(AdministrativeDivision.geom, 4326),
                        func.ST_GeomFromText(polygon.wkt, 4326)))

            current = 0
            outputs = 0
            total = admindivs.count()
            for admindiv in admindivs:
                # print ' ', admindiv.id, admindiv.code, admindiv.name

                current += 1
                if admindiv.geom is None:
                    print '   ', ('{}-{} has null geometry'
                                  .format(admindiv.code, admindiv.name))
                    continue

                reprojected = transform(
                    project,
                    to_shape(admindiv.geom))

                output = Output()
                output.hazardset = hazardset
                output.administrativedivision = admindiv
                output.hazardlevel = None

                # TODO: calculate coverage ratio
                output.coverage_ratio = 100

                for level in (u'HIG', u'MED', u'LOW'):
                    layer = layers[level]
                    src = readers[level]

                    if not reprojected.intersects(polygon):
                        continue

                    window = src.window(*reprojected.bounds)
                    data = src.read(1, window=window, masked=True)
                    if data.shape[0] * data.shape[1] == 0:
                        continue

                    threshold = thresholds[layer.hazardunit]
                    positive_data = (data > threshold).astype(rasterio.uint8)

                    division = features.rasterize(
                        ((g, 1) for g in [reprojected]),
                        out_shape=data.shape,
                        transform=src.window_transform(window),
                        all_touched=True)

                    masked = numpy.ma.masked_array(positive_data,
                                                   mask=~division.astype(bool))

                    if str(numpy.max(masked)) == str(numpy.ma.masked):
                        break
                    else:
                        if output.hazardlevel is None:
                            output.hazardlevel = level_VLO

                    if numpy.max(masked) > 0:
                        output.hazardlevel = layer.hazardlevel
                        break

                if output.hazardlevel is not None:
                    # print '    hazardlevel :', output.hazardlevel.mnemonic
                    DBSession.add(output)
                    outputs += 1

                percent = int(100.0 * current / total)
                if percent % 10 == 0 and percent != last_percent:
                    print '  ... processed {}%'.format(percent)
                    last_percent = percent
                    pass

    hazardset.processed = True

    DBSession.flush()
    transaction.commit()

    print ('Successfully processed {} divisions, {} outputs generated in {}'
           .format(total, outputs, datetime.datetime.now() - chrono))
Beispiel #35
0
def report(request):
    try:
        division_code = request.matchdict.get('divisioncode')
    except:
        raise HTTPBadRequest(detail='incorrect value for parameter '
                                    '"divisioncode"')

    hazard = request.matchdict.get('hazardtype', None)

    # Get all the hazard types.
    hazardtype_query = DBSession.query(HazardType).order_by(HazardType.order)

    # Get the hazard categories corresponding to the administrative
    # division whose code is division_code.
    hazardcategories_query = DBSession.query(HazardCategory) \
        .join(HazardCategory.administrativedivisions) \
        .join(HazardType) \
        .join(HazardLevel) \
        .filter(AdministrativeDivision.code == division_code)

    # Create a dict with the categories. Keys are the hazard type mnemonic.
    hazardcategories = {d.hazardtype.mnemonic: d
                        for d in hazardcategories_query}

    hazard_types = []
    for hazardtype in hazardtype_query:
        cat = _hazardlevel_nodata
        if hazardtype.mnemonic in hazardcategories:
            cat = hazardcategories[hazardtype.mnemonic].hazardlevel
        hazard_types.append({
            'hazardtype': hazardtype,
            'hazardlevel': cat
        })

    association = None
    technical_recommendations = None
    further_resources = None
    climate_change_recommendation = None

    # Get the administrative division whose code is division_code.
    _alias = aliased(AdministrativeDivision)
    division = DBSession.query(AdministrativeDivision) \
        .outerjoin(_alias, _alias.code == AdministrativeDivision.parent_code) \
        .filter(AdministrativeDivision.code == division_code).one()

    if hazard is not None:
        try:
            association = DBSession.query(
                HazardCategoryAdministrativeDivisionAssociation) \
                .join(AdministrativeDivision) \
                .join(HazardCategory) \
                .join(HazardLevel) \
                .join(HazardType) \
                .filter(HazardType.mnemonic == hazard) \
                .filter(AdministrativeDivision.code == division_code) \
                .one()
        except NoResultFound:
            url = request.route_url('report_overview',
                                    divisioncode=division_code)
            return HTTPFound(location=url)

        try:
            # get the code for level 0 division
            code = division.code
            if division.leveltype_id == 2:
                code = division.parent.code
            if division.leveltype_id == 3:
                code = division.parent.parent.code
            climate_change_recommendation = DBSession.query(
                    ClimateChangeRecommendation) \
                .join(AdministrativeDivision) \
                .join(HazardType) \
                .filter(AdministrativeDivision.code == code) \
                .filter(HazardType.mnemonic == hazard) \
                .one()
        except NoResultFound:
            pass

        technical_recommendations = DBSession.query(TechnicalRecommendation) \
            .join(TechnicalRecommendation.hazardcategory_associations) \
            .join(HazardCategory) \
            .filter(HazardCategory.id == association.hazardcategory.id) \
            .all()

        further_resources = DBSession.query(FurtherResource) \
            .join(FurtherResource.hazardcategory_associations) \
            .join(HazardCategory) \
            .outerjoin(AdministrativeDivision) \
            .filter(HazardCategory.id == association.hazardcategory.id) \
            .filter(or_(AdministrativeDivision.code == division_code,
                        AdministrativeDivision.code == null())) \
            .all()

    # Get the geometry for division and compute its extent
    cte = select([AdministrativeDivision.geom]) \
        .where(AdministrativeDivision.code == division_code) \
        .cte('bounds')
    bounds = list(DBSession.query(
        func.ST_XMIN(cte.c.geom),
        func.ST_YMIN(cte.c.geom),
        func.ST_XMAX(cte.c.geom),
        func.ST_YMAX(cte.c.geom))
        .one())
    division_bounds = bounds

    # compute a 0-360 version of the extent
    cte = select([
        func.ST_Shift_Longitude(AdministrativeDivision.geom).label('shift')]) \
        .where(AdministrativeDivision.code == division_code) \
        .cte('bounds')
    bounds_shifted = list(DBSession.query(
        func.ST_XMIN(cte.c.shift),
        func.ST_YMIN(cte.c.shift),
        func.ST_XMAX(cte.c.shift),
        func.ST_YMAX(cte.c.shift))
        .one())

    # Use the 0-360 if it's smaller
    if bounds_shifted[2] - bounds_shifted[0] < bounds[2] - bounds[0]:
        division_bounds = bounds_shifted

    parents = []
    if division.leveltype_id >= 2:
        parents.append(division.parent)
    if division.leveltype_id == 3:
        parents.append(division.parent.parent)

    return {'hazards': hazard_types,
            'hazards_sorted': sorted(hazard_types,
                                     key=lambda a: a['hazardlevel'].order),
            'hazard_category': association.hazardcategory
            if association else '',
            'source': association.source
            if association else '',
            'climate_change_recommendation': climate_change_recommendation,
            'recommendations': technical_recommendations,
            'resources': further_resources,
            'division': division,
            'bounds': division_bounds,
            'parents': parents,
            'parent_division': division.parent}
 def layerByLevel(self, level):
     hazardlevel = HazardLevel.get(level)
     return DBSession.query(Layer) \
         .filter(Layer.hazardset_id == self.id) \
         .filter(Layer.hazardlevel_id == hazardlevel.id) \
         .one_or_none()
def clearall():
    logger.info('Reset all hazardsets to incomplete state')
    DBSession.query(HazardSet).update({
        HazardSet.processed: False
    })
    DBSession.flush()
def create_outputs(hazardset, layers, readers):
    type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic]
    adminlevel_REG = AdminLevelType.get(u'REG')

    bbox = None
    for reader in readers.itervalues():
        polygon = polygon_from_boundingbox(reader.bounds)
        if bbox is None:
            bbox = polygon
        else:
            bbox = bbox.intersection(polygon)

    admindivs = DBSession.query(AdministrativeDivision) \
        .filter(AdministrativeDivision.leveltype_id == adminlevel_REG.id) \
        .filter(func.ST_Intersects(AdministrativeDivision.geom,
                func.ST_GeomFromText(bbox.wkt, 4326))) \
        .order_by(AdministrativeDivision.id)  # Needed for windowed querying

    current = 0
    last_percent = 0
    outputs = []
    total = admindivs.count()
    logger.info('  Iterating over {} administrative divisions'.format(total))

    # Windowed querying to limit memory usage
    limit = 1000  # 1000 records <=> 10 Mo
    admindivs = admindivs.limit(limit)
    for offset in xrange(0, total, limit):
        admindivs = admindivs.offset(offset)

        for admindiv in admindivs:
            current += 1

            if admindiv.geom is None:
                logger.warning('    {}-{} has null geometry'
                               .format(admindiv.code, admindiv.name))
                continue

            shape = to_shape(admindiv.geom)

            # Try block to include admindiv.code in exception message
            try:
                if 'values' in type_settings.keys():
                    # preprocessed layer
                    hazardlevel = preprocessed_hazardlevel(
                        type_settings,
                        layers[0], readers[0],
                        shape)
                else:
                    hazardlevel = notpreprocessed_hazardlevel(
                        hazardset.hazardtype.mnemonic, type_settings, layers,
                        readers, shape)

            except Exception as e:
                e.message = ("{}-{} raises an exception :\n{}"
                             .format(admindiv.code, admindiv.name, e.message))
                raise

            # Create output record
            if hazardlevel is not None:
                output = Output()
                output.hazardset = hazardset
                output.admin_id = admindiv.id
                output.hazardlevel = hazardlevel
                # TODO: calculate coverage ratio
                output.coverage_ratio = 100
                outputs.append(output)

            # Remove admindiv from memory
            DBSession.expunge(admindiv)

            percent = int(100.0 * current / total)
            if percent % 10 == 0 and percent != last_percent:
                logger.info('  ... processed {}%'.format(percent))
                last_percent = percent

    return outputs
def process_hazardset(hazardset_id, force=False):
    hazardset = DBSession.query(HazardSet).get(hazardset_id)
    if hazardset is None:
        raise ProcessException('Hazardset {} does not exist.'
                               .format(hazardset_id))

    chrono = datetime.datetime.now()

    if hazardset.processed:
        if force:
            hazardset.processed = False
        else:
            raise ProcessException('Hazardset {} has already been processed.'
                                   .format(hazardset.id))

    logger.info("  Cleaning previous outputs")
    DBSession.query(Output) \
        .filter(Output.hazardset_id == hazardset.id) \
        .delete()
    DBSession.flush()

    type_settings = settings['hazard_types'][hazardset.hazardtype.mnemonic]

    with rasterio.drivers():
        try:
            logger.info("  Opening raster files")
            # Open rasters
            layers = {}
            readers = {}
            if 'values' in type_settings.keys():
                # preprocessed layer
                layer = DBSession.query(Layer) \
                    .filter(Layer.hazardset_id == hazardset.id) \
                    .one()
                reader = rasterio.open(layer_path(layer))

                layers[0] = layer
                readers[0] = reader

            else:
                for level in (u'HIG', u'MED', u'LOW'):
                    hazardlevel = HazardLevel.get(level)
                    layer = DBSession.query(Layer) \
                        .filter(Layer.hazardset_id == hazardset.id) \
                        .filter(Layer.hazardlevel_id == hazardlevel.id) \
                        .one()
                    reader = rasterio.open(layer_path(layer))

                    layers[level] = layer
                    readers[level] = reader
                if ('mask_return_period' in type_settings):
                    layer = DBSession.query(Layer) \
                        .filter(Layer.hazardset_id == hazardset.id) \
                        .filter(Layer.mask.is_(True)) \
                        .one()
                    reader = rasterio.open(layer_path(layer))
                    layers['mask'] = layer
                    readers['mask'] = reader

            outputs = create_outputs(hazardset, layers, readers)
            if outputs:
                DBSession.add_all(outputs)

        finally:
            logger.info("  Closing raster files")
            for key, reader in readers.iteritems():
                if reader and not reader.closed:
                    reader.close()

    hazardset.processed = True
    DBSession.flush()

    logger.info('  Successfully processed {}, {} outputs generated in {}'
                .format(hazardset.id,
                        len(outputs),
                        datetime.datetime.now() - chrono))

    return True
def complete_hazardset(hazardset_id, dry_run=False):
    hazardset = DBSession.query(HazardSet).get(hazardset_id)
    if hazardset is None:
        raise Exception('Hazardset {} does not exist.'
                        .format(hazardset_id))

    hazardtype = hazardset.hazardtype
    type_settings = settings['hazard_types'][hazardtype.mnemonic]
    preprocessed = 'values' in type_settings

    layers = []
    if preprocessed:
        if len(hazardset.layers) == 0:
            logger.info('  No layer found')
            return False
        layers.append(hazardset.layers[0])
    else:
        for level in (u'LOW', u'MED', u'HIG'):
            layer = hazardset.layerByLevel(level)
            if layer is None:
                logger.info('  No layer for level {}'.format(level))
                return False
            layers.append(layer)
        if ('mask_return_period' in type_settings):
            layer = DBSession.query(Layer) \
                .filter(Layer.hazardset_id == hazardset_id) \
                .filter(Layer.mask.is_(True))
            if layer.count() == 0:
                logger.info('  No mask layer')
                return False
            layers.append(layer.one())

    for layer in layers:
        if not layer.downloaded:
            logger.info('  No data for layer {}'.format(layer.name()))
            return False

    stats = DBSession.query(
        Layer.local,
        func.min(Layer.data_lastupdated_date),
        func.min(Layer.metadata_lastupdated_date),
        func.min(Layer.calculation_method_quality),
        func.min(Layer.scientific_quality)) \
        .filter(Layer.hazardset_id == hazardset.id) \
        .group_by(Layer.local)

    if stats.count() > 1:
        logger.warning('  Mixed local and global layers')
        return False

    stat = stats.one()

    hazardset.local = stat[0]
    hazardset.data_lastupdated_date = stat[1]
    hazardset.metadata_lastupdated_date = stat[2]
    hazardset.calculation_method_quality = stat[3]
    hazardset.scientific_quality = stat[4]
    hazardset.complete = True
    DBSession.flush()

    logger.info('  Completed')
    return True
def main(argv=sys.argv):
    engine = engine_from_config(settings, 'sqlalchemy.')
    DBSession.configure(bind=engine)
    process_outputs()
def populate_processing():
    hazardtypeEQ = HazardType.get(u'EQ')
    hazardtypeFL = HazardType.get(u'FL')

    hazardset_id = u'hazardset1'
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset1 = HazardSet()
    hazardset1.id = hazardset_id
    hazardset1.hazardtype = hazardtypeEQ
    hazardset1.local = False
    hazardset1.calculation_method_quality = 0
    hazardset1.data_lastupdated_date = datetime.now()
    hazardset1.metadata_lastupdated_date = datetime.now()
    hazardset1.complete = True
    hazardset1.layers.extend(make_layers())
    DBSession.add(hazardset1)

    hazardset_id = u'hazardset1b'
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset1b = HazardSet()
    hazardset1b.id = hazardset_id
    hazardset1b.hazardtype = hazardtypeFL
    hazardset1b.local = True
    hazardset1b.calculation_method_quality = 3
    hazardset1b.data_lastupdated_date = datetime.now()
    hazardset1b.metadata_lastupdated_date = datetime.now()
    hazardset1b.complete = True
    hazardset1b.layers.extend(make_layers())
    DBSession.add(hazardset1b)

    # create hazardsets 2, 3 ... 5
    hazardset_id = u'hazardset2'
    # calculation_method_quality = 1 / scientific_quality = 0
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset2 = HazardSet()
    hazardset2.id = hazardset_id
    hazardset2.hazardtype = hazardtypeEQ
    hazardset2.local = True
    hazardset2.calculation_method_quality = 1
    hazardset2.scientific_quality = 0
    hazardset2.data_lastupdated_date = datetime.now()
    hazardset2.metadata_lastupdated_date = datetime.now()
    hazardset2.complete = True
    hazardset2.layers.extend(make_layers())
    DBSession.add(hazardset2)

    hazardset_id = u'hazardset3'
    # date = 2015-01-01 / global /
    # calculation_method_quality = 1 / scientific_quality = 2
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset3 = HazardSet()
    hazardset3.id = hazardset_id
    hazardset3.hazardtype = hazardtypeEQ
    hazardset3.local = False
    hazardset3.calculation_method_quality = 1
    hazardset3.scientific_quality = 2
    hazardset3.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
    hazardset3.metadata_lastupdated_date = datetime.now()
    hazardset3.complete = True
    hazardset3.layers.extend(make_layers())
    DBSession.add(hazardset3)

    hazardset_id = u'hazardset4'
    # date = 2015-01-01 / local /
    # calculation_method_quality = 1 / scientific_quality = 2
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset4 = HazardSet()
    hazardset4.id = hazardset_id
    hazardset4.hazardtype = hazardtypeEQ
    hazardset4.local = True
    hazardset4.calculation_method_quality = 1
    hazardset4.scientific_quality = 2
    hazardset4.data_lastupdated_date = datetime(2015, 1, 1, 0, 0)
    hazardset4.metadata_lastupdated_date = datetime.now()
    hazardset4.complete = True
    hazardset4.layers.extend(make_layers())
    DBSession.add(hazardset4)

    hazardset_id = u'hazardset5'
    # date = now() / local /
    # calculation_method_quality = 1 / scientific_quality = 2
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset5 = HazardSet()
    hazardset5.id = hazardset_id
    hazardset5.hazardtype = hazardtypeEQ
    hazardset5.local = True
    hazardset5.calculation_method_quality = 1
    hazardset5.scientific_quality = 2
    hazardset5.data_lastupdated_date = datetime.now()
    hazardset5.metadata_lastupdated_date = datetime.now()
    hazardset5.complete = True
    hazardset5.layers.extend(make_layers())
    DBSession.add(hazardset5)

    hazardset_id = u'hazardset6'
    # date = now() / global /
    # calculation_method_quality = 1 / scientific_quality = 2
    print 'Populating hazardset {}'.format(hazardset_id)
    hazardset6 = HazardSet()
    hazardset6.id = hazardset_id
    hazardset6.hazardtype = hazardtypeEQ
    hazardset6.local = False
    hazardset6.calculation_method_quality = 1
    hazardset6.scientific_quality = 2
    hazardset6.data_lastupdated_date = datetime.now()
    hazardset6.metadata_lastupdated_date = datetime.now()
    hazardset6.complete = True
    hazardset6.layers.extend(make_layers())
    DBSession.add(hazardset6)

    # populate output table

    # admin div (code 30) has only one hazardset for EQ
    admin30 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 30).one()
    # => test outcome = hazardset1
    output1 = Output()
    output1.coverage_ratio = 10
    output1.hazardset = hazardset1
    output1.administrativedivision = admin30
    output1.hazardlevel = HazardLevel.get(u'HIG')
    DBSession.add(output1)

    # admin div (code 30) also has another hazardset
    # but this one is for FL
    # => test outcome = hazardset1b
    output1b = Output()
    output1b.coverage_ratio = 11
    output1b.hazardset = hazardset1b
    output1b.administrativedivision = admin30
    output1b.hazardlevel = HazardLevel.get(u'NPR')
    DBSession.add(output1b)

    # admin div (code 31) has 2 hazardsets,
    # one with a higher calculation_method_quality
    # => test outcome = hazardset2
    admin31 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 31).one()
    output2 = Output()
    output2.coverage_ratio = 20
    output2.hazardset = hazardset1  # calculation_method_quality = 0
    output2.administrativedivision = admin31
    output2.hazardlevel = HazardLevel.get(u'MED')
    DBSession.add(output2)
    output3 = Output()
    output3.coverage_ratio = 30
    output3.hazardset = hazardset2  # calculation_method_quality = 1
    output3.administrativedivision = admin31
    output3.hazardlevel = HazardLevel.get(u'LOW')
    DBSession.add(output3)

    # admin div (code 32) has 2 hazardsets,
    # both share the same calculation_method_quality,
    # one with a higher scientific_quality
    # => test outcome = hazardset3
    admin32 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 32).one()
    output4 = Output()
    output4.coverage_ratio = 40
    output4.hazardset = hazardset2
    # calculation_method_quality = 1 / scientific_quality = 0
    output4.administrativedivision = admin32
    output4.hazardlevel = HazardLevel.get(u'MED')
    DBSession.add(output4)
    output5 = Output()
    output5.coverage_ratio = 50
    output5.hazardset = hazardset3
    # calculation_method_quality = 1 / scientific_quality = 2
    output5.administrativedivision = admin32
    output5.hazardlevel = HazardLevel.get(u'LOW')
    DBSession.add(output5)

    # admin div (code 33) has 2 hazardsets,
    # both share the same ratings, one is global, one local
    # => test outcome = hazardset4
    admin33 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 33).one()
    output6 = Output()
    output6.coverage_ratio = 60
    output6.hazardset = hazardset3
    # global / calculation_method_quality = 1 / scientific_quality = 2
    output6.administrativedivision = admin33
    output6.hazardlevel = HazardLevel.get(u'MED')
    DBSession.add(output6)
    output7 = Output()
    output7.coverage_ratio = 70
    output7.hazardset = hazardset4
    # local / calculation_method_quality = 1 / scientific_quality = 2
    output7.administrativedivision = admin33
    output7.hazardlevel = HazardLevel.get(u'LOW')
    DBSession.add(output7)

    # admin div (code 34) has 2 hazardsets,
    # both share the same ratings, are local, one is more recent
    # => test outcome = hazardset5
    admin34 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 34).one()
    output8 = Output()
    output8.coverage_ratio = 80
    output8.hazardset = hazardset4
    # date = 2015-01-01 / local /
    # calculation_method_quality = 1 / scientific_quality = 2
    output8.administrativedivision = admin34
    output8.hazardlevel = HazardLevel.get(u'MED')
    DBSession.add(output8)
    output9 = Output()
    output9.coverage_ratio = 90
    output9.hazardset = hazardset5
    # date = now() / local /
    # calculation_method_quality = 1 / scientific_quality = 2
    output9.administrativedivision = admin34
    output9.hazardlevel = HazardLevel.get(u'LOW')
    DBSession.add(output9)

    # admin div (code 35) has 2 hazardsets,
    # both share the same ratings, are global, one is more recent
    # => test outcome = hazardset6
    admin35 = DBSession.query(AdministrativeDivision)\
        .filter(AdministrativeDivision.code == 35).one()
    output10 = Output()
    output10.coverage_ratio = 95
    output10.hazardset = hazardset3
    # date = 2015-01-01 / global /
    # calculation_method_quality = 1 / scientific_quality = 2
    output10.administrativedivision = admin35
    output10.hazardlevel = HazardLevel.get(u'MED')
    DBSession.add(output10)
    output11 = Output()
    output11.coverage_ratio = 99
    output11.hazardset = hazardset6
    # date = now() / global /
    # calculation_method_quality = 1 / scientific_quality = 2
    output11.administrativedivision = admin35
    output11.hazardlevel = HazardLevel.get(u'LOW')
    DBSession.add(output11)

    DBSession.flush()
def populate_datamart():
    shape = Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
    geometry = from_shape(shape, 3857)

    div_level_1 = AdministrativeDivision(**{
        'code': 10,
        'leveltype_id': 1,
        'name': u'Division level 1'
    })
    div_level_1.geom = geometry
    DBSession.add(div_level_1)

    div_level_2 = AdministrativeDivision(**{
        'code': 20,
        'leveltype_id': 2,
        'name': u'Division level 2'
    })
    div_level_2.parent_code = div_level_1.code
    div_level_2.geom = geometry
    DBSession.add(div_level_2)

    div_level_3_0 = AdministrativeDivision(**{
        'code': 30,
        'leveltype_id': 3,
        'name': u'Division level 3 - 1'
    })
    div_level_3_0.parent_code = div_level_2.code
    div_level_3_0.geom = geometry
    div_level_3_0.hazardcategories = []
    DBSession.add(div_level_3_0)

    div_level_3_1 = AdministrativeDivision(**{
        'code': 31,
        'leveltype_id': 3,
        'name': u'Division level 3 - 1'
    })
    div_level_3_1.parent_code = div_level_2.code
    div_level_3_1.geom = geometry
    div_level_3_1.hazardcategories = []
    DBSession.add(div_level_3_1)

    div_level_3_2 = AdministrativeDivision(**{
        'code': 32,
        'leveltype_id': 3,
        'name': u'Division level 3 - 2'
    })
    div_level_3_2.parent_code = div_level_2.code
    div_level_3_2.geom = geometry
    div_level_3_2.hazardcategories = []
    DBSession.add(div_level_3_2)

    div_level_3_3 = AdministrativeDivision(**{
        'code': 33,
        'leveltype_id': 3,
        'name': u'Division level 3 - 3'
    })
    div_level_3_3.parent_code = div_level_2.code
    div_level_3_3.geom = geometry
    div_level_3_3.hazardcategories = []
    DBSession.add(div_level_3_3)

    div_level_3_4 = AdministrativeDivision(**{
        'code': 34,
        'leveltype_id': 3,
        'name': u'Division level 3 - 4'
    })
    div_level_3_4.parent_code = div_level_2.code
    div_level_3_4.geom = geometry
    div_level_3_4.hazardcategories = []
    DBSession.add(div_level_3_4)

    div_level_3_5 = AdministrativeDivision(**{
        'code': 35,
        'leveltype_id': 3,
        'name': u'Division level 3 - 5'
    })
    div_level_3_5.parent_code = div_level_2.code
    div_level_3_5.geom = geometry
    div_level_3_5.hazardcategories = []
    DBSession.add(div_level_3_5)

    div_level_3_6 = AdministrativeDivision(**{
        'code': 36,
        'leveltype_id': 3,
        'name': u'Division level 3 - 6'
    })
    div_level_3_6.parent_code = div_level_2.code
    div_level_3_6.geom = geometry
    div_level_3_6.hazardcategories = []
    DBSession.add(div_level_3_6)

    DBSession.flush()
Beispiel #44
0
def populate_db():
    config = ConfigParser.ConfigParser()
    config.read(local_settings_path)
    db_url = config.get('app:main', 'sqlalchemy.url')

    from sqlalchemy import create_engine
    engine = create_engine(db_url)

    from ..scripts.initializedb import populate_db as populate
    populate(engine, drop_all=True)

    shape = MultiPolygon([
        Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])
    ])
    geometry = from_shape(shape, 4326)

    div_level_1 = AdministrativeDivision(**{
        'code': 10,
        'leveltype_id': 1,
        'name': u'Division level 1'
    })
    div_level_1.geom = geometry
    DBSession.add(div_level_1)

    div_level_2 = AdministrativeDivision(**{
        'code': 20,
        'leveltype_id': 2,
        'name': u'Division level 2'
    })
    div_level_2.parent_code = div_level_1.code
    div_level_2.geom = geometry
    DBSession.add(div_level_2)

    shape = MultiPolygon([
        Polygon([(0, 0), (0, 1), (.5, 1), (.5, 0), (0, 0)])
    ])
    geometry = from_shape(shape, 4326)

    div_level_3_1 = AdministrativeDivision(**{
        'code': 30,
        'leveltype_id': 3,
        'name': u'Division level 3 - 1'
    })
    div_level_3_1.parent_code = div_level_2.code
    div_level_3_1.geom = geometry
    div_level_3_1.hazardcategories = []

    shape = MultiPolygon([
        Polygon([(.5, 0), (.5, 1), (1, 1), (1, 0), (.5, 0)])
    ])
    geometry = from_shape(shape, 4326)

    div_level_3_2 = AdministrativeDivision(**{
        'code': 31,
        'leveltype_id': 3,
        'name': u'Division level 3 - 2'
    })
    div_level_3_2.parent_code = div_level_2.code
    div_level_3_2.geom = geometry
    div_level_3_2.hazardcategories = []

    category_eq_hig = HazardCategory(**{
        'general_recommendation': u'General recommendation for EQ HIG',
    })
    category_eq_hig.hazardtype = DBSession.query(HazardType) \
        .filter(HazardType.mnemonic == u'EQ').one()
    category_eq_hig.hazardlevel = DBSession.query(HazardLevel) \
        .filter(HazardLevel.mnemonic == u'HIG').one()

    associate_admindiv_hazardcategory(div_level_3_1, category_eq_hig, u'test')
    associate_admindiv_hazardcategory(div_level_3_2, category_eq_hig, u'test')

    climate_rec = ClimateChangeRecommendation()
    climate_rec.text = u'Climate change recommendation'
    climate_rec.administrativedivision = div_level_1
    climate_rec.hazardtype = DBSession.query(HazardType) \
        .filter(HazardType.mnemonic == u'EQ').one()
    DBSession.add(climate_rec)

    technical_rec = TechnicalRecommendation(**{
        'text': u'Recommendation #1 for earthquake, applied to'
                ' hazard categories HIG, MED and LOW'
    })
    association = HazardCategoryTechnicalRecommendationAssociation(order=1)
    association.hazardcategory = category_eq_hig
    technical_rec.hazardcategory_associations.append(association)
    DBSession.add(technical_rec)

    technical_rec = TechnicalRecommendation(**{
        'text': u'Educational web resources on earthquakes and'
                ' seismic hazard'
    })
    association = HazardCategoryTechnicalRecommendationAssociation(order=1)
    association.hazardcategory = category_eq_hig
    technical_rec.hazardcategory_associations.append(association)
    DBSession.add(technical_rec)

    category_fl_med = HazardCategory(**{
        'general_recommendation': u'General recommendation for FL MED',
    })
    category_fl_med.hazardtype = DBSession.query(HazardType) \
        .filter(HazardType.mnemonic == u'FL').one()
    category_fl_med.hazardlevel = DBSession.query(HazardLevel) \
        .filter(HazardLevel.mnemonic == u'MED').one()

    associate_admindiv_hazardcategory(div_level_3_1, category_fl_med, u'test')
    DBSession.add(div_level_3_1)
    associate_admindiv_hazardcategory(div_level_3_2, category_fl_med, u'test')
    DBSession.add(div_level_3_2)

    further_resource = FurtherResource(**{
        'text': u'Educational web resources on earthquakes and' +
                ' seismic hazard',
        'url': u'http://earthquake.usgs.gov/learn/?source=sitemap'
    })
    association = HazardCategoryFurtherResourceAssociation(order=1)
    association.hazardcategory = category_eq_hig
    further_resource.hazardcategory_associations.append(association)
    DBSession.add(further_resource)

    # Add further resource for one division only
    further_resource = FurtherResource(**{
        'text': u'Further resource specific to division level 3 - 2',
        'url': u'http://domain.com/the/document/url.txt'
    })
    association = HazardCategoryFurtherResourceAssociation(order=2)
    association.hazardcategory = category_eq_hig
    association.administrativedivision = div_level_3_2
    further_resource.hazardcategory_associations.append(association)
    DBSession.add(further_resource)
Beispiel #45
0
def new_geonode_id():
    row = DBSession.query(func.max(Layer.geonode_id)).one_or_none()
    if row[0] is None:
        return 1
    return row[0] + 1