Example #1
0
    def harvest_layer(self, object):
        logger.info("Harvesting layer {id} - {title}".format(**object))
        title = object["title"]

        # we need to retrieve more information on this layer
        # since the regions array is not advertised by the main
        # regions listing from GeoNode
        o = self.fetch("api/layers/{id}/".format(**object))

        if "regions" not in list(o.keys()):
            warning(object, 'Attribute "regions" is missing')

        region_ids = []
        for r in o.get("regions", []):
            # r is like "/api/regions/1/"
            region_ids.append(r.split("/")[3])

        if len(region_ids) == 0:
            regions = []
        else:
            regions = self.dbsession.query(Region).filter(Region.id.in_(region_ids)).all()

        hazardset_id = o['hazard_set']
        if not hazardset_id:
            logger.info("  hazard_set is empty")
            return False

        # FIXME: temporary override
        if hazardset_id in excluded_hazardsets:
            logger.info("  hazard_set {} is excluded, skipping")
            return False

        hazardtype = self.check_hazard_type(o)
        if not hazardtype:
            return False

        type_settings = self.settings["hazard_types"][hazardtype.mnemonic]
        preprocessed = "values" in type_settings

        local = "GLOBAL" not in hazardset_id

        mask = False
        if preprocessed is True:
            hazardlevel = None
            # harvest hazard_unit for preprocessed layers
            hazard_unit = o['hazard_unit']
            if o['hazard_period']:
                logger.info('  return period found in preprocessed hazardset')
                return False
            hazard_period = None

        else:
            try:
                hazard_period = int(o['hazard_period'])
            except:
                hazard_period = None
            if hazard_period is None:
                logger.info('  no return period found')
                return False
            hazardlevel = None
            for level in ("LOW", "MED", "HIG"):
                if between(hazard_period, type_settings["return_periods"][level]):
                    hazardlevel = HazardLevel.get(self.dbsession, level)
                    break

            if "mask_return_period" in type_settings and between(
                hazard_period, type_settings["mask_return_period"]
            ):
                mask = True

            if hazardlevel is None and not mask:
                logger.info("  No corresponding hazard_level")
                return False

            hazard_unit = o['hazard_unit']
            if hazard_unit == '':
                logger.info('  hazard_unit is empty')
                return False

        if o['srid'] != 'EPSG:4326':
            logger.info('  srid is different from "EPSG:4326"')
            return False

        data_update_date = parse_date(o['data_update_date'])
        if not data_update_date:
            warning(o, 'data_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
            # We use a very old date for good comparison in decision tree
            data_update_date = datetime.fromtimestamp(0)

        metadata_update_date = parse_date(o['metadata_update_date'])
        if not metadata_update_date:
            warning(o, 'metadata_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
            # We use a very old date for good comparison in decision tree
            metadata_update_date = datetime.fromtimestamp(0)

        calculation_method_quality = o['calculation_method_quality']
        if not calculation_method_quality:
            warning(o, 'calculation_method_quality is empty: skip layer')
            return False
        calculation_method_quality = int(float(calculation_method_quality))

        scientific_quality = o['scientific_quality']
        if not scientific_quality:
            warning(o, 'scientific_quality is empty')
            return False
        scientific_quality = int(float(scientific_quality))

        download_url = o['download_url']
        if not download_url:
            warning(o, 'download_url is empty')
            return False

        hazardset = self.dbsession.query(HazardSet).get(hazardset_id)

        # Create hazardset before layer
        if hazardset is None:
            logger.info("  Create new hazardset {}".format(hazardset_id))
            hazardset = HazardSet()
            hazardset.id = hazardset_id
            hazardset.hazardtype = hazardtype
            self.dbsession.add(hazardset)

        # get detail_url and owner_organization from last updated layer
        geonode = self.settings["geonode"]
        geonode_base_url = geonode["url"]

        if o['detail_url'] and not mask:
            hazardset.detail_url = geonode_base_url + o['detail_url']
        if o['owner']['organization'] and not mask:
            hazardset.owner_organization = o['owner']['organization']
        if not mask:
            hazardset.regions = regions

        layer = self.dbsession.query(Layer).get(o['id'])
        if layer is None:
            logger.info("  Create new Layer {}".format(title))
            layer = Layer()
            layer.geonode_id = o['id']
            layer.hazardset = hazardset
            layer.mask = False

        else:
            # If data has changed
            if (
                layer.data_lastupdated_date != data_update_date
                or layer.download_url != download_url
            ):
                logger.info("  Invalidate downloaded")
                layer.downloaded = False
                hazardset.complete = False
                hazardset.processed = None
                # Remove file from cache
                layer.download_url = download_url
                path = self.layer_path(layer)
                if os.path.isfile(path):
                    os.unlink(path)

            # Some hazardset fields are calculated during completing
            if (
                layer.calculation_method_quality != calculation_method_quality
                or layer.scientific_quality != scientific_quality
                or layer.metadata_lastupdated_date != metadata_update_date
            ):
                logger.info("  Invalidate complete")
                hazardset.complete = False

            # Some fields invalidate outputs
            if layer.hazardunit != hazard_unit:
                logger.info("  Invalidate processed")
                hazardset.processed = None

        typename = o.get("typename", None)
        if typename is None:
            warning(o, 'Attribute "typename" is missing')
        layer.typename = typename

        layer.return_period = hazard_period
        layer.hazardunit = hazard_unit
        layer.data_lastupdated_date = data_update_date
        layer.metadata_lastupdated_date = metadata_update_date
        layer.download_url = download_url

        # TODO: retrieve quality attributes
        layer.calculation_method_quality = calculation_method_quality
        layer.scientific_quality = scientific_quality
        layer.local = local

        layer.set_harvested(True)
        self.dbsession.flush()
        return True
Example #2
0
def populate_db():
    DBSession.query(Output).delete()
    DBSession.query(Layer).delete()
    DBSession.query(HazardSet).delete()

    DBSession.query(HazardTypeFurtherResourceAssociation).delete()
    DBSession.query(FurtherResource).delete()
    DBSession.query(HazardCategoryTechnicalRecommendationAssociation).delete()
    DBSession.query(TechnicalRecommendation).delete()
    DBSession.query(ClimateChangeRecommendation).delete()
    DBSession.query(HazardCategoryAdministrativeDivisionAssociation).delete()
    DBSession.query(CAdHt).delete()
    DBSession.query(Contact).delete()
    DBSession.query(Region).delete()
    DBSession.query(AdministrativeDivision).delete()

    hazardtype_eq = (DBSession.query(HazardType).filter(
        HazardType.mnemonic == "EQ").one())
    hazardset1 = HazardSet()
    hazardset1.id = "hazardset1"
    hazardset1.hazardtype = hazardtype_eq
    hazardset1.data_lastupdated_date = datetime.now()
    hazardset1.metadata_lastupdated_date = datetime.now()
    hazardset1.detail_url = "http://domain.com/path/"
    hazardset1.owner_organization = "data_provider"
    DBSession.add(hazardset1)

    shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])])
    geometry = from_shape(shape, 4326)

    # admin_div_10 is a country (division level 1)
    admin_div_10 = AdministrativeDivision(**{
        "code": 10,
        "leveltype_id": 1,
        "name": "Division level 1"
    })
    admin_div_10.geom = geometry
    DBSession.add(admin_div_10)

    # admin_div_11 is another country (division level 1)
    admin_div_11 = AdministrativeDivision(**{
        "code": 11,
        "leveltype_id": 1,
        "name": "Division level 1 2"
    })
    admin_div_11.geom = geometry
    DBSession.add(admin_div_11)

    # admin_div_12 is another country (division level 1)
    admin_div_12 = AdministrativeDivision(**{
        "code": 12,
        "leveltype_id": 1,
        "name": "Division level 1 3"
    })
    admin_div_12.geom = geometry
    DBSession.add(admin_div_12)

    # admin_div_13 is another country (division level 1)
    admin_div_13 = AdministrativeDivision(**{
        "code": 13,
        "leveltype_id": 1,
        "name": "Division level 1 4"
    })
    admin_div_13.geom = geometry
    DBSession.add(admin_div_13)

    # admin_div_20 is a province (division level 2)
    # its parent is admin_div_10
    admin_div_20 = AdministrativeDivision(**{
        "code": 20,
        "leveltype_id": 2,
        "name": "Division level 2"
    })
    admin_div_20.parent_code = admin_div_10.code
    admin_div_20.geom = geometry
    DBSession.add(admin_div_20)

    shape = MultiPolygon(
        [Polygon([(0, 0), (0, 1), (0.5, 1), (0.5, 0), (0, 0)])])
    geometry = from_shape(shape, 4326)

    # admin_div_31 is a region (division level 3)
    # its parent is admin_div_20
    admin_div_31 = AdministrativeDivision(**{
        "code": 31,
        "leveltype_id": 3,
        "name": "Division level 3 - 1"
    })
    admin_div_31.parent_code = admin_div_20.code
    admin_div_31.geom = geometry
    admin_div_31.hazardcategories = []

    shape = MultiPolygon(
        [Polygon([(0.5, 0), (0.5, 1), (1, 1), (1, 0), (0.5, 0)])])
    geometry = from_shape(shape, 4326)

    # admin_div_32 is a region (division level 3)
    # its parent is also admin_div_20
    admin_div_32 = AdministrativeDivision(**{
        "code": 32,
        "leveltype_id": 3,
        "name": "Division level 3 - 2"
    })
    admin_div_32.parent_code = admin_div_20.code
    admin_div_32.geom = geometry
    admin_div_32.hazardcategories = []

    # Here's a quick, graphical recap:
    #
    # admin_div_10 -> admin_div_20 -> admin_div_31
    #                             `-> admin_div_32
    # admin_div_11
    # admin_div_12
    # admin_div_13

    # GeoNode Regions
    # global_region contains all countries, **except admin_div_12**
    global_region = Region(**{"id": 1, "level": 0, "name": "Global region"})
    global_region.administrativedivisions.append(admin_div_10)
    global_region.administrativedivisions.append(admin_div_11)
    global_region.administrativedivisions.append(admin_div_13)

    # region_1 is a country
    # it matches GAULS's admin_div_10
    region_1 = Region(**{"id": 2, "level": 3, "name": "Country 1"})
    region_1.administrativedivisions.append(admin_div_10)

    # region_2 is another country
    # it matches GAULS's admin_div_11
    region_2 = Region(**{"id": 3, "level": 3, "name": "Country 2"})
    region_2.administrativedivisions.append(admin_div_11)

    # region_3 is another country
    # it matches GAULS's admin_div_12
    region_3 = Region(**{"id": 4, "level": 3, "name": "Country 3"})
    region_3.administrativedivisions.append(admin_div_12)

    # Here's a quick, graphical recap:
    #
    # global_region  -> admin_div_10 (region_1) -> admin_div_20 -> admin_div_31
    #            `                                             `-> admin_div_32
    #             `
    #              ` -> admin_div_11 (region_2)
    #               `-> admin_div_13
    #
    # region_3 = admin_div_12

    category_eq_hig = HazardCategory.get(DBSession, "EQ", "HIG")
    category_eq_hig.general_recommendation = "General recommendation for EQ HIG"

    category_fl_hig = HazardCategory.get(DBSession, "FL", "HIG")

    # admin_div_31 has (EQ, HIGH)
    association = HazardCategoryAdministrativeDivisionAssociation(
        **{"hazardcategory": category_eq_hig})
    association.hazardsets.append(hazardset1)
    admin_div_31.hazardcategories.append(association)

    # admin_div_31 has (RF, HIGH)
    admin_div_32.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_fl_hig}))

    # admin_div_32 has (EQ, HIGH)
    admin_div_32.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_eq_hig}))

    # admin_div_10 has (EQ, HIGH)
    admin_div_10.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_eq_hig}))

    # admin_div_11 has no category (this is tested)
    # admin_div_12 has (EQ, HIGH)
    admin_div_12.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_eq_hig}))

    # admin_div_13 has (EQ, HIGH)
    admin_div_13.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_eq_hig}))

    climate_rec = ClimateChangeRecommendation(
        text="Climate change recommendation",
        hazardtype=HazardType.get(DBSession, "EQ"))
    climate_rec.associations.append(
        CcrAd(administrativedivision=admin_div_10,
              hazardtype=HazardType.get(DBSession, "EQ")))
    DBSession.add(climate_rec)

    climate_rec = ClimateChangeRecommendation(
        text="Climate change recommendation 2",
        hazardtype=HazardType.get(DBSession, "EQ"))
    climate_rec.associations.append(
        CcrAd(administrativedivision=admin_div_11,
              hazardtype=HazardType.get(DBSession, "EQ")))
    DBSession.add(climate_rec)

    technical_rec = TechnicalRecommendation(
        **{
            "text":
            "Recommendation #1 for earthquake, applied to"
            " hazard categories HIG, MED and LOW"
        })
    association = HazardCategoryTechnicalRecommendationAssociation(order=1)
    association.hazardcategory = category_eq_hig
    technical_rec.hazardcategory_associations.append(association)
    DBSession.add(technical_rec)

    technical_rec = TechnicalRecommendation(**{
        "text":
        "Educational web resources on earthquakes and"
        " seismic hazard"
    })
    association = HazardCategoryTechnicalRecommendationAssociation(order=1)
    association.hazardcategory = category_eq_hig
    technical_rec.hazardcategory_associations.append(association)
    DBSession.add(technical_rec)

    category_fl_med = HazardCategory.get(DBSession, "FL", "MED")
    category_fl_med.general_recommendation = "General recommendation for FL MED"

    admin_div_31.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_fl_med}))
    DBSession.add(admin_div_31)
    admin_div_32.hazardcategories.append(
        HazardCategoryAdministrativeDivisionAssociation(
            **{"hazardcategory": category_fl_med}))
    DBSession.add(admin_div_32)

    # generic further resource for EQ:
    # it should be found on every EQ report page
    # (except admin_div_12 which is not linked with global region)
    further_resource = FurtherResource(
        **{
            "text": "Educational web resources on earthquakes and" +
            " seismic hazard",
            "id": 3,
        })
    association = HazardTypeFurtherResourceAssociation()
    association.hazardtype = hazardtype_eq
    association.region = global_region
    further_resource.hazardtype_associations.append(association)
    DBSession.add(further_resource)

    # further resource for EQ & region 1:
    # it should be found only on region 1 (and sub-divisions) page
    further_resource = FurtherResource(**{
        "text": "Further resource for earthquake for region 1",
        "id": 5
    })
    association = HazardTypeFurtherResourceAssociation()
    association.hazardtype = hazardtype_eq
    association.region = region_1
    further_resource.hazardtype_associations.append(association)
    DBSession.add(further_resource)

    # contact for EQ & admin_div_11:
    contact1 = Contact(
        **{
            "name": "Contact name",
            "url": "http://domain.com",
            "phone": "0123456789",
            "email": "*****@*****.**",
        })
    DBSession.add(contact1)
    association = CAdHt()
    association.hazardtype = hazardtype_eq
    association.administrativedivision = admin_div_10
    association.contact = contact1
    DBSession.add(association)

    # contact for EQ & admin_div_11:
    contact2 = Contact(
        **{
            "name": "Contact name 2",
            "url": "http://domain.com",
            "phone": "0123456789",
            "email": "*****@*****.**",
        })
    DBSession.add(contact1)
    association = CAdHt()
    association.hazardtype = hazardtype_eq
    association.administrativedivision = admin_div_10
    association.contact = contact2
    DBSession.add(association)

    Publication.new(DBSession)

    DBSession.flush()