def harvest_layer(self, object):
logger.info("Harvesting layer {id} - {title}".format(**object))
title = object["title"]
# we need to retrieve more information on this layer
# since the regions array is not advertised by the main
# regions listing from GeoNode
o = self.fetch("api/layers/{id}/".format(**object))
if "regions" not in list(o.keys()):
warning(object, 'Attribute "regions" is missing')
region_ids = []
for r in o.get("regions", []):
# r is like "/api/regions/1/"
region_ids.append(r.split("/")[3])
if len(region_ids) == 0:
regions = []
else:
regions = self.dbsession.query(Region).filter(Region.id.in_(region_ids)).all()
hazardset_id = o['hazard_set']
if not hazardset_id:
logger.info(" hazard_set is empty")
return False
# FIXME: temporary override
if hazardset_id in excluded_hazardsets:
logger.info(" hazard_set {} is excluded, skipping")
return False
hazardtype = self.check_hazard_type(o)
if not hazardtype:
return False
type_settings = self.settings["hazard_types"][hazardtype.mnemonic]
preprocessed = "values" in type_settings
local = "GLOBAL" not in hazardset_id
mask = False
if preprocessed is True:
hazardlevel = None
# harvest hazard_unit for preprocessed layers
hazard_unit = o['hazard_unit']
if o['hazard_period']:
logger.info(' return period found in preprocessed hazardset')
return False
hazard_period = None
else:
try:
hazard_period = int(o['hazard_period'])
except:
hazard_period = None
if hazard_period is None:
logger.info(' no return period found')
return False
hazardlevel = None
for level in ("LOW", "MED", "HIG"):
if between(hazard_period, type_settings["return_periods"][level]):
hazardlevel = HazardLevel.get(self.dbsession, level)
break
if "mask_return_period" in type_settings and between(
hazard_period, type_settings["mask_return_period"]
):
mask = True
if hazardlevel is None and not mask:
logger.info(" No corresponding hazard_level")
return False
hazard_unit = o['hazard_unit']
if hazard_unit == '':
logger.info(' hazard_unit is empty')
return False
if o['srid'] != 'EPSG:4326':
logger.info(' srid is different from "EPSG:4326"')
return False
data_update_date = parse_date(o['data_update_date'])
if not data_update_date:
warning(o, 'data_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
# We use a very old date for good comparison in decision tree
data_update_date = datetime.fromtimestamp(0)
metadata_update_date = parse_date(o['metadata_update_date'])
if not metadata_update_date:
warning(o, 'metadata_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
# We use a very old date for good comparison in decision tree
metadata_update_date = datetime.fromtimestamp(0)
calculation_method_quality = o['calculation_method_quality']
if not calculation_method_quality:
warning(o, 'calculation_method_quality is empty: skip layer')
return False
calculation_method_quality = int(float(calculation_method_quality))
scientific_quality = o['scientific_quality']
if not scientific_quality:
warning(o, 'scientific_quality is empty')
return False
scientific_quality = int(float(scientific_quality))
download_url = o['download_url']
if not download_url:
warning(o, 'download_url is empty')
return False
hazardset = self.dbsession.query(HazardSet).get(hazardset_id)
# Create hazardset before layer
if hazardset is None:
logger.info(" Create new hazardset {}".format(hazardset_id))
hazardset = HazardSet()
hazardset.id = hazardset_id
hazardset.hazardtype = hazardtype
self.dbsession.add(hazardset)
# get detail_url and owner_organization from last updated layer
geonode = self.settings["geonode"]
geonode_base_url = geonode["url"]
if o['detail_url'] and not mask:
hazardset.detail_url = geonode_base_url + o['detail_url']
if o['owner']['organization'] and not mask:
hazardset.owner_organization = o['owner']['organization']
if not mask:
hazardset.regions = regions
layer = self.dbsession.query(Layer).get(o['id'])
if layer is None:
logger.info(" Create new Layer {}".format(title))
layer = Layer()
layer.geonode_id = o['id']
layer.hazardset = hazardset
layer.mask = False
else:
# If data has changed
if (
layer.data_lastupdated_date != data_update_date
or layer.download_url != download_url
):
logger.info(" Invalidate downloaded")
layer.downloaded = False
hazardset.complete = False
hazardset.processed = None
# Remove file from cache
layer.download_url = download_url
path = self.layer_path(layer)
if os.path.isfile(path):
os.unlink(path)
# Some hazardset fields are calculated during completing
if (
layer.calculation_method_quality != calculation_method_quality
or layer.scientific_quality != scientific_quality
or layer.metadata_lastupdated_date != metadata_update_date
):
logger.info(" Invalidate complete")
hazardset.complete = False
# Some fields invalidate outputs
if layer.hazardunit != hazard_unit:
logger.info(" Invalidate processed")
hazardset.processed = None
typename = o.get("typename", None)
if typename is None:
warning(o, 'Attribute "typename" is missing')
layer.typename = typename
layer.return_period = hazard_period
layer.hazardunit = hazard_unit
layer.data_lastupdated_date = data_update_date
layer.metadata_lastupdated_date = metadata_update_date
layer.download_url = download_url
# TODO: retrieve quality attributes
layer.calculation_method_quality = calculation_method_quality
layer.scientific_quality = scientific_quality
layer.local = local
layer.set_harvested(True)
self.dbsession.flush()
return True
def populate_db():
DBSession.query(Output).delete()
DBSession.query(Layer).delete()
DBSession.query(HazardSet).delete()
DBSession.query(HazardTypeFurtherResourceAssociation).delete()
DBSession.query(FurtherResource).delete()
DBSession.query(HazardCategoryTechnicalRecommendationAssociation).delete()
DBSession.query(TechnicalRecommendation).delete()
DBSession.query(ClimateChangeRecommendation).delete()
DBSession.query(HazardCategoryAdministrativeDivisionAssociation).delete()
DBSession.query(CAdHt).delete()
DBSession.query(Contact).delete()
DBSession.query(Region).delete()
DBSession.query(AdministrativeDivision).delete()
hazardtype_eq = (DBSession.query(HazardType).filter(
HazardType.mnemonic == "EQ").one())
hazardset1 = HazardSet()
hazardset1.id = "hazardset1"
hazardset1.hazardtype = hazardtype_eq
hazardset1.data_lastupdated_date = datetime.now()
hazardset1.metadata_lastupdated_date = datetime.now()
hazardset1.detail_url = "http://domain.com/path/"
hazardset1.owner_organization = "data_provider"
DBSession.add(hazardset1)
shape = MultiPolygon([Polygon([(0, 0), (0, 1), (1, 1), (1, 0), (0, 0)])])
geometry = from_shape(shape, 4326)
# admin_div_10 is a country (division level 1)
admin_div_10 = AdministrativeDivision(**{
"code": 10,
"leveltype_id": 1,
"name": "Division level 1"
})
admin_div_10.geom = geometry
DBSession.add(admin_div_10)
# admin_div_11 is another country (division level 1)
admin_div_11 = AdministrativeDivision(**{
"code": 11,
"leveltype_id": 1,
"name": "Division level 1 2"
})
admin_div_11.geom = geometry
DBSession.add(admin_div_11)
# admin_div_12 is another country (division level 1)
admin_div_12 = AdministrativeDivision(**{
"code": 12,
"leveltype_id": 1,
"name": "Division level 1 3"
})
admin_div_12.geom = geometry
DBSession.add(admin_div_12)
# admin_div_13 is another country (division level 1)
admin_div_13 = AdministrativeDivision(**{
"code": 13,
"leveltype_id": 1,
"name": "Division level 1 4"
})
admin_div_13.geom = geometry
DBSession.add(admin_div_13)
# admin_div_20 is a province (division level 2)
# its parent is admin_div_10
admin_div_20 = AdministrativeDivision(**{
"code": 20,
"leveltype_id": 2,
"name": "Division level 2"
})
admin_div_20.parent_code = admin_div_10.code
admin_div_20.geom = geometry
DBSession.add(admin_div_20)
shape = MultiPolygon(
[Polygon([(0, 0), (0, 1), (0.5, 1), (0.5, 0), (0, 0)])])
geometry = from_shape(shape, 4326)
# admin_div_31 is a region (division level 3)
# its parent is admin_div_20
admin_div_31 = AdministrativeDivision(**{
"code": 31,
"leveltype_id": 3,
"name": "Division level 3 - 1"
})
admin_div_31.parent_code = admin_div_20.code
admin_div_31.geom = geometry
admin_div_31.hazardcategories = []
shape = MultiPolygon(
[Polygon([(0.5, 0), (0.5, 1), (1, 1), (1, 0), (0.5, 0)])])
geometry = from_shape(shape, 4326)
# admin_div_32 is a region (division level 3)
# its parent is also admin_div_20
admin_div_32 = AdministrativeDivision(**{
"code": 32,
"leveltype_id": 3,
"name": "Division level 3 - 2"
})
admin_div_32.parent_code = admin_div_20.code
admin_div_32.geom = geometry
admin_div_32.hazardcategories = []
# Here's a quick, graphical recap:
#
# admin_div_10 -> admin_div_20 -> admin_div_31
# `-> admin_div_32
# admin_div_11
# admin_div_12
# admin_div_13
# GeoNode Regions
# global_region contains all countries, **except admin_div_12**
global_region = Region(**{"id": 1, "level": 0, "name": "Global region"})
global_region.administrativedivisions.append(admin_div_10)
global_region.administrativedivisions.append(admin_div_11)
global_region.administrativedivisions.append(admin_div_13)
# region_1 is a country
# it matches GAULS's admin_div_10
region_1 = Region(**{"id": 2, "level": 3, "name": "Country 1"})
region_1.administrativedivisions.append(admin_div_10)
# region_2 is another country
# it matches GAULS's admin_div_11
region_2 = Region(**{"id": 3, "level": 3, "name": "Country 2"})
region_2.administrativedivisions.append(admin_div_11)
# region_3 is another country
# it matches GAULS's admin_div_12
region_3 = Region(**{"id": 4, "level": 3, "name": "Country 3"})
region_3.administrativedivisions.append(admin_div_12)
# Here's a quick, graphical recap:
#
# global_region -> admin_div_10 (region_1) -> admin_div_20 -> admin_div_31
# ` `-> admin_div_32
# `
# ` -> admin_div_11 (region_2)
# `-> admin_div_13
#
# region_3 = admin_div_12
category_eq_hig = HazardCategory.get(DBSession, "EQ", "HIG")
category_eq_hig.general_recommendation = "General recommendation for EQ HIG"
category_fl_hig = HazardCategory.get(DBSession, "FL", "HIG")
# admin_div_31 has (EQ, HIGH)
association = HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_eq_hig})
association.hazardsets.append(hazardset1)
admin_div_31.hazardcategories.append(association)
# admin_div_31 has (RF, HIGH)
admin_div_32.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_fl_hig}))
# admin_div_32 has (EQ, HIGH)
admin_div_32.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_eq_hig}))
# admin_div_10 has (EQ, HIGH)
admin_div_10.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_eq_hig}))
# admin_div_11 has no category (this is tested)
# admin_div_12 has (EQ, HIGH)
admin_div_12.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_eq_hig}))
# admin_div_13 has (EQ, HIGH)
admin_div_13.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_eq_hig}))
climate_rec = ClimateChangeRecommendation(
text="Climate change recommendation",
hazardtype=HazardType.get(DBSession, "EQ"))
climate_rec.associations.append(
CcrAd(administrativedivision=admin_div_10,
hazardtype=HazardType.get(DBSession, "EQ")))
DBSession.add(climate_rec)
climate_rec = ClimateChangeRecommendation(
text="Climate change recommendation 2",
hazardtype=HazardType.get(DBSession, "EQ"))
climate_rec.associations.append(
CcrAd(administrativedivision=admin_div_11,
hazardtype=HazardType.get(DBSession, "EQ")))
DBSession.add(climate_rec)
technical_rec = TechnicalRecommendation(
**{
"text":
"Recommendation #1 for earthquake, applied to"
" hazard categories HIG, MED and LOW"
})
association = HazardCategoryTechnicalRecommendationAssociation(order=1)
association.hazardcategory = category_eq_hig
technical_rec.hazardcategory_associations.append(association)
DBSession.add(technical_rec)
technical_rec = TechnicalRecommendation(**{
"text":
"Educational web resources on earthquakes and"
" seismic hazard"
})
association = HazardCategoryTechnicalRecommendationAssociation(order=1)
association.hazardcategory = category_eq_hig
technical_rec.hazardcategory_associations.append(association)
DBSession.add(technical_rec)
category_fl_med = HazardCategory.get(DBSession, "FL", "MED")
category_fl_med.general_recommendation = "General recommendation for FL MED"
admin_div_31.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_fl_med}))
DBSession.add(admin_div_31)
admin_div_32.hazardcategories.append(
HazardCategoryAdministrativeDivisionAssociation(
**{"hazardcategory": category_fl_med}))
DBSession.add(admin_div_32)
# generic further resource for EQ:
# it should be found on every EQ report page
# (except admin_div_12 which is not linked with global region)
further_resource = FurtherResource(
**{
"text": "Educational web resources on earthquakes and" +
" seismic hazard",
"id": 3,
})
association = HazardTypeFurtherResourceAssociation()
association.hazardtype = hazardtype_eq
association.region = global_region
further_resource.hazardtype_associations.append(association)
DBSession.add(further_resource)
# further resource for EQ & region 1:
# it should be found only on region 1 (and sub-divisions) page
further_resource = FurtherResource(**{
"text": "Further resource for earthquake for region 1",
"id": 5
})
association = HazardTypeFurtherResourceAssociation()
association.hazardtype = hazardtype_eq
association.region = region_1
further_resource.hazardtype_associations.append(association)
DBSession.add(further_resource)
# contact for EQ & admin_div_11:
contact1 = Contact(
**{
"name": "Contact name",
"url": "http://domain.com",
"phone": "0123456789",
"email": "*****@*****.**",
})
DBSession.add(contact1)
association = CAdHt()
association.hazardtype = hazardtype_eq
association.administrativedivision = admin_div_10
association.contact = contact1
DBSession.add(association)
# contact for EQ & admin_div_11:
contact2 = Contact(
**{
"name": "Contact name 2",
"url": "http://domain.com",
"phone": "0123456789",
"email": "*****@*****.**",
})
DBSession.add(contact1)
association = CAdHt()
association.hazardtype = hazardtype_eq
association.administrativedivision = admin_div_10
association.contact = contact2
DBSession.add(association)
Publication.new(DBSession)
DBSession.flush()