def test_not_corresponding_rasters(self, open_mock):
"""Difference in origin, resolution or size must not complete"""
hazardset_id = "notpreprocessed"
hazardtype = HazardType.get(DBSession, "FL")
regions = DBSession.query(Region).all()
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
regions=regions,
)
DBSession.add(hazardset)
for level in ["HIG", "MED", "LOW"]:
layer = Layer(
hazardlevel=HazardLevel.get(DBSession, level),
mask=False,
return_period=None,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
mask_layer = Layer(
hazardlevel=None,
mask=True,
return_period=None,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(mask_layer)
DBSession.flush()
self.completer().execute()
hazardset = DBSession.query(HazardSet).one()
self.assertEqual(
hazardset.complete_error,
"All layers should have the same origin, resolution and size",
)
self.assertEqual(hazardset.complete, False)
def populate_preprocessed(type):
hazardset_id = "preprocessed"
hazardtype = HazardType.get(DBSession, type)
regions = DBSession.query(Region).all()
print("Populating hazardset {}".format(hazardset_id))
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
regions=regions,
)
DBSession.add(hazardset)
layer = Layer(
hazardlevel=None,
mask=False,
return_period=None,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
hazardset.complete = True
DBSession.flush()
def test_open_exception(self, open_mock):
"""Test handling of open exception"""
hazardset_id = "notpreprocessed"
hazardtype = HazardType.get(DBSession, "EQ")
regions = DBSession.query(Region).all()
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
regions=regions,
)
DBSession.add(hazardset)
for level in ["HIG", "MED", "LOW"]:
layer = Layer(
hazardlevel=HazardLevel.get(DBSession, level),
mask=False,
return_period=None,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
DBSession.flush()
self.completer().execute()
hazardset = DBSession.query(HazardSet).one()
self.assertEqual(hazardset.complete_error,
"Error opening layer notpreprocessed")
self.assertEqual(hazardset.complete, False)
def test_complete_preprocessed(self, open_mock):
"""Test complete preprocessed hazardset"""
hazardset_id = "preprocessed"
hazardtype = HazardType.get(DBSession, "VA")
regions = DBSession.query(Region).all()
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
regions=regions,
)
DBSession.add(hazardset)
layer = Layer(
hazardlevel=None,
mask=False,
return_period=None,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
DBSession.flush()
self.completer().execute()
hazardset = DBSession.query(HazardSet).one()
self.assertEqual(hazardset.complete_error, None)
self.assertEqual(hazardset.complete, True)
def harvest_layer(self, object):
logger.info("Harvesting layer {id} - {title}".format(**object))
title = object["title"]
# we need to retrieve more information on this layer
# since the regions array is not advertised by the main
# regions listing from GeoNode
o = self.fetch("api/layers/{id}/".format(**object))
if "regions" not in list(o.keys()):
warning(object, 'Attribute "regions" is missing')
region_ids = []
for r in o.get("regions", []):
# r is like "/api/regions/1/"
region_ids.append(r.split("/")[3])
if len(region_ids) == 0:
regions = []
else:
regions = self.dbsession.query(Region).filter(Region.id.in_(region_ids)).all()
hazardset_id = o['hazard_set']
if not hazardset_id:
logger.info(" hazard_set is empty")
return False
# FIXME: temporary override
if hazardset_id in excluded_hazardsets:
logger.info(" hazard_set {} is excluded, skipping")
return False
hazardtype = self.check_hazard_type(o)
if not hazardtype:
return False
type_settings = self.settings["hazard_types"][hazardtype.mnemonic]
preprocessed = "values" in type_settings
local = "GLOBAL" not in hazardset_id
mask = False
if preprocessed is True:
hazardlevel = None
# harvest hazard_unit for preprocessed layers
hazard_unit = o['hazard_unit']
if o['hazard_period']:
logger.info(' return period found in preprocessed hazardset')
return False
hazard_period = None
else:
try:
hazard_period = int(o['hazard_period'])
except:
hazard_period = None
if hazard_period is None:
logger.info(' no return period found')
return False
hazardlevel = None
for level in ("LOW", "MED", "HIG"):
if between(hazard_period, type_settings["return_periods"][level]):
hazardlevel = HazardLevel.get(self.dbsession, level)
break
if "mask_return_period" in type_settings and between(
hazard_period, type_settings["mask_return_period"]
):
mask = True
if hazardlevel is None and not mask:
logger.info(" No corresponding hazard_level")
return False
hazard_unit = o['hazard_unit']
if hazard_unit == '':
logger.info(' hazard_unit is empty')
return False
if o['srid'] != 'EPSG:4326':
logger.info(' srid is different from "EPSG:4326"')
return False
data_update_date = parse_date(o['data_update_date'])
if not data_update_date:
warning(o, 'data_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
# We use a very old date for good comparison in decision tree
data_update_date = datetime.fromtimestamp(0)
metadata_update_date = parse_date(o['metadata_update_date'])
if not metadata_update_date:
warning(o, 'metadata_update_date is empty: set to {}'.format(datetime.fromtimestamp(0)))
# We use a very old date for good comparison in decision tree
metadata_update_date = datetime.fromtimestamp(0)
calculation_method_quality = o['calculation_method_quality']
if not calculation_method_quality:
warning(o, 'calculation_method_quality is empty: skip layer')
return False
calculation_method_quality = int(float(calculation_method_quality))
scientific_quality = o['scientific_quality']
if not scientific_quality:
warning(o, 'scientific_quality is empty')
return False
scientific_quality = int(float(scientific_quality))
download_url = o['download_url']
if not download_url:
warning(o, 'download_url is empty')
return False
hazardset = self.dbsession.query(HazardSet).get(hazardset_id)
# Create hazardset before layer
if hazardset is None:
logger.info(" Create new hazardset {}".format(hazardset_id))
hazardset = HazardSet()
hazardset.id = hazardset_id
hazardset.hazardtype = hazardtype
self.dbsession.add(hazardset)
# get detail_url and owner_organization from last updated layer
geonode = self.settings["geonode"]
geonode_base_url = geonode["url"]
if o['detail_url'] and not mask:
hazardset.detail_url = geonode_base_url + o['detail_url']
if o['owner']['organization'] and not mask:
hazardset.owner_organization = o['owner']['organization']
if not mask:
hazardset.regions = regions
layer = self.dbsession.query(Layer).get(o['id'])
if layer is None:
logger.info(" Create new Layer {}".format(title))
layer = Layer()
layer.geonode_id = o['id']
layer.hazardset = hazardset
layer.mask = False
else:
# If data has changed
if (
layer.data_lastupdated_date != data_update_date
or layer.download_url != download_url
):
logger.info(" Invalidate downloaded")
layer.downloaded = False
hazardset.complete = False
hazardset.processed = None
# Remove file from cache
layer.download_url = download_url
path = self.layer_path(layer)
if os.path.isfile(path):
os.unlink(path)
# Some hazardset fields are calculated during completing
if (
layer.calculation_method_quality != calculation_method_quality
or layer.scientific_quality != scientific_quality
or layer.metadata_lastupdated_date != metadata_update_date
):
logger.info(" Invalidate complete")
hazardset.complete = False
# Some fields invalidate outputs
if layer.hazardunit != hazard_unit:
logger.info(" Invalidate processed")
hazardset.processed = None
typename = o.get("typename", None)
if typename is None:
warning(o, 'Attribute "typename" is missing')
layer.typename = typename
layer.return_period = hazard_period
layer.hazardunit = hazard_unit
layer.data_lastupdated_date = data_update_date
layer.metadata_lastupdated_date = metadata_update_date
layer.download_url = download_url
# TODO: retrieve quality attributes
layer.calculation_method_quality = calculation_method_quality
layer.scientific_quality = scientific_quality
layer.local = local
layer.set_harvested(True)
self.dbsession.flush()
return True
def populate_notpreprocessed(type, unit):
hazardset_id = "notpreprocessed"
hazardtype = HazardType.get(DBSession, type)
hazardtype_settings = settings["hazard_types"][hazardtype.mnemonic]
regions = DBSession.query(Region).all()
print("Populating hazardset {}".format(hazardset_id))
hazardset = HazardSet(
id=hazardset_id,
hazardtype=hazardtype,
local=False,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
regions=regions,
)
DBSession.add(hazardset)
return_periods = hazardtype_settings["return_periods"]
for level in ("HIG", "MED", "LOW"):
hazardlevel = HazardLevel.get(DBSession, level)
level_return_periods = return_periods[level]
if isinstance(level_return_periods, list):
return_period = level_return_periods[0]
else:
return_period = level_return_periods
layer = Layer(
hazardlevel=hazardlevel,
mask=False,
return_period=return_period,
hazardunit=unit,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
mask_return_periods = hazardtype_settings["mask_return_period"]
if isinstance(mask_return_periods, list):
mask_return_period = mask_return_periods[0]
else:
mask_return_period = mask_return_periods
layer = Layer(
hazardlevel=None,
mask=True,
return_period=mask_return_period,
hazardunit=unit,
data_lastupdated_date=datetime.now(),
metadata_lastupdated_date=datetime.now(),
geonode_id=new_geonode_id(DBSession),
download_url="test",
calculation_method_quality=5,
scientific_quality=1,
local=False,
downloaded=True,
)
hazardset.layers.append(layer)
hazardset.complete = True
DBSession.flush()
def make_layers():
layers = []
for level in ("HIG", "MED", "LOW"):
layer = Layer()
layer.hazardlevel = HazardLevel.get(level)
layer.return_period = 1
layer.hazardunit = "m"
layer.data_lastupdated_date = datetime.now()
layer.metadata_lastupdated_date = datetime.now()
layer.geonode_id = new_geonode_id()
layer.download_url = "http://something"
layer.calculation_method_quality = 5
layer.scientific_quality = 1
layer.local = False
layer.downloaded = True
layers.append(layer)
return layers