def test_plugin_compatibility(self):
"""Default plugins perform as expected
"""
# Upload a raster and a vector data set
hazard_filename = os.path.join(UNITDATA, 'hazard',
'jakarta_flood_design.tif')
hazard_layer = save_to_geonode(hazard_filename)
check_layer(hazard_layer, full=True)
exposure_filename = os.path.join(UNITDATA, 'exposure',
'buildings_osm_4326.shp')
exposure_layer = save_to_geonode(exposure_filename)
check_layer(exposure_layer, full=True)
# Test
plugin_list = get_admissible_plugins()
assert len(plugin_list) > 0
geoserver = {'url': settings.GEOSERVER_BASE_URL + 'ows',
'name': 'Local Geoserver',
'version': '1.0.0',
'id': 0}
metadata = get_metadata(geoserver['url'])
msg = 'There were no layers in test geoserver'
assert len(metadata) > 0, msg
# Characterisation test to preserve the behaviour of
# get_layer_descriptors. FIXME: I think we should change this to be
# a dictionary of metadata entries (ticket #126).
reference = {'geonode:buildings_osm_4326':
{'layertype': 'vector',
'category': 'exposure',
'subcategory': 'structure',
'title': 'buildings_osm_4326'},
'geonode:jakarta_flood_like_2007_with_structural_improvements':
{'layertype': 'raster',
'category': 'hazard',
'subcategory': 'flood',
'title': 'Jakarta flood like 2007 with structural improvements'}
}
for name, keywords in reference.items():
msg = 'Expected layer %s in %s' % (name, metadata.keys())
assert name in metadata.keys(), msg
values = metadata[name]['keywords']
for key in keywords.keys():
refval = keywords[key]
val = values[key]
msg = ('Got value "%s" for key "%s" '
'Expected "%s"' % (val, key, refval))
assert refval == val, msg
plugins = get_admissible_plugins(keywords=reference.values())
msg = 'No compatible layers returned'
assert len(plugins) > 0, msg
def test_shapefile(self):
"""Shapefile can be uploaded
"""
thefile = os.path.join(UNITDATA, 'exposure', 'buildings_osm_4326.shp')
layer = save_to_geonode(thefile, user=self.user, overwrite=True)
check_layer(layer, full=True)
assert isinstance(layer.geographic_bounding_box, basestring)
def test_plugin_selection_http(self):
"""Verify the plugins can recognize compatible layers (HTTP).
"""
# Upload a raster and a vector data set
hazard_filename = os.path.join(UNITDATA, 'hazard',
'jakarta_flood_design.tif')
hazard_layer = save_to_geonode(hazard_filename,
user=self.user,
overwrite=True)
check_layer(hazard_layer, full=True)
msg = 'No keywords found in layer %s' % hazard_layer.name
assert hazard_layer.keywords.count() > 0, msg
exposure_filename = os.path.join(UNITDATA, 'exposure',
'buildings_osm_4326.shp')
exposure_layer = save_to_geonode(exposure_filename)
check_layer(exposure_layer, full=True)
msg = 'No keywords found in layer %s' % exposure_layer.name
assert exposure_layer.keywords.count() > 0, msg
# make sure the keywords for the flood building impact
# function are present.
hazard_keywords = [k.name for k in hazard_layer.keywords.all()]
exposure_keywords = [k.name for k in exposure_layer.keywords.all()]
assert 'category:hazard' in hazard_keywords
assert 'subcategory:flood' in hazard_keywords
assert 'category:exposure' in exposure_keywords
assert 'subcategory:building' in exposure_keywords
c = Client()
functions_url = reverse('safe-functions')
rv = c.get(functions_url)
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
data = json.loads(rv.content)
assert 'functions' in data
functions = data['functions']
for function in functions:
if function['name'] == 'Flood Building Impact Function':
layers = function['layers']
msg_tmpl = 'Expected layer %s in list of compatible layers: %s'
hazard_msg = msg_tmpl % (hazard_layer.typename, layers)
assert hazard_layer.typename in layers, hazard_msg
exposure_msg = msg_tmpl % (exposure_layer.typename, layers)
assert exposure_layer.typename in layers, exposure_msg
def test_repeated_upload(self):
"""The same file can be uploaded more than once
"""
thefile = os.path.join(UNITDATA, 'hazard', 'jakarta_flood_design.tif')
uploaded1 = save_to_geonode(thefile, overwrite=True,
user=self.user)
check_layer(uploaded1, full=True)
uploaded2 = save_to_geonode(thefile, overwrite=True,
user=self.user)
check_layer(uploaded2, full=True)
uploaded3 = save_to_geonode(thefile, overwrite=False,
user=self.user)
check_layer(uploaded3, full=True)
msg = ('Expected %s but got %s' % (uploaded1.name, uploaded2.name))
assert uploaded1.name == uploaded2.name, msg
msg = ('Expected a different name when uploading %s using '
'overwrite=False but got %s' % (thefile, uploaded3.name))
assert uploaded1.name != uploaded3.name, msg
def test_tiff(self):
"""GeoTIF file can be uploaded
"""
thefile = os.path.join(UNITDATA, 'hazard', 'jakarta_flood_design.tif')
uploaded = save_to_geonode(thefile, user=self.user, overwrite=True)
check_layer(uploaded, full=True)
def test_the_earthquake_fatality_estimation_allen(self):
"""Fatality computation computed correctly with GeoServer Data
"""
# Simulate bounding box from application
viewport_bbox_string = '104.3,-8.2,110.04,-5.17'
# Upload exposure data for this test
name = 'Population_2010'
exposure_filename = '%s/%s.asc' % (TESTDATA, name)
exposure_layer = save_to_geonode(exposure_filename,
user=self.user, overwrite=True)
workspace = exposure_layer.workspace
layer_name = exposure_layer.name
msg = 'Expected layer name to be "%s". Got %s' % (name, layer_name)
assert layer_name == name.lower(), msg
exposure_name = '%s:%s' % (workspace, layer_name)
# Check metadata
assert_bounding_box_matches(exposure_layer, exposure_filename)
exp_bbox_string = get_bounding_box_string(exposure_filename)
check_layer(exposure_layer, full=True)
# Now we know that exposure layer is good, lets upload some
# hazard layers and do the calculations
filename = 'lembang_mmi_hazmap.asc'
# Save
hazard_filename = '%s/%s' % (TESTDATA, filename)
hazard_layer = save_to_geonode(hazard_filename,
user=self.user, overwrite=True)
hazard_name = '%s:%s' % (hazard_layer.workspace,
hazard_layer.name)
# Check metadata
assert_bounding_box_matches(hazard_layer, hazard_filename)
haz_bbox_string = get_bounding_box_string(hazard_filename)
check_layer(hazard_layer, full=True)
calculate_url = reverse('safe-calculate')
# Run calculation
c = Client()
rv = c.post(calculate_url, data=dict(
hazard_server=INTERNAL_SERVER_URL,
hazard=hazard_name,
exposure_server=INTERNAL_SERVER_URL,
exposure=exposure_name,
#bbox=viewport_bbox_string,
bbox=exp_bbox_string, # This one reproduced the
# crash for lembang
impact_function='I T B Fatality Function',
keywords='test,shakemap,usgs'))
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
data = json.loads(rv.content)
if 'errors' in data:
errors = data['errors']
if errors is not None:
msg = ('The server returned the error message: %s'
% str(errors))
raise Exception(msg)
assert 'success' in data
assert 'hazard_layer' in data
assert 'exposure_layer' in data
assert 'run_duration' in data
assert 'run_date' in data
assert 'layer' in data
assert data['success']
# Download result and check
layer_name = data['layer'].split('/')[-1]
result_layer = download(INTERNAL_SERVER_URL,
layer_name,
get_bounding_box_string(hazard_filename))
assert os.path.exists(result_layer.filename)
def test_linked_datasets(self):
"""Linked datesets can be pulled in e.g. to include gender break down
"""
# Upload exposure data for this test. This will automatically
# pull in female_pct_yogya.asc through its "associates" keyword
name = 'population_yogya'
exposure_filename = '%s/%s.asc' % (TESTDATA, name)
exposure_layer = save_to_geonode(exposure_filename,
user=self.user, overwrite=True)
exposure_name = '%s:%s' % (exposure_layer.workspace,
exposure_layer.name)
# Check metadata
assert_bounding_box_matches(exposure_layer, exposure_filename)
exp_bbox_string = get_bounding_box_string(exposure_filename)
check_layer(exposure_layer, full=True)
# Upload hazard data
filename = 'eq_yogya_2006.asc'
hazard_filename = '%s/%s' % (TESTDATA, filename)
hazard_layer = save_to_geonode(hazard_filename,
user=self.user, overwrite=True)
hazard_name = '%s:%s' % (hazard_layer.workspace,
hazard_layer.name)
# Check metadata
assert_bounding_box_matches(hazard_layer, hazard_filename)
haz_bbox_string = get_bounding_box_string(hazard_filename)
check_layer(hazard_layer, full=True)
calculate_url = reverse('safe-calculate')
# Run calculation
c = Client()
rv = c.post(calculate_url, data=dict(
hazard_server=INTERNAL_SERVER_URL,
hazard=hazard_name,
exposure_server=INTERNAL_SERVER_URL,
exposure=exposure_name,
bbox=haz_bbox_string,
impact_function='Empirical Fatality Function',
keywords='test,fatalities,population,usgs'))
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
data = json.loads(rv.content)
if 'errors' in data:
errors = data['errors']
if errors is not None:
msg = ('The server returned the error message: %s'
% str(errors))
raise Exception(msg)
assert 'success' in data
assert 'hazard_layer' in data
assert 'exposure_layer' in data
assert 'run_duration' in data
assert 'run_date' in data
assert 'layer' in data
assert data['success']
# Download result and check
layer_name = data['layer'].split('/')[-1]
result_layer = download(INTERNAL_SERVER_URL,
layer_name,
get_bounding_box_string(hazard_filename))
assert os.path.exists(result_layer.filename)
# Check calculated values
keywords = result_layer.get_keywords()
assert 'caption' in keywords
def test_earthquake_exposure_plugin(self):
"""Population exposure to individual MMI levels can be computed
"""
# Upload exposure data for this test
# FIXME (Ole): While this dataset is ok for testing,
# note that is has been resampled without scaling
# so numbers are about 25 times too large.
# Consider replacing test populations dataset for good measures,
# just in case any one accidentally started using this dataset
# for real.
name = 'Population_2010'
exposure_filename = '%s/%s.asc' % (TESTDATA, name)
exposure_layer = save_to_geonode(exposure_filename,
user=self.user, overwrite=True)
exposure_name = '%s:%s' % (exposure_layer.workspace,
exposure_layer.name)
# Check metadata
assert_bounding_box_matches(exposure_layer, exposure_filename)
exp_bbox_string = get_bounding_box_string(exposure_filename)
check_layer(exposure_layer, full=True)
# Upload hazard data
filename = 'lembang_mmi_hazmap.asc'
hazard_filename = '%s/%s' % (TESTDATA, filename)
hazard_layer = save_to_geonode(hazard_filename,
user=self.user, overwrite=True)
hazard_name = '%s:%s' % (hazard_layer.workspace,
hazard_layer.name)
# Check metadata
assert_bounding_box_matches(hazard_layer, hazard_filename)
haz_bbox_string = get_bounding_box_string(hazard_filename)
check_layer(hazard_layer, full=True)
calculate_url = reverse('safe-calculate')
# Run calculation
c = Client()
rv = c.post(calculate_url, data=dict(
hazard_server=INTERNAL_SERVER_URL,
hazard=hazard_name,
exposure_server=INTERNAL_SERVER_URL,
exposure=exposure_name,
bbox=haz_bbox_string,
impact_function='Earthquake Building Damage Function',
keywords='test,population,exposure,usgs'))
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
data = json.loads(rv.content)
if 'errors' in data:
errors = data['errors']
if errors is not None:
msg = ('The server returned the error message: %s'
% str(errors))
raise Exception(msg)
assert 'success' in data
assert 'hazard_layer' in data
assert 'exposure_layer' in data
assert 'run_duration' in data
assert 'run_date' in data
assert 'layer' in data
assert data['success']
# Download result and check
layer_name = data['layer'].split('/')[-1]
result_layer = download(INTERNAL_SERVER_URL,
layer_name,
get_bounding_box_string(hazard_filename))
assert os.path.exists(result_layer.filename)
# Check calculated values
keywords = result_layer.get_keywords()
assert 'mmi-classes' in keywords
assert 'affected-population' in keywords
mmi_classes = [int(x) for x in keywords['mmi-classes'].split('_')]
count = [float(x) for x in keywords['affected-population'].split('_')]
# Brute force count for each population level
population = download(INTERNAL_SERVER_URL,
exposure_name,
get_bounding_box_string(hazard_filename))
intensity = download(INTERNAL_SERVER_URL,
hazard_name,
get_bounding_box_string(hazard_filename))
# Extract data
H = intensity.get_data(nan=0)
P = population.get_data(nan=0)
brutecount = {}
for mmi in mmi_classes:
brutecount[mmi] = 0
for i in range(P.shape[0]):
for j in range(P.shape[1]):
mmi = H[i, j]
if not numpy.isnan(mmi):
mmi_class = int(round(mmi))
pop = P[i, j]
if not numpy.isnan(pop):
brutecount[mmi_class] += pop
for i, mmi in enumerate(mmi_classes):
assert numpy.allclose(count[i], brutecount[mmi], rtol=1.0e-6)
def test_jakarta_flood_study(self):
"""HKV Jakarta flood study calculated correctly using the API
"""
# FIXME (Ole): Redo with population as shapefile later
# Expected values from HKV
expected_value = 1537920
# Name files for hazard level, exposure and expected fatalities
population = 'Population_Jakarta_geographic'
plugin_name = 'Flood Evacuation Function'
# Upload exposure data for this test
exposure_filename = '%s/%s.asc' % (TESTDATA, population)
exposure_layer = save_to_geonode(exposure_filename,
user=self.user, overwrite=True)
workspace = exposure_layer.workspace
exposure_name = '%s:%s' % (workspace, exposure_layer.name)
# Check metadata
assert_bounding_box_matches(exposure_layer, exposure_filename)
exp_bbox_string = get_bounding_box_string(exposure_filename)
check_layer(exposure_layer, full=True)
# Now we know that exposure layer is good, lets upload some
# hazard layers and do the calculations
filename = 'jakarta_flood_design.tif'
hazard_filename = os.path.join(UNITDATA, 'hazard', filename)
exposure_filename = os.path.join(TESTDATA, population)
# Save
hazard_layer = save_to_geonode(hazard_filename,
user=self.user, overwrite=True)
hazard_name = '%s:%s' % (hazard_layer.workspace,
hazard_layer.name)
# Check metadata
assert_bounding_box_matches(hazard_layer, hazard_filename)
haz_bbox_string = get_bounding_box_string(hazard_filename)
check_layer(hazard_layer, full=True)
calculate_url = reverse('safe-calculate')
# Run calculation
c = Client()
rv = c.post(calculate_url, data=dict(
hazard_server=INTERNAL_SERVER_URL,
hazard=hazard_name,
exposure_server=INTERNAL_SERVER_URL,
exposure=exposure_name,
bbox=exp_bbox_string,
impact_function=plugin_name,
keywords='test,flood,HKV'))
self.assertEqual(rv.status_code, 200)
self.assertEqual(rv['Content-Type'], 'application/json')
data = json.loads(rv.content)
if 'errors' in data:
errors = data['errors']
if errors is not None:
raise Exception(errors)
assert 'hazard_layer' in data
assert 'exposure_layer' in data
assert 'run_duration' in data
assert 'run_date' in data
assert 'layer' in data