def get_bounding_boxes(haz_metadata, exp_metadata, req_bbox):
"""Check and get appropriate bounding boxes for input layers
Input
haz_metadata: Metadata for hazard layer
exp_metadata: Metadata for exposure layer
req_bbox: Bounding box (string) as requested by HTML POST.
Output
haz_bbox: Bounding box to be used for hazard layer.
exp_bbox: Bounding box to be used for exposure layer
imp_bbox: Bounding box to be used for resulting impact layer
Note exp_bbox and imp_bbox are the same and calculated as the
intersection among hazard, exposure and viewport bounds.
haz_bbox may be grown by one pixel size in case exposure data
is vector data to make sure points always can be interpolated
"""
# Input checks
msg = ('Invalid bounding box %s (%s). '
'It must be a string' % (str(req_bbox), type(req_bbox)))
assert isinstance(req_bbox, basestring), msg
check_bbox_string(req_bbox)
# Get bounding boxes for layers and viewport
haz_bbox = haz_metadata['bounding_box']
exp_bbox = exp_metadata['bounding_box']
vpt_bbox = bboxstring2list(req_bbox)
# New bounding box for data common to hazard, exposure and viewport
# Download only data within this intersection
intersection_bbox = bbox_intersection(vpt_bbox, haz_bbox, exp_bbox)
if intersection_bbox is None:
# Bounding boxes did not overlap
msg = ('Bounding boxes of hazard data [%s], exposure data [%s] '
'and viewport [%s] did not overlap, so no computation was '
'done. Please make sure you pan to where the data is and '
'that hazard and exposure data overlaps.'
% (bboxlist2string(haz_bbox, decimals=3),
bboxlist2string(exp_bbox, decimals=3),
bboxlist2string(vpt_bbox, decimals=3)))
logger.info(msg)
raise Exception(msg)
# Grow hazard bbox to buffer this common bbox in case where
# hazard is raster and exposure is vector
if (haz_metadata['layer_type'] == 'raster' and
exp_metadata['layer_type'] == 'vector'):
haz_res = haz_metadata['resolution']
haz_bbox = buffered_bounding_box(intersection_bbox, haz_res)
else:
haz_bbox = intersection_bbox
# Usually the intersection bbox is used for both exposure layer and result
exp_bbox = imp_bbox = intersection_bbox
return haz_bbox, exp_bbox, imp_bbox
def get_bounding_boxes(haz_metadata, exp_metadata, req_bbox):
"""Check and get appropriate bounding boxes for input layers
Input
haz_metadata: Metadata for hazard layer
exp_metadata: Metadata for exposure layer
req_bbox: Bounding box (string) as requested by HTML POST.
Output
haz_bbox: Bounding box to be used for hazard layer.
exp_bbox: Bounding box to be used for exposure layer
imp_bbox: Bounding box to be used for resulting impact layer
Note exp_bbox and imp_bbox are the same and calculated as the
intersection among hazard, exposure and viewport bounds.
haz_bbox may be grown by one pixel size in case exposure data
is vector data to make sure points always can be interpolated
"""
# Input checks
msg = ('Invalid bounding box %s (%s). '
'It must be a string' % (str(req_bbox), type(req_bbox)))
assert isinstance(req_bbox, basestring), msg
check_bbox_string(req_bbox)
# Get bounding boxes for layers and viewport
haz_bbox = haz_metadata['bounding_box']
exp_bbox = exp_metadata['bounding_box']
vpt_bbox = bboxstring2list(req_bbox)
# New bounding box for data common to hazard, exposure and viewport
# Download only data within this intersection
intersection_bbox = bbox_intersection(vpt_bbox, haz_bbox, exp_bbox)
if intersection_bbox is None:
# Bounding boxes did not overlap
msg = ('Bounding boxes of hazard data [%s], exposure data [%s] '
'and viewport [%s] did not overlap, so no computation was '
'done. Please make sure you pan to where the data is and '
'that hazard and exposure data overlaps.' %
(bboxlist2string(haz_bbox, decimals=3),
bboxlist2string(exp_bbox, decimals=3),
bboxlist2string(vpt_bbox, decimals=3)))
logger.info(msg)
raise Exception(msg)
# Grow hazard bbox to buffer this common bbox in case where
# hazard is raster and exposure is vector
if (haz_metadata['layer_type'] == 'raster'
and exp_metadata['layer_type'] == 'vector'):
haz_res = haz_metadata['resolution']
haz_bbox = buffered_bounding_box(intersection_bbox, haz_res)
else:
haz_bbox = intersection_bbox
# Usually the intersection bbox is used for both exposure layer and result
exp_bbox = imp_bbox = intersection_bbox
return haz_bbox, exp_bbox, imp_bbox
def test_bounding_box_conversions(self):
"""Bounding boxes can be converted between list and string
"""
# Good ones
for x in [[105, -7, 108, -5],
[106.5, -6.5, 107, -6],
[94.972335, -11.009721, 141.014, 6.073612333333],
[105.3, -8.5, 110.0, -5.5],
[105.6, -7.8, 110.5, -5.1]]:
bbox_string = bboxlist2string(x)
bbox_list = bboxstring2list(bbox_string)
assert numpy.allclose(x, bbox_list, rtol=1.0e-6, atol=1.0e-6)
for x in ['105,-7,108,-5',
'106.5, -6.5, 107,-6',
'94.972335,-11.009721,141.014,6.073612333333']:
bbox_list = bboxstring2list(x)
# Check that numbers are numerically consistent
assert numpy.allclose([float(z) for z in x.split(',')],
bbox_list, rtol=1.0e-6, atol=1.0e-6)
# Bad ones
for bbox in [[105, -7, 'x', -5],
[106.5, -6.5, -6],
[94.972335, 0, -11.009721, 141.014, 6]]:
try:
bbox_string = bboxlist2string(bbox)
except:
pass
else:
msg = 'Should have raised exception'
raise Exception(msg)
for x in ['106.5,-6.5,-6',
'106.5,-6.5,-6,4,10',
'94.972335,x,141.014,6.07']:
try:
bbox_list = bboxstring2list(x)
except:
pass
else:
msg = 'Should have raised exception: %s' % x
raise Exception(msg)
def calculate(request, save_output=save_to_geonode):
start = datetime.datetime.now()
if request.method == 'GET':
# FIXME: Add a basic form here to be able to generate the POST request.
return HttpResponse('This should be accessed by robots, not humans.'
'In other words using HTTP POST instead of GET.')
elif request.method == 'POST':
data = request.POST
impact_function_name = data['impact_function']
hazard_server = data['hazard_server']
hazard_layer = data['hazard']
exposure_server = data['exposure_server']
exposure_layer = data['exposure']
bbox = data['bbox']
keywords = data['keywords']
if request.user.is_anonymous():
theuser = get_valid_user()
else:
theuser = request.user
# Create entry in database
calculation = Calculation(user=theuser,
run_date=start,
hazard_server=hazard_server,
hazard_layer=hazard_layer,
exposure_server=exposure_server,
exposure_layer=exposure_layer,
impact_function=impact_function_name,
success=False)
try:
# Input checks
msg = 'This cannot happen :-)'
assert isinstance(bbox, basestring), msg
check_bbox_string(bbox)
# Find the intersection of bounding boxes for viewport,
# hazard and exposure.
vpt_bbox = bboxstring2list(bbox)
haz_bbox = get_metadata(hazard_server,
hazard_layer)['bounding_box']
exp_bbox = get_metadata(exposure_server,
exposure_layer)['bounding_box']
# Impose minimum bounding box size (as per issue #101).
# FIXME (Ole): This will need to be revisited in conjunction with
# raster resolutions at some point.
min_res = 0.00833334
eps = 1.0e-1
vpt_bbox = minimal_bounding_box(vpt_bbox, min_res, eps=eps)
haz_bbox = minimal_bounding_box(haz_bbox, min_res, eps=eps)
exp_bbox = minimal_bounding_box(exp_bbox, min_res, eps=eps)
# New bounding box for data common to hazard, exposure and viewport
# Download only data within this intersection
intersection = bbox_intersection(vpt_bbox, haz_bbox, exp_bbox)
if intersection is None:
# Bounding boxes did not overlap
msg = ('Bounding boxes of hazard data, exposure data and '
'viewport did not overlap, so no computation was '
'done. Please try again.')
logger.info(msg)
raise Exception(msg)
bbox = bboxlist2string(intersection)
plugin_list = get_plugins(impact_function_name)
_, impact_function = plugin_list[0].items()[0]
impact_function_source = inspect.getsource(impact_function)
calculation.impact_function_source = impact_function_source
calculation.bbox = bbox
calculation.save()
msg = 'Performing requested calculation'
logger.info(msg)
# Download selected layer objects
msg = ('- Downloading hazard layer %s from %s' % (hazard_layer,
hazard_server))
logger.info(msg)
H = download(hazard_server, hazard_layer, bbox)
msg = ('- Downloading exposure layer %s from %s' % (exposure_layer,
exposure_server))
logger.info(msg)
E = download(exposure_server, exposure_layer, bbox)
# Calculate result using specified impact function
msg = ('- Calculating impact using %s' % impact_function)
logger.info(msg)
impact_filename = calculate_impact(layers=[H, E],
impact_fcn=impact_function)
# Upload result to internal GeoServer
msg = ('- Uploading impact layer %s' % impact_filename)
logger.info(msg)
result = save_output(impact_filename,
title='output_%s' % start.isoformat(),
user=theuser)
except Exception, e:
#FIXME: Reimplement error saving for calculation
logger.error(e)
errors = e.__str__()
trace = exception_format(e)
calculation.errors = errors
calculation.stacktrace = trace
calculation.save()
jsondata = json.dumps({'errors': errors, 'stacktrace': trace})
return HttpResponse(jsondata, mimetype='application/json')
def test_specified_raster_resolution(self):
"""Raster layers can be downloaded with specific resolution
This is another test for ticket #103
Native test data:
maumere....asc
ncols 931
nrows 463
cellsize 0.00018
Population_Jakarta
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Here we download it at a range of fixed resolutions that
are both coarser and finer, and check that the dimensions
of the downloaded matrix are as expected.
We also check that the extrema of the subsampled matrix are sane
"""
for test_filename in [
'maumere_aos_depth_20m_land_wgs84.asc',
'Population_Jakarta_geographic.asc', 'Population_2010.asc'
]:
hazard_filename = ('%s/%s' % (TESTDATA, test_filename))
# Get reference values
H = read_layer(hazard_filename)
depth_min_ref, depth_max_ref = H.get_extrema()
native_resolution = H.get_resolution()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Test for a range of resolutions
for res in [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002,
0.0001,
0.00006,
0.00003
]: # Finer
# To save time don't do finest resolution for the
# two population sets
if test_filename.startswith('Population') and res < 0.00006:
break
# Set bounding box
bbox = get_bounding_box_string(hazard_filename)
compare_extrema = True
if test_filename == 'Population_2010.asc':
# Make bbox small for finer resolutions to
# save time and to test that as well.
# However, extrema obviously won't match those
# of the full dataset. Once we can clip
# datasets, we can remove this restriction.
if res < 0.005:
bbox = '106.685974,-6.373421,106.974534,-6.079886'
compare_extrema = False
bb = bboxstring2list(bbox)
# Download data at specified resolution
H = download(INTERNAL_SERVER_URL,
hazard_name,
bbox,
resolution=res)
A = H.get_data()
# Verify that data has the requested bobx and resolution
actual_bbox = H.get_bounding_box()
msg = ('Bounding box for %s was not as requested. I got %s '
'but '
'expected %s' % (hazard_name, actual_bbox, bb))
assert numpy.allclose(actual_bbox, bb, rtol=1.0e-6)
# FIXME (Ole): How do we sensibly resolve the issue with
# resx, resy vs one resolution (issue #173)
actual_resolution = H.get_resolution()[0]
# FIXME (Ole): Resolution is often far from the requested
# see issue #102
# Here we have to accept up to 5%
tolerance102 = 5.0e-2
msg = ('Resolution of %s was not as requested. I got %s but '
'expected %s' % (hazard_name, actual_resolution, res))
assert numpy.allclose(actual_resolution,
res,
rtol=tolerance102), msg
# Determine expected shape from bbox (W, S, E, N)
ref_rows = int(round((bb[3] - bb[1]) / res))
ref_cols = int(round((bb[2] - bb[0]) / res))
# Compare shapes (generally, this may differ by 1)
msg = ('Shape of downloaded raster was [%i, %i]. '
'Expected [%i, %i].' %
(A.shape[0], A.shape[1], ref_rows, ref_cols))
assert (ref_rows == A.shape[0] and ref_cols == A.shape[1]), msg
# Assess that the range of the interpolated data is sane
if not compare_extrema:
continue
# For these test sets we get exact match of the minimum
msg = (
'Minimum of %s resampled at resolution %f '
'was %f. Expected %f.' %
(hazard_layer.name, res, numpy.nanmin(A), depth_min_ref))
assert numpy.allclose(depth_min_ref,
numpy.nanmin(A),
rtol=0.0,
atol=0.0), msg
# At the maximum it depends on the subsampling
msg = (
'Maximum of %s resampled at resolution %f '
'was %f. Expected %f.' %
(hazard_layer.name, res, numpy.nanmax(A), depth_max_ref))
if res < native_resolution[0]:
# When subsampling to finer resolutions we expect a
# close match
assert numpy.allclose(depth_max_ref,
numpy.nanmax(A),
rtol=1.0e-10,
atol=1.0e-8), msg
elif res < native_resolution[0] * 10:
# When upsampling to coarser resolutions we expect
# ballpark match (~20%)
assert numpy.allclose(depth_max_ref,
numpy.nanmax(A),
rtol=0.17,
atol=0.0), msg
else:
# Upsampling to very coarse resolutions, just want sanity
assert 0 < numpy.nanmax(A) <= depth_max_ref
