def test_native_raster_resolution(self):
"""Raster layer retains native resolution through Geoserver
Raster layer can be uploaded and downloaded again with
native resolution. This is one test for ticket #103
"""
hazard_filename = os.path.join(UNITDATA, 'hazard',
'jakarta_flood_design.tif')
# Get reference values
H = read_layer(hazard_filename)
A_ref = H.get_data(nan=True)
depth_min_ref, depth_max_ref = H.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again with native resolution
bbox = get_bounding_box_string(hazard_filename)
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data(nan=True)
# Compare shapes
msg = ('Shape of downloaded raster was [%i, %i]. '
'Expected [%i, %i].' %
(A.shape[0], A.shape[1], A_ref.shape[0], A_ref.shape[1]))
assert numpy.allclose(A_ref.shape, A.shape, rtol=0, atol=0), msg
# Compare extrema to values reference values (which have also been
# verified by QGIS for this layer and tested in test_engine.py)
depth_min, depth_max = H.get_extrema()
msg = ('Extrema of downloaded file were [%f, %f] but '
'expected [%f, %f]' %
(depth_min, depth_max, depth_min_ref, depth_max_ref))
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-6,
atol=1.0e-10), msg
# Compare data number by number
assert nanallclose(A, A_ref, rtol=1.0e-8)
def test_native_raster_resolution(self):
"""Raster layer retains native resolution through Geoserver
Raster layer can be uploaded and downloaded again with
native resolution. This is one test for ticket #103
"""
hazard_filename = os.path.join(UNITDATA, 'hazard', 'jakarta_flood_design.tif')
# Get reference values
H = read_layer(hazard_filename)
A_ref = H.get_data(nan=True)
depth_min_ref, depth_max_ref = H.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again with native resolution
bbox = get_bounding_box_string(hazard_filename)
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data(nan=True)
# Compare shapes
msg = ('Shape of downloaded raster was [%i, %i]. '
'Expected [%i, %i].' % (A.shape[0], A.shape[1],
A_ref.shape[0], A_ref.shape[1]))
assert numpy.allclose(A_ref.shape, A.shape, rtol=0, atol=0), msg
# Compare extrema to values reference values (which have also been
# verified by QGIS for this layer and tested in test_engine.py)
depth_min, depth_max = H.get_extrema()
msg = ('Extrema of downloaded file were [%f, %f] but '
'expected [%f, %f]' % (depth_min, depth_max,
depth_min_ref, depth_max_ref))
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-6, atol=1.0e-10), msg
# Compare data number by number
assert nanallclose(A, A_ref, rtol=1.0e-8)
def test_data_resampling_example(self):
"""Raster data is unchanged when going through geonode
"""
# Name file names for hazard level, exposure and expected fatalities
hazard_filename = os.path.join(TESTDATA, '..', 'hazard', 'maumere_aos_depth_20m_land_wgs84.asc')
exposure_filename = os.path.join(TESTDATA, 'maumere_pop_prj.shp')
#------------
# Hazard data
#------------
# Read hazard input data for reference
H_ref = read_layer(hazard_filename)
A_ref = H_ref.get_data()
depth_min_ref, depth_max_ref = H_ref.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again
bbox = get_bounding_box_string(hazard_filename) # The biggest
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data()
depth_min, depth_max = H.get_extrema()
# FIXME (Ole): The layer read from file is single precision only:
# Issue #17
# Here's the explanation why interpolation below produce slightly
# different results (but why?)
# The layer read from file is single precision which may be due to
# the way it is converted from ASC to TIF. In other words the
# problem may be in raster.write_to_file. Float64 is
# specified there, so this is a mystery.
#print 'A', A.dtype # Double precision
#print 'A_ref', A_ref.dtype # Single precision
# Compare extrema to values from numpy array
assert numpy.allclose(depth_max, numpy.nanmax(A),
rtol=1.0e-12, atol=1.0e-12)
assert numpy.allclose(depth_max_ref, numpy.nanmax(A_ref),
rtol=1.0e-12, atol=1.0e-12)
# Compare to reference
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-12, atol=1.0e-12)
# Compare extrema to values read off QGIS for this layer
assert numpy.allclose([depth_min, depth_max], [0.0, 16.68],
rtol=1.0e-6, atol=1.0e-10)
# Investigate difference visually
#from matplotlib.pyplot import matshow, show
#matshow(A)
#matshow(A_ref)
#matshow(A - A_ref)
#show()
#print
for i in range(A.shape[0]):
for j in range(A.shape[1]):
if not numpy.isnan(A[i, j]):
err = abs(A[i, j] - A_ref[i, j])
if err > 0:
msg = ('%i, %i: %.15f, %.15f, %.15f'
% (i, j, A[i, j], A_ref[i, j], err))
raise Exception(msg)
#if A[i,j] > 16:
# print i, j, A[i, j], A_ref[i, j]
# Compare elements (nan & numbers)
id_nan = numpy.isnan(A)
id_nan_ref = numpy.isnan(A_ref)
assert numpy.all(id_nan == id_nan_ref)
assert numpy.allclose(A[-id_nan], A_ref[-id_nan],
rtol=1.0e-15, atol=1.0e-15)
#print 'MAX', A[245, 283], A_ref[245, 283]
#print 'MAX: %.15f %.15f %.15f' %(A[245, 283], A_ref[245, 283])
assert numpy.allclose(A[245, 283], A_ref[245, 283],
rtol=1.0e-15, atol=1.0e-15)
#--------------
# Exposure data
#--------------
# Read exposure input data for reference
E_ref = read_layer(exposure_filename)
# Upload to internal geonode
exposure_layer = save_to_geonode(exposure_filename, user=self.user)
exposure_name = '%s:%s' % (exposure_layer.workspace,
exposure_layer.name)
# Download data again
E = download(INTERNAL_SERVER_URL, exposure_name, bbox)
# Check exposure data against reference
coordinates = E.get_geometry()
coordinates_ref = E_ref.get_geometry()
assert numpy.allclose(coordinates, coordinates_ref,
rtol=1.0e-12, atol=1.0e-12)
attributes = E.get_data()
attributes_ref = E_ref.get_data()
for i, att in enumerate(attributes):
att_ref = attributes_ref[i]
for key in att:
assert att[key] == att_ref[key]
# Test safe's interpolation function
I = assign_hazard_values_to_exposure_data(H, E, attribute_name='depth')
icoordinates = I.get_geometry()
I_ref = assign_hazard_values_to_exposure_data(H_ref, E_ref, attribute_name='depth')
icoordinates_ref = I_ref.get_geometry()
assert numpy.allclose(coordinates,
icoordinates,
rtol=1.0e-12, atol=1.0e-12)
assert numpy.allclose(coordinates,
icoordinates_ref,
rtol=1.0e-12, atol=1.0e-12)
iattributes = I.get_data()
assert numpy.allclose(icoordinates, coordinates)
N = len(icoordinates)
assert N == 891
# Set tolerance for single precision until issue #17 has been fixed
# It appears that the single precision leads to larger interpolation
# errors
rtol_issue17 = 2.0e-3
atol_issue17 = 1.0e-4
# Verify interpolated values with test result
for i in range(N):
interpolated_depth_ref = I_ref.get_data()[i]['depth']
interpolated_depth = iattributes[i]['depth']
assert nanallclose(interpolated_depth,
interpolated_depth_ref,
rtol=rtol_issue17, atol=atol_issue17)
pointid = attributes[i]['POINTID']
if pointid == 263:
#print i, pointid, attributes[i],
#print interpolated_depth, coordinates[i]
# Check that location is correct
assert numpy.allclose(coordinates[i],
[122.20367299, -8.61300358],
rtol=1.0e-7, atol=1.0e-12)
# This is known to be outside inundation area so should
# near zero
assert numpy.allclose(interpolated_depth, 0.0,
rtol=1.0e-12, atol=1.0e-12)
if pointid == 148:
# Check that location is correct
#print coordinates[i]
assert numpy.allclose(coordinates[i],
[122.2045912, -8.608483265],
rtol=1.0e-7, atol=1.0e-12)
# This is in an inundated area with a surrounding depths of
# 4.531, 3.911
# 2.675, 2.583
assert interpolated_depth < 4.531
assert interpolated_depth < 3.911
assert interpolated_depth > 2.583
assert interpolated_depth > 2.675
#print interpolated_depth
# This is a characterisation test for bilinear interpolation
assert numpy.allclose(interpolated_depth, 3.62477215491,
rtol=rtol_issue17, atol=1.0e-12)
# Check that interpolated points are within range
msg = ('Interpolated depth %f at point %i was outside extrema: '
'[%f, %f]. ' % (interpolated_depth, i,
depth_min, depth_max))
if not numpy.isnan(interpolated_depth):
assert depth_min <= interpolated_depth <= depth_max, msg
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled
This is a test for ticket #168
Native test .asc data has
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
for test_filename in ['jakarta_flood_design.tif']:
raster_filename = os.path.join(UNITDATA, 'hazard', test_filename)
# Get reference values
R = read_layer(raster_filename)
R_min_ref, R_max_ref = R.get_extrema()
native_resolution = R.get_resolution()
# Upload to internal geonode
raster_layer = save_to_geonode(raster_filename, user=self.user)
raster_name = '%s:%s' % (raster_layer.workspace,
raster_layer.name)
# Test for a range of resolutions
for res in [0.01, 0.005, 0.002, 0.001, 0.0005, 0.0002]:
bbox = get_bounding_box_string(raster_filename)
R = download(INTERNAL_SERVER_URL, raster_name,
bbox, resolution=res)
A_native = R.get_data(scaling=False)
A_scaled = R.get_data(scaling=True)
sigma = (R.get_resolution()[0] / native_resolution[0]) ** 2
# Compare extrema
expected_scaled_max = sigma * numpy.nanmax(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'max(A_scaled) was %f but expected %f'
% (numpy.nanmax(A_scaled), expected_scaled_max))
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(A_scaled),
rtol=1.0e0, atol=1.0e-1), msg
expected_scaled_min = sigma * numpy.nanmin(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'min(A_scaled) was %f but expected %f'
% (numpy.nanmin(A_scaled), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(A_scaled),
rtol=1.0e0, atol=1.0e-1), msg
# Compare elementwise
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_native * sigma, A_scaled,
rtol=1.0e0, atol=1.0e-1), msg
# Check that it also works with manual scaling
A_manual = R.get_data(scaling=sigma)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_manual, A_scaled,
rtol=1.0e0, atol=1.0e-1), msg
# Check that an exception is raised for bad arguments
try:
R.get_data(scaling='bad')
except:
pass
else:
msg = 'String argument should have raised exception'
raise Exception(msg)
try:
R.get_data(scaling='(1, 3)')
except:
pass
else:
msg = 'Tuple argument should have raised exception'
raise Exception(msg)
# Check None option without existence of density keyword
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none,
rtol=1.0e-0, atol=1.0e-1), msg
# Try with None and density keyword
R.keywords['density'] = 'true'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled, A_none,
rtol=1.0e2, atol=1.0e2), msg
R.keywords['density'] = 'Yes'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled, A_none,
rtol=1.0e2, atol=1.0e2), msg
R.keywords['density'] = 'False'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none,
rtol=1.0e2, atol=1.0e2), msg
R.keywords['density'] = 'no'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none,
rtol=1.0e2, atol=1.0e2), msg
def test_data_resampling_example(self):
"""Raster data is unchanged when going through geonode
"""
# Name file names for hazard level, exposure and expected fatalities
hazard_filename = os.path.join(TESTDATA, '..', 'hazard',
'maumere_aos_depth_20m_land_wgs84.asc')
exposure_filename = os.path.join(TESTDATA, 'maumere_pop_prj.shp')
#------------
# Hazard data
#------------
# Read hazard input data for reference
H_ref = read_layer(hazard_filename)
A_ref = H_ref.get_data()
depth_min_ref, depth_max_ref = H_ref.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again
bbox = get_bounding_box_string(hazard_filename) # The biggest
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data()
depth_min, depth_max = H.get_extrema()
# FIXME (Ole): The layer read from file is single precision only:
# Issue #17
# Here's the explanation why interpolation below produce slightly
# different results (but why?)
# The layer read from file is single precision which may be due to
# the way it is converted from ASC to TIF. In other words the
# problem may be in raster.write_to_file. Float64 is
# specified there, so this is a mystery.
#print 'A', A.dtype # Double precision
#print 'A_ref', A_ref.dtype # Single precision
# Compare extrema to values from numpy array
assert numpy.allclose(depth_max,
numpy.nanmax(A),
rtol=1.0e-12,
atol=1.0e-12)
assert numpy.allclose(depth_max_ref,
numpy.nanmax(A_ref),
rtol=1.0e-12,
atol=1.0e-12)
# Compare to reference
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-12,
atol=1.0e-12)
# Compare extrema to values read off QGIS for this layer
assert numpy.allclose([depth_min, depth_max], [0.0, 16.68],
rtol=1.0e-6,
atol=1.0e-10)
# Investigate difference visually
#from matplotlib.pyplot import matshow, show
#matshow(A)
#matshow(A_ref)
#matshow(A - A_ref)
#show()
#print
for i in range(A.shape[0]):
for j in range(A.shape[1]):
if not numpy.isnan(A[i, j]):
err = abs(A[i, j] - A_ref[i, j])
if err > 0:
msg = ('%i, %i: %.15f, %.15f, %.15f' %
(i, j, A[i, j], A_ref[i, j], err))
raise Exception(msg)
#if A[i,j] > 16:
# print i, j, A[i, j], A_ref[i, j]
# Compare elements (nan & numbers)
id_nan = numpy.isnan(A)
id_nan_ref = numpy.isnan(A_ref)
assert numpy.all(id_nan == id_nan_ref)
assert numpy.allclose(A[-id_nan],
A_ref[-id_nan],
rtol=1.0e-15,
atol=1.0e-15)
#print 'MAX', A[245, 283], A_ref[245, 283]
#print 'MAX: %.15f %.15f %.15f' %(A[245, 283], A_ref[245, 283])
assert numpy.allclose(A[245, 283],
A_ref[245, 283],
rtol=1.0e-15,
atol=1.0e-15)
#--------------
# Exposure data
#--------------
# Read exposure input data for reference
E_ref = read_layer(exposure_filename)
# Upload to internal geonode
exposure_layer = save_to_geonode(exposure_filename, user=self.user)
exposure_name = '%s:%s' % (exposure_layer.workspace,
exposure_layer.name)
# Download data again
E = download(INTERNAL_SERVER_URL, exposure_name, bbox)
# Check exposure data against reference
coordinates = E.get_geometry()
coordinates_ref = E_ref.get_geometry()
assert numpy.allclose(coordinates,
coordinates_ref,
rtol=1.0e-12,
atol=1.0e-12)
attributes = E.get_data()
attributes_ref = E_ref.get_data()
for i, att in enumerate(attributes):
att_ref = attributes_ref[i]
for key in att:
assert att[key] == att_ref[key]
# Test safe's interpolation function
I = assign_hazard_values_to_exposure_data(H, E, attribute_name='depth')
icoordinates = I.get_geometry()
I_ref = assign_hazard_values_to_exposure_data(H_ref,
E_ref,
attribute_name='depth')
icoordinates_ref = I_ref.get_geometry()
assert numpy.allclose(coordinates,
icoordinates,
rtol=1.0e-12,
atol=1.0e-12)
assert numpy.allclose(coordinates,
icoordinates_ref,
rtol=1.0e-12,
atol=1.0e-12)
iattributes = I.get_data()
assert numpy.allclose(icoordinates, coordinates)
N = len(icoordinates)
assert N == 891
# Set tolerance for single precision until issue #17 has been fixed
# It appears that the single precision leads to larger interpolation
# errors
rtol_issue17 = 2.0e-3
atol_issue17 = 1.0e-4
# Verify interpolated values with test result
for i in range(N):
interpolated_depth_ref = I_ref.get_data()[i]['depth']
interpolated_depth = iattributes[i]['depth']
assert nanallclose(interpolated_depth,
interpolated_depth_ref,
rtol=rtol_issue17,
atol=atol_issue17)
pointid = attributes[i]['POINTID']
if pointid == 263:
#print i, pointid, attributes[i],
#print interpolated_depth, coordinates[i]
# Check that location is correct
assert numpy.allclose(coordinates[i],
[122.20367299, -8.61300358],
rtol=1.0e-7,
atol=1.0e-12)
# This is known to be outside inundation area so should
# near zero
assert numpy.allclose(interpolated_depth,
0.0,
rtol=1.0e-12,
atol=1.0e-12)
if pointid == 148:
# Check that location is correct
#print coordinates[i]
assert numpy.allclose(coordinates[i],
[122.2045912, -8.608483265],
rtol=1.0e-7,
atol=1.0e-12)
# This is in an inundated area with a surrounding depths of
# 4.531, 3.911
# 2.675, 2.583
assert interpolated_depth < 4.531
assert interpolated_depth < 3.911
assert interpolated_depth > 2.583
assert interpolated_depth > 2.675
#print interpolated_depth
# This is a characterisation test for bilinear interpolation
assert numpy.allclose(interpolated_depth,
3.62477215491,
rtol=rtol_issue17,
atol=1.0e-12)
# Check that interpolated points are within range
msg = ('Interpolated depth %f at point %i was outside extrema: '
'[%f, %f]. ' %
(interpolated_depth, i, depth_min, depth_max))
if not numpy.isnan(interpolated_depth):
assert depth_min <= interpolated_depth <= depth_max, msg
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled
This is a test for ticket #168
Native test .asc data has
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
for test_filename in ['jakarta_flood_design.tif']:
raster_filename = os.path.join(UNITDATA, 'hazard', test_filename)
# Get reference values
R = read_layer(raster_filename)
R_min_ref, R_max_ref = R.get_extrema()
native_resolution = R.get_resolution()
# Upload to internal geonode
raster_layer = save_to_geonode(raster_filename, user=self.user)
raster_name = '%s:%s' % (raster_layer.workspace, raster_layer.name)
# Test for a range of resolutions
for res in [0.01, 0.005, 0.002, 0.001, 0.0005, 0.0002]:
bbox = get_bounding_box_string(raster_filename)
R = download(INTERNAL_SERVER_URL,
raster_name,
bbox,
resolution=res)
A_native = R.get_data(scaling=False)
A_scaled = R.get_data(scaling=True)
sigma = (R.get_resolution()[0] / native_resolution[0])**2
# Compare extrema
expected_scaled_max = sigma * numpy.nanmax(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'max(A_scaled) was %f but expected %f' %
(numpy.nanmax(A_scaled), expected_scaled_max))
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(A_scaled),
rtol=1.0e0,
atol=1.0e-1), msg
expected_scaled_min = sigma * numpy.nanmin(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'min(A_scaled) was %f but expected %f' %
(numpy.nanmin(A_scaled), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(A_scaled),
rtol=1.0e0,
atol=1.0e-1), msg
# Compare elementwise
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_native * sigma,
A_scaled,
rtol=1.0e0,
atol=1.0e-1), msg
# Check that it also works with manual scaling
A_manual = R.get_data(scaling=sigma)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_manual, A_scaled, rtol=1.0e0,
atol=1.0e-1), msg
# Check that an exception is raised for bad arguments
try:
R.get_data(scaling='bad')
except:
pass
else:
msg = 'String argument should have raised exception'
raise Exception(msg)
try:
R.get_data(scaling='(1, 3)')
except:
pass
else:
msg = 'Tuple argument should have raised exception'
raise Exception(msg)
# Check None option without existence of density keyword
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none, rtol=1.0e-0,
atol=1.0e-1), msg
# Try with None and density keyword
R.keywords['density'] = 'true'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled, A_none, rtol=1.0e2,
atol=1.0e2), msg
R.keywords['density'] = 'Yes'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled, A_none, rtol=1.0e2,
atol=1.0e2), msg
R.keywords['density'] = 'False'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none, rtol=1.0e2,
atol=1.0e2), msg
R.keywords['density'] = 'no'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native, A_none, rtol=1.0e2,
atol=1.0e2), msg