def __eq__(self, other, rtol=1.0e-5, atol=1.0e-8):
"""Override '==' to allow comparison with other raster objecs
Input
other: Raster instance to compare to
rtol, atol: Relative and absolute tolerance.
See numpy.allclose for details
"""
# Check type
if not isinstance(other, Raster):
msg = ('Raster instance cannot be compared to %s'
' as its type is %s ' % (str(other), type(other)))
raise TypeError(msg)
# Check projection
if self.projection != other.projection:
return False
# Check geotransform
if self.get_geotransform() != other.get_geotransform():
return False
# Check data
if not nanallclose(self.get_data(),
other.get_data(),
rtol=rtol, atol=atol):
return False
# Check keywords
if self.keywords != other.keywords:
return False
# Raster layers are identical up to the specified tolerance
return True
def test_linear_interpolation_nan_array(self):
"""Interpolation library works (linear mode) with grid points being NaN
"""
# Define pixel centers along each direction
x = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
y = [4.0, 5.0, 7.0, 9.0, 11.0, 13.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
A[2, 3] = numpy.nan # (x=2.0, y=9.0): NaN
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 12)
etas = numpy.linspace(y[0], y[-1], 10)
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
# Set reference result with expected NaNs and compare
for i, (xi, eta) in enumerate(points):
if (1.0 < xi <= 3.0) and (7.0 < eta <= 11.0):
refs[i] = numpy.nan
assert nanallclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_linear_interpolation_nan_array(self):
"""Interpolation library works (linear mode) with grid points being NaN
"""
# Define pixel centers along each direction
x = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
y = [4.0, 5.0, 7.0, 9.0, 11.0, 13.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
A[2, 3] = numpy.nan # (x=2.0, y=9.0): NaN
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 12)
etas = numpy.linspace(y[0], y[-1], 10)
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
# Set reference result with expected NaNs and compare
for i, (xi, eta) in enumerate(points):
if (1.0 < xi <= 3.0) and (7.0 < eta <= 11.0):
refs[i] = numpy.nan
assert nanallclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_linear_interpolation_nan_points(self):
"""Interpolation library works with interpolation points being NaN
This is was the reason for bug reported in:
https://github.com/AIFDR/riab/issues/155
"""
# Define pixel centers along each direction
x = [1.0, 2.0, 4.0]
y = [5.0, 9.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 10)
etas = numpy.linspace(y[0], y[-1], 10)
xis[6:7] = numpy.nan
etas[3] = numpy.nan
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
assert nanallclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_linear_interpolation_nan_points(self):
"""Interpolation library works with interpolation points being NaN
This is was the reason for bug reported in:
https://github.com/AIFDR/riab/issues/155
"""
# Define pixel centers along each direction
x = [1.0, 2.0, 4.0]
y = [5.0, 9.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 10)
etas = numpy.linspace(y[0], y[-1], 10)
xis[6:7] = numpy.nan
etas[3] = numpy.nan
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
assert nanallclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_native_raster_resolution(self):
"""Raster layer retains native resolution through Geoserver
Raster layer can be uploaded and downloaded again with
native resolution. This is one test for ticket #103
"""
hazard_filename = ('%s/maumere_aos_depth_20m_land_wgs84.asc' %
TESTDATA)
# Get reference values
H = read_layer(hazard_filename)
A_ref = H.get_data(nan=True)
depth_min_ref, depth_max_ref = H.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again with native resolution
bbox = get_bounding_box_string(hazard_filename)
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data(nan=True)
# Compare shapes
msg = ('Shape of downloaded raster was [%i, %i]. '
'Expected [%i, %i].' %
(A.shape[0], A.shape[1], A_ref.shape[0], A_ref.shape[1]))
assert numpy.allclose(A_ref.shape, A.shape, rtol=0, atol=0), msg
# Compare extrema to values reference values (which have also been
# verified by QGIS for this layer and tested in test_engine.py)
depth_min, depth_max = H.get_extrema()
msg = ('Extrema of downloaded file were [%f, %f] but '
'expected [%f, %f]' %
(depth_min, depth_max, depth_min_ref, depth_max_ref))
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-6,
atol=1.0e-10), msg
# Compare data number by number
assert nanallclose(A, A_ref, rtol=1.0e-8)
def test_interpolation_random_array_and_nan(self):
"""Interpolation library (constant and linear) works with NaN
"""
# Define pixel centers along each direction
x = numpy.arange(20) * 1.0
y = numpy.arange(25) * 1.0
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define arbitrary values for each x, y pair
numpy.random.seed(17)
A = numpy.random.random((len(x), len(y))) * 10
# Create islands of NaN
A[5, 13] = numpy.nan
A[6, 14] = A[6, 18] = numpy.nan
A[7, 14:18] = numpy.nan
A[8, 13:18] = numpy.nan
A[9, 12:19] = numpy.nan
A[10, 14:17] = numpy.nan
A[11, 15] = numpy.nan
A[15, 5:6] = numpy.nan
# Creat interpolation points
xis = numpy.linspace(x[0], x[-1], 39) # Hit all mid points
etas = numpy.linspace(y[0], y[-1], 73) # Hit thirds
points = combine_coordinates(xis, etas)
for mode in ['linear', 'constant']:
vals = interpolate2d(x, y, A, points, mode=mode)
# Calculate reference result with expected NaNs and compare
i = j = 0
for k, (xi, eta) in enumerate(points):
# Find indices of nearest higher value in x and y
i = numpy.searchsorted(x, xi)
j = numpy.searchsorted(y, eta)
if i > 0 and j > 0:
# Get four neigbours
A00 = A[i - 1, j - 1]
A01 = A[i - 1, j]
A10 = A[i, j - 1]
A11 = A[i, j]
if numpy.allclose(xi, x[i]):
alpha = 1.0
else:
alpha = 0.5
if numpy.allclose(eta, y[j]):
beta = 1.0
else:
beta = eta - y[j - 1]
if mode == 'linear':
if numpy.any(numpy.isnan([A00, A01, A10, A11])):
ref = numpy.nan
else:
ref = (A00 * (1 - alpha) * (1 - beta) + A01 *
(1 - alpha) * beta + A10 * alpha *
(1 - beta) + A11 * alpha * beta)
elif mode == 'constant':
assert alpha >= 0.5 # Only case in this test
if beta < 0.5:
ref = A10
else:
ref = A11
else:
msg = 'Unknown mode: %s' % mode
raise Exception(msg)
#print i, j, xi, eta, alpha, beta, vals[k], ref
assert nanallclose(vals[k], ref, rtol=1e-12, atol=1e-12)
def test_interpolation_random_array_and_nan(self):
"""Interpolation library (constant and linear) works with NaN
"""
# Define pixel centers along each direction
x = numpy.arange(20) * 1.0
y = numpy.arange(25) * 1.0
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define arbitrary values for each x, y pair
numpy.random.seed(17)
A = numpy.random.random((len(x), len(y))) * 10
# Create islands of NaN
A[5, 13] = numpy.nan
A[6, 14] = A[6, 18] = numpy.nan
A[7, 14:18] = numpy.nan
A[8, 13:18] = numpy.nan
A[9, 12:19] = numpy.nan
A[10, 14:17] = numpy.nan
A[11, 15] = numpy.nan
A[15, 5:6] = numpy.nan
# Creat interpolation points
xis = numpy.linspace(x[0], x[-1], 39) # Hit all mid points
etas = numpy.linspace(y[0], y[-1], 73) # Hit thirds
points = combine_coordinates(xis, etas)
for mode in ['linear', 'constant']:
vals = interpolate2d(x, y, A, points, mode=mode)
# Calculate reference result with expected NaNs and compare
i = j = 0
for k, (xi, eta) in enumerate(points):
# Find indices of nearest higher value in x and y
i = numpy.searchsorted(x, xi)
j = numpy.searchsorted(y, eta)
if i > 0 and j > 0:
# Get four neigbours
A00 = A[i - 1, j - 1]
A01 = A[i - 1, j]
A10 = A[i, j - 1]
A11 = A[i, j]
if numpy.allclose(xi, x[i]):
alpha = 1.0
else:
alpha = 0.5
if numpy.allclose(eta, y[j]):
beta = 1.0
else:
beta = eta - y[j - 1]
if mode == 'linear':
if numpy.any(numpy.isnan([A00, A01, A10, A11])):
ref = numpy.nan
else:
ref = (A00 * (1 - alpha) * (1 - beta) +
A01 * (1 - alpha) * beta +
A10 * alpha * (1 - beta) +
A11 * alpha * beta)
elif mode == 'constant':
assert alpha >= 0.5 # Only case in this test
if beta < 0.5:
ref = A10
else:
ref = A11
else:
msg = 'Unknown mode: %s' % mode
raise Exception(msg)
#print i, j, xi, eta, alpha, beta, vals[k], ref
assert nanallclose(vals[k], ref, rtol=1e-12, atol=1e-12)
def test_data_resampling_example(self):
"""Raster data is unchanged when going through geonode
"""
# Name file names for hazard level, exposure and expected fatalities
hazard_filename = ('%s/maumere_aos_depth_20m_land_wgs84.asc'
% TESTDATA)
exposure_filename = ('%s/maumere_pop_prj.shp' % TESTDATA)
#------------
# Hazard data
#------------
# Read hazard input data for reference
H_ref = read_layer(hazard_filename)
A_ref = H_ref.get_data()
depth_min_ref, depth_max_ref = H_ref.get_extrema()
# Upload to internal geonode
hazard_layer = save_to_geonode(hazard_filename, user=self.user)
hazard_name = '%s:%s' % (hazard_layer.workspace, hazard_layer.name)
# Download data again
bbox = get_bounding_box_string(hazard_filename) # The biggest
H = download(INTERNAL_SERVER_URL, hazard_name, bbox)
A = H.get_data()
depth_min, depth_max = H.get_extrema()
# FIXME (Ole): The layer read from file is single precision only:
# Issue #17
# Here's the explanation why interpolation below produce slightly
# different results (but why?)
# The layer read from file is single precision which may be due to
# the way it is converted from ASC to TIF. In other words the
# problem may be in raster.write_to_file. Float64 is
# specified there, so this is a mystery.
#print 'A', A.dtype # Double precision
#print 'A_ref', A_ref.dtype # Single precision
# Compare extrema to values from numpy array
assert numpy.allclose(depth_max, numpy.nanmax(A),
rtol=1.0e-12, atol=1.0e-12)
assert numpy.allclose(depth_max_ref, numpy.nanmax(A_ref),
rtol=1.0e-12, atol=1.0e-12)
# Compare to reference
assert numpy.allclose([depth_min, depth_max],
[depth_min_ref, depth_max_ref],
rtol=1.0e-12, atol=1.0e-12)
# Compare extrema to values read off QGIS for this layer
assert numpy.allclose([depth_min, depth_max], [0.0, 16.68],
rtol=1.0e-6, atol=1.0e-10)
# Investigate difference visually
#from matplotlib.pyplot import matshow, show
#matshow(A)
#matshow(A_ref)
#matshow(A - A_ref)
#show()
#print
for i in range(A.shape[0]):
for j in range(A.shape[1]):
if not numpy.isnan(A[i, j]):
err = abs(A[i, j] - A_ref[i, j])
if err > 0:
msg = ('%i, %i: %.15f, %.15f, %.15f'
% (i, j, A[i, j], A_ref[i, j], err))
raise Exception(msg)
#if A[i,j] > 16:
# print i, j, A[i, j], A_ref[i, j]
# Compare elements (nan & numbers)
id_nan = numpy.isnan(A)
id_nan_ref = numpy.isnan(A_ref)
assert numpy.all(id_nan == id_nan_ref)
assert numpy.allclose(A[-id_nan], A_ref[-id_nan],
rtol=1.0e-15, atol=1.0e-15)
#print 'MAX', A[245, 283], A_ref[245, 283]
#print 'MAX: %.15f %.15f %.15f' %(A[245, 283], A_ref[245, 283])
assert numpy.allclose(A[245, 283], A_ref[245, 283],
rtol=1.0e-15, atol=1.0e-15)
#--------------
# Exposure data
#--------------
# Read exposure input data for reference
E_ref = read_layer(exposure_filename)
# Upload to internal geonode
exposure_layer = save_to_geonode(exposure_filename, user=self.user)
exposure_name = '%s:%s' % (exposure_layer.workspace,
exposure_layer.name)
# Download data again
E = download(INTERNAL_SERVER_URL, exposure_name, bbox)
# Check exposure data against reference
coordinates = E.get_geometry()
coordinates_ref = E_ref.get_geometry()
assert numpy.allclose(coordinates, coordinates_ref,
rtol=1.0e-12, atol=1.0e-12)
attributes = E.get_data()
attributes_ref = E_ref.get_data()
for i, att in enumerate(attributes):
att_ref = attributes_ref[i]
for key in att:
assert att[key] == att_ref[key]
# Test riab's interpolation function
I = H.interpolate(E, name='depth')
icoordinates = I.get_geometry()
I_ref = H_ref.interpolate(E_ref, name='depth')
icoordinates_ref = I_ref.get_geometry()
assert numpy.allclose(coordinates,
icoordinates,
rtol=1.0e-12, atol=1.0e-12)
assert numpy.allclose(coordinates,
icoordinates_ref,
rtol=1.0e-12, atol=1.0e-12)
iattributes = I.get_data()
assert numpy.allclose(icoordinates, coordinates)
N = len(icoordinates)
assert N == 891
# Set tolerance for single precision until issue #17 has been fixed
# It appears that the single precision leads to larger interpolation
# errors
rtol_issue17 = 2.0e-3
atol_issue17 = 1.0e-4
# Verify interpolated values with test result
for i in range(N):
interpolated_depth_ref = I_ref.get_data()[i]['depth']
interpolated_depth = iattributes[i]['depth']
assert nanallclose(interpolated_depth,
interpolated_depth_ref,
rtol=rtol_issue17, atol=atol_issue17)
pointid = attributes[i]['POINTID']
if pointid == 263:
#print i, pointid, attributes[i],
#print interpolated_depth, coordinates[i]
# Check that location is correct
assert numpy.allclose(coordinates[i],
[122.20367299, -8.61300358],
rtol=1.0e-7, atol=1.0e-12)
# This is known to be outside inundation area so should
# near zero
assert numpy.allclose(interpolated_depth, 0.0,
rtol=1.0e-12, atol=1.0e-12)
if pointid == 148:
# Check that location is correct
#print coordinates[i]
assert numpy.allclose(coordinates[i],
[122.2045912, -8.608483265],
rtol=1.0e-7, atol=1.0e-12)
# This is in an inundated area with a surrounding depths of
# 4.531, 3.911
# 2.675, 2.583
assert interpolated_depth < 4.531
assert interpolated_depth < 3.911
assert interpolated_depth > 2.583
assert interpolated_depth > 2.675
#print interpolated_depth
# This is a characterisation test for bilinear interpolation
assert numpy.allclose(interpolated_depth, 3.62477215491,
rtol=rtol_issue17, atol=1.0e-12)
# Check that interpolated points are within range
msg = ('Interpolated depth %f at point %i was outside extrema: '
'[%f, %f]. ' % (interpolated_depth, i,
depth_min, depth_max))
if not numpy.isnan(interpolated_depth):
assert depth_min <= interpolated_depth <= depth_max, msg
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled
This is a test for ticket #168
Native test .asc data has
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
for test_filename in [
'Population_Jakarta_geographic.asc', 'Population_2010.asc'
]:
raster_filename = ('%s/%s' % (TESTDATA, test_filename))
# Get reference values
R = read_layer(raster_filename)
R_min_ref, R_max_ref = R.get_extrema()
native_resolution = R.get_resolution()
# Upload to internal geonode
raster_layer = save_to_geonode(raster_filename, user=self.user)
raster_name = '%s:%s' % (raster_layer.workspace, raster_layer.name)
# Test for a range of resolutions
for res in [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002
]: # Finer
# To save time don't do finest resolution for the
# large population set
if test_filename.startswith('Population_2010') and res < 0.005:
break
bbox = get_bounding_box_string(raster_filename)
R = download(INTERNAL_SERVER_URL,
raster_name,
bbox,
resolution=res)
A_native = R.get_data(scaling=False)
A_scaled = R.get_data(scaling=True)
sigma = (R.get_resolution()[0] / native_resolution[0])**2
# Compare extrema
expected_scaled_max = sigma * numpy.nanmax(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'max(A_scaled) was %f but expected %f' %
(numpy.nanmax(A_scaled), expected_scaled_max))
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(A_scaled),
rtol=1.0e-8,
atol=1.0e-8), msg
expected_scaled_min = sigma * numpy.nanmin(A_native)
msg = ('Resampled raster was not rescaled correctly: '
'min(A_scaled) was %f but expected %f' %
(numpy.nanmin(A_scaled), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(A_scaled),
rtol=1.0e-8,
atol=1.0e-12), msg
# Compare elementwise
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_native * sigma,
A_scaled,
rtol=1.0e-8,
atol=1.0e-8), msg
# Check that it also works with manual scaling
A_manual = R.get_data(scaling=sigma)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_manual,
A_scaled,
rtol=1.0e-8,
atol=1.0e-8), msg
# Check that an exception is raised for bad arguments
try:
R.get_data(scaling='bad')
except:
pass
else:
msg = 'String argument should have raised exception'
raise Exception(msg)
try:
R.get_data(scaling='(1, 3)')
except:
pass
else:
msg = 'Tuple argument should have raised exception'
raise Exception(msg)
# Check None option without existence of density keyword
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native,
A_none,
rtol=1.0e-12,
atol=1.0e-12), msg
# Try with None and density keyword
R.keywords['density'] = 'true'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled,
A_none,
rtol=1.0e-12,
atol=1.0e-12), msg
R.keywords['density'] = 'Yes'
A_none = R.get_data(scaling=None)
msg = 'Resampled raster was not rescaled correctly'
assert nanallclose(A_scaled,
A_none,
rtol=1.0e-12,
atol=1.0e-12), msg
R.keywords['density'] = 'False'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native,
A_none,
rtol=1.0e-12,
atol=1.0e-12), msg
R.keywords['density'] = 'no'
A_none = R.get_data(scaling=None)
msg = 'Data should not have changed'
assert nanallclose(A_native,
A_none,
rtol=1.0e-12,
atol=1.0e-12), msg