def test_linear_interpolation_nan_points(self):
"""Interpolation library works with interpolation points being NaN
This is was the reason for bug reported in:
https://github.com/AIFDR/riab/issues/155
"""
# Define pixel centers along each direction
x = [1.0, 2.0, 4.0]
y = [5.0, 9.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 10)
etas = numpy.linspace(y[0], y[-1], 10)
xis[6:7] = numpy.nan
etas[3] = numpy.nan
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
assert nan_allclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_linear_interpolation_nan_array(self):
"""Interpolation library works (linear mode) with grid points being NaN
"""
# Define pixel centers along each direction
x = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
y = [4.0, 5.0, 7.0, 9.0, 11.0, 13.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
A[2, 3] = numpy.nan # (x=2.0, y=9.0): NaN
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 12)
etas = numpy.linspace(y[0], y[-1], 10)
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
# Set reference result with expected NaNs and compare
for i, (xi, eta) in enumerate(points):
if (1.0 < xi <= 3.0) and (7.0 < eta <= 11.0):
refs[i] = numpy.nan
assert nan_allclose(vals, refs, rtol=1e-12, atol=1e-12)
def __eq__(self, other, rtol=1.0e-5, atol=1.0e-8):
"""Override '==' to allow comparison with other raster objecs
Args:
* other: Raster instance to compare to
* rtol, atol: Relative and absolute tolerance.
See numpy.allclose for details
"""
# Check type
if not isinstance(other, Raster):
msg = ('Raster instance cannot be compared to %s'
' as its type is %s ' % (str(other), type(other)))
raise TypeError(msg)
# Check projection
if self.projection != other.projection:
return False
# Check geotransform
if self.get_geotransform() != other.get_geotransform():
return False
# Check data
if not nan_allclose(
self.get_data(), other.get_data(), rtol=rtol, atol=atol):
return False
# Check keywords
if self.keywords != other.keywords:
return False
# Raster layers are identical up to the specified tolerance
return True
def test_linear_interpolation_nan_array(self):
"""Interpolation library works (linear mode) with grid points being NaN
"""
# Define pixel centers along each direction
x = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
y = [4.0, 5.0, 7.0, 9.0, 11.0, 13.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
A[2, 3] = numpy.nan # (x=2.0, y=9.0): NaN
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 12)
etas = numpy.linspace(y[0], y[-1], 10)
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
# Set reference result with expected NaNs and compare
for i, (xi, eta) in enumerate(points):
if (1.0 < xi <= 3.0) and (7.0 < eta <= 11.0):
refs[i] = numpy.nan
assert nan_allclose(vals, refs, rtol=1e-12, atol=1e-12)
def test_linear_interpolation_nan_points(self):
"""Interpolation library works with interpolation points being NaN
This is was the reason for bug reported in:
https://github.com/AIFDR/riab/issues/155
"""
# Define pixel centers along each direction
x = [1.0, 2.0, 4.0]
y = [5.0, 9.0]
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define values for each x, y pair as a linear function
for i in range(len(x)):
for j in range(len(y)):
A[i, j] = linear_function(x[i], y[j])
# Then test that interpolated points can contain NaN
xis = numpy.linspace(x[0], x[-1], 10)
etas = numpy.linspace(y[0], y[-1], 10)
xis[6:7] = numpy.nan
etas[3] = numpy.nan
points = combine_coordinates(xis, etas)
vals = interpolate2d(x, y, A, points, mode='linear')
refs = linear_function(points[:, 0], points[:, 1])
assert nan_allclose(vals, refs, rtol=1e-12, atol=1e-12)
def __eq__(self, other, rtol=1.0e-5, atol=1.0e-8):
"""Override '==' to allow comparison with other raster objecs
Args:
* other: Raster instance to compare to
* rtol, atol: Relative and absolute tolerance.
See numpy.allclose for details
"""
# Check type
if not isinstance(other, Raster):
msg = ('Raster instance cannot be compared to %s'
' as its type is %s ' % (str(other), type(other)))
raise TypeError(msg)
# Check projection
if self.projection != other.projection:
return False
# Check geotransform
if self.get_geotransform() != other.get_geotransform():
return False
# Check data
if not nan_allclose(
self.get_data(),
other.get_data(),
rtol=rtol, atol=atol):
return False
# Check keywords
if self.keywords != other.keywords:
return False
# Raster layers are identical up to the specified tolerance
return True
def test_interpolation_random_array_and_nan(self):
"""Interpolation library (constant and linear) works with NaN
"""
# Define pixel centers along each direction
x = numpy.arange(20) * 1.0
y = numpy.arange(25) * 1.0
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define arbitrary values for each x, y pair
numpy.random.seed(17)
A = numpy.random.random((len(x), len(y))) * 10
# Create islands of NaN
A[5, 13] = numpy.nan
A[6, 14] = A[6, 18] = numpy.nan
A[7, 14:18] = numpy.nan
A[8, 13:18] = numpy.nan
A[9, 12:19] = numpy.nan
A[10, 14:17] = numpy.nan
A[11, 15] = numpy.nan
A[15, 5:6] = numpy.nan
# Creat interpolation points
xis = numpy.linspace(x[0], x[-1], 39) # Hit all mid points
etas = numpy.linspace(y[0], y[-1], 73) # Hit thirds
points = combine_coordinates(xis, etas)
for mode in ['linear', 'constant']:
vals = interpolate2d(x, y, A, points, mode=mode)
# Calculate reference result with expected NaNs and compare
i = j = 0
for k, (xi, eta) in enumerate(points):
# Find indices of nearest higher value in x and y
i = numpy.searchsorted(x, xi)
j = numpy.searchsorted(y, eta)
if i > 0 and j > 0:
# Get four neigbours
A00 = A[i - 1, j - 1]
A01 = A[i - 1, j]
A10 = A[i, j - 1]
A11 = A[i, j]
if numpy.allclose(xi, x[i]):
alpha = 1.0
else:
alpha = 0.5
if numpy.allclose(eta, y[j]):
beta = 1.0
else:
beta = eta - y[j - 1]
if mode == 'linear':
if numpy.any(numpy.isnan([A00, A01, A10, A11])):
ref = numpy.nan
else:
ref = (A00 * (1 - alpha) * (1 - beta) +
A01 * (1 - alpha) * beta +
A10 * alpha * (1 - beta) +
A11 * alpha * beta)
elif mode == 'constant':
assert alpha >= 0.5 # Only case in this test
if beta < 0.5:
ref = A10
else:
ref = A11
else:
msg = 'Unknown mode: %s' % mode
raise Exception(msg)
# print i, j, xi, eta, alpha, beta, vals[k], ref
assert nan_allclose(vals[k], ref, rtol=1e-12, atol=1e-12)
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled.
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
filenames = [
'Population_Jakarta_geographic.asc',
'Population_2010.asc'
]
for filename in filenames:
raster_path = ('%s/%s' % (TESTDATA, filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
del max_value
del min_value
native_resolution = safe_layer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
bounding_box = safe_layer.get_bounding_box()
resolutions = [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002 # Finer
]
# Test for a range of resolutions
for resolution in resolutions: # Finer
# To save time only do two resolutions for the
# large population set
if filename.startswith('Population_2010'):
if resolution > 0.01 or resolution < 0.005:
break
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
result = clip_layer(
raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords
)
safe_layer = read_safe_layer(result.source())
native_data = safe_layer.get_data(scaling=False)
scaled_data = safe_layer.get_data(scaling=True)
sigma_value = (safe_layer.get_resolution()[0] /
native_resolution[0]) ** 2
# Compare extrema
expected_scaled_max = sigma_value * numpy.nanmax(native_data)
message = (
'Resampled raster was not rescaled correctly: '
'max(scaled_data) was %f but expected %f' %
(numpy.nanmax(scaled_data), expected_scaled_max))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(scaled_data) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(scaled_data),
rtol=1.0e-6, atol=1.0e-8), message
expected_scaled_min = sigma_value * numpy.nanmin(native_data)
message = (
'Resampled raster was not rescaled correctly: '
'min(scaled_data) was %f but expected %f' %
(numpy.nanmin(scaled_data), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(scaled_data),
rtol=1.0e-8, atol=1.0e-12), message
# Compare element-wise
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(native_data * sigma_value, scaled_data,
rtol=1.0e-8, atol=1.0e-8), message
# Check that it also works with manual scaling
manual_data = safe_layer.get_data(scaling=sigma_value)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(manual_data, scaled_data,
rtol=1.0e-8, atol=1.0e-8), message
# Check that an exception is raised for bad arguments
try:
safe_layer.get_data(scaling='bad')
except GetDataError:
pass
else:
message = 'String argument should have raised exception'
raise Exception(message)
try:
safe_layer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
message = 'Tuple argument should have raised exception'
raise Exception(message)
# Check None option without keyword datatype == 'density'
safe_layer.keywords['datatype'] = 'undefined'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
# Try with None and density keyword
safe_layer.keywords['datatype'] = 'density'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(scaled_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
safe_layer.keywords['datatype'] = 'counts'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled.
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
filenames = [
'Population_Jakarta_geographic.asc', 'Population_2010.asc'
]
for filename in filenames:
raster_path = ('%s/%s' % (TESTDATA, filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
del max_value
del min_value
native_resolution = safe_layer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
bounding_box = safe_layer.get_bounding_box()
resolutions = [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002 # Finer
]
# Test for a range of resolutions
for resolution in resolutions: # Finer
# To save time only do two resolutions for the
# large population set
if filename.startswith('Population_2010'):
if resolution > 0.01 or resolution < 0.005:
break
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
result = clip_layer(raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords)
safe_layer = read_safe_layer(result.source())
native_data = safe_layer.get_data(scaling=False)
scaled_data = safe_layer.get_data(scaling=True)
sigma_value = (safe_layer.get_resolution()[0] /
native_resolution[0])**2
# Compare extrema
expected_scaled_max = sigma_value * numpy.nanmax(native_data)
message = ('Resampled raster was not rescaled correctly: '
'max(scaled_data) was %f but expected %f' %
(numpy.nanmax(scaled_data), expected_scaled_max))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(scaled_data) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(scaled_data),
rtol=1.0e-6,
atol=1.0e-8), message
expected_scaled_min = sigma_value * numpy.nanmin(native_data)
message = ('Resampled raster was not rescaled correctly: '
'min(scaled_data) was %f but expected %f' %
(numpy.nanmin(scaled_data), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(scaled_data),
rtol=1.0e-8,
atol=1.0e-12), message
# Compare element-wise
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(native_data * sigma_value,
scaled_data,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that it also works with manual scaling
manual_data = safe_layer.get_data(scaling=sigma_value)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(manual_data,
scaled_data,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that an exception is raised for bad arguments
try:
safe_layer.get_data(scaling='bad')
except GetDataError:
pass
else:
message = 'String argument should have raised exception'
raise Exception(message)
try:
safe_layer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
message = 'Tuple argument should have raised exception'
raise Exception(message)
# Check None option without keyword datatype == 'density'
safe_layer.keywords['datatype'] = 'undefined'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
# Try with None and density keyword
safe_layer.keywords['datatype'] = 'density'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(scaled_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
safe_layer.keywords['datatype'] = 'counts'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
def test_interpolation_random_array_and_nan(self):
"""Interpolation library (constant and linear) works with NaN
"""
# Define pixel centers along each direction
x = numpy.arange(20) * 1.0
y = numpy.arange(25) * 1.0
# Define ny by nx array with corresponding values
A = numpy.zeros((len(x), len(y)))
# Define arbitrary values for each x, y pair
numpy.random.seed(17)
A = numpy.random.random((len(x), len(y))) * 10
# Create islands of NaN
A[5, 13] = numpy.nan
A[6, 14] = A[6, 18] = numpy.nan
A[7, 14:18] = numpy.nan
A[8, 13:18] = numpy.nan
A[9, 12:19] = numpy.nan
A[10, 14:17] = numpy.nan
A[11, 15] = numpy.nan
A[15, 5:6] = numpy.nan
# Creat interpolation points
xis = numpy.linspace(x[0], x[-1], 39) # Hit all mid points
etas = numpy.linspace(y[0], y[-1], 73) # Hit thirds
points = combine_coordinates(xis, etas)
for mode in ['linear', 'constant']:
vals = interpolate2d(x, y, A, points, mode=mode)
# Calculate reference result with expected NaNs and compare
i = j = 0
for k, (xi, eta) in enumerate(points):
# Find indices of nearest higher value in x and y
i = numpy.searchsorted(x, xi)
j = numpy.searchsorted(y, eta)
if i > 0 and j > 0:
# Get four neigbours
A00 = A[i - 1, j - 1]
A01 = A[i - 1, j]
A10 = A[i, j - 1]
A11 = A[i, j]
if numpy.allclose(xi, x[i]):
alpha = 1.0
else:
alpha = 0.5
if numpy.allclose(eta, y[j]):
beta = 1.0
else:
beta = eta - y[j - 1]
if mode == 'linear':
if numpy.any(numpy.isnan([A00, A01, A10, A11])):
ref = numpy.nan
else:
ref = (A00 * (1 - alpha) * (1 - beta) + A01 *
(1 - alpha) * beta + A10 * alpha *
(1 - beta) + A11 * alpha * beta)
elif mode == 'constant':
assert alpha >= 0.5 # Only case in this test
if beta < 0.5:
ref = A10
else:
ref = A11
else:
msg = 'Unknown mode: %s' % mode
raise Exception(msg)
# print i, j, xi, eta, alpha, beta, vals[k], ref
assert nan_allclose(vals[k], ref, rtol=1e-12, atol=1e-12)
