def test_clipRaster(self):
"""Raster layers can be clipped
"""
# Create a raster layer
myName = 'shake'
myRasterLayer = QgsRasterLayer(RASTERPATH, myName)
message = 'Did not find layer "%s" in path "%s"' % \
(myName, RASTERPATH)
assert myRasterLayer is not None, message
# Create a bounding box
myRect = [97, -3, 104, 1]
# Clip the vector to the bbox
myResult = clip_layer(myRasterLayer, myRect)
# Check the output is valid
assert os.path.exists(myResult.source())
# Clip and give a desired resolution for the output
mySize = 0.05
myResult = clip_layer(myRasterLayer, myRect, mySize)
myNewRasterLayer = QgsRasterLayer(myResult.source(), myName)
assert myNewRasterLayer.isValid(), 'Resampled raster is not valid'
message = ('Resampled raster has incorrect pixel size.'
'Expected: %f, Actual: %f' %
(mySize, myNewRasterLayer.rasterUnitsPerPixelX()))
assert myNewRasterLayer.rasterUnitsPerPixelX() == mySize, message
def test_clip_raster(self):
"""Raster layers can be clipped."""
# Create a raster layer
layer_name = 'shake'
raster_layer = QgsRasterLayer(RASTERPATH, layer_name)
message = ('Did not find layer "%s" in path "%s"' %
(layer_name, RASTERPATH))
assert raster_layer is not None, message
# Create a bounding box
bounding_box = [97, -3, 104, 1]
# Clip the vector to the bbox
result = clip_layer(raster_layer, bounding_box)
# Check the output is valid
assert os.path.exists(result.source())
# Clip and give a desired resolution for the output
# big pixel size
size = 0.05
result = clip_layer(raster_layer, bounding_box, size)
new_raster_layer = QgsRasterLayer(result.source(), layer_name)
assert new_raster_layer.isValid(), 'Resampled raster is not valid'
message = ('Resampled raster has incorrect pixel size. Expected: %5f, '
'Actual: %5f' %
(size, new_raster_layer.rasterUnitsPerPixelX()))
assert new_raster_layer.rasterUnitsPerPixelX() == size, message
def test_clip_raster_small(self):
"""Raster layers can be clipped in small and precise size. For #710."""
# Create a raster layer
layer_name = 'shake'
raster_layer = QgsRasterLayer(RASTERPATH, layer_name)
message = (
'Did not find layer "%s" in path "%s"' % (layer_name, RASTERPATH))
assert raster_layer is not None, message
# Create a bounding box
bounding_box = [97, -3, 104, 1]
# Clip the vector to the bbox
result = clip_layer(raster_layer, bounding_box)
# Check the output is valid
assert os.path.exists(result.source())
# Clip and give a desired resolution for the output
# small pixel size and high precision
# based on pixel size of Flood_Current_Depth_Jakarta_geographic.asc
size = 0.00045228819716
result = clip_layer(raster_layer, bounding_box, size)
new_raster_layer = QgsRasterLayer(result.source(), layer_name)
assert new_raster_layer.isValid(), 'Resampled raster is not valid'
message = (
'Resampled raster has incorrect pixel size. Expected: %.14f, '
'Actual: %.14f' % (
size, new_raster_layer.rasterUnitsPerPixelX()))
result_size = new_raster_layer.rasterUnitsPerPixelX()
self.assertAlmostEqual(result_size, size, places=13, msg=message)
def test_clip_raster(self):
"""Raster layers can be clipped."""
# Create a raster layer
layer_name = 'shake'
raster_layer = QgsRasterLayer(RASTERPATH, layer_name)
message = (
'Did not find layer "%s" in path "%s"' % (layer_name, RASTERPATH))
assert raster_layer is not None, message
# Create a bounding box
bounding_box = [97, -3, 104, 1]
# Clip the vector to the bbox
result = clip_layer(raster_layer, bounding_box)
# Check the output is valid
assert os.path.exists(result.source())
# Clip and give a desired resolution for the output
# big pixel size
size = 0.05
result = clip_layer(raster_layer, bounding_box, size)
new_raster_layer = QgsRasterLayer(result.source(), layer_name)
assert new_raster_layer.isValid(), 'Resampled raster is not valid'
message = (
'Resampled raster has incorrect pixel size. Expected: %5f, '
'Actual: %5f' % (size, new_raster_layer.rasterUnitsPerPixelX()))
assert new_raster_layer.rasterUnitsPerPixelX() == size, message
def test_clipRaster(self):
"""Raster layers can be clipped
"""
# Create a raster layer
myName = 'shake'
myRasterLayer = QgsRasterLayer(RASTERPATH, myName)
myMessage = 'Did not find layer "%s" in path "%s"' % \
(myName, RASTERPATH)
assert myRasterLayer is not None, myMessage
# Create a bounding box
myRect = [97, -3, 104, 1]
# Clip the vector to the bbox
myResult = clip_layer(myRasterLayer, myRect)
# Check the output is valid
assert os.path.exists(myResult.source())
# Clip and give a desired resolution for the output
mySize = 0.05
myResult = clip_layer(myRasterLayer, myRect, mySize)
myNewRasterLayer = QgsRasterLayer(myResult.source(), myName)
assert myNewRasterLayer.isValid(), 'Resampled raster is not valid'
myMessage = ('Resampled raster has incorrect pixel size.'
'Expected: %f, Actual: %f' %
(mySize, myNewRasterLayer.rasterUnitsPerPixelX()))
assert myNewRasterLayer.rasterUnitsPerPixelX() == mySize, myMessage
def test_raster_scaling_projected(self):
"""Attempt to scale projected density raster layers raise exception.
Automatic scaling when resampling density data
does not currently work for projected layers. See issue #123.
For the time being this test checks that an exception is raised
when scaling is attempted on projected layers.
When we resolve issue #123, this test should be rewritten.
"""
test_filename = 'Population_Jakarta_UTM48N.tif'
raster_path = ('%s/%s' % (TESTDATA, test_filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
native_resolution = safe_layer.get_resolution()
print min_value, max_value
print native_resolution
# Define bounding box in EPSG:4326
bounding_box = [106.61, -6.38, 107.05, -6.07]
resolutions = [0.02, 0.01, 0.005, 0.002, 0.001]
# Test for a range of resolutions
for resolution in resolutions:
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
try:
clip_layer(
raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords
)
except InvalidProjectionError:
pass
else:
message = 'Should have raised InvalidProjectionError'
raise Exception(message)
def test_raster_scaling_projected(self):
"""Attempt to scale projected density raster layers raise exception.
Automatic scaling when resampling density data
does not currently work for projected layers. See issue #123.
For the time being this test checks that an exception is raised
when scaling is attempted on projected layers.
When we resolve issue #123, this test should be rewritten.
"""
test_filename = 'Population_Jakarta_UTM48N.tif'
raster_path = ('%s/%s' % (TESTDATA, test_filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
native_resolution = safe_layer.get_resolution()
print min_value, max_value
print native_resolution
# Define bounding box in EPSG:4326
bounding_box = [106.61, -6.38, 107.05, -6.07]
resolutions = [0.02, 0.01, 0.005, 0.002, 0.001]
# Test for a range of resolutions
for resolution in resolutions:
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
try:
clip_layer(raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords)
except InvalidProjectionError:
pass
else:
message = 'Should have raised InvalidProjectionError'
raise Exception(message)
def testRasterScaling_projected(self):
"""Attempt to scale projected density raster layers raise exception
Automatic scaling when resampling density data
does not currently work for projected layers. See issue #123.
For the time being this test checks that an exception is raised
when scaling is attempted on projected layers.
When we resolve issue #123, this test should be rewritten.
"""
myTestFilename = 'Population_Jakarta_UTM48N.tif'
myRasterPath = ('%s/%s' % (TESTDATA, myTestFilename))
# Get reference values
mySafeLayer = readSafeLayer(myRasterPath)
myMinimum, myMaximum = mySafeLayer.get_extrema()
myNativeResolution = mySafeLayer.get_resolution()
print
print myMinimum, myMaximum
print myNativeResolution
# Define bounding box in EPSG:4326
myBoundingBox = [106.61, -6.38, 107.05, -6.07]
# Test for a range of resolutions
for myResolution in [0.02, 0.01, 0.005, 0.002, 0.001]:
# Clip the raster to the bbox
myExtraKeywords = {'resolution': myNativeResolution}
myRasterLayer = QgsRasterLayer(myRasterPath, 'xxx')
try:
clip_layer(myRasterLayer,
myBoundingBox,
myResolution,
extra_keywords=myExtraKeywords)
except InvalidProjectionError:
pass
else:
myMessage = 'Should have raised InvalidProjectionError'
raise Exception(myMessage)
def testRasterScaling_projected(self):
"""Attempt to scale projected density raster layers raise exception
Automatic scaling when resampling density data
does not currently work for projected layers. See issue #123.
For the time being this test checks that an exception is raised
when scaling is attempted on projected layers.
When we resolve issue #123, this test should be rewritten.
"""
myTestFilename = 'Population_Jakarta_UTM48N.tif'
myRasterPath = ('%s/%s' % (TESTDATA, myTestFilename))
# Get reference values
mySafeLayer = readSafeLayer(myRasterPath)
myMinimum, myMaximum = mySafeLayer.get_extrema()
myNativeResolution = mySafeLayer.get_resolution()
print
print myMinimum, myMaximum
print myNativeResolution
# Define bounding box in EPSG:4326
myBoundingBox = [106.61, -6.38, 107.05, -6.07]
# Test for a range of resolutions
for myResolution in [0.02, 0.01, 0.005, 0.002, 0.001]:
# Clip the raster to the bbox
myExtraKeywords = {'resolution': myNativeResolution}
myRasterLayer = QgsRasterLayer(myRasterPath, 'xxx')
try:
clip_layer(myRasterLayer,
myBoundingBox,
myResolution,
extra_keywords=myExtraKeywords)
except InvalidProjectionError:
pass
else:
message = 'Should have raised InvalidProjectionError'
raise Exception(message)
def test_clipOnePixel(self):
# Create a raster layer
myRasterPath = os.path.join(EXPDATA, 'glp10ag.asc')
myTitle = 'people'
myRasterLayer = QgsRasterLayer(myRasterPath, myTitle)
# Create very small bounding box
mySmallRect = [100.3430, -0.9089, 100.3588, -0.9022]
# Clip the raster to the bbox
try:
_ = clip_layer(myRasterLayer, mySmallRect)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
def test_clipOnePixel(self):
# Create a raster layer
myRasterPath = os.path.join(EXPDATA, 'glp10ag.asc')
myTitle = 'people'
myRasterLayer = QgsRasterLayer(myRasterPath, myTitle)
# Create very small bounding box
mySmallRect = [100.3430, -0.9089, 100.3588, -0.9022]
# Clip the raster to the bbox
try:
_ = clip_layer(myRasterLayer, mySmallRect)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
def test_vectorProjections(self):
"""Test that vector input data is reprojected properly during clip"""
# Input data is OSM in GOOGLE CRS
# We are reprojecting to GEO and expecting the output shp to be in GEO
# see https://github.com/AIFDR/inasafe/issues/119
# and https://github.com/AIFDR/inasafe/issues/95
myVectorLayer = QgsVectorLayer(VECTOR_PATH2, 'OSM Buildings', 'ogr')
message = 'Failed to load osm buildings'
assert myVectorLayer is not None, message
assert myVectorLayer.isValid()
set_canvas_crs(GEOCRS, True)
set_jakarta_extent()
myClipRect = [106.52, -6.38, 107.14, -6.07]
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myClipRect)
assert (os.path.exists(myResult.source()))
def test_clip_one_pixel(self):
"""Test for clipping less than one pixel."""
# Create a raster layer
raster_path = os.path.join(EXPDATA, 'glp10ag.asc')
title = 'people'
raster_layer = QgsRasterLayer(raster_path, title)
# Create very small bounding box
small_bounding_box = [100.3430, -0.9089, 100.3588, -0.9022]
# Clip the raster to the bbox
try:
_ = clip_layer(raster_layer, small_bounding_box)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
def test_vector_projections(self):
"""Test that vector input data is reprojected properly during clip."""
# Input data is OSM in GOOGLE CRS
# We are reprojecting to GEO and expecting the output shp to be in GEO
# see https://github.com/AIFDR/inasafe/issues/119
# and https://github.com/AIFDR/inasafe/issues/95
vector_layer = QgsVectorLayer(
VECTOR_PATH2, 'OSM Buildings', 'ogr')
message = 'Failed to load osm buildings'
assert vector_layer is not None, message
assert vector_layer.isValid()
set_canvas_crs(GEOCRS, True)
set_jakarta_extent()
clip_rectangle = [106.52, -6.38, 107.14, -6.07]
# Clip the vector to the bbox
result = clip_layer(vector_layer, clip_rectangle)
assert(os.path.exists(result.source()))
def test_clip_one_pixel(self):
"""Test for clipping less than one pixel."""
# Create a raster layer
raster_path = os.path.join(EXPDATA, 'glp10ag.asc')
title = 'people'
raster_layer = QgsRasterLayer(raster_path, title)
# Create very small bounding box
small_bounding_box = [100.3430, -0.9089, 100.3588, -0.9022]
# Clip the raster to the bbox
try:
_ = clip_layer(raster_layer, small_bounding_box)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
def test_clipVectorHard(self):
"""Vector layers can be hard clipped.
Hard clipping will remove any dangling, non intersecting elements.
"""
myVectorLayer = QgsVectorLayer(VECTOR_PATH3, 'OSM Buildings', 'ogr')
message = 'Failed to load osm buildings'
assert myVectorLayer is not None, message
assert myVectorLayer.isValid()
set_canvas_crs(GEOCRS, True)
set_jakarta_extent()
myClipRect = [106.8218, -6.1842, 106.8232, -6.1830]
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myClipRect, hard_clip_flag=True)
# Check the output is valid
assert (os.path.exists(myResult.source()))
def test_clip_vector(self):
"""Vector layers can be clipped."""
# Create a vector layer
layer_name = 'padang'
vector_layer = QgsVectorLayer(VECTOR_PATH, layer_name, 'ogr')
message = ('Did not find layer "%s" in path "%s"' %
(layer_name, VECTOR_PATH))
assert vector_layer is not None, message
assert vector_layer.isValid()
# Create a bounding box
bounding_box = [100.03, -1.14, 100.81, -0.73]
# Clip the vector to the bbox
result = clip_layer(vector_layer, bounding_box)
# Check the output is valid
assert (os.path.exists(result.source()))
def test_clip_vector_hard(self):
"""Vector layers can be hard clipped.
Hard clipping will remove any dangling, non intersecting elements.
"""
vector_layer = QgsVectorLayer(
VECTOR_PATH3, 'OSM Buildings', 'ogr')
message = 'Failed to load osm buildings'
assert vector_layer is not None, message
assert vector_layer.isValid()
set_canvas_crs(GEOCRS, True)
set_jakarta_extent()
clip_rectangle = [106.8218, -6.1842, 106.8232, -6.1830]
# Clip the vector to the bbox
result = clip_layer(vector_layer, clip_rectangle, hard_clip_flag=True)
# Check the output is valid
assert(os.path.exists(result.source()))
def test_clip_vector(self):
"""Vector layers can be clipped."""
# Create a vector layer
layer_name = 'padang'
vector_layer = QgsVectorLayer(VECTOR_PATH, layer_name, 'ogr')
message = (
'Did not find layer "%s" in path "%s"' % (layer_name, VECTOR_PATH))
assert vector_layer is not None, message
assert vector_layer.isValid()
# Create a bounding box
bounding_box = [100.03, -1.14, 100.81, -0.73]
# Clip the vector to the bbox
result = clip_layer(vector_layer, bounding_box)
# Check the output is valid
assert(os.path.exists(result.source()))
def test_clipVector(self):
"""Vector layers can be clipped.
"""
# Create a vector layer
myName = 'padang'
myVectorLayer = QgsVectorLayer(VECTOR_PATH, myName, 'ogr')
myMessage = 'Did not find layer "%s" in path "%s"' % \
(myName, VECTOR_PATH)
assert myVectorLayer is not None, myMessage
assert myVectorLayer.isValid()
# Create a bounding box
myRect = [100.03, -1.14, 100.81, -0.73]
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myRect)
# Check the output is valid
assert(os.path.exists(myResult.source()))
def test_clipVector(self):
"""Vector layers can be clipped.
"""
# Create a vector layer
myName = 'padang'
myVectorLayer = QgsVectorLayer(VECTOR_PATH, myName, 'ogr')
message = 'Did not find layer "%s" in path "%s"' % \
(myName, VECTOR_PATH)
assert myVectorLayer is not None, message
assert myVectorLayer.isValid()
# Create a bounding box
myRect = [100.03, -1.14, 100.81, -0.73]
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myRect)
# Check the output is valid
assert (os.path.exists(myResult.source()))
def _aggregate(self, myImpactLayer, myExpectedResults):
myAggregationLayer = QgsVectorLayer(
os.path.join(BOUNDDATA, 'kabupaten_jakarta.shp'),
'test aggregation',
'ogr')
# create a copy of aggregation layer
myGeoExtent = extent_to_geo_array(
myAggregationLayer.extent(),
myAggregationLayer.crs())
myAggrAttribute = self.keywordIO.read_keywords(
myAggregationLayer, self.defaults['AGGR_ATTR_KEY'])
# noinspection PyArgumentEqualDefault
myAggregationLayer = clip_layer(
layer=myAggregationLayer,
extent=myGeoExtent,
explode_flag=True,
explode_attribute=myAggrAttribute)
myAggregator = Aggregator(None, myAggregationLayer)
# setting up
myAggregator.isValid = True
myAggregator.layer = myAggregationLayer
myAggregator.safeLayer = safe_read_layer(
str(myAggregator.layer.source()))
myAggregator.aoiMode = False
myAggregator.aggregate(myImpactLayer)
myProvider = myAggregator.layer.dataProvider()
myProvider.select(myProvider.attributeIndexes())
myFeature = QgsFeature()
myResults = []
while myProvider.nextFeature(myFeature):
myFeatureResults = {}
myAtMap = myFeature.attributeMap()
for (k, attr) in myAtMap.iteritems():
myFeatureResults[k] = str(attr.toString())
myResults.append(myFeatureResults)
self.assertEqual(myExpectedResults, myResults)
def test_clip_raster_with_no_extension(self):
"""Test we can clip a raster with no extension - see #659."""
# Create a raster layer
source_file = os.path.join(UNITDATA, 'other', 'tenbytenraster.asc')
test_file = unique_filename(prefix='tenbytenraster-')
shutil.copyfile(source_file, test_file)
# Create a keywords file
source_file = os.path.join(UNITDATA, 'other',
'tenbytenraster.keywords')
keywords_file = test_file + '.keywords'
shutil.copyfile(source_file, keywords_file)
# Test the raster layer
raster_layer = QgsRasterLayer(test_file, 'ten by ten')
# Create a bounding box
rectangle = [1535380, 5083260, 1535380 + 40, 5083260 + 40]
# Clip the vector to the bounding box
result = clip_layer(raster_layer, rectangle)
# Check the output is valid
assert os.path.exists(result.source())
def test_clip_raster_with_no_extension(self):
"""Test we can clip a raster with no extension - see #659."""
# Create a raster layer
source_file = os.path.join(UNITDATA, 'other', 'tenbytenraster.asc')
test_file = unique_filename(prefix='tenbytenraster-')
shutil.copyfile(source_file, test_file)
# Create a keywords file
source_file = os.path.join(
UNITDATA, 'other', 'tenbytenraster.keywords')
keywords_file = test_file + '.keywords'
shutil.copyfile(source_file, keywords_file)
# Test the raster layer
raster_layer = QgsRasterLayer(test_file, 'ten by ten')
# Create a bounding box
rectangle = [1535380, 5083260, 1535380 + 40, 5083260 + 40]
# Clip the vector to the bounding box
result = clip_layer(raster_layer, rectangle)
# Check the output is valid
assert os.path.exists(result.source())
def _prepareLayer(self):
"""Prepare the aggregation layer to match analysis extents."""
myMessage = m.Message(
m.Heading(
self.tr('Preparing aggregation layer'),
**PROGRESS_UPDATE_STYLE),
m.Paragraph(self.tr(
'We are clipping the aggregation layer to match the '
'intersection of the hazard and exposure layer extents.')))
self._sendMessage(myMessage)
# This is used to hold an *in memory copy* of the aggregation layer
# or a in memory layer with the clip extents as a feature.
if self.aoiMode:
self.layer = self._extentsToLayer()
# Area Of Interest (AOI) mode flag
else:
# we use only the exposure extent, because both exposure and hazard
# have the same extent at this point.
myGeoExtent = extent_to_geo_array(
self.exposureLayer.extent(),
self.exposureLayer.crs())
myAggrAttribute = self.keywordIO.read_keywords(
self.layer, self.defaults['AGGR_ATTR_KEY'])
myClippedLayer = clip_layer(
layer=self.layer,
extent=myGeoExtent,
explode_flag=True,
explode_attribute=myAggrAttribute)
myName = '%s %s' % (self.layer.name(), self.tr('aggregation'))
self.layer = myClippedLayer
self.layer.setLayerName(myName)
if self.showIntermediateLayers:
self.keywordIO.update_keywords(self.layer, {'title': myName})
QgsMapLayerRegistry.instance().addMapLayer(self.layer)
def testRasterScaling(self):
"""Raster layers can be scaled when resampled
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
for myFilename in ['Population_Jakarta_geographic.asc',
'Population_2010.asc']:
myRasterPath = ('%s/%s' % (TESTDATA, myFilename))
# Get reference values
mySafeLayer = readSafeLayer(myRasterPath)
myMinimum, myMaximum = mySafeLayer.get_extrema()
del myMaximum
del myMinimum
myNativeResolution = mySafeLayer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
myBoundingBox = mySafeLayer.get_bounding_box()
# Test for a range of resolutions
for myResolution in [0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002]: # Finer
# To save time only do two resolutions for the
# large population set
if myFilename.startswith('Population_2010'):
if myResolution > 0.01 or myResolution < 0.005:
break
# Clip the raster to the bbox
myExtraKeywords = {'resolution': myNativeResolution}
myRasterLayer = QgsRasterLayer(myRasterPath, 'xxx')
myResult = clip_layer(myRasterLayer,
myBoundingBox,
myResolution,
extra_keywords=myExtraKeywords)
mySafeLayer = readSafeLayer(myResult.source())
myNativeData = mySafeLayer.get_data(scaling=False)
myScaledData = mySafeLayer.get_data(scaling=True)
mySigma = (mySafeLayer.get_resolution()[0] /
myNativeResolution[0]) ** 2
# Compare extrema
myExpectedScaledMax = mySigma * numpy.nanmax(myNativeData)
myMessage = ('Resampled raster was not rescaled correctly: '
'max(myScaledData) was %f but expected %f'
% (numpy.nanmax(myScaledData),
myExpectedScaledMax))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(myScaledData) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(myExpectedScaledMax,
numpy.nanmax(myScaledData),
rtol=1.0e-6, atol=1.0e-8), myMessage
myExpectedScaledMin = mySigma * numpy.nanmin(myNativeData)
myMessage = ('Resampled raster was not rescaled correctly: '
'min(myScaledData) was %f but expected %f'
% (numpy.nanmin(myScaledData),
myExpectedScaledMin))
assert numpy.allclose(myExpectedScaledMin,
numpy.nanmin(myScaledData),
rtol=1.0e-8, atol=1.0e-12), myMessage
# Compare elementwise
myMessage = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myNativeData * mySigma, myScaledData,
rtol=1.0e-8, atol=1.0e-8), myMessage
# Check that it also works with manual scaling
myManualData = mySafeLayer.get_data(scaling=mySigma)
myMessage = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myManualData, myScaledData,
rtol=1.0e-8, atol=1.0e-8), myMessage
# Check that an exception is raised for bad arguments
try:
mySafeLayer.get_data(scaling='bad')
except GetDataError:
pass
else:
myMessage = 'String argument should have raised exception'
raise Exception(myMessage)
try:
mySafeLayer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
myMessage = 'Tuple argument should have raised exception'
raise Exception(myMessage)
# Check None option without keyword datatype == 'density'
mySafeLayer.keywords['datatype'] = 'undefined'
myUnscaledData = mySafeLayer.get_data(scaling=None)
myMessage = 'Data should not have changed'
assert nan_allclose(myNativeData, myUnscaledData,
rtol=1.0e-12, atol=1.0e-12), myMessage
# Try with None and density keyword
mySafeLayer.keywords['datatype'] = 'density'
myUnscaledData = mySafeLayer.get_data(scaling=None)
myMessage = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myScaledData, myUnscaledData,
rtol=1.0e-12, atol=1.0e-12), myMessage
mySafeLayer.keywords['datatype'] = 'counts'
myUnscaledData = mySafeLayer.get_data(scaling=None)
myMessage = 'Data should not have changed'
assert nan_allclose(myNativeData, myUnscaledData,
rtol=1.0e-12, atol=1.0e-12), myMessage
def test_clip_both(self):
"""Raster and Vector layers can be clipped."""
# Create a vector layer
layer_name = 'padang'
vector_layer = QgsVectorLayer(VECTOR_PATH, layer_name, 'ogr')
message = (
'Did not find layer "%s" in path "%s"' % (layer_name, VECTOR_PATH))
assert vector_layer.isValid(), message
# Create a raster layer
layer_name = 'shake'
raster_layer = QgsRasterLayer(RASTERPATH, layer_name)
message = (
'Did not find layer "%s" in path "%s"' % (layer_name, RASTERPATH))
assert raster_layer.isValid(), message
# Create a bounding box
view_port_geo_extent = [99.53, -1.22, 101.20, -0.36]
# Get the Hazard extents as an array in EPSG:4326
hazard_geo_extent = [
raster_layer.extent().xMinimum(),
raster_layer.extent().yMinimum(),
raster_layer.extent().xMaximum(),
raster_layer.extent().yMaximum()
]
# Get the Exposure extents as an array in EPSG:4326
exposure_geo_extent = [
vector_layer.extent().xMinimum(),
vector_layer.extent().yMinimum(),
vector_layer.extent().xMaximum(),
vector_layer.extent().yMaximum()
]
# Now work out the optimal extent between the two layers and
# the current view extent. The optimal extent is the intersection
# between the two layers and the viewport.
# Extent is returned as an array [xmin,ymin,xmax,ymax]
geo_extent = get_optimal_extent(
hazard_geo_extent, exposure_geo_extent, view_port_geo_extent)
# Clip the vector to the bbox
result = clip_layer(vector_layer, geo_extent)
# Check the output is valid
assert os.path.exists(result.source())
read_safe_layer(result.source())
# Clip the raster to the bbox
result = clip_layer(raster_layer, geo_extent)
# Check the output is valid
assert os.path.exists(result.source())
read_safe_layer(result.source())
# -------------------------------
# Check the extra keywords option
# -------------------------------
# Clip the vector to the bbox
result = clip_layer(
vector_layer, geo_extent, extra_keywords={'kermit': 'piggy'})
# Check the output is valid
assert os.path.exists(result.source())
safe_layer = read_safe_layer(result.source())
keywords = safe_layer.get_keywords()
# message = 'Extra keyword was not found in %s: %s' % (myResult,
# keywords)
assert keywords['kermit'] == 'piggy'
# Clip the raster to the bbox
result = clip_layer(
raster_layer, geo_extent, extra_keywords={'zoot': 'animal'})
# Check the output is valid
assert os.path.exists(result.source())
safe_layer = read_safe_layer(result.source())
keywords = safe_layer.get_keywords()
message = ('Extra keyword was not found in %s: %s' %
(result.source(), keywords))
assert keywords['zoot'] == 'animal', message
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled.
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
filenames = [
'Population_Jakarta_geographic.asc',
'Population_2010.asc'
]
for filename in filenames:
raster_path = ('%s/%s' % (TESTDATA, filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
del max_value
del min_value
native_resolution = safe_layer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
bounding_box = safe_layer.get_bounding_box()
resolutions = [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002 # Finer
]
# Test for a range of resolutions
for resolution in resolutions: # Finer
# To save time only do two resolutions for the
# large population set
if filename.startswith('Population_2010'):
if resolution > 0.01 or resolution < 0.005:
break
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
result = clip_layer(
raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords
)
safe_layer = read_safe_layer(result.source())
native_data = safe_layer.get_data(scaling=False)
scaled_data = safe_layer.get_data(scaling=True)
sigma_value = (safe_layer.get_resolution()[0] /
native_resolution[0]) ** 2
# Compare extrema
expected_scaled_max = sigma_value * numpy.nanmax(native_data)
message = (
'Resampled raster was not rescaled correctly: '
'max(scaled_data) was %f but expected %f' %
(numpy.nanmax(scaled_data), expected_scaled_max))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(scaled_data) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(scaled_data),
rtol=1.0e-6, atol=1.0e-8), message
expected_scaled_min = sigma_value * numpy.nanmin(native_data)
message = (
'Resampled raster was not rescaled correctly: '
'min(scaled_data) was %f but expected %f' %
(numpy.nanmin(scaled_data), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(scaled_data),
rtol=1.0e-8, atol=1.0e-12), message
# Compare element-wise
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(native_data * sigma_value, scaled_data,
rtol=1.0e-8, atol=1.0e-8), message
# Check that it also works with manual scaling
manual_data = safe_layer.get_data(scaling=sigma_value)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(manual_data, scaled_data,
rtol=1.0e-8, atol=1.0e-8), message
# Check that an exception is raised for bad arguments
try:
safe_layer.get_data(scaling='bad')
except GetDataError:
pass
else:
message = 'String argument should have raised exception'
raise Exception(message)
try:
safe_layer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
message = 'Tuple argument should have raised exception'
raise Exception(message)
# Check None option without keyword datatype == 'density'
safe_layer.keywords['datatype'] = 'undefined'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
# Try with None and density keyword
safe_layer.keywords['datatype'] = 'density'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(scaled_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
safe_layer.keywords['datatype'] = 'counts'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data, unscaled_data,
rtol=1.0e-12, atol=1.0e-12), message
def test_clipBoth(self):
"""Raster and Vector layers can be clipped
"""
# Create a vector layer
myName = 'padang'
myVectorLayer = QgsVectorLayer(VECTOR_PATH, myName, 'ogr')
myMessage = 'Did not find layer "%s" in path "%s"' % \
(myName, VECTOR_PATH)
assert myVectorLayer.isValid(), myMessage
# Create a raster layer
myName = 'shake'
myRasterLayer = QgsRasterLayer(RASTERPATH, myName)
myMessage = 'Did not find layer "%s" in path "%s"' % \
(myName, RASTERPATH)
assert myRasterLayer.isValid(), myMessage
# Create a bounding box
myViewportGeoExtent = [99.53, -1.22, 101.20, -0.36]
# Get the Hazard extents as an array in EPSG:4326
myHazardGeoExtent = [myRasterLayer.extent().xMinimum(),
myRasterLayer.extent().yMinimum(),
myRasterLayer.extent().xMaximum(),
myRasterLayer.extent().yMaximum()]
# Get the Exposure extents as an array in EPSG:4326
myExposureGeoExtent = [myVectorLayer.extent().xMinimum(),
myVectorLayer.extent().yMinimum(),
myVectorLayer.extent().xMaximum(),
myVectorLayer.extent().yMaximum()]
# Now work out the optimal extent between the two layers and
# the current view extent. The optimal extent is the intersection
# between the two layers and the viewport.
# Extent is returned as an array [xmin,ymin,xmax,ymax]
myGeoExtent = getOptimalExtent(myHazardGeoExtent,
myExposureGeoExtent,
myViewportGeoExtent)
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myGeoExtent)
# Check the output is valid
assert os.path.exists(myResult.source())
readSafeLayer(myResult.source())
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer, myGeoExtent)
# Check the output is valid
assert os.path.exists(myResult.source())
readSafeLayer(myResult.source())
# -------------------------------
# Check the extra keywords option
# -------------------------------
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myGeoExtent,
extra_keywords={'kermit': 'piggy'})
# Check the output is valid
assert os.path.exists(myResult.source())
L = readSafeLayer(myResult.source())
kwds = L.get_keywords()
# myMessage = 'Extra keyword wasn\'t found in %s: %s' % (myResult,
# kwds)
assert kwds['kermit'] == 'piggy'
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer, myGeoExtent,
extra_keywords={'zoot': 'animal'})
# Check the output is valid
assert os.path.exists(myResult.source())
L = readSafeLayer(myResult.source())
kwds = L.get_keywords()
myMessage = 'Extra keyword was not found in %s: %s' % (
myResult.source(), kwds)
assert kwds['zoot'] == 'animal', myMessage
# Clip the raster to the bbox
try:
_ = clip_layer(myRasterLayer, mySmallRect)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
else:
myMsg = "Failed, does not raise exception"
raise Exception(myMsg)
# Create not very small bounding box
myNotSmallRect = [100.3430, -0.9089, 101.3588, -1.9022]
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer, myNotSmallRect)
assert myResult is not None, 'Should be a success clipping'
def test_invalidFilenamesCaught(self):
"""Invalid filenames raise appropriate exceptions
Wrote this test because test_clipBoth raised the wrong error
when file was missing. Instead of reporting that, it gave
Western boundary must be less than eastern. I got [0.0, 0.0, 0.0, 0.0]
See issue #170
"""
# Try to create a vector layer from non-existing filename
myName = 'stnhaoeu_78oeukqjkrcgA'
myPath = 'OEk_tnshoeu_439_kstnhoe'
def _aggregate(
self,
impact_layer,
expected_results,
use_native_zonal_stats=False):
"""Helper to calculate aggregation.
Expected results is split into two lists - one list contains numeric
attributes, the other strings. This is done so that we can use numpy
.testing.assert_allclose which doesn't work on strings
"""
expected_string_results = []
expected_numeric_results = []
for item in expected_results:
string_results = []
numeric_results = []
for field in item:
try:
value = float(field)
numeric_results.append(value)
except ValueError:
string_results.append(str(field))
expected_numeric_results.append(numeric_results)
expected_string_results.append(string_results)
aggregation_layer = QgsVectorLayer(
os.path.join(BOUNDDATA, 'kabupaten_jakarta.shp'),
'test aggregation',
'ogr')
# create a copy of aggregation layer
geo_extent = extent_to_geo_array(
aggregation_layer.extent(),
aggregation_layer.crs())
aggregation_attribute = self._keywordIO.read_keywords(
aggregation_layer, self._defaults['AGGR_ATTR_KEY'])
# noinspection PyArgumentEqualDefault
aggregation_layer = clip_layer(
layer=aggregation_layer,
extent=geo_extent,
explode_flag=True,
explode_attribute=aggregation_attribute)
aggregator = Aggregator(None, aggregation_layer)
# setting up
aggregator.is_valid = True
aggregator.layer = aggregation_layer
aggregator.safe_layer = safe_read_layer(
str(aggregator.layer.source()))
aggregator.aoi_mode = False
aggregator.use_native_zonal_stats = use_native_zonal_stats
aggregator.aggregate(impact_layer)
provider = aggregator.layer.dataProvider()
string_results = []
numeric_results = []
for feature in provider.getFeatures():
feature_string_results = []
feature_numeric_results = []
attributes = feature.attributes()
for attr in attributes:
if isinstance(attr, (int, float)):
feature_numeric_results.append(attr)
else:
feature_string_results.append(attr)
numeric_results.append(feature_numeric_results)
string_results.append(feature_string_results)
# check string attributes
self.assertEqual(expected_string_results, string_results)
# check numeric attributes with a 0.01% tolerance compared to the
# native QGIS stats
numpy.testing.assert_allclose(expected_numeric_results,
numeric_results,
rtol=0.01)
def test_clip_both(self):
"""Raster and Vector layers can be clipped."""
# Create a vector layer
layer_name = 'padang'
vector_layer = QgsVectorLayer(VECTOR_PATH, layer_name, 'ogr')
message = ('Did not find layer "%s" in path "%s"' %
(layer_name, VECTOR_PATH))
assert vector_layer.isValid(), message
# Create a raster layer
layer_name = 'shake'
raster_layer = QgsRasterLayer(RASTERPATH, layer_name)
message = ('Did not find layer "%s" in path "%s"' %
(layer_name, RASTERPATH))
assert raster_layer.isValid(), message
# Create a bounding box
view_port_geo_extent = [99.53, -1.22, 101.20, -0.36]
# Get the Hazard extents as an array in EPSG:4326
hazard_geo_extent = [
raster_layer.extent().xMinimum(),
raster_layer.extent().yMinimum(),
raster_layer.extent().xMaximum(),
raster_layer.extent().yMaximum()
]
# Get the Exposure extents as an array in EPSG:4326
exposure_geo_extent = [
vector_layer.extent().xMinimum(),
vector_layer.extent().yMinimum(),
vector_layer.extent().xMaximum(),
vector_layer.extent().yMaximum()
]
# Now work out the optimal extent between the two layers and
# the current view extent. The optimal extent is the intersection
# between the two layers and the viewport.
# Extent is returned as an array [xmin,ymin,xmax,ymax]
geo_extent = get_optimal_extent(hazard_geo_extent, exposure_geo_extent,
view_port_geo_extent)
# Clip the vector to the bbox
result = clip_layer(vector_layer, geo_extent)
# Check the output is valid
assert os.path.exists(result.source())
read_safe_layer(result.source())
# Clip the raster to the bbox
result = clip_layer(raster_layer, geo_extent)
# Check the output is valid
assert os.path.exists(result.source())
read_safe_layer(result.source())
# -------------------------------
# Check the extra keywords option
# -------------------------------
# Clip the vector to the bbox
result = clip_layer(vector_layer,
geo_extent,
extra_keywords={'kermit': 'piggy'})
# Check the output is valid
assert os.path.exists(result.source())
safe_layer = read_safe_layer(result.source())
keywords = safe_layer.get_keywords()
# message = 'Extra keyword was not found in %s: %s' % (myResult,
# keywords)
assert keywords['kermit'] == 'piggy'
# Clip the raster to the bbox
result = clip_layer(raster_layer,
geo_extent,
extra_keywords={'zoot': 'animal'})
# Check the output is valid
assert os.path.exists(result.source())
safe_layer = read_safe_layer(result.source())
keywords = safe_layer.get_keywords()
message = ('Extra keyword was not found in %s: %s' %
(result.source(), keywords))
assert keywords['zoot'] == 'animal', message
# Clip the raster to the bbox
try:
_ = clip_layer(raster_layer, small_bounding_box)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
else:
message = "Failed, does not raise exception"
raise Exception(message)
# Create not very small bounding box
not_small_bounding_box = [100.3430, -0.9089, 101.3588, -1.9022]
# Clip the raster to the bbox
result = clip_layer(raster_layer, not_small_bounding_box)
assert result is not None, 'Should be a success clipping'
def test_invalid_filenames_caught(self):
"""Invalid filenames raise appropriate exceptions.
Wrote this test because test_clipBoth raised the wrong error
when file was missing. Instead of reporting that, it gave
Western boundary must be less than eastern. I got [0.0, 0.0, 0.0, 0.0]
See issue #170
"""
# Try to create a vector layer from non-existing filename
name = 'stnhaoeu_78oeukqjkrcgA'
path = 'OEk_tnshoeu_439_kstnhoe'
def test_raster_scaling(self):
"""Raster layers can be scaled when resampled.
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
filenames = [
'Population_Jakarta_geographic.asc', 'Population_2010.asc'
]
for filename in filenames:
raster_path = ('%s/%s' % (TESTDATA, filename))
# Get reference values
safe_layer = read_safe_layer(raster_path)
min_value, max_value = safe_layer.get_extrema()
del max_value
del min_value
native_resolution = safe_layer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
bounding_box = safe_layer.get_bounding_box()
resolutions = [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002 # Finer
]
# Test for a range of resolutions
for resolution in resolutions: # Finer
# To save time only do two resolutions for the
# large population set
if filename.startswith('Population_2010'):
if resolution > 0.01 or resolution < 0.005:
break
# Clip the raster to the bbox
extra_keywords = {'resolution': native_resolution}
raster_layer = QgsRasterLayer(raster_path, 'xxx')
result = clip_layer(raster_layer,
bounding_box,
resolution,
extra_keywords=extra_keywords)
safe_layer = read_safe_layer(result.source())
native_data = safe_layer.get_data(scaling=False)
scaled_data = safe_layer.get_data(scaling=True)
sigma_value = (safe_layer.get_resolution()[0] /
native_resolution[0])**2
# Compare extrema
expected_scaled_max = sigma_value * numpy.nanmax(native_data)
message = ('Resampled raster was not rescaled correctly: '
'max(scaled_data) was %f but expected %f' %
(numpy.nanmax(scaled_data), expected_scaled_max))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(scaled_data) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(expected_scaled_max,
numpy.nanmax(scaled_data),
rtol=1.0e-6,
atol=1.0e-8), message
expected_scaled_min = sigma_value * numpy.nanmin(native_data)
message = ('Resampled raster was not rescaled correctly: '
'min(scaled_data) was %f but expected %f' %
(numpy.nanmin(scaled_data), expected_scaled_min))
assert numpy.allclose(expected_scaled_min,
numpy.nanmin(scaled_data),
rtol=1.0e-8,
atol=1.0e-12), message
# Compare element-wise
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(native_data * sigma_value,
scaled_data,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that it also works with manual scaling
manual_data = safe_layer.get_data(scaling=sigma_value)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(manual_data,
scaled_data,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that an exception is raised for bad arguments
try:
safe_layer.get_data(scaling='bad')
except GetDataError:
pass
else:
message = 'String argument should have raised exception'
raise Exception(message)
try:
safe_layer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
message = 'Tuple argument should have raised exception'
raise Exception(message)
# Check None option without keyword datatype == 'density'
safe_layer.keywords['datatype'] = 'undefined'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
# Try with None and density keyword
safe_layer.keywords['datatype'] = 'density'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(scaled_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
safe_layer.keywords['datatype'] = 'counts'
unscaled_data = safe_layer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(native_data,
unscaled_data,
rtol=1.0e-12,
atol=1.0e-12), message
def _aggregate(self,
impact_layer,
expected_results,
use_native_zonal_stats=False):
"""Helper to calculate aggregation.
Expected results is split into two lists - one list contains numeric
attributes, the other strings. This is done so that we can use numpy
.testing.assert_allclose which doesn't work on strings
"""
expected_string_results = []
expected_numeric_results = []
for item in expected_results:
string_results = []
numeric_results = []
for field in item:
try:
value = float(field)
numeric_results.append(value)
except ValueError:
string_results.append(str(field))
expected_numeric_results.append(numeric_results)
expected_string_results.append(string_results)
aggregation_layer = QgsVectorLayer(
os.path.join(BOUNDDATA, 'kabupaten_jakarta.shp'),
'test aggregation', 'ogr')
# create a copy of aggregation layer
geo_extent = extent_to_geo_array(aggregation_layer.extent(),
aggregation_layer.crs())
aggregation_attribute = self.keywordIO.read_keywords(
aggregation_layer, self.defaults['AGGR_ATTR_KEY'])
# noinspection PyArgumentEqualDefault
aggregation_layer = clip_layer(layer=aggregation_layer,
extent=geo_extent,
explode_flag=True,
explode_attribute=aggregation_attribute)
aggregator = Aggregator(None, aggregation_layer)
# setting up
aggregator.is_valid = True
aggregator.layer = aggregation_layer
aggregator.safe_layer = safe_read_layer(str(aggregator.layer.source()))
aggregator.aoi_mode = False
aggregator.use_native_zonal_stats = use_native_zonal_stats
aggregator.aggregate(impact_layer)
provider = aggregator.layer.dataProvider()
string_results = []
numeric_results = []
for feature in provider.getFeatures():
feature_string_results = []
feature_numeric_results = []
attributes = feature.attributes()
for attr in attributes:
if isinstance(attr, (int, float)):
feature_numeric_results.append(attr)
else:
feature_string_results.append(attr)
numeric_results.append(feature_numeric_results)
string_results.append(feature_string_results)
# check string attributes
self.assertEqual(expected_string_results, string_results)
# check numeric attributes with a 0.01% tolerance compared to the
# native QGIS stats
numpy.testing.assert_allclose(expected_numeric_results,
numeric_results,
rtol=0.01)
# Clip the raster to the bbox
try:
_ = clip_layer(raster_layer, small_bounding_box)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
else:
message = "Failed, does not raise exception"
raise Exception(message)
# Create not very small bounding box
not_small_bounding_box = [100.3430, -0.9089, 101.3588, -1.9022]
# Clip the raster to the bbox
result = clip_layer(raster_layer, not_small_bounding_box)
assert result is not None, 'Should be a success clipping'
def test_invalid_filenames_caught(self):
"""Invalid filenames raise appropriate exceptions.
Wrote this test because test_clipBoth raised the wrong error
when file was missing. Instead of reporting that, it gave
Western boundary must be less than eastern. I got [0.0, 0.0, 0.0, 0.0]
See issue #170
"""
# Try to create a vector layer from non-existing filename
name = 'stnhaoeu_78oeukqjkrcgA'
path = 'OEk_tnshoeu_439_kstnhoe'
def test_clipBoth(self):
"""Raster and Vector layers can be clipped
"""
# Create a vector layer
myName = 'padang'
myVectorLayer = QgsVectorLayer(VECTOR_PATH, myName, 'ogr')
message = 'Did not find layer "%s" in path "%s"' % \
(myName, VECTOR_PATH)
assert myVectorLayer.isValid(), message
# Create a raster layer
myName = 'shake'
myRasterLayer = QgsRasterLayer(RASTERPATH, myName)
message = 'Did not find layer "%s" in path "%s"' % \
(myName, RASTERPATH)
assert myRasterLayer.isValid(), message
# Create a bounding box
myViewportGeoExtent = [99.53, -1.22, 101.20, -0.36]
# Get the Hazard extents as an array in EPSG:4326
myHazardGeoExtent = [
myRasterLayer.extent().xMinimum(),
myRasterLayer.extent().yMinimum(),
myRasterLayer.extent().xMaximum(),
myRasterLayer.extent().yMaximum()
]
# Get the Exposure extents as an array in EPSG:4326
myExposureGeoExtent = [
myVectorLayer.extent().xMinimum(),
myVectorLayer.extent().yMinimum(),
myVectorLayer.extent().xMaximum(),
myVectorLayer.extent().yMaximum()
]
# Now work out the optimal extent between the two layers and
# the current view extent. The optimal extent is the intersection
# between the two layers and the viewport.
# Extent is returned as an array [xmin,ymin,xmax,ymax]
myGeoExtent = getOptimalExtent(myHazardGeoExtent, myExposureGeoExtent,
myViewportGeoExtent)
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer, myGeoExtent)
# Check the output is valid
assert os.path.exists(myResult.source())
readSafeLayer(myResult.source())
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer, myGeoExtent)
# Check the output is valid
assert os.path.exists(myResult.source())
readSafeLayer(myResult.source())
# -------------------------------
# Check the extra keywords option
# -------------------------------
# Clip the vector to the bbox
myResult = clip_layer(myVectorLayer,
myGeoExtent,
extra_keywords={'kermit': 'piggy'})
# Check the output is valid
assert os.path.exists(myResult.source())
L = readSafeLayer(myResult.source())
kwds = L.get_keywords()
# message = 'Extra keyword wasn\'t found in %s: %s' % (myResult,
# kwds)
assert kwds['kermit'] == 'piggy'
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer,
myGeoExtent,
extra_keywords={'zoot': 'animal'})
# Check the output is valid
assert os.path.exists(myResult.source())
L = readSafeLayer(myResult.source())
kwds = L.get_keywords()
message = 'Extra keyword was not found in %s: %s' % (myResult.source(),
kwds)
assert kwds['zoot'] == 'animal', message
def testRasterScaling(self):
"""Raster layers can be scaled when resampled
This is a test for ticket #52
Native test .asc data has
Population_Jakarta_geographic.asc
ncols 638
nrows 649
cellsize 0.00045228819716044
Population_2010.asc
ncols 5525
nrows 2050
cellsize 0.0083333333333333
Scaling is necessary for raster data that represents density
such as population per km^2
"""
for filename in [
'Population_Jakarta_geographic.asc', 'Population_2010.asc'
]:
myRasterPath = ('%s/%s' % (TESTDATA, filename))
# Get reference values
mySafeLayer = readSafeLayer(myRasterPath)
myMinimum, myMaximum = mySafeLayer.get_extrema()
del myMaximum
del myMinimum
myNativeResolution = mySafeLayer.get_resolution()
# Get the Hazard extents as an array in EPSG:4326
myBoundingBox = mySafeLayer.get_bounding_box()
# Test for a range of resolutions
for myResolution in [
0.02,
0.01,
0.005,
0.002,
0.001,
0.0005, # Coarser
0.0002
]: # Finer
# To save time only do two resolutions for the
# large population set
if filename.startswith('Population_2010'):
if myResolution > 0.01 or myResolution < 0.005:
break
# Clip the raster to the bbox
myExtraKeywords = {'resolution': myNativeResolution}
myRasterLayer = QgsRasterLayer(myRasterPath, 'xxx')
myResult = clip_layer(myRasterLayer,
myBoundingBox,
myResolution,
extra_keywords=myExtraKeywords)
mySafeLayer = readSafeLayer(myResult.source())
myNativeData = mySafeLayer.get_data(scaling=False)
myScaledData = mySafeLayer.get_data(scaling=True)
mySigma = (mySafeLayer.get_resolution()[0] /
myNativeResolution[0])**2
# Compare extrema
myExpectedScaledMax = mySigma * numpy.nanmax(myNativeData)
message = ('Resampled raster was not rescaled correctly: '
'max(myScaledData) was %f but expected %f' %
(numpy.nanmax(myScaledData), myExpectedScaledMax))
# FIXME (Ole): The rtol used to be 1.0e-8 -
# now it has to be 1.0e-6, otherwise we get
# max(myScaledData) was 12083021.000000 but
# expected 12083020.414316
# Is something being rounded to the nearest
# integer?
assert numpy.allclose(myExpectedScaledMax,
numpy.nanmax(myScaledData),
rtol=1.0e-6,
atol=1.0e-8), message
myExpectedScaledMin = mySigma * numpy.nanmin(myNativeData)
message = ('Resampled raster was not rescaled correctly: '
'min(myScaledData) was %f but expected %f' %
(numpy.nanmin(myScaledData), myExpectedScaledMin))
assert numpy.allclose(myExpectedScaledMin,
numpy.nanmin(myScaledData),
rtol=1.0e-8,
atol=1.0e-12), message
# Compare elementwise
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myNativeData * mySigma,
myScaledData,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that it also works with manual scaling
myManualData = mySafeLayer.get_data(scaling=mySigma)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myManualData,
myScaledData,
rtol=1.0e-8,
atol=1.0e-8), message
# Check that an exception is raised for bad arguments
try:
mySafeLayer.get_data(scaling='bad')
except GetDataError:
pass
else:
message = 'String argument should have raised exception'
raise Exception(message)
try:
mySafeLayer.get_data(scaling='(1, 3)')
except GetDataError:
pass
else:
message = 'Tuple argument should have raised exception'
raise Exception(message)
# Check None option without keyword datatype == 'density'
mySafeLayer.keywords['datatype'] = 'undefined'
myUnscaledData = mySafeLayer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(myNativeData,
myUnscaledData,
rtol=1.0e-12,
atol=1.0e-12), message
# Try with None and density keyword
mySafeLayer.keywords['datatype'] = 'density'
myUnscaledData = mySafeLayer.get_data(scaling=None)
message = 'Resampled raster was not rescaled correctly'
assert nan_allclose(myScaledData,
myUnscaledData,
rtol=1.0e-12,
atol=1.0e-12), message
mySafeLayer.keywords['datatype'] = 'counts'
myUnscaledData = mySafeLayer.get_data(scaling=None)
message = 'Data should not have changed'
assert nan_allclose(myNativeData,
myUnscaledData,
rtol=1.0e-12,
atol=1.0e-12), message
def _aggregate(self,
myImpactLayer,
myExpectedResults,
useNativeZonalStats=False):
"""Helper to calculate aggregation.
Expected results is split into two lists - one list contains numeric
attributes, the other strings. This is done so that we can use numpy
.testing.assert_allclose which doesn't work on strings
"""
myExpectedStringResults = []
myExpectedNumericResults = []
for item in myExpectedResults:
myItemNumResults = []
myItemStrResults = []
for field in item:
try:
value = float(field)
myItemNumResults.append(value)
except ValueError:
myItemStrResults.append(str(field))
myExpectedNumericResults.append(myItemNumResults)
myExpectedStringResults.append(myItemStrResults)
myAggregationLayer = QgsVectorLayer(
os.path.join(BOUNDDATA, 'kabupaten_jakarta.shp'),
'test aggregation',
'ogr')
# create a copy of aggregation layer
myGeoExtent = extent_to_geo_array(
myAggregationLayer.extent(),
myAggregationLayer.crs())
myAggrAttribute = self.keywordIO.read_keywords(
myAggregationLayer, self.defaults['AGGR_ATTR_KEY'])
# noinspection PyArgumentEqualDefault
myAggregationLayer = clip_layer(
layer=myAggregationLayer,
extent=myGeoExtent,
explode_flag=True,
explode_attribute=myAggrAttribute)
myAggregator = Aggregator(None, myAggregationLayer)
# setting up
myAggregator.isValid = True
myAggregator.layer = myAggregationLayer
myAggregator.safeLayer = safe_read_layer(
str(myAggregator.layer.source()))
myAggregator.aoiMode = False
myAggregator.useNativeZonalStats = useNativeZonalStats
myAggregator.aggregate(myImpactLayer)
myProvider = myAggregator.layer.dataProvider()
myNumericResults = []
myStringResults = []
for myFeature in myProvider.getFeatures():
myFeatureNumResults = []
myFeatureStrResults = []
myAttrs = myFeature.attributes()
for attr in myAttrs:
if isinstance(attr, (int, float)):
myFeatureNumResults.append(attr)
else:
myFeatureStrResults.append(attr)
myNumericResults.append(myFeatureNumResults)
myStringResults.append(myFeatureStrResults)
# check string attributes
self.assertEqual(myExpectedStringResults, myStringResults)
# check numeric attributes with a 0.01% tolerance compared to the
# native QGIS stats
numpy.testing.assert_allclose(myExpectedNumericResults,
myNumericResults,
rtol=0.01)
# Clip the raster to the bbox
try:
_ = clip_layer(myRasterLayer, mySmallRect)
except CallGDALError:
pass
except Exception, e:
raise Exception('Exception is not expected, %s' % e)
else:
myMsg = "Failed, does not raise exception"
raise Exception(myMsg)
# Create not very small bounding box
myNotSmallRect = [100.3430, -0.9089, 101.3588, -1.9022]
# Clip the raster to the bbox
myResult = clip_layer(myRasterLayer, myNotSmallRect)
assert myResult is not None, 'Should be a success clipping'
def test_invalidFilenamesCaught(self):
"""Invalid filenames raise appropriate exceptions
Wrote this test because test_clipBoth raised the wrong error
when file was missing. Instead of reporting that, it gave
Western boundary must be less than eastern. I got [0.0, 0.0, 0.0, 0.0]
See issue #170
"""
# Try to create a vector layer from non-existing filename
myName = 'stnhaoeu_78oeukqjkrcgA'
path = 'OEk_tnshoeu_439_kstnhoe'
