def test_write_keywords(self):
"""Test writing keywords with no sublayer, no existing file."""
keywords = DKI_KEYWORDS
filename = self.make_temp_file()
write_keywords(keywords, filename)
# Read back contents and check against control dataset
control_keywords = read_keywords(SIMPLE_PATH, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords(self):
"""Test writing keywords with no sublayer, no existing file."""
keywords = DKI_KEYWORDS
filename = self.make_temp_file()
write_keywords(keywords, filename)
# Read back contents and check against control dataset
control_keywords = read_keywords(SIMPLE_PATH, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_singlesublayer(self):
"""Test writing keywords for single sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(OSM_KEYWORDS, filename=filename, sublayer='osm')
# read back contents and check against control dataset
control_path = test_data_path('other', 'expected_singlelayer.keywords')
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_singlesublayer(self):
"""Test writing keywords for single sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(OSM_KEYWORDS, filename=filename, sublayer='osm')
# read back contents and check against control dataset
control_path = test_data_path('other', 'expected_singlelayer.keywords')
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_singlesublayer(self):
"""Test writing keywords for single sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(OSM_KEYWORDS, filename=filename, sublayer="osm")
# read back contents and check against control dataset
control_path = os.path.abspath(os.path.join(UNITDATA, "other", "expected_singlelayer.keywords"))
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = "Expected:\n%s\nGot:\n%s\n" % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_multisublayer(self):
"""Test writing keywords for named sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(DKI_KEYWORDS, filename=filename, sublayer="dki")
write_keywords(OSM_KEYWORDS, filename=filename, sublayer="osm")
# Read back contents and check against control dataset
control_path = test_data_path("other", "expected_multilayer.keywords")
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = "Expected:\n%s\nGot:\n%s\n" % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_multisublayer(self):
"""Test writing keywords for named sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(DKI_KEYWORDS, filename=filename, sublayer='dki')
write_keywords(OSM_KEYWORDS, filename=filename, sublayer='osm')
# Read back contents and check against control dataset
control_path = os.path.abspath(
os.path.join(UNITDATA, 'other', 'expected_multilayer.keywords'))
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_write_keywords_multisublayer(self):
"""Test writing keywords for named sublayer, no existing file."""
filename = self.make_temp_file()
write_keywords(DKI_KEYWORDS, filename=filename, sublayer='dki')
write_keywords(OSM_KEYWORDS, filename=filename, sublayer='osm')
# Read back contents and check against control dataset
control_path = os.path.abspath(
os.path.join(UNITDATA, 'other', 'expected_multilayer.keywords'))
control_keywords = read_keywords(control_path, all_blocks=True)
actual_keywords = read_keywords(filename, all_blocks=True)
msg = 'Expected:\n%s\nGot:\n%s\n' % (control_keywords, actual_keywords)
assert control_keywords == actual_keywords, msg
def test_read_all_keywords(self):
"""Test reading all keywords for all layers"""
keywords = read_keywords(KEYWORD_PATH, all_blocks=True)
expected_keywords = {"osm_buildings": DKI_KEYWORDS, "osm_flood": OSM_KEYWORDS}
msg = "Expected:\n%s\nGot:\n%s\n" % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def testSqliteWriting(self):
"""Test that writing a dataset to sqlite works."""
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
test_dir = temp_dir(sub_dir='test')
test_file = unique_filename(suffix='.sqlite', dir=test_dir)
layer.write_to_file(test_file, sublayer='foo')
def testSqliteWriting(self):
"""Test that writing a dataset to sqlite works."""
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
test_dir = temp_dir(sub_dir='test')
test_file = unique_filename(suffix='.sqlite', dir=test_dir)
layer.write_to_file(test_file, sublayer='foo')
def test_read_keywords(self):
"""Test reading keywords - get first block from kwds as default."""
keywords = read_keywords(KEYWORD_PATH)
expected_keywords = DKI_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_read_keywords_for_sublayer(self):
"""Test reading keywords for specific sublayer."""
keywords = read_keywords(KEYWORD_PATH, sublayer='osm_flood')
expected_keywords = OSM_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_read_keywords_simple(self):
"""Test reading keywords from keywords file with no sublayers"""
keywords = read_keywords(SIMPLE_PATH)
expected_keywords = DKI_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_read_keywords(self):
"""Test reading keywords - get first block from kwds as default."""
keywords = read_keywords(KEYWORD_PATH)
expected_keywords = DKI_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_read_keywords_simple(self):
"""Test reading keywords from keywords file with no sublayers"""
keywords = read_keywords(SIMPLE_PATH)
expected_keywords = DKI_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_read_keywords_for_sublayer(self):
"""Test reading keywords for specific sublayer."""
keywords = read_keywords(KEYWORD_PATH, sublayer='osm_flood')
expected_keywords = OSM_KEYWORDS
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_sublayer_loading(self):
"""Test if we can load sublayers."""
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
layer = Vector(data=SQLITE_PATH, keywords=keywords, sublayer=EXPOSURE_SUBLAYER_NAME)
msg = "Expected layer to be a polygon layer, got a %s" % layer.geometry_type
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250, "Expected 250 features, got %s" % count)
def test_shapefile_loading(self):
"""Test that loading a dataset with no sublayers works."""
keywords = read_keywords(SHP_BASE + ".keywords")
layer = Vector(data=SHP_BASE + ".shp", keywords=keywords)
msg = "Expected layer to be a polygon layer, got a %s" % layer.geometry_type
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250, "Expected 250 features, got %s" % count)
def test_read_all_keywords(self):
"""Test reading all keywords for all layers"""
keywords = read_keywords(KEYWORD_PATH, all_blocks=True)
expected_keywords = {'osm_buildings': DKI_KEYWORDS,
'osm_flood': OSM_KEYWORDS}
msg = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords)
self.assertEquals(keywords, expected_keywords, msg)
LOGGER.debug(keywords)
def test_sqlite_writing(self):
"""Test that writing a dataset to sqlite works."""
keywords = read_keywords(SHP_BASE + ".keywords")
layer = Vector(data=SHP_BASE + ".shp", keywords=keywords)
test_dir = temp_dir(sub_dir="test")
test_file = unique_filename(suffix=".sqlite", dir=test_dir)
layer.write_to_file(test_file, sublayer="foo")
self.assertTrue(os.path.exists(test_file))
def test_shapefile_loading(self):
"""Test that loading a dataset with no sublayers works."""
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250, 'Expected 250 features, got %s' % count)
def testShpLoading(self):
"""Test that loading a dataset with no sublayers works."""
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
assert layer.is_polygon_data, msg
count = len(layer)
assert count == 250, 'Expected 250 features, got %s' % count
def read_file_keywords(layer_path, keyword=None):
"""Get metadata from the keywords file associated with a local
file in the file system.
.. note:: Requires a str representing a file path instance
as parameter As opposed to read_keywords_from_layer which
takes a inasafe file object as parameter.
.. seealso:: read_keywords_from_layer
:param: layer_path: a string representing a path to a layer
(e.g. '/tmp/foo.shp', '/tmp/foo.tif')
:type layer_path: str
:param keyword: optional - the metadata keyword to retrieve e.g. 'title'
:type keyword: str
:return: A string containing the retrieved value for the keyword if
the keyword argument is specified, otherwise the
complete keywords dictionary is returned.
:raises: KeywordNotFoundError, NoKeywordsFoundError, InvalidParameterError
Note:
* KeywordNotFoundError - occurs when the keyword is not recognised.
* NoKeywordsFoundError - occurs when no keyword file exists.
* InvalidParameterError - occurs when the layer does not exist.
"""
# check the source layer path is valid
if not os.path.isfile(layer_path):
message = tr('Cannot get keywords from a non-existent file. File '
'%s does not exist.' % layer_path)
raise InvalidParameterError(message)
# check there really is a keywords file for this layer
# priority for iso path first
keyword_file_path = os.path.splitext(layer_path)[0]
keyword_file_path += '.keywords'
keyword_iso_path = os.path.splitext(layer_path)[0]
keyword_iso_path += '.xml'
if not os.path.isfile(keyword_file_path)\
and not os.path.isfile(keyword_iso_path):
message = tr('No keywords file found for %s' % keyword_file_path)
raise NoKeywordsFoundError(message)
elif os.path.isfile(keyword_file_path) \
and not os.path.isfile(keyword_iso_path):
# switch to .keywords file if iso xml file didn't exist
keyword_iso_path = keyword_file_path
# now get the requested keyword using the inasafe library
try:
dictionary = read_keywords(keyword_iso_path)
except Exception, e:
message = tr(
'Keyword retrieval failed for %s (%s) \n %s' % (
keyword_file_path, keyword, str(e)))
raise KeywordNotFoundError(message)
def testShpLoading(self):
"""Test that loading a dataset with no sublayers works."""
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
assert layer.is_polygon_data, msg
count = len(layer)
assert count == 250, 'Expected 250 features, got %s' % count
def testSublayerLoading(self):
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
layer = Vector(data=SQLITE_PATH, keywords=keywords,
sublayer=EXPOSURE_SUBLAYER_NAME)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
assert layer.is_polygon_data, msg
count = len(layer)
assert count == 250, 'Expected 250 features, got %s' % count
def testSublayerLoading(self):
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
layer = Vector(data=SQLITE_PATH, keywords=keywords,
sublayer=EXPOSURE_SUBLAYER_NAME)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
assert layer.is_polygon_data, msg
count = len(layer)
assert count == 250, 'Expected 250 features, got %s' % count
def read_file_keywords(layer_path, keyword=None):
"""Get metadata from the keywords file associated with a local
file in the file system.
.. note:: Requires a str representing a file path instance
as parameter As opposed to read_keywords_from_layer which
takes a inasafe file object as parameter.
.. seealso:: read_keywords_from_layer
:param: layer_path: a string representing a path to a layer
(e.g. '/tmp/foo.shp', '/tmp/foo.tif')
:type layer_path: str
:param keyword: optional - the metadata keyword to retrieve e.g. 'title'
:type keyword: str
:return: A string containing the retrieved value for the keyword if
the keyword argument is specified, otherwise the
complete keywords dictionary is returned.
:raises: KeywordNotFoundError, NoKeywordsFoundError, InvalidParameterError
Note:
* KeywordNotFoundError - occurs when the keyword is not recognised.
* NoKeywordsFoundError - occurs when no keyword file exists.
* InvalidParameterError - occurs when the layer does not exist.
"""
# check the source layer path is valid
if not os.path.isfile(layer_path):
message = tr('Cannot get keywords from a non-existent file. File '
'%s does not exist.' % layer_path)
raise InvalidParameterError(message)
# check there really is a keywords file for this layer
# priority for iso path first
keyword_file_path = os.path.splitext(layer_path)[0]
keyword_file_path += '.keywords'
keyword_iso_path = os.path.splitext(layer_path)[0]
keyword_iso_path += '.xml'
if not os.path.isfile(keyword_file_path)\
and not os.path.isfile(keyword_iso_path):
message = tr('No keywords file found for %s' % keyword_file_path)
raise NoKeywordsFoundError(message)
elif os.path.isfile(keyword_file_path) \
and not os.path.isfile(keyword_iso_path):
# switch to .keywords file if iso xml file didn't exist
keyword_iso_path = keyword_file_path
# now get the requested keyword using the inasafe library
try:
dictionary = read_keywords(keyword_iso_path)
except Exception, e:
message = tr('Keyword retrieval failed for %s (%s) \n %s' %
(keyword_file_path, keyword, str(e)))
raise KeywordNotFoundError(message)
def read_from_file(self, filename):
# Open data file for reading
# File must be kept open, otherwise GDAL methods segfault.
fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
if fid is None:
msg = 'Could not open file %s' % filename
raise Exception(msg)
# Record raster metadata from file
basename, ext = os.path.splitext(filename)
# If file is ASCII, check that projection is around.
# GDAL does not check this nicely, so it is worth an
# error message
if ext == '.asc':
try:
open(basename + '.prj')
except IOError:
msg = ('Projection file not found for %s. You must supply '
'a projection file with extension .prj' % filename)
raise RuntimeError(msg)
# Look for any keywords
self.keywords = read_keywords(basename + '.keywords')
# Determine name
if 'title' in self.keywords:
rastername = self.keywords['title']
else:
# Use basename without leading directories as name
rastername = os.path.split(basename)[-1]
self.name = rastername
self.filename = filename
self.projection = Projection(self.fid.GetProjection())
self.geotransform = self.fid.GetGeoTransform()
self.columns = fid.RasterXSize
self.rows = fid.RasterYSize
self.number_of_bands = fid.RasterCount
# Assume that file contains all data in one band
msg = 'Only one raster band currently allowed'
if self.number_of_bands > 1:
msg = ('WARNING: Number of bands in %s are %i. '
'Only the first band will currently be '
'used.' % (filename, self.number_of_bands))
# FIXME(Ole): Let us use python warnings here
raise Exception(msg)
# Get first band.
band = self.band = fid.GetRasterBand(1)
if band is None:
msg = 'Could not read raster band from %s' % filename
raise Exception(msg)
def read_from_file(self, filename):
# Open data file for reading
# File must be kept open, otherwise GDAL methods segfault.
fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
if fid is None:
msg = 'Could not open file %s' % filename
raise Exception(msg)
# Record raster metadata from file
basename, ext = os.path.splitext(filename)
# If file is ASCII, check that projection is around.
# GDAL does not check this nicely, so it is worth an
# error message
if ext == '.asc':
try:
open(basename + '.prj')
except IOError:
msg = ('Projection file not found for %s. You must supply '
'a projection file with extension .prj' % filename)
raise RuntimeError(msg)
# Look for any keywords
self.keywords = read_keywords(basename + '.keywords')
# Determine name
if 'title' in self.keywords:
rastername = self.keywords['title']
else:
# Use basename without leading directories as name
rastername = os.path.split(basename)[-1]
self.name = rastername
self.filename = filename
self.projection = Projection(self.fid.GetProjection())
self.geotransform = self.fid.GetGeoTransform()
self.columns = fid.RasterXSize
self.rows = fid.RasterYSize
self.number_of_bands = fid.RasterCount
# Assume that file contains all data in one band
msg = 'Only one raster band currently allowed'
if self.number_of_bands > 1:
msg = ('WARNING: Number of bands in %s are %i. '
'Only the first band will currently be '
'used.' % (filename, self.number_of_bands))
# FIXME(Ole): Let us use python warnings here
raise Exception(msg)
# Get first band.
band = self.band = fid.GetRasterBand(1)
if band is None:
msg = 'Could not read raster band from %s' % filename
raise Exception(msg)
def test_sublayer_loading(self):
"""Test if we can load sublayers."""
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
layer = Vector(data=SQLITE_PATH, keywords=keywords,
sublayer=EXPOSURE_SUBLAYER_NAME)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250, 'Expected 250 features, got %s' % count)
def test_qgis_vector_layer_loading(self):
"""Test that reading from QgsVectorLayer works."""
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
if QGIS_IS_AVAILABLE:
qgis_layer = QgsVectorLayer(SHP_BASE + ".shp", "test", "ogr")
layer = Vector(data=qgis_layer, keywords=keywords)
msg = "Expected layer to be a polygon layer, got a %s" % layer.geometry_type
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250, "Expected 250 features, got %s" % count)
def test_convert_to_qgis_vector_layer(self):
"""Test that converting to QgsVectorLayer works."""
if QGIS_IS_AVAILABLE:
# Create vector layer
keywords = read_keywords(SHP_BASE + ".keywords")
layer = Vector(data=SHP_BASE + ".shp", keywords=keywords)
# Convert to QgsVectorLayer
qgis_layer = layer.as_qgis_native()
provider = qgis_layer.dataProvider()
count = provider.featureCount()
self.assertEqual(count, 250, "Expected 250 features, got %s" % count)
def test_qgis_vector_layer_loading(self):
"""Test that reading from QgsVectorLayer works."""
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
if qgis_imported:
qgis_layer = QgsVectorLayer(SHP_BASE + '.shp', 'test', 'ogr')
layer = Vector(data=qgis_layer, keywords=keywords)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(
count, 250, 'Expected 250 features, got %s' % count)
def test_convert_to_qgis_vector_layer(self):
"""Test that converting to QgsVectorLayer works."""
if qgis_imported:
# Create vector layer
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
# Convert to QgsVectorLayer
qgis_layer = layer.as_qgis_native()
provider = qgis_layer.dataProvider()
count = provider.featureCount()
self.assertEqual(
count, 250, 'Expected 250 features, got %s' % count)
def test_qgis_vector_layer_loading(self):
"""Test that reading from QgsVectorLayer works."""
keywords = read_keywords(KEYWORD_PATH, EXPOSURE_SUBLAYER_NAME)
if QGIS_IS_AVAILABLE:
qgis_layer = QgsVectorLayer(SHP_BASE + '.shp', 'test', 'ogr')
layer = Vector(data=qgis_layer, keywords=keywords)
msg = ('Expected layer to be a polygon layer, got a %s' %
layer.geometry_type)
self.assertTrue(layer.is_polygon_data, msg)
count = len(layer)
self.assertEqual(count, 250,
'Expected 250 features, got %s' % count)
def test_convert_to_qgis_vector_layer(self):
"""Test that converting to QgsVectorLayer works."""
if QGIS_IS_AVAILABLE:
# Create vector layer
keywords = read_keywords(SHP_BASE + '.keywords')
layer = Vector(data=SHP_BASE + '.shp', keywords=keywords)
# Convert to QgsVectorLayer
qgis_layer = layer.as_qgis_native()
provider = qgis_layer.dataProvider()
count = provider.featureCount()
self.assertEqual(count, 250,
'Expected 250 features, got %s' % count)
def test_convert_to_qgis_raster_layer(self):
"""Test that converting to QgsVectorLayer works."""
# Create vector layer
keywords = read_keywords(RASTER_BASE + '.keywords')
layer = Raster(data=RASTER_BASE + '.tif', keywords=keywords)
# Convert to QgsRasterLayer
qgis_layer = layer.as_qgis_native()
qgis_extent = qgis_layer.dataProvider().extent()
qgis_extent = [qgis_extent.xMinimum(), qgis_extent.yMinimum(),
qgis_extent.xMaximum(), qgis_extent.yMaximum()]
layer_exent = layer.get_bounding_box()
self.assertListEqual(
layer_exent, qgis_extent,
'Expected %s extent, got %s' % (qgis_extent, layer_exent))
def test_convert_to_qgis_raster_layer(self):
"""Test that converting to QgsVectorLayer works."""
# Create vector layer
keywords = read_keywords(RASTER_BASE + '.keywords')
layer = Raster(data=RASTER_BASE + '.tif', keywords=keywords)
# Convert to QgsRasterLayer
qgis_layer = layer.as_qgis_native()
qgis_extent = qgis_layer.dataProvider().extent()
qgis_extent = [qgis_extent.xMinimum(), qgis_extent.yMinimum(),
qgis_extent.xMaximum(), qgis_extent.yMaximum()]
layer_exent = layer.get_bounding_box()
self.assertListEqual(
layer_exent, qgis_extent,
'Expected %s extent, got %s' % (qgis_extent, layer_exent))
def read_from_file(self, filename):
""" Read and unpack vector data.
It is assumed that the file contains only one layer with the
pertinent features. Further it is assumed for the moment that
all geometries are points.
* A feature is a geometry and a set of attributes.
* A geometry refers to location and can be point, line, polygon or
combinations thereof.
* The attributes or obtained through GetField()
The full OGR architecture is documented at
* http://www.gdal.org/ogr/ogr_arch.html
* http://www.gdal.org/ogr/ogr_apitut.html
Examples are at
* danieljlewis.org/files/2010/09/basicpythonmap.pdf
* http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
* http://www.packtpub.com/article/geospatial-data-python-geometry
"""
basename, _ = os.path.splitext(filename)
# Look for any keywords
self.keywords = read_keywords(basename + '.keywords')
# FIXME (Ole): Should also look for style file to populate style_info
# Determine name
if 'title' in self.keywords:
vectorname = self.keywords['title']
else:
# Use basename without leading directories as name
vectorname = os.path.split(basename)[-1]
self.name = vectorname
self.filename = filename
self.geometry_type = None # In case there are no features
fid = ogr.Open(filename)
if fid is None:
msg = 'Could not open %s' % filename
raise IOError(msg)
# Assume that file contains all data in one layer
msg = 'Only one vector layer currently allowed'
if fid.GetLayerCount() > 1:
msg = ('WARNING: Number of layers in %s are %i. '
'Only the first layer will currently be '
'used.' % (filename, fid.GetLayerCount()))
raise Exception(msg)
layer = fid.GetLayerByIndex(0)
# Get spatial extent
self.extent = layer.GetExtent()
# Get projection
p = layer.GetSpatialRef()
self.projection = Projection(p)
# Get number of features
N = layer.GetFeatureCount()
# Extract coordinates and attributes for all features
geometry = []
data = []
for i in range(N):
feature = layer.GetFeature(i)
if feature is None:
msg = 'Could not get feature %i from %s' % (i, filename)
raise Exception(msg)
# Record coordinates ordered as Longitude, Latitude
G = feature.GetGeometryRef()
if G is None:
msg = ('Geometry was None in filename %s ' % filename)
raise Exception(msg)
else:
self.geometry_type = G.GetGeometryType()
if self.geometry_type == ogr.wkbPoint:
geometry.append((G.GetX(), G.GetY()))
elif self.geometry_type == ogr.wkbLineString:
M = G.GetPointCount()
coordinates = []
for j in range(M):
coordinates.append((G.GetX(j), G.GetY(j)))
# Record entire line as an Mx2 numpy array
geometry.append(numpy.array(coordinates,
dtype='d',
copy=False))
elif self.geometry_type == ogr.wkbPolygon:
ring = G.GetGeometryRef(0)
M = ring.GetPointCount()
coordinates = []
for j in range(M):
coordinates.append((ring.GetX(j), ring.GetY(j)))
# Record entire polygon ring as an Mx2 numpy array
geometry.append(numpy.array(coordinates,
dtype='d',
copy=False))
#elif self.geometry_type == ogr.wkbMultiPolygon:
# # FIXME: Unpact multiple polygons to simple polygons
# # For hints on how to unpack see
#http://osgeo-org.1803224.n2.nabble.com/
#gdal-dev-Shapefile-Multipolygon-with-interior-rings-td5391090.html
# ring = G.GetGeometryRef(0)
# M = ring.GetPointCount()
# coordinates = []
# for j in range(M):
# coordinates.append((ring.GetX(j), ring.GetY(j)))
# # Record entire polygon ring as an Mx2 numpy array
# geometry.append(numpy.array(coordinates,
# dtype='d',
# copy=False))
else:
msg = ('Only point, line and polygon geometries are '
'supported. '
'Geometry type in filename %s '
'was %s.' % (filename,
self.geometry_type))
raise Exception(msg)
# Record attributes by name
number_of_fields = feature.GetFieldCount()
fields = {}
for j in range(number_of_fields):
name = feature.GetFieldDefnRef(j).GetName()
# FIXME (Ole): Ascertain the type of each field?
# We need to cast each appropriately?
# This is issue #66
# (https://github.com/AIFDR/riab/issues/66)
#feature_type = feature.GetFieldDefnRef(j).GetType()
fields[name] = feature.GetField(j)
#print 'Field', name, feature_type, j, fields[name]
data.append(fields)
# Store geometry coordinates as a compact numeric array
self.geometry = geometry
self.data = data
def read_from_file(self, filename):
""" Read and unpack vector data.
It is assumed that the file contains only one layer with the
pertinent features. Further it is assumed for the moment that
all geometries are points.
* A feature is a geometry and a set of attributes.
* A geometry refers to location and can be point, line, polygon or
combinations thereof.
* The attributes or obtained through GetField()
The full OGR architecture is documented at
* http://www.gdal.org/ogr/ogr_arch.html
* http://www.gdal.org/ogr/ogr_apitut.html
Examples are at
* danieljlewis.org/files/2010/09/basicpythonmap.pdf
* http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
* http://www.packtpub.com/article/geospatial-data-python-geometry
"""
basename, _ = os.path.splitext(filename)
# Look for any keywords
self.keywords = read_keywords(basename + '.keywords')
# FIXME (Ole): Should also look for style file to populate style_info
# Determine name
if 'title' in self.keywords:
vectorname = self.keywords['title']
else:
# Use basename without leading directories as name
vectorname = os.path.split(basename)[-1]
self.name = vectorname
self.filename = filename
self.geometry_type = None # In case there are no features
fid = ogr.Open(filename)
if fid is None:
msg = 'Could not open %s' % filename
raise IOError(msg)
# Assume that file contains all data in one layer
msg = 'Only one vector layer currently allowed'
if fid.GetLayerCount() > 1:
msg = ('WARNING: Number of layers in %s are %i. '
'Only the first layer will currently be '
'used.' % (filename, fid.GetLayerCount()))
raise Exception(msg)
layer = fid.GetLayerByIndex(0)
# Get spatial extent
self.extent = layer.GetExtent()
# Get projection
p = layer.GetSpatialRef()
self.projection = Projection(p)
# Get number of features
N = layer.GetFeatureCount()
# Extract coordinates and attributes for all features
geometry = []
data = []
for i in range(N):
feature = layer.GetFeature(i)
if feature is None:
msg = 'Could not get feature %i from %s' % (i, filename)
raise Exception(msg)
# Record coordinates ordered as Longitude, Latitude
G = feature.GetGeometryRef()
if G is None:
msg = ('Geometry was None in filename %s ' % filename)
raise Exception(msg)
else:
self.geometry_type = G.GetGeometryType()
if self.geometry_type == ogr.wkbPoint:
geometry.append((G.GetX(), G.GetY()))
elif self.geometry_type == ogr.wkbLineString:
M = G.GetPointCount()
coordinates = []
for j in range(M):
coordinates.append((G.GetX(j), G.GetY(j)))
# Record entire line as an Mx2 numpy array
geometry.append(
numpy.array(coordinates, dtype='d', copy=False))
elif self.geometry_type == ogr.wkbPolygon:
ring = G.GetGeometryRef(0)
M = ring.GetPointCount()
coordinates = []
for j in range(M):
coordinates.append((ring.GetX(j), ring.GetY(j)))
# Record entire polygon ring as an Mx2 numpy array
geometry.append(
numpy.array(coordinates, dtype='d', copy=False))
#elif self.geometry_type == ogr.wkbMultiPolygon:
# # FIXME: Unpact multiple polygons to simple polygons
# # For hints on how to unpack see
#http://osgeo-org.1803224.n2.nabble.com/
#gdal-dev-Shapefile-Multipolygon-with-interior-rings-td5391090.html
# ring = G.GetGeometryRef(0)
# M = ring.GetPointCount()
# coordinates = []
# for j in range(M):
# coordinates.append((ring.GetX(j), ring.GetY(j)))
# # Record entire polygon ring as an Mx2 numpy array
# geometry.append(numpy.array(coordinates,
# dtype='d',
# copy=False))
else:
msg = ('Only point, line and polygon geometries are '
'supported. '
'Geometry type in filename %s '
'was %s.' % (filename, self.geometry_type))
raise Exception(msg)
# Record attributes by name
number_of_fields = feature.GetFieldCount()
fields = {}
for j in range(number_of_fields):
name = feature.GetFieldDefnRef(j).GetName()
# FIXME (Ole): Ascertain the type of each field?
# We need to cast each appropriately?
# This is issue #66
# (https://github.com/AIFDR/riab/issues/66)
#feature_type = feature.GetFieldDefnRef(j).GetType()
fields[name] = feature.GetField(j)
#print 'Field', name, feature_type, j, fields[name]
data.append(fields)
# Store geometry coordinates as a compact numeric array
self.geometry = geometry
self.data = data
def _clip_raster_layer(layer, extent, cell_size=None, extra_keywords=None):
"""Clip a Hazard or Exposure raster layer to the extents provided.
The layer must be a raster layer or an exception will be thrown.
.. note:: The extent *must* be in EPSG:4326.
The output layer will always be in WGS84/Geographic.
:param layer: A valid QGIS raster layer in EPSG:4326
:type layer: QgsRasterLayer
:param extent: An array representing the exposure layer
extents in the form [xmin, ymin, xmax, ymax]. It is assumed
that the coordinates are in EPSG:4326 although currently
no checks are made to enforce this.
or:
A QgsGeometry of type polygon.
**Polygon clipping currently only supported for vector datasets.**
:type extent: list(float), QgsGeometry
:param cell_size: Cell size (in GeoCRS) which the layer should
be resampled to. If not provided for a raster layer (i.e.
theCellSize=None), the native raster cell size will be used.
:type cell_size: float
:returns: Output clipped layer (placed in the system temp dir).
:rtype: QgsRasterLayer
:raises: InvalidProjectionError - if input layer is a density
layer in projected coordinates. See issue #123.
"""
if not layer or not extent:
message = tr('Layer or Extent passed to clip is None.')
raise InvalidParameterError(message)
if layer.type() != QgsMapLayer.RasterLayer:
message = tr('Expected a raster layer but received a %s.' %
str(layer.type()))
raise InvalidParameterError(message)
working_layer = layer.source()
# Check for existence of keywords file
base, _ = os.path.splitext(working_layer)
keywords_path = base + '.xml'
message = tr('Input file to be clipped "%s" does not have the '
'expected keywords file %s' % (working_layer, keywords_path))
verify(os.path.isfile(keywords_path), message)
# Raise exception if layer is projected and refers to density (issue #123)
# FIXME (Ole): Need to deal with it - e.g. by automatically reprojecting
# the layer at this point and setting the native resolution accordingly
# in its keywords.
try:
keywords = read_iso19115_metadata(working_layer)
except (MetadataReadError, NoKeywordsFoundError):
keywords = read_keywords(base + '.keywords')
keywords = write_read_iso_19115_metadata(working_layer, keywords)
if 'datatype' in keywords and keywords['datatype'] == 'count':
if str(layer.crs().authid()) != 'EPSG:4326':
# This layer is not WGS84 geographic
message = (
'Layer %s represents count but has spatial reference "%s". '
'Count layers must be given in WGS84 geographic coordinates, '
'so please reproject and try again. For more information, see '
'issue https://github.com/AIFDR/inasafe/issues/123' %
(working_layer, layer.crs().toProj4()))
raise InvalidProjectionError(message)
# We need to provide gdalwarp with a dataset for the clip
# because unlike gdal_translate, it does not take projwin.
clip_kml = extent_to_kml(extent)
# Create a filename for the clipped, resampled and reprojected layer
handle, filename = tempfile.mkstemp('.tif', 'clip_', temp_dir())
os.close(handle)
os.remove(filename)
# If no cell size is specified, we need to run gdalwarp without
# specifying the output pixel size to ensure the raster dims
# remain consistent.
binary_list = which('gdalwarp')
LOGGER.debug('Path for gdalwarp: %s' % binary_list)
if len(binary_list) < 1:
raise CallGDALError(tr('gdalwarp could not be found on your computer'))
# Use the first matching gdalwarp found
binary = binary_list[0]
if cell_size is None:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -cutline %s -crop_to_cutline '
'-ot Float64 -of GTiff "%s" "%s"' %
(binary, clip_kml, working_layer, filename))
else:
command = ('"%s" -q -t_srs EPSG:4326 -r near -tr %s %s -cutline %s '
'-crop_to_cutline -ot Float64 -of GTiff "%s" "%s"' %
(binary, repr(cell_size), repr(cell_size), clip_kml,
working_layer, filename))
LOGGER.debug(command)
result = QProcess().execute(command)
# For QProcess exit codes see
# http://qt-project.org/doc/qt-4.8/qprocess.html#execute
if result == -2: # cannot be started
message_detail = tr('Process could not be started.')
message = tr('<p>Error while executing the following shell command:'
'</p><pre>%s</pre><p>Error message: %s' %
(command, message_detail))
raise CallGDALError(message)
elif result == -1: # process crashed
message_detail = tr('Process crashed.')
message = tr(
'<p>Error while executing the following shell command:</p>'
'<pre>%s</pre><p>Error message: %s' % (command, message_detail))
raise CallGDALError(message)
# .. todo:: Check the result of the shell call is ok
keyword_io = KeywordIO()
keyword_io.copy_keywords(layer, filename, extra_keywords=extra_keywords)
base_name = '%s clipped' % layer.name()
layer = QgsRasterLayer(filename, base_name)
return layer
def _clip_raster_layer(
layer, extent, cell_size=None, extra_keywords=None):
"""Clip a Hazard or Exposure raster layer to the extents provided.
The layer must be a raster layer or an exception will be thrown.
.. note:: The extent *must* be in EPSG:4326.
The output layer will always be in WGS84/Geographic.
:param layer: A valid QGIS raster layer in EPSG:4326
:type layer: QgsRasterLayer
:param extent: An array representing the exposure layer
extents in the form [xmin, ymin, xmax, ymax]. It is assumed
that the coordinates are in EPSG:4326 although currently
no checks are made to enforce this.
or:
A QgsGeometry of type polygon.
**Polygon clipping currently only supported for vector datasets.**
:type extent: list(float), QgsGeometry
:param cell_size: Cell size (in GeoCRS) which the layer should
be resampled to. If not provided for a raster layer (i.e.
theCellSize=None), the native raster cell size will be used.
:type cell_size: float
:returns: Output clipped layer (placed in the system temp dir).
:rtype: QgsRasterLayer
:raises: InvalidProjectionError - if input layer is a density
layer in projected coordinates. See issue #123.
"""
if not layer or not extent:
message = tr('Layer or Extent passed to clip is None.')
raise InvalidParameterError(message)
if layer.type() != QgsMapLayer.RasterLayer:
message = tr(
'Expected a raster layer but received a %s.' %
str(layer.type()))
raise InvalidParameterError(message)
working_layer = layer.source()
# Check for existence of keywords file
base, _ = os.path.splitext(working_layer)
keywords_path = base + '.xml'
message = tr(
'Input file to be clipped "%s" does not have the '
'expected keywords file %s' % (
working_layer,
keywords_path
))
verify(os.path.isfile(keywords_path), message)
# Raise exception if layer is projected and refers to density (issue #123)
# FIXME (Ole): Need to deal with it - e.g. by automatically reprojecting
# the layer at this point and setting the native resolution accordingly
# in its keywords.
try:
keywords = read_iso19115_metadata(working_layer)
except (MetadataReadError, NoKeywordsFoundError):
keywords = read_keywords(base + '.keywords')
keywords = write_read_iso_19115_metadata(working_layer, keywords)
if 'datatype' in keywords and keywords['datatype'] == 'count':
if str(layer.crs().authid()) != 'EPSG:4326':
# This layer is not WGS84 geographic
message = (
'Layer %s represents count but has spatial reference "%s". '
'Count layers must be given in WGS84 geographic coordinates, '
'so please reproject and try again. For more information, see '
'issue https://github.com/AIFDR/inasafe/issues/123' % (
working_layer,
layer.crs().toProj4()
))
raise InvalidProjectionError(message)
# We need to provide gdalwarp with a dataset for the clip
# because unlike gdal_translate, it does not take projwin.
clip_kml = extent_to_kml(extent)
# Create a filename for the clipped, resampled and reprojected layer
handle, filename = tempfile.mkstemp('.tif', 'clip_', temp_dir())
os.close(handle)
os.remove(filename)
# If no cell size is specified, we need to run gdalwarp without
# specifying the output pixel size to ensure the raster dims
# remain consistent.
binary_list = which('gdalwarp')
LOGGER.debug('Path for gdalwarp: %s' % binary_list)
if len(binary_list) < 1:
raise CallGDALError(
tr('gdalwarp could not be found on your computer'))
# Use the first matching gdalwarp found
binary = binary_list[0]
if cell_size is None:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -cutline %s -crop_to_cutline '
'-ot Float64 -of GTiff "%s" "%s"' % (
binary,
clip_kml,
working_layer,
filename))
else:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -tr %s %s -cutline %s '
'-crop_to_cutline -ot Float64 -of GTiff "%s" "%s"' % (
binary,
repr(cell_size),
repr(cell_size),
clip_kml,
working_layer,
filename))
LOGGER.debug(command)
result = QProcess().execute(command)
# For QProcess exit codes see
# http://qt-project.org/doc/qt-4.8/qprocess.html#execute
if result == -2: # cannot be started
message_detail = tr('Process could not be started.')
message = tr(
'<p>Error while executing the following shell command:'
'</p><pre>%s</pre><p>Error message: %s'
% (command, message_detail))
raise CallGDALError(message)
elif result == -1: # process crashed
message_detail = tr('Process crashed.')
message = tr(
'<p>Error while executing the following shell command:</p>'
'<pre>%s</pre><p>Error message: %s' % (command, message_detail))
raise CallGDALError(message)
# .. todo:: Check the result of the shell call is ok
keyword_io = KeywordIO()
keyword_io.copy_keywords(layer, filename, extra_keywords=extra_keywords)
base_name = '%s clipped' % layer.name()
layer = QgsRasterLayer(filename, base_name)
return layer
