def test_write_read_iso_19115_metadata(self):
"""Test for write_read_iso_19115_metadata."""
keywords = {
# 'date': '26-03-2015 14:03',
'exposure': 'structure',
'keyword_version': inasafe_keyword_version,
'layer_geometry': 'polygon',
'layer_mode': 'classified',
'layer_purpose': 'exposure',
'license': 'Open Data Commons Open Database License (ODbL)',
'source': 'OpenStreetMap - www.openstreetmap.org',
'title': 'Buildings',
'inasafe_fields': {
'youth_count_field': [
'POP_F_0-4',
'POP_F_5-6',
'POP_F_7-12',
]
}
}
layer = clone_shp_layer(
name='buildings',
include_keywords=False,
source_directory=standard_data_path('exposure'))
write_iso19115_metadata(layer.source(), keywords)
read_metadata = read_iso19115_metadata(layer.source())
self.assertDictEqual(keywords, read_metadata)
# Version 3.5
keywords = {
# 'date': '26-03-2015 14:03',
'exposure': 'structure',
'keyword_version': '3.2',
'layer_geometry': 'polygon',
'layer_mode': 'classified',
'layer_purpose': 'exposure',
'license': 'Open Data Commons Open Database License (ODbL)',
'source': 'OpenStreetMap - www.openstreetmap.org',
'structure_class_field': 'TYPE',
'title': 'Buildings'
}
layer = clone_shp_layer(
name='buildings',
include_keywords=False,
source_directory=standard_data_path('exposure'))
write_iso19115_metadata(layer.source(), keywords, version_35=True)
read_metadata = read_iso19115_metadata(layer.source(), version_35=True)
self.assertDictEqual(keywords, read_metadata)
def test_write_read_iso_19115_metadata(self):
"""Test for write_read_iso_19115_metadata."""
keywords = {
# 'date': '26-03-2015 14:03',
'exposure': 'structure',
'keyword_version': inasafe_keyword_version,
'layer_geometry': 'polygon',
'layer_mode': 'classified',
'layer_purpose': 'exposure',
'license': 'Open Data Commons Open Database License (ODbL)',
'source': 'OpenStreetMap - www.openstreetmap.org',
'title': 'Buildings',
'inasafe_fields': {
'youth_count_field': [
'POP_F_0-4', 'POP_F_5-6', 'POP_F_7-12',
]
}
}
layer = clone_shp_layer(
name='buildings',
include_keywords=False,
source_directory=standard_data_path('exposure'))
write_iso19115_metadata(layer.source(), keywords)
read_metadata = read_iso19115_metadata(layer.source())
self.assertDictEqual(keywords, read_metadata)
def test_read_iso19115_metadata(self):
"""Test for read_iso19115_metadata method."""
exposure_layer = clone_shp_layer(
name='buildings',
include_keywords=False,
source_directory=standard_data_path('exposure'))
keywords = {
# 'date': '26-03-2015 14:03',
'exposure': 'structure',
'keyword_version': inasafe_keyword_version,
'layer_geometry': 'polygon',
'layer_mode': 'classified',
'layer_purpose': 'exposure',
'license': 'Open Data Commons Open Database License (ODbL)',
'source': 'OpenStreetMap - www.openstreetmap.org',
'title': 'Buildings'
}
write_iso19115_metadata(exposure_layer.source(), keywords)
read_metadata = read_iso19115_metadata(exposure_layer.source())
for key in set(keywords.keys()) & set(read_metadata.keys()):
self.assertEqual(read_metadata[key], keywords[key])
for key in set(keywords.keys()) - set(read_metadata.keys()):
message = 'key %s is not found in ISO metadata' % key
self.assertEqual(read_metadata[key], keywords[key], message)
for key in set(read_metadata.keys()) - set(keywords.keys()):
message = 'key %s is not found in old keywords' % key
self.assertEqual(read_metadata[key], keywords[key], message)
def read_keywords(layer, keyword=None):
"""Read keywords for a datasource and return them as a dictionary.
This is a wrapper method that will 'do the right thing' to fetch
keywords for the given datasource. In particular, if the datasource
is remote (e.g. a database connection) it will fetch the keywords from
the keywords store.
:param layer: A QGIS QgsMapLayer instance that you want to obtain
the keywords for.
:type layer: QgsMapLayer, QgsRasterLayer, QgsVectorLayer,
QgsPluginLayer
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
TODO: Don't raise generic exceptions.
:raises: HashNotFoundError, Exception, OperationalError,
NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError,
UnsupportedProviderError
"""
source = layer.source()
# Try to read from ISO metadata first.
return read_iso19115_metadata(source, keyword)
def read_keywords(layer, keyword=None):
"""Read keywords for a datasource and return them as a dictionary.
This is a wrapper method that will 'do the right thing' to fetch
keywords for the given datasource. In particular, if the datasource
is remote (e.g. a database connection) it will fetch the keywords from
the keywords store.
:param layer: A QGIS QgsMapLayer instance that you want to obtain
the keywords for.
:type layer: QgsMapLayer, QgsRasterLayer, QgsVectorLayer,
QgsPluginLayer
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
TODO: Don't raise generic exceptions.
:raises: HashNotFoundError, Exception, OperationalError,
NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError,
UnsupportedProviderError
"""
source = layer.source()
# Try to read from ISO metadata first.
return read_iso19115_metadata(source, keyword)
def test_read_iso19115_metadata(self):
"""Test for read_iso19115_metadata method."""
exposure_layer = clone_shp_layer(
name='buildings',
include_keywords=False,
source_directory=standard_data_path('exposure'))
keywords = {
# 'date': '26-03-2015 14:03',
'exposure': 'structure',
'keyword_version': inasafe_keyword_version,
'layer_geometry': 'polygon',
'layer_mode': 'classified',
'layer_purpose': 'exposure',
'license': 'Open Data Commons Open Database License (ODbL)',
'source': 'OpenStreetMap - www.openstreetmap.org',
'title': 'Buildings'
}
write_iso19115_metadata(exposure_layer.source(), keywords)
read_metadata = read_iso19115_metadata(exposure_layer.source())
for key in set(keywords.keys()) & set(read_metadata.keys()):
self.assertEqual(read_metadata[key], keywords[key])
for key in set(keywords.keys()) - set(read_metadata.keys()):
message = 'key %s is not found in ISO metadata' % key
self.assertEqual(read_metadata[key], keywords[key], message)
for key in set(read_metadata.keys()) - set(keywords.keys()):
message = 'key %s is not found in old keywords' % key
self.assertEqual(read_metadata[key], keywords[key], message)
def read_keywords_file(cls, filename, keyword=None):
"""Read keywords from a keywords file and return as dictionary
This serves as a wrapper function that should be provided by Keyword
IO. Use this if you are sure that the filename is a keyword file.
:param filename: The filename of the keyword, typically with .xml or
.keywords extension. If not, will raise exceptions
:type filename: str
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
:raises: KeywordNotFoundError, InvalidParameterError
"""
# Try to read from ISO metadata first.
_, ext = os.path.splitext(filename)
dictionary = {}
if ext == '.xml':
try:
dictionary = read_iso19115_metadata(filename)
except (MetadataReadError, NoKeywordsFoundError):
pass
elif ext == '.keywords':
try:
dictionary = read_file_keywords(filename)
# update to xml based metadata
write_read_iso_19115_metadata(filename, dictionary)
except (HashNotFoundError,
Exception,
OperationalError,
NoKeywordsFoundError,
KeywordNotFoundError,
InvalidParameterError,
UnsupportedProviderError):
raise
else:
raise InvalidParameterError(
'Keywords file have .xml or .keywords extension')
# if no keyword was supplied, just return the dict
if keyword is None:
return dictionary
if keyword not in dictionary:
message = tr('No value was found in file %s for keyword %s' % (
filename, keyword))
raise KeywordNotFoundError(message)
return dictionary[keyword]
def read_keywords_file(cls, filename, keyword=None):
"""Read keywords from a keywords file and return as dictionary
This serves as a wrapper function that should be provided by Keyword
IO. Use this if you are sure that the filename is a keyword file.
:param filename: The filename of the keyword, typically with .xml or
.keywords extension. If not, will raise exceptions
:type filename: str
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
:raises: KeywordNotFoundError, InvalidParameterError
"""
# Try to read from ISO metadata first.
_, ext = os.path.splitext(filename)
dictionary = {}
if ext == '.xml':
try:
dictionary = read_iso19115_metadata(filename)
except (MetadataReadError, NoKeywordsFoundError):
pass
elif ext == '.keywords':
try:
dictionary = read_file_keywords(filename)
# update to xml based metadata
write_read_iso_19115_metadata(filename, dictionary)
except (HashNotFoundError,
Exception,
OperationalError,
NoKeywordsFoundError,
KeywordNotFoundError,
InvalidParameterError,
UnsupportedProviderError):
raise
else:
raise InvalidParameterError(
'Keywords file have .xml or .keywords extension')
# if no keyword was supplied, just return the dict
if keyword is None:
return dictionary
if keyword not in dictionary:
message = tr('No value was found in file %s for keyword %s' % (
filename, keyword))
raise KeywordNotFoundError(message)
return dictionary[keyword]
def test_mmi_to_raster(self):
"""Check we can convert the shake event to a raster."""
# Check the tif file
raster_path = SMOOTHED_SHAKE_GRID.mmi_to_raster(force_flag=True)
self.assertTrue(os.path.exists(raster_path))
# Check the qml file
expected_qml = raster_path.replace('tif', 'qml')
self.assertTrue(os.path.exists(expected_qml))
# Check the keywords file
expected_keywords = raster_path.replace('tif', 'xml')
self.assertTrue(os.path.exists(expected_keywords))
# Check that extra_keywords exists
keywords = read_iso19115_metadata(raster_path)
self.assertIn('extra_keywords', keywords.keys())
def test_mmi_to_raster(self):
"""Check we can convert the shake event to a raster."""
# Check the tif file
raster_path = SMOOTHED_SHAKE_GRID.mmi_to_raster(force_flag=True)
self.assertTrue(os.path.exists(raster_path))
# Check the qml file
expected_qml = raster_path.replace('tif', 'qml')
self.assertTrue(os.path.exists(expected_qml))
# Check the keywords file
expected_keywords = raster_path.replace('tif', 'xml')
self.assertTrue(os.path.exists(expected_keywords))
# Check that extra_keywords exists
keywords = read_iso19115_metadata(raster_path)
self.assertIn('extra_keywords', list(keywords.keys()))
def test_copy_keywords(self):
"""Test we can copy the keywords."""
out_path = unique_filename(prefix='test_copy_keywords', suffix='.shp')
layer = clone_raster_layer(name='generic_continuous_flood',
extension='.asc',
include_keywords=True,
source_directory=test_data_path('hazard'))
self.keyword_io.copy_keywords(layer, out_path)
# copied_keywords = read_file_keywords(out_path.split('.')[0] + 'xml')
copied_keywords = read_iso19115_metadata(out_path)
expected_keywords = self.expected_raster_keywords
expected_keywords['keyword_version'] = inasafe_keyword_version
self.maxDiff = None
self.assertDictEqual(copied_keywords, expected_keywords)
def load_layer(layer_path):
"""Helper to load and return a single QGIS layer
:param layer_path: Path name to raster or vector file.
:type layer_path: str
:returns: tuple containing layer and its layer_purpose.
:rtype: (QgsMapLayer, str)
"""
# Extract basename and absolute path
file_name = os.path.split(layer_path)[-1] # In case path was absolute
base_name, extension = os.path.splitext(file_name)
# Determine if layer is hazard or exposure
layer_purpose = 'undefined'
try:
try:
keywords = read_iso19115_metadata(layer_path)
except:
try:
keywords = read_file_keywords(layer_path)
keywords = write_read_iso_19115_metadata(layer_path, keywords)
except NoKeywordsFoundError:
keywords = {}
if 'layer_purpose' in keywords:
layer_purpose = keywords['layer_purpose']
except NoKeywordsFoundError:
pass
# Create QGis Layer Instance
if extension in ['.asc', '.tif']:
layer = QgsRasterLayer(layer_path, base_name)
elif extension in ['.shp']:
layer = QgsVectorLayer(layer_path, base_name, 'ogr')
else:
message = 'File %s had illegal extension' % layer_path
raise Exception(message)
# noinspection PyUnresolvedReferences
message = 'Layer "%s" is not valid' % layer.source()
# noinspection PyUnresolvedReferences
if not layer.isValid():
print message
# noinspection PyUnresolvedReferences
if not layer.isValid():
raise Exception(message)
return layer, layer_purpose
def load_layer(layer_path):
"""Helper to load and return a single QGIS layer
:param layer_path: Path name to raster or vector file.
:type layer_path: str
:returns: tuple containing layer and its layer_purpose.
:rtype: (QgsMapLayer, str)
"""
# Extract basename and absolute path
file_name = os.path.split(layer_path)[-1] # In case path was absolute
base_name, extension = os.path.splitext(file_name)
# Determine if layer is hazard or exposure
layer_purpose = 'undefined'
try:
try:
keywords = read_iso19115_metadata(layer_path)
except:
try:
keywords = read_file_keywords(layer_path)
keywords = write_read_iso_19115_metadata(layer_path, keywords)
except NoKeywordsFoundError:
keywords = {}
if 'layer_purpose' in keywords:
layer_purpose = keywords['layer_purpose']
except NoKeywordsFoundError:
pass
# Create QGis Layer Instance
if extension in ['.asc', '.tif']:
layer = QgsRasterLayer(layer_path, base_name)
elif extension in ['.shp']:
layer = QgsVectorLayer(layer_path, base_name, 'ogr')
else:
message = 'File %s had illegal extension' % layer_path
raise Exception(message)
# noinspection PyUnresolvedReferences
message = 'Layer "%s" is not valid' % layer.source()
# noinspection PyUnresolvedReferences
if not layer.isValid():
print message
# noinspection PyUnresolvedReferences
if not layer.isValid():
raise Exception(message)
return layer, layer_purpose
def test_copy_keywords(self):
"""Test we can copy the keywords."""
self.maxDiff = None
out_path = unique_filename(
prefix='test_copy_keywords', suffix='.shp')
layer = clone_raster_layer(
name='generic_continuous_flood',
extension='.asc',
include_keywords=True,
source_directory=standard_data_path('hazard'))
self.keyword_io.copy_keywords(layer, out_path)
# copied_keywords = read_file_keywords(out_path.split('.')[0] + 'xml')
copied_keywords = read_iso19115_metadata(out_path)
expected_keywords = self.expected_raster_keywords
expected_keywords['keyword_version'] = inasafe_keyword_version
self.assertDictEqual(copied_keywords, expected_keywords)
def load_layer(layer_path):
"""Helper to load and return a single QGIS layer
:param layer_path: Path name to raster or vector file.
:type layer_path: str
:returns: tuple containing layer and its layer_purpose.
:rtype: (QgsMapLayer, str)
"""
# Extract basename and absolute path
file_name = os.path.split(layer_path)[-1] # In case path was absolute
base_name, extension = os.path.splitext(file_name)
# Determine if layer is hazard or exposure
layer_purpose = "undefined"
try:
keywords = read_iso19115_metadata(layer_path)
if "layer_purpose" in keywords:
layer_purpose = keywords["layer_purpose"]
except NoKeywordsFoundError:
pass
# Create QGis Layer Instance
if extension in [".asc", ".tif"]:
layer = QgsRasterLayer(layer_path, base_name)
elif extension in [".shp"]:
layer = QgsVectorLayer(layer_path, base_name, "ogr")
else:
message = "File %s had illegal extension" % layer_path
raise Exception(message)
# noinspection PyUnresolvedReferences
message = 'Layer "%s" is not valid' % layer.source()
# noinspection PyUnresolvedReferences
if not layer.isValid():
LOGGER.log(message)
# noinspection PyUnresolvedReferences
if not layer.isValid():
raise Exception(message)
return layer, layer_purpose
def read_keywords_iso_metadata(metadata_url, keyword=None):
"""Read xml metadata of a layer
:param keyword: Can be string or tuple containing keywords to search for
:type keyword: str, (str, )
:return: the keywords, or a dictionary with key-value pair
"""
filename = download_file(metadata_url)
# add xml extension
new_filename = filename + '.xml'
shutil.move(filename, new_filename)
keywords = read_iso19115_metadata(new_filename)
if keyword:
if isinstance(keyword, tuple) or isinstance(keyword, list):
ret_val = {}
for key in keyword:
ret_val[key] = keywords.get(key, None)
return ret_val
else:
return keywords.get(keyword, None)
return keywords
def read_keywords_iso_metadata(metadata_url, keyword=None):
"""Read xml metadata of a layer
:param keyword: Can be string or tuple containing keywords to search for
:type keyword: str, (str, )
:return: the keywords, or a dictionary with key-value pair
"""
filename = download_file(metadata_url)
# add xml extension
new_filename = filename + '.xml'
shutil.move(filename, new_filename)
keywords = read_iso19115_metadata(new_filename)
if keyword:
if isinstance(keyword, tuple) or isinstance(keyword, list):
ret_val = {}
for key in keyword:
ret_val[key] = keywords.get(key, None)
return ret_val
else:
return keywords.get(keyword, None)
return keywords
def write_to_file(self, filename, sublayer=None):
"""Save vector data to file
:param filename: filename with extension .shp or .gml
:type filename: str
:param sublayer: Optional parameter for writing a sublayer. Ignored
unless we are writing to an sqlite file.
:type sublayer: str
:raises: WriteLayerError
Note:
Shp limitation, if attribute names are longer than 10
characters they will be truncated. This is due to limitations in
the shp file driver and has to be done here since gdal v1.7 onwards
has changed its handling of this issue:
http://www.gdal.org/ogr/drv_shapefile.html
**For this reason we recommend writing to spatialite.**
"""
# Check file format
base_name, extension = os.path.splitext(filename)
msg = ('Invalid file type for file %s. Only extensions '
'sqlite, shp or gml allowed.' % filename)
verify(extension in ['.sqlite', '.shp', '.gml'], msg)
driver = DRIVER_MAP[extension]
# FIXME (Ole): Tempory flagging of GML issue (ticket #18)
if extension == '.gml':
msg = ('OGR GML driver does not store geospatial reference.'
'This format is disabled for the time being. See '
'https://github.com/AIFDR/riab/issues/18')
raise WriteLayerError(msg)
# Derive layer_name from filename (excluding preceding dirs)
if sublayer is None or extension == '.shp':
layer_name = os.path.split(base_name)[-1]
else:
layer_name = sublayer
# Get vector data
if self.is_polygon_data:
geometry = self.get_geometry(as_geometry_objects=True)
else:
geometry = self.get_geometry()
data = self.get_data()
N = len(geometry)
# Clear any previous file of this name (ogr does not overwrite)
try:
os.remove(filename)
except OSError:
pass
# Create new file with one layer
drv = ogr.GetDriverByName(driver)
if drv is None:
msg = 'OGR driver %s not available' % driver
raise WriteLayerError(msg)
ds = drv.CreateDataSource(get_string(filename))
if ds is None:
msg = 'Creation of output file %s failed' % filename
raise WriteLayerError(msg)
lyr = ds.CreateLayer(
get_string(layer_name),
self.projection.spatial_reference,
self.geometry_type)
if lyr is None:
msg = 'Could not create layer %s' % layer_name
raise WriteLayerError(msg)
# Define attributes if any
store_attributes = False
fields = []
if data is not None:
if len(data) > 0:
try:
fields = data[0].keys()
except:
msg = ('Input parameter "attributes" was specified '
'but it does not contain list of dictionaries '
'with field information as expected. The first '
'element is %s' % data[0])
raise WriteLayerError(msg)
else:
# Establish OGR types for each element
ogr_types = {}
for name in fields:
att = data[0][name]
py_type = type(att)
msg = ('Unknown type for storing vector '
'data: %s, %s' % (name, str(py_type)[1:-1]))
verify(py_type in TYPE_MAP, msg)
ogr_types[name] = TYPE_MAP[py_type]
else:
# msg = ('Input parameter "data" was specified '
# 'but appears to be empty')
# raise InaSAFEError(msg)
pass
# Create attribute fields in layer
store_attributes = True
for name in fields:
# Rizky : OGR can't handle unicode field name, thus we
# convert it to ASCII
fd = ogr.FieldDefn(str(name), ogr_types[name])
# FIXME (Ole): Trying to address issue #16
# But it doesn't work and
# somehow changes the values of MMI in test
# width = max(128, len(name))
# print name, width
# fd.SetWidth(width)
# Silent handling of warnings like
# Warning 6: Normalized/laundered field name:
# 'CONTENTS_LOSS_AUD' to 'CONTENTS_L'
gdal.PushErrorHandler('CPLQuietErrorHandler')
if lyr.CreateField(fd) != 0:
msg = 'Could not create field %s' % name
raise WriteLayerError(msg)
# Restore error handler
gdal.PopErrorHandler()
# Store geometry
geom = ogr.Geometry(self.geometry_type)
layer_def = lyr.GetLayerDefn()
for i in range(N):
# Create new feature instance
feature = ogr.Feature(layer_def)
# Store geometry and check
if self.is_point_data:
x = float(geometry[i][0])
y = float(geometry[i][1])
geom.SetPoint_2D(0, x, y)
elif self.is_line_data:
geom = array_to_line(
geometry[i], geometry_type=ogr.wkbLineString)
elif self.is_polygon_data:
# Create polygon geometry
geom = ogr.Geometry(ogr.wkbPolygon)
# Add outer ring
linear_ring = array_to_line(
geometry[i].outer_ring, geometry_type=ogr.wkbLinearRing)
geom.AddGeometry(linear_ring)
# Add inner rings if any
for A in geometry[i].inner_rings:
geom.AddGeometry(array_to_line(
A, geometry_type=ogr.wkbLinearRing))
else:
msg = 'Geometry type %s not implemented' % self.geometry_type
raise WriteLayerError(msg)
feature.SetGeometry(geom)
G = feature.GetGeometryRef()
if G is None:
msg = 'Could not create GeometryRef for file %s' % filename
raise WriteLayerError(msg)
# Store attributes
if store_attributes:
for j, name in enumerate(fields):
actual_field_name = layer_def.GetFieldDefn(j).GetNameRef()
val = data[i][name]
if isinstance(val, numpy.ndarray):
# A singleton of type <type 'numpy.ndarray'> works
# for gdal version 1.6 but fails for version 1.8
# in SetField with error: NotImplementedError:
# Wrong number of arguments for overloaded function
val = float(val)
elif val is None:
val = ''
# We do this because there is NaN problem on windows
# NaN value must be converted to _pseudo_in to solve the
# problem. But, when InaSAFE read the file, it'll be
# converted back to NaN value, so that NaN in InaSAFE is a
# numpy.nan
# please check https://github.com/AIFDR/inasafe/issues/269
# for more information
if val != val:
val = _pseudo_inf
feature.SetField(actual_field_name, val)
# Save this feature
if lyr.CreateFeature(feature) != 0:
msg = 'Failed to create feature %i in file %s' % (i, filename)
raise WriteLayerError(msg)
feature.Destroy()
# Write keywords if any
# write_keywords(self.keywords, base_name + '.keywords')
write_iso19115_metadata(filename, self.keywords)
self.keywords = read_iso19115_metadata(filename)
def read_from_file(self, filename):
"""Read and unpack vector data.
It is assumed that the file contains only one layer with the
pertinent features. Further it is assumed for the moment that
all geometries are points.
* A feature is a geometry and a set of attributes.
* A geometry refers to location and can be point, line, polygon or
combinations thereof.
* The attributes or obtained through GetField()
The full OGR architecture is documented at
* http://www.gdal.org/ogr/ogr_arch.html
* http://www.gdal.org/ogr/ogr_apitut.html
Examples are at
* danieljlewis.org/files/2010/09/basicpythonmap.pdf
* http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
* http://www.packtpub.com/article/geospatial-data-python-geometry
Limitation of the Shapefile are documented in
http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/
geoprocessing_tool_reference/
geoprocessing_considerations_for_shapefile_output.htm
:param filename: a fully qualified location to the file
:type filename: str
:raises: ReadLayerError
"""
base_name = os.path.splitext(filename)[0]
# Look for any keywords
try:
self.keywords = read_iso19115_metadata(filename)
except (NoKeywordsFoundError, MetadataReadError):
keywords = read_keywords(base_name + '.keywords')
self.keywords = write_read_iso_19115_metadata(filename, keywords)
# FIXME (Ole): Should also look for style file to populate style_info
# Determine name
if 'title' in self.keywords:
title = self.keywords['title']
# Lookup internationalised title if available
title = tr(title)
vector_name = title
else:
# Use base_name without leading directories as name
vector_name = os.path.split(base_name)[-1]
if self.name is None:
self.name = vector_name
self.filename = filename
self.geometry_type = None # In case there are no features
fid = ogr.Open(filename)
if fid is None:
msg = 'Could not open %s' % filename
raise ReadLayerError(msg)
# Assume that file contains all data in one layer
msg = 'Only one vector layer currently allowed'
if fid.GetLayerCount() > 1 and self.sublayer is None:
msg = ('WARNING: Number of layers in %s are %i. '
'Only the first layer will currently be '
'used. Specify sublayer when creating '
'the Vector if you wish to use a different layer.'
% (filename, fid.GetLayerCount()))
LOGGER.warn(msg)
# Why do we raise an exception if it is only a warning? TS
raise ReadLayerError(msg)
if self.sublayer is not None:
layer = fid.GetLayerByName(self.sublayer)
else:
layer = fid.GetLayerByIndex(0)
# Get spatial extent
self.extent = layer.GetExtent()
# Get projection
p = layer.GetSpatialRef()
self.projection = Projection(p)
layer.ResetReading()
# Extract coordinates and attributes for all features
geometry = []
data = []
# Use feature iterator
for feature in layer:
# Record coordinates ordered as Longitude, Latitude
G = feature.GetGeometryRef()
if G is None:
msg = ('Geometry was None in filename %s ' % filename)
raise ReadLayerError(msg)
else:
self.geometry_type = G.GetGeometryType()
if self.is_point_data:
geometry.append((G.GetX(), G.GetY()))
elif self.is_line_data:
ring = get_ring_data(G)
geometry.append(ring)
elif self.is_polygon_data:
polygon = get_polygon_data(G)
geometry.append(polygon)
elif self.is_multi_polygon_data:
try:
G = ogr.ForceToPolygon(G)
except:
msg = ('Got geometry type Multipolygon (%s) for '
'filename %s and could not convert it to '
'singlepart. However, you can use QGIS '
'functionality to convert multipart vector '
'data to singlepart (Vector -> Geometry Tools '
'-> Multipart to Singleparts and use the '
'resulting dataset.'
% (ogr.wkbMultiPolygon, filename))
raise ReadLayerError(msg)
else:
# Read polygon data as single part
self.geometry_type = ogr.wkbPolygon
polygon = get_polygon_data(G)
geometry.append(polygon)
else:
msg = ('Only point, line and polygon geometries are '
'supported. '
'Geometry type in filename %s '
'was %s.' % (filename,
self.geometry_type))
raise ReadLayerError(msg)
# Record attributes by name
number_of_fields = feature.GetFieldCount()
fields = {}
for j in range(number_of_fields):
name = feature.GetFieldDefnRef(j).GetName()
# FIXME (Ole): Ascertain the type of each field?
# We need to cast each appropriately?
# This is issue #66
# (https://github.com/AIFDR/riab/issues/66)
# feature_type = feature.GetFieldDefnRef(j).GetType()
fields[name] = feature.GetField(j)
# We do this because there is NaN problem on windows
# NaN value must be converted to _pseudo_in to solve the
# problem. But, when InaSAFE read the file, it'll be
# converted back to NaN value, so that NaN in InaSAFE is a
# numpy.nan
# please check https://github.com/AIFDR/inasafe/issues/269
# for more information
if fields[name] == _pseudo_inf:
fields[name] = float('nan')
# print 'Field', name, feature_type, j, fields[name]
data.append(fields)
# Store geometry coordinates as a compact numeric array
self.geometry = geometry
self.data = data
def write_to_file(self, filename, sublayer=None):
"""Save vector data to file
:param filename: filename with extension .shp or .gml
:type filename: str
:param sublayer: Optional parameter for writing a sublayer. Ignored
unless we are writing to an sqlite file.
:type sublayer: str
:raises: WriteLayerError
Note:
Shp limitation, if attribute names are longer than 10
characters they will be truncated. This is due to limitations in
the shp file driver and has to be done here since gdal v1.7 onwards
has changed its handling of this issue:
http://www.gdal.org/ogr/drv_shapefile.html
**For this reason we recommend writing to spatialite.**
"""
# Check file format
base_name, extension = os.path.splitext(filename)
msg = ('Invalid file type for file %s. Only extensions '
'sqlite, shp or gml allowed.' % filename)
verify(extension in ['.sqlite', '.shp', '.gml'], msg)
driver = DRIVER_MAP[extension]
# FIXME (Ole): Tempory flagging of GML issue (ticket #18)
if extension == '.gml':
msg = ('OGR GML driver does not store geospatial reference.'
'This format is disabled for the time being. See '
'https://github.com/AIFDR/riab/issues/18')
raise WriteLayerError(msg)
# Derive layer_name from filename (excluding preceding dirs)
if sublayer is None or extension == '.shp':
layer_name = os.path.split(base_name)[-1]
else:
layer_name = sublayer
# Get vector data
if self.is_polygon_data:
geometry = self.get_geometry(as_geometry_objects=True)
else:
geometry = self.get_geometry()
data = self.get_data()
N = len(geometry)
# Clear any previous file of this name (ogr does not overwrite)
try:
os.remove(filename)
except OSError:
pass
# Create new file with one layer
drv = ogr.GetDriverByName(driver)
if drv is None:
msg = 'OGR driver %s not available' % driver
raise WriteLayerError(msg)
ds = drv.CreateDataSource(get_string(filename))
if ds is None:
msg = 'Creation of output file %s failed' % filename
raise WriteLayerError(msg)
lyr = ds.CreateLayer(get_string(layer_name),
self.projection.spatial_reference,
self.geometry_type)
if lyr is None:
msg = 'Could not create layer %s' % layer_name
raise WriteLayerError(msg)
# Define attributes if any
store_attributes = False
fields = []
if data is not None:
if len(data) > 0:
try:
fields = data[0].keys()
except:
msg = ('Input parameter "attributes" was specified '
'but it does not contain list of dictionaries '
'with field information as expected. The first '
'element is %s' % data[0])
raise WriteLayerError(msg)
else:
# Establish OGR types for each element
ogr_types = {}
for name in fields:
att = data[0][name]
py_type = type(att)
msg = ('Unknown type for storing vector '
'data: %s, %s' % (name, str(py_type)[1:-1]))
verify(py_type in TYPE_MAP, msg)
ogr_types[name] = TYPE_MAP[py_type]
else:
# msg = ('Input parameter "data" was specified '
# 'but appears to be empty')
# raise InaSAFEError(msg)
pass
# Create attribute fields in layer
store_attributes = True
for name in fields:
# Rizky : OGR can't handle unicode field name, thus we
# convert it to ASCII
fd = ogr.FieldDefn(str(name), ogr_types[name])
# FIXME (Ole): Trying to address issue #16
# But it doesn't work and
# somehow changes the values of MMI in test
# width = max(128, len(name))
# print name, width
# fd.SetWidth(width)
# Silent handling of warnings like
# Warning 6: Normalized/laundered field name:
# 'CONTENTS_LOSS_AUD' to 'CONTENTS_L'
gdal.PushErrorHandler('CPLQuietErrorHandler')
if lyr.CreateField(fd) != 0:
msg = 'Could not create field %s' % name
raise WriteLayerError(msg)
# Restore error handler
gdal.PopErrorHandler()
# Store geometry
geom = ogr.Geometry(self.geometry_type)
layer_def = lyr.GetLayerDefn()
for i in range(N):
# Create new feature instance
feature = ogr.Feature(layer_def)
# Store geometry and check
if self.is_point_data:
x = float(geometry[i][0])
y = float(geometry[i][1])
geom.SetPoint_2D(0, x, y)
elif self.is_line_data:
geom = array_to_line(geometry[i],
geometry_type=ogr.wkbLineString)
elif self.is_polygon_data:
# Create polygon geometry
geom = ogr.Geometry(ogr.wkbPolygon)
# Add outer ring
linear_ring = array_to_line(geometry[i].outer_ring,
geometry_type=ogr.wkbLinearRing)
geom.AddGeometry(linear_ring)
# Add inner rings if any
for A in geometry[i].inner_rings:
geom.AddGeometry(
array_to_line(A, geometry_type=ogr.wkbLinearRing))
else:
msg = 'Geometry type %s not implemented' % self.geometry_type
raise WriteLayerError(msg)
feature.SetGeometry(geom)
G = feature.GetGeometryRef()
if G is None:
msg = 'Could not create GeometryRef for file %s' % filename
raise WriteLayerError(msg)
# Store attributes
if store_attributes:
for j, name in enumerate(fields):
actual_field_name = layer_def.GetFieldDefn(j).GetNameRef()
val = data[i][name]
if isinstance(val, numpy.ndarray):
# A singleton of type <type 'numpy.ndarray'> works
# for gdal version 1.6 but fails for version 1.8
# in SetField with error: NotImplementedError:
# Wrong number of arguments for overloaded function
val = float(val)
elif val is None:
val = ''
# We do this because there is NaN problem on windows
# NaN value must be converted to _pseudo_in to solve the
# problem. But, when InaSAFE read the file, it'll be
# converted back to NaN value, so that NaN in InaSAFE is a
# numpy.nan
# please check https://github.com/AIFDR/inasafe/issues/269
# for more information
if val != val:
val = _pseudo_inf
feature.SetField(actual_field_name, val)
# Save this feature
if lyr.CreateFeature(feature) != 0:
msg = 'Failed to create feature %i in file %s' % (i, filename)
raise WriteLayerError(msg)
feature.Destroy()
# Write keywords if any
# write_keywords(self.keywords, base_name + '.keywords')
write_iso19115_metadata(filename, self.keywords)
self.keywords = read_iso19115_metadata(filename)
def read_from_file(self, filename):
"""Read and unpack vector data.
It is assumed that the file contains only one layer with the
pertinent features. Further it is assumed for the moment that
all geometries are points.
* A feature is a geometry and a set of attributes.
* A geometry refers to location and can be point, line, polygon or
combinations thereof.
* The attributes or obtained through GetField()
The full OGR architecture is documented at
* http://www.gdal.org/ogr/ogr_arch.html
* http://www.gdal.org/ogr/ogr_apitut.html
Examples are at
* danieljlewis.org/files/2010/09/basicpythonmap.pdf
* http://invisibleroads.com/tutorials/gdal-shapefile-points-save.html
* http://www.packtpub.com/article/geospatial-data-python-geometry
Limitation of the Shapefile are documented in
http://resources.esri.com/help/9.3/ArcGISDesktop/com/Gp_ToolRef/
geoprocessing_tool_reference/
geoprocessing_considerations_for_shapefile_output.htm
:param filename: a fully qualified location to the file
:type filename: str
:raises: ReadLayerError
"""
base_name = os.path.splitext(filename)[0]
# Look for any keywords
try:
self.keywords = read_iso19115_metadata(filename)
except (NoKeywordsFoundError, MetadataReadError):
keywords = read_keywords(base_name + '.keywords')
self.keywords = write_read_iso_19115_metadata(filename, keywords)
# FIXME (Ole): Should also look for style file to populate style_info
# Determine name
if 'title' in self.keywords:
title = self.keywords['title']
# Lookup internationalised title if available
title = tr(title)
vector_name = title
else:
# Use base_name without leading directories as name
vector_name = os.path.split(base_name)[-1]
if self.name is None:
self.name = vector_name
self.filename = filename
self.geometry_type = None # In case there are no features
fid = ogr.Open(filename)
if fid is None:
msg = 'Could not open %s' % filename
raise ReadLayerError(msg)
# Assume that file contains all data in one layer
msg = 'Only one vector layer currently allowed'
if fid.GetLayerCount() > 1 and self.sublayer is None:
msg = ('WARNING: Number of layers in %s are %i. '
'Only the first layer will currently be '
'used. Specify sublayer when creating '
'the Vector if you wish to use a different layer.' %
(filename, fid.GetLayerCount()))
LOGGER.warn(msg)
# Why do we raise an exception if it is only a warning? TS
raise ReadLayerError(msg)
if self.sublayer is not None:
layer = fid.GetLayerByName(self.sublayer)
else:
layer = fid.GetLayerByIndex(0)
# Get spatial extent
self.extent = layer.GetExtent()
# Get projection
p = layer.GetSpatialRef()
self.projection = Projection(p)
layer.ResetReading()
# Extract coordinates and attributes for all features
geometry = []
data = []
# Use feature iterator
for feature in layer:
# Record coordinates ordered as Longitude, Latitude
G = feature.GetGeometryRef()
if G is None:
msg = ('Geometry was None in filename %s ' % filename)
raise ReadLayerError(msg)
else:
self.geometry_type = G.GetGeometryType()
if self.is_point_data:
geometry.append((G.GetX(), G.GetY()))
elif self.is_line_data:
ring = get_ring_data(G)
geometry.append(ring)
elif self.is_polygon_data:
polygon = get_polygon_data(G)
geometry.append(polygon)
elif self.is_multi_polygon_data:
try:
G = ogr.ForceToPolygon(G)
except:
msg = ('Got geometry type Multipolygon (%s) for '
'filename %s and could not convert it to '
'singlepart. However, you can use QGIS '
'functionality to convert multipart vector '
'data to singlepart (Vector -> Geometry Tools '
'-> Multipart to Singleparts and use the '
'resulting dataset.' %
(ogr.wkbMultiPolygon, filename))
raise ReadLayerError(msg)
else:
# Read polygon data as single part
self.geometry_type = ogr.wkbPolygon
polygon = get_polygon_data(G)
geometry.append(polygon)
else:
msg = ('Only point, line and polygon geometries are '
'supported. '
'Geometry type in filename %s '
'was %s.' % (filename, self.geometry_type))
raise ReadLayerError(msg)
# Record attributes by name
number_of_fields = feature.GetFieldCount()
fields = {}
for j in range(number_of_fields):
name = feature.GetFieldDefnRef(j).GetName()
# FIXME (Ole): Ascertain the type of each field?
# We need to cast each appropriately?
# This is issue #66
# (https://github.com/AIFDR/riab/issues/66)
# feature_type = feature.GetFieldDefnRef(j).GetType()
fields[name] = feature.GetField(j)
# We do this because there is NaN problem on windows
# NaN value must be converted to _pseudo_in to solve the
# problem. But, when InaSAFE read the file, it'll be
# converted back to NaN value, so that NaN in InaSAFE is a
# numpy.nan
# please check https://github.com/AIFDR/inasafe/issues/269
# for more information
if fields[name] == _pseudo_inf:
fields[name] = float('nan')
# print 'Field', name, feature_type, j, fields[name]
data.append(fields)
# Store geometry coordinates as a compact numeric array
self.geometry = geometry
self.data = data
def read_from_file(self, filename):
"""Read and unpack raster data
"""
# Open data file for reading
# File must be kept open, otherwise GDAL methods segfault.
fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
if fid is None:
# As gdal doesn't return to us what the problem is we have to
# figure it out ourselves. Maybe capture stderr?
if not os.path.exists(filename):
msg = 'Could not find file %s' % filename
else:
msg = ('File %s exists, but could not be read. '
'Please check if the file can be opened with '
'e.g. qgis or gdalinfo' % filename)
raise ReadLayerError(msg)
# Record raster metadata from file
basename, ext = os.path.splitext(filename)
# If file is ASCII, check that projection is around.
# GDAL does not check this nicely, so it is worth an
# error message
if ext == '.asc':
try:
open(basename + '.prj')
except IOError:
msg = ('Projection file not found for %s. You must supply '
'a projection file with extension .prj' % filename)
raise ReadLayerError(msg)
# Look for any keywords
self.keywords = read_iso19115_metadata(filename)
# Determine name
if 'title' in self.keywords:
title = self.keywords['title']
# Lookup internationalised title if available
title = tr(title)
rastername = title
else:
# Use basename without leading directories as name
rastername = os.path.split(basename)[-1]
if self.name is None:
self.name = rastername
self.filename = filename
self.projection = Projection(self.fid.GetProjection())
self.geotransform = self.fid.GetGeoTransform()
self.columns = fid.RasterXSize
self.rows = fid.RasterYSize
self.number_of_bands = fid.RasterCount
# Assume that file contains all data in one band
msg = 'Only one raster band currently allowed'
if self.number_of_bands > 1:
msg = ('WARNING: Number of bands in %s are %i. '
'Only the first band will currently be '
'used.' % (filename, self.number_of_bands))
# FIXME(Ole): Let us use python warnings here
raise ReadLayerError(msg)
# Get first band.
band = self.band = fid.GetRasterBand(1)
if band is None:
msg = 'Could not read raster band from %s' % filename
raise ReadLayerError(msg)
# Force garbage collection to free up any memory we can (TS)
gc.collect()
# Read from raster file
data = band.ReadAsArray()
# Convert to double precision (issue #75)
data = numpy.array(data, dtype=numpy.float64)
# Self check
M, N = data.shape
msg = (
'Dimensions of raster array do not match those of '
'raster file %s' % self.filename)
verify(M == self.rows, msg)
verify(N == self.columns, msg)
nodata = self.band.GetNoDataValue()
if nodata is None:
nodata = -9999
if nodata is not numpy.nan:
NaN = numpy.ones((M, N), numpy.float64) * numpy.nan
data = numpy.where(data == nodata, NaN, data)
self.data = data
def read_from_file(self, filename):
"""Read and unpack raster data
"""
# Open data file for reading
# File must be kept open, otherwise GDAL methods segfault.
fid = self.fid = gdal.Open(filename, gdal.GA_ReadOnly)
if fid is None:
# As gdal doesn't return to us what the problem is we have to
# figure it out ourselves. Maybe capture stderr?
if not os.path.exists(filename):
msg = 'Could not find file %s' % filename
else:
msg = (
'File %s exists, but could not be read. '
'Please check if the file can be opened with '
'e.g. qgis or gdalinfo' % filename)
raise ReadLayerError(msg)
# Record raster metadata from file
basename, ext = os.path.splitext(filename)
# If file is ASCII, check that projection is around.
# GDAL does not check this nicely, so it is worth an
# error message
if ext == '.asc':
try:
open(basename + '.prj')
except IOError:
msg = ('Projection file not found for %s. You must supply '
'a projection file with extension .prj' % filename)
raise ReadLayerError(msg)
# Look for any keywords
self.keywords = read_iso19115_metadata(filename)
# Determine name
if 'title' in self.keywords:
title = self.keywords['title']
# Lookup internationalised title if available
title = tr(title)
rastername = title
else:
# Use basename without leading directories as name
rastername = os.path.split(basename)[-1]
if self.name is None:
self.name = rastername
self.filename = filename
self.projection = Projection(self.fid.GetProjection())
self.geotransform = self.fid.GetGeoTransform()
self.columns = fid.RasterXSize
self.rows = fid.RasterYSize
self.number_of_bands = fid.RasterCount
# Assume that file contains all data in one band
msg = 'Only one raster band currently allowed'
if self.number_of_bands > 1:
msg = ('WARNING: Number of bands in %s are %i. '
'Only the first band will currently be '
'used.' % (filename, self.number_of_bands))
# FIXME(Ole): Let us use python warnings here
raise ReadLayerError(msg)
# Get first band.
band = self.band = fid.GetRasterBand(1)
if band is None:
msg = 'Could not read raster band from %s' % filename
raise ReadLayerError(msg)
# Force garbage collection to free up any memory we can (TS)
gc.collect()
def load_layer(full_layer_uri_string, name=None, provider=None):
"""Helper to load and return a single QGIS layer based on our layer URI.
:param provider: The provider name to use if known to open the layer.
Default to None, we will try to guess it, but it's much better if you
can provide it.
:type provider:
:param name: The name of the layer. If not provided, it will be computed
based on the URI.
:type name: basestring
:param full_layer_uri_string: Layer URI, with provider type.
:type full_layer_uri_string: str
:returns: tuple containing layer and its layer_purpose.
:rtype: (QgsMapLayer, str)
"""
if provider:
# Cool !
layer_path = full_layer_uri_string
else:
# Let's check if the driver is included in the path
layer_path, provider = decode_full_layer_uri(full_layer_uri_string)
if not provider:
# Let's try to check if it's file based and look for a extension
if '|' in layer_path:
clean_uri = layer_path.split('|')[0]
else:
clean_uri = layer_path
is_file_based = os.path.exists(clean_uri)
if is_file_based:
# Extract basename and absolute path
file_name = os.path.split(layer_path)[
-1] # If path was absolute
extension = os.path.splitext(file_name)[1]
if extension in OGR_EXTENSIONS:
provider = 'ogr'
elif extension in GDAL_EXTENSIONS:
provider = 'gdal'
else:
provider = None
if not provider:
layer = load_layer_without_provider(layer_path)
else:
layer = load_layer_with_provider(layer_path, provider)
if not layer or not layer.isValid():
message = 'Layer "%s" is not valid' % layer_path
LOGGER.debug(message)
raise InvalidLayerError(message)
# Define the name
if not name:
source = layer.source()
if '|' in source:
clean_uri = source.split('|')[0]
else:
clean_uri = source
is_file_based = os.path.exists(clean_uri)
if is_file_based:
# Extract basename and absolute path
file_name = os.path.split(layer_path)[-1] # If path was absolute
name = os.path.splitext(file_name)[0]
else:
# Might be a DB, take the DB name
source = QgsDataSourceURI(source)
name = source.table()
if not name:
name = 'default'
if qgis_version() >= 21800:
layer.setName(name)
else:
layer.setLayerName(name)
# Determine if layer is hazard or exposure
layer_purpose = 'undefined'
try:
keywords = read_iso19115_metadata(layer.source())
if 'layer_purpose' in keywords:
layer_purpose = keywords['layer_purpose']
except NoKeywordsFoundError:
pass
monkey_patch_keywords(layer)
return layer, layer_purpose
def _clip_raster_layer(
layer, extent, cell_size=None, extra_keywords=None):
"""Clip a Hazard or Exposure raster layer to the extents provided.
The layer must be a raster layer or an exception will be thrown.
.. note:: The extent *must* be in EPSG:4326.
The output layer will always be in WGS84/Geographic.
:param layer: A valid QGIS raster layer in EPSG:4326
:type layer: QgsRasterLayer
:param extent: An array representing the exposure layer
extents in the form [xmin, ymin, xmax, ymax]. It is assumed
that the coordinates are in EPSG:4326 although currently
no checks are made to enforce this.
or:
A QgsGeometry of type polygon.
**Polygon clipping currently only supported for vector datasets.**
:type extent: list(float), QgsGeometry
:param cell_size: Cell size (in GeoCRS) which the layer should
be resampled to. If not provided for a raster layer (i.e.
theCellSize=None), the native raster cell size will be used.
:type cell_size: float
:returns: Output clipped layer (placed in the system temp dir).
:rtype: QgsRasterLayer
:raises: InvalidProjectionError - if input layer is a density
layer in projected coordinates. See issue #123.
"""
if not layer or not extent:
message = tr('Layer or Extent passed to clip is None.')
raise InvalidParameterError(message)
if layer.type() != QgsMapLayer.RasterLayer:
message = tr(
'Expected a raster layer but received a %s.' %
str(layer.type()))
raise InvalidParameterError(message)
working_layer = layer.source()
# Check for existence of keywords file
base, _ = os.path.splitext(working_layer)
keywords_path = base + '.xml'
message = tr(
'Input file to be clipped "%s" does not have the '
'expected keywords file %s' % (
working_layer,
keywords_path
))
verify(os.path.isfile(keywords_path), message)
# Raise exception if layer is projected and refers to density (issue #123)
# FIXME (Ole): Need to deal with it - e.g. by automatically reprojecting
# the layer at this point and setting the native resolution accordingly
# in its keywords.
keywords = read_iso19115_metadata(working_layer)
if 'datatype' in keywords and keywords['datatype'] == 'count':
if str(layer.crs().authid()) != 'EPSG:4326':
# This layer is not WGS84 geographic
message = (
'Layer %s represents count but has spatial reference "%s". '
'Count layers must be given in WGS84 geographic coordinates, '
'so please reproject and try again. For more information, see '
'issue https://github.com/AIFDR/inasafe/issues/123' % (
working_layer,
layer.crs().toProj4()
))
raise InvalidProjectionError(message)
# We need to provide gdalwarp with a dataset for the clip
# because unlike gdal_translate, it does not take projwin.
clip_kml = extent_to_kml(extent)
# Create a filename for the clipped, resampled and reprojected layer
handle, filename = tempfile.mkstemp('.tif', 'clip_', temp_dir())
os.close(handle)
os.remove(filename)
# If no cell size is specified, we need to run gdalwarp without
# specifying the output pixel size to ensure the raster dims
# remain consistent.
binary_list = which('gdalwarp')
LOGGER.debug('Path for gdalwarp: %s' % binary_list)
if len(binary_list) < 1:
raise CallGDALError(
tr('gdalwarp could not be found on your computer'))
# Use the first matching gdalwarp found
binary = binary_list[0]
if cell_size is None:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -cutline %s -crop_to_cutline '
'-ot Float64 -of GTiff "%s" "%s"' % (
binary,
clip_kml,
working_layer,
filename))
else:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -tr %s %s -cutline %s '
'-crop_to_cutline -ot Float64 -of GTiff "%s" "%s"' % (
binary,
repr(cell_size),
repr(cell_size),
clip_kml,
working_layer,
filename))
LOGGER.debug(command)
result = QProcess().execute(command)
# For QProcess exit codes see
# http://qt-project.org/doc/qt-4.8/qprocess.html#execute
if result == -2: # cannot be started
message_detail = tr('Process could not be started.')
message = tr(
'<p>Error while executing the following shell command:'
'</p><pre>%s</pre><p>Error message: %s'
% (command, message_detail))
raise CallGDALError(message)
elif result == -1: # process crashed
message_detail = tr('Process crashed.')
message = tr(
'<p>Error while executing the following shell command:</p>'
'<pre>%s</pre><p>Error message: %s' % (command, message_detail))
raise CallGDALError(message)
# .. todo:: Check the result of the shell call is ok
keyword_io = KeywordIO()
keyword_io.copy_keywords(layer, filename, extra_keywords=extra_keywords)
base_name = '%s clipped' % layer.name()
layer = QgsRasterLayer(filename, base_name)
return layer
def _clip_raster_layer(
layer, extent, cell_size=None, extra_keywords=None):
"""Clip a Hazard or Exposure raster layer to the extents provided.
The layer must be a raster layer or an exception will be thrown.
.. note:: The extent *must* be in EPSG:4326.
The output layer will always be in WGS84/Geographic.
:param layer: A valid QGIS raster layer in EPSG:4326
:type layer: QgsRasterLayer
:param extent: An array representing the exposure layer
extents in the form [xmin, ymin, xmax, ymax]. It is assumed
that the coordinates are in EPSG:4326 although currently
no checks are made to enforce this.
or:
A QgsGeometry of type polygon.
**Polygon clipping currently only supported for vector datasets.**
:type extent: list(float), QgsGeometry
:param cell_size: Cell size (in GeoCRS) which the layer should
be resampled to. If not provided for a raster layer (i.e.
theCellSize=None), the native raster cell size will be used.
:type cell_size: float
:returns: Output clipped layer (placed in the system temp dir).
:rtype: QgsRasterLayer
:raises: InvalidProjectionError - if input layer is a density
layer in projected coordinates. See issue #123.
"""
if not layer or not extent:
message = tr('Layer or Extent passed to clip is None.')
raise InvalidParameterError(message)
if layer.type() != QgsMapLayer.RasterLayer:
message = tr(
'Expected a raster layer but received a %s.' %
str(layer.type()))
raise InvalidParameterError(message)
working_layer = layer.source()
# Check for existence of keywords file
base, _ = os.path.splitext(working_layer)
keywords_path = base + '.xml'
message = tr(
'Input file to be clipped "%s" does not have the '
'expected keywords file %s' % (
working_layer,
keywords_path
))
verify(os.path.isfile(keywords_path), message)
# Raise exception if layer is projected and refers to density (issue #123)
# FIXME (Ole): Need to deal with it - e.g. by automatically reprojecting
# the layer at this point and setting the native resolution accordingly
# in its keywords.
try:
keywords = read_iso19115_metadata(working_layer)
except (MetadataReadError, NoKeywordsFoundError):
keywords = read_keywords(base + '.keywords')
keywords = write_read_iso_19115_metadata(working_layer, keywords)
if 'datatype' in keywords and keywords['datatype'] == 'count':
if str(layer.crs().authid()) != 'EPSG:4326':
# This layer is not WGS84 geographic
message = (
'Layer %s represents count but has spatial reference "%s". '
'Count layers must be given in WGS84 geographic coordinates, '
'so please reproject and try again. For more information, see '
'issue https://github.com/AIFDR/inasafe/issues/123' % (
working_layer,
layer.crs().toProj4()
))
raise InvalidProjectionError(message)
# We need to provide gdalwarp with a dataset for the clip
# because unlike gdal_translate, it does not take projwin.
clip_kml = extent_to_kml(extent)
# Create a filename for the clipped, resampled and reprojected layer
handle, filename = tempfile.mkstemp('.tif', 'clip_', temp_dir())
os.close(handle)
os.remove(filename)
# If no cell size is specified, we need to run gdalwarp without
# specifying the output pixel size to ensure the raster dims
# remain consistent.
binary_list = which('gdalwarp')
LOGGER.debug('Path for gdalwarp: %s' % binary_list)
if len(binary_list) < 1:
raise CallGDALError(
tr('gdalwarp could not be found on your computer'))
# Use the first matching gdalwarp found
binary = binary_list[0]
if cell_size is None:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -cutline %s -crop_to_cutline '
'-ot Float64 -of GTiff "%s" "%s"' % (
binary,
clip_kml,
working_layer,
filename))
else:
command = (
'"%s" -q -t_srs EPSG:4326 -r near -tr %s %s -cutline %s '
'-crop_to_cutline -ot Float64 -of GTiff "%s" "%s"' % (
binary,
repr(cell_size),
repr(cell_size),
clip_kml,
working_layer,
filename))
LOGGER.debug(command)
result = QProcess().execute(command)
# For QProcess exit codes see
# http://qt-project.org/doc/qt-4.8/qprocess.html#execute
if result == -2: # cannot be started
message_detail = tr('Process could not be started.')
message = tr(
'<p>Error while executing the following shell command:'
'</p><pre>%s</pre><p>Error message: %s'
% (command, message_detail))
raise CallGDALError(message)
elif result == -1: # process crashed
message_detail = tr('Process crashed.')
message = tr(
'<p>Error while executing the following shell command:</p>'
'<pre>%s</pre><p>Error message: %s' % (command, message_detail))
raise CallGDALError(message)
# .. todo:: Check the result of the shell call is ok
keyword_io = KeywordIO()
keyword_io.copy_keywords(layer, filename, extra_keywords=extra_keywords)
base_name = '%s clipped' % layer.name()
layer = QgsRasterLayer(filename, base_name)
return layer
def read_keywords(self, layer, keyword=None):
"""Read keywords for a datasource and return them as a dictionary.
This is a wrapper method that will 'do the right thing' to fetch
keywords for the given datasource. In particular, if the datasource
is remote (e.g. a database connection) it will fetch the keywords from
the keywords store.
:param layer: A QGIS QgsMapLayer instance that you want to obtain
the keywords for.
:type layer: QgsMapLayer, QgsRasterLayer, QgsVectorLayer,
QgsPluginLayer
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
TODO: Don't raise generic exceptions.
:raises: HashNotFoundError, Exception, OperationalError,
NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError,
UnsupportedProviderError
"""
source = layer.source()
# Try to read from ISO metadata first.
try:
return read_iso19115_metadata(source, keyword)
except (MetadataReadError, NoKeywordsFoundError):
pass
try:
flag = self.are_keywords_file_based(layer)
except UnsupportedProviderError:
raise
try:
if flag:
keywords = read_file_keywords(source)
mandatory_keywords = [
'layer_purpose',
'keyword_version'
]
for mandatory_keyword in mandatory_keywords:
if mandatory_keyword not in keywords.keys():
message = tr(
'The layer does not have keyword %s in its '
'keywords.' % mandatory_keyword)
raise KeywordNotFoundError(message)
else:
uri = self.normalize_uri(layer)
keywords = self.read_keyword_from_uri(uri)
return write_read_iso_19115_metadata(source, keywords, keyword)
except (HashNotFoundError,
Exception,
OperationalError,
NoKeywordsFoundError,
KeywordNotFoundError,
InvalidParameterError,
UnsupportedProviderError):
raise
