def check_inasafe_fields(layer):
"""Helper to check inasafe_fields.
:param layer: The layer to check.
:type layer: QgsVectorLayer
:return: Return True if the layer is valid.
:rtype: bool
:raises: Exception with a message if the layer is not correct.
"""
inasafe_fields = layer.keywords['inasafe_fields']
real_fields = [field.name() for field in layer.fields().toList()]
difference = set(inasafe_fields.values()).difference(real_fields)
if len(difference):
message = tr(
'inasafe_fields has more fields than the layer %s itself : %s' %
(layer.keywords['layer_purpose'], difference))
raise InvalidLayerError(message)
difference = set(real_fields).difference(inasafe_fields.values())
if len(difference):
message = tr('The layer %s has more fields than inasafe_fields : %s' %
(layer.title(), difference))
raise InvalidLayerError(message)
return True
def prepare_input(self):
"""Fetch all the input from dialog, validate, and store it.
Consider this as a bridge between dialog interface and our logical
stored data in this class
:raises: InvalidLayerError, CanceledImportDialogError
"""
# Validate The combobox impact layers (they should be different)
first_layer_index = self.first_layer.currentIndex()
second_layer_index = self.second_layer.currentIndex()
if first_layer_index < 0:
raise InvalidLayerError(self.tr('First layer is not valid.'))
if second_layer_index < 0:
raise InvalidLayerError(self.tr('Second layer is not valid.'))
if first_layer_index == second_layer_index:
raise InvalidLayerError(
self.tr('First layer must be different to second layer'
'.'))
# Get all chosen layers
self.first_impact['layer'] = self.first_layer.itemData(
self.first_layer.currentIndex(), QtCore.Qt.UserRole)
self.second_impact['layer'] = self.second_layer.itemData(
self.second_layer.currentIndex(), QtCore.Qt.UserRole)
self.aggregation['layer'] = self.aggregation_layer.itemData(
self.aggregation_layer.currentIndex(), QtCore.Qt.UserRole)
# Validate the output directory
self.require_directory()
# Get output directory
self.out_dir = self.output_directory.text()
# Whether to use own report template:
if self.report_template_checkbox.isChecked():
own_template_path = self.report_template_le.text()
if os.path.isfile(own_template_path):
self.template_path = own_template_path
else:
raise FileNotFoundError(
self.tr('Template file does not exist.'))
# Flag whether to merge entire area or based on aggregation unit
# Rizky: Fix nasty bug where the dialog stuck in entire_area_mode
# the mode should be rechecked based on selected aggregation layer
self.entire_area_mode = True
if (self.aggregation_layer.currentIndex() > 0
and not self.aggregation['layer'] is None):
self.entire_area_mode = False
def check_inasafe_fields(layer, keywords_only=False):
"""Helper to check inasafe_fields.
:param layer: The layer to check.
:type layer: QgsVectorLayer
:param keywords_only: If we should check from the keywords only. False by
default, we will check also from the layer.
:type keywords_only: bool
:return: Return True if the layer is valid.
:rtype: bool
:raises: Exception with a message if the layer is not correct.
"""
inasafe_fields = layer.keywords['inasafe_fields']
real_fields = [field.name() for field in layer.fields().toList()]
inasafe_fields_flat = []
for value in list(inasafe_fields.values()):
if isinstance(value, list):
inasafe_fields_flat.extend(value)
else:
inasafe_fields_flat.append(value)
difference = set(inasafe_fields_flat).difference(real_fields)
if len(difference):
message = tr(
'inasafe_fields has more fields than the layer %s itself : %s'
% (layer.keywords['layer_purpose'], difference))
raise InvalidLayerError(message)
if keywords_only:
# We don't check if it's valid from the layer. The layer may have more
# fields than inasafe_fields.
return True
difference = set(real_fields).difference(inasafe_fields_flat)
if len(difference):
message = tr(
'The layer %s has more fields than inasafe_fields : %s'
% (layer.title(), difference))
raise InvalidLayerError(message)
return True
def create_keyword_file(self, algorithm):
"""Create keyword file for the raster file created.
Basically copy a template from keyword file in converter data
and add extra keyword (usually a title)
:param algorithm: Which re-sampling algorithm to use.
valid options are 'nearest' (for nearest neighbour), 'invdist'
(for inverse distance), 'average' (for moving average). Defaults
to 'nearest' if not specified. Note that passing re-sampling alg
parameters is currently not supported. If None is passed it will
be replaced with 'nearest'.
:type algorithm: str
"""
keyword_io = KeywordIO()
classes = {}
for item in earthquake_mmi_scale['classes']:
classes[item['key']] = [
item['numeric_default_min'], item['numeric_default_max']]
keywords = {
'hazard': hazard_earthquake['key'],
'hazard_category': hazard_category_single_event['key'],
'keyword_version': inasafe_keyword_version,
'layer_geometry': layer_geometry_raster['key'],
'layer_mode': layer_mode_continuous['key'],
'layer_purpose': layer_purpose_hazard['key'],
'continuous_hazard_unit': unit_mmi['key'],
'classification': earthquake_mmi_scale['key'],
'thresholds': classes
}
if self.algorithm_name:
layer_path = os.path.join(
self.output_dir, '%s-%s.tif' % (
self.output_basename, algorithm))
else:
layer_path = os.path.join(
self.output_dir, '%s.tif' % self.output_basename)
# append title and source to the keywords file
if len(self.title.strip()) == 0:
keyword_title = self.output_basename
else:
keyword_title = self.title
keywords['title'] = keyword_title
hazard_layer = QgsRasterLayer(layer_path, keyword_title)
if not hazard_layer.isValid():
raise InvalidLayerError()
keyword_io.write_keywords(hazard_layer, keywords)
def check_layer(layer, has_geometry=True):
"""Helper to check layer validity.
This function wil; raise InvalidLayerError if the layer is invalid.
:param layer: The layer to check.
:type layer: QgsMapLayer
:param has_geometry: If the layer must have a geometry. True by default.
If it's a raster layer, we will no check this parameter. If we do not
want to check the geometry type, we can set it to None.
:type has_geometry: bool,None
:raise: InvalidLayerError
:return: Return True if the layer is valid.
:rtype: bool
"""
if is_vector_layer(layer) or is_raster_layer(layer):
if not layer.isValid():
raise InvalidLayerError('The layer is invalid : %s' %
layer.publicSource())
if is_vector_layer(layer):
sub_layers = layer.dataProvider().subLayers()
if len(sub_layers) > 1:
names = ';'.join(sub_layers)
source = layer.source()
raise InvalidLayerError(
tr('The layer should not have many sublayers : {source} : '
'{names}').format(source=source, names=names))
# We only check the geometry if we have at least one feature.
if layer.geometryType() == QgsWkbTypes.UnknownGeometry and (
layer.featureCount() != 0):
raise InvalidLayerError(
tr('The layer has not a valid geometry type.'))
if layer.wkbType() == QgsWkbTypes.Unknown and (
layer.featureCount() != 0):
raise InvalidLayerError(
tr('The layer has not a valid geometry type.'))
if isinstance(has_geometry, bool) and layer.featureCount() != 0:
if layer.isSpatial() != has_geometry:
raise InvalidLayerError(
tr('The layer has not a correct geometry type.'))
else:
raise InvalidLayerError(
tr('The layer is neither a raster nor a vector : {type}').format(
type=type(layer)))
return True
def layer(self, layer):
"""Setter for layer property.
:param layer: The actual layer.
:type layer: QgsMapLayer, Layer
"""
if isinstance(layer, QgsMapLayer) or isinstance(layer, Layer):
self._layer = layer
else:
message = tr('SafeLayer only accept QgsMapLayer or '
'safe.storage.layer.Layer.')
raise InvalidLayerError(message)
if isinstance(layer, Layer):
self.keywords = layer.keywords
elif isinstance(layer, QgsMapLayer):
keyword_io = KeywordIO()
self.keywords = keyword_io.read_keywords(layer)
else:
self.keywords = {}
def set_contour_properties(self, input_file):
"""Set the X, Y, RGB, ROMAN attributes of the contour layer.
:param input_file: (Required) Name of the contour layer.
:type input_file: str
:raise: InvalidLayerError if anything is amiss with the layer.
"""
LOGGER.debug('set_contour_properties requested for %s.' % input_file)
layer = QgsVectorLayer(input_file, 'mmi-contours', "ogr")
if not layer.isValid():
raise InvalidLayerError(input_file)
layer.startEditing()
# Now loop through the db adding selected features to mem layer
request = QgsFeatureRequest()
fields = layer.dataProvider().fields()
for feature in layer.getFeatures(request):
if not feature.isValid():
LOGGER.debug('Skipping feature')
continue
# Work out x and y
line = feature.geometry().asPolyline()
y = line[0].y()
x_max = line[0].x()
x_min = x_max
for point in line:
if point.y() < y:
y = point.y()
x = point.x()
if x < x_min:
x_min = x
if x > x_max:
x_max = x
x = x_min + ((x_max - x_min) / 2)
# Get length
length = feature.geometry().length()
mmi_value = float(feature['MMI'])
# We only want labels on the whole number contours
if mmi_value != round(mmi_value):
roman = ''
else:
roman = romanise(mmi_value)
# RGB from http://en.wikipedia.org/wiki/Mercalli_intensity_scale
rgb = mmi_colour(mmi_value)
# Now update the feature
feature_id = feature.id()
layer.changeAttributeValue(feature_id, fields.indexFromName('X'),
x)
layer.changeAttributeValue(feature_id, fields.indexFromName('Y'),
y)
layer.changeAttributeValue(feature_id, fields.indexFromName('RGB'),
rgb)
layer.changeAttributeValue(feature_id,
fields.indexFromName('ROMAN'), roman)
layer.changeAttributeValue(feature_id,
fields.indexFromName('ALIGN'), 'Center')
layer.changeAttributeValue(feature_id,
fields.indexFromName('VALIGN'), 'HALF')
layer.changeAttributeValue(feature_id, fields.indexFromName('LEN'),
length)
layer.commitChanges()
def create_keyword_file(self, algorithm):
"""Create keyword file for the raster file created.
Basically copy a template from keyword file in converter data
and add extra keyword (usually a title)
:param algorithm: Which re-sampling algorithm to use.
valid options are 'nearest' (for nearest neighbour), 'invdist'
(for inverse distance), 'average' (for moving average). Defaults
to 'nearest' if not specified. Note that passing re-sampling alg
parameters is currently not supported. If None is passed it will
be replaced with 'nearest'.
:type algorithm: str
"""
keyword_io = KeywordIO()
# Set thresholds for each exposure
mmi_default_classes = default_classification_thresholds(
earthquake_mmi_scale)
mmi_default_threshold = {
earthquake_mmi_scale['key']: {
'active': True,
'classes': mmi_default_classes
}
}
generic_default_classes = default_classification_thresholds(
generic_hazard_classes)
generic_default_threshold = {
generic_hazard_classes['key']: {
'active': True,
'classes': generic_default_classes
}
}
threshold_keyword = {}
for exposure in exposure_all:
# Not all exposure is supported by earthquake_mmi_scale
if exposure in earthquake_mmi_scale['exposures']:
threshold_keyword[exposure['key']] = mmi_default_threshold
else:
threshold_keyword[exposure['key']] = generic_default_threshold
extra_keywords = {
extra_keyword_earthquake_latitude['key']:
self.latitude,
extra_keyword_earthquake_longitude['key']:
self.longitude,
extra_keyword_earthquake_magnitude['key']:
self.magnitude,
extra_keyword_earthquake_depth['key']:
self.depth,
extra_keyword_earthquake_description['key']:
self.description,
extra_keyword_earthquake_location['key']:
self.location,
extra_keyword_earthquake_event_time['key']:
self.time.strftime('%Y-%m-%dT%H:%M:%S'),
extra_keyword_time_zone['key']:
self.time_zone,
extra_keyword_earthquake_x_minimum['key']:
self.x_minimum,
extra_keyword_earthquake_x_maximum['key']:
self.x_maximum,
extra_keyword_earthquake_y_minimum['key']:
self.y_minimum,
extra_keyword_earthquake_y_maximum['key']:
self.y_maximum,
extra_keyword_earthquake_event_id['key']:
self.event_id
}
for key, value in self.extra_keywords.items():
extra_keywords[key] = value
# Delete empty element.
empty_keys = []
for key, value in extra_keywords.items():
if value is None:
empty_keys.append(key)
for empty_key in empty_keys:
extra_keywords.pop(empty_key)
keywords = {
'hazard': hazard_earthquake['key'],
'hazard_category': hazard_category_single_event['key'],
'keyword_version': inasafe_keyword_version,
'layer_geometry': layer_geometry_raster['key'],
'layer_mode': layer_mode_continuous['key'],
'layer_purpose': layer_purpose_hazard['key'],
'continuous_hazard_unit': unit_mmi['key'],
'classification': earthquake_mmi_scale['key'],
'thresholds': threshold_keyword,
'extra_keywords': extra_keywords,
'active_band': 1
}
if self.algorithm_name:
layer_path = os.path.join(
self.output_dir,
'%s-%s.tif' % (self.output_basename, algorithm))
else:
layer_path = os.path.join(self.output_dir,
'%s.tif' % self.output_basename)
# append title and source to the keywords file
if len(self.title.strip()) == 0:
keyword_title = self.output_basename
else:
keyword_title = self.title
keywords['title'] = keyword_title
hazard_layer = QgsRasterLayer(layer_path, keyword_title)
if not hazard_layer.isValid():
raise InvalidLayerError()
keyword_io.write_keywords(hazard_layer, keywords)
def merge(self):
"""Merge the postprocessing_report from each impact."""
# Ensure there is always only a single root element or minidom moans
first_postprocessing_report = \
self.first_impact['postprocessing_report']
second_postprocessing_report = \
self.second_impact['postprocessing_report']
# noinspection PyTypeChecker
first_report = '<body>' + first_postprocessing_report + '</body>'
# noinspection PyTypeChecker
second_report = '<body>' + second_postprocessing_report + '</body>'
# Now create a dom document for each
first_document = minidom.parseString(get_string(first_report))
second_document = minidom.parseString(get_string(second_report))
first_impact_tables = first_document.getElementsByTagName('table')
second_impact_tables = second_document.getElementsByTagName('table')
# Now create dictionary report from DOM
first_report_dict = self.generate_report_dictionary_from_dom(
first_impact_tables)
second_report_dict = self.generate_report_dictionary_from_dom(
second_impact_tables)
# Rizky: Consistency checks with aggregation
# Make sure the aggregation layer both presents in both layers
# We shouldn't have problems with Entire Area mode. It just means
# the impact layer and summary is merged into single report.
# We can have 3 cases:
# 1. If both of them were not aggregated, we can just merge the map
# only
# 2. If both of them were aggregated, we can just merge the map, and
# merge postprocessor report using 'Total aggregation in areas' key
# 3. If only one of them were aggregated, we can just merge the map,
# and uses postprocessor report from the one who has.
if self.entire_area_mode:
# We won't be bothered with the map, it will be merged anyway in
# all 3 cases. We should bother with the postprocessor report.
# If one of them has the report, it means it will contain more
# than one report keys. We can just swap the first report if they
# have one key, and the second have more than one
if (len(first_report_dict.keys()) == 1
and len(second_report_dict.keys()) > 1):
swap_var = first_report_dict
first_report_dict = second_report_dict
second_report_dict = swap_var
# This condition will covers aggregated mode
# For this case, we should make sure both layers are aggregated with
# the same aggregation layer of the chosen aggregation layer
else:
# check that both layers must have aggregated postprocessor.
# aggregated postprocessor means the report_dict must have minimum
# 2 keys
if not (len(first_report_dict.keys()) > 1
and len(second_report_dict.keys()) > 1):
raise InvalidLayerError(
self.tr(
'Please choose impact layers with aggregated '
'postprocessor if you want to use aggregation layer.'))
# collect all report keys (will contain aggregation areas in the
# report)
report_keys = first_report_dict.keys()
# Discard the last keys. It will always contains total area, not
# aggregated area
if len(report_keys) > 0:
del report_keys[-1]
sec_report = second_report_dict.keys()
if len(sec_report) > 0:
del sec_report[-1]
for k in sec_report:
if k not in report_keys:
report_keys.append(k)
# collect all aggregation areas in aggregation layer
layer = self.aggregation['layer']
aggregation_attr = self.aggregation['aggregation_attribute']
aggregation_attr_index = layer.fieldNameIndex(aggregation_attr)
aggregation_keys = []
for f in layer.getFeatures():
area = f[aggregation_attr_index]
if area not in aggregation_keys:
aggregation_keys.append(area)
is_subset = True
for k in report_keys:
if k not in aggregation_keys:
is_subset = False
if not is_subset:
# This means report keys contains area keys that is not in
# aggregation layer. Which means possibly it is using the
# wrong aggregation layer.
raise InvalidLayerError(
self.tr('First and Second layer does not use chosen '
'Aggregation layer'))
# Generate report summary for all aggregation unit
self.generate_report_summary(first_report_dict, second_report_dict)
# Generate html reports file from merged dictionary
self.generate_html_reports(first_report_dict, second_report_dict)
# Generate PDF Reports using composer and/or atlas generation:
self.generate_reports()
# Delete html report files:
for area in self.html_reports:
report_path = self.html_reports[area]
# Rizky : Fix possible bugs in Windows related to issue:
# https://github.com/AIFDR/inasafe/issues/1862
try:
# avoid race condition using with statement
with open(report_path, 'w') as report_file:
report_file.close()
os.remove(report_path)
except OSError:
pass
def validate_all_layers(self):
"""Validate all layers based on the keywords.
When we do the validation, we also fetch the information we need:
1. 'map_title' from each impact layer
2. 'exposure_title' from each impact layer
3. 'postprocessing_report' from each impact layer
4. 'aggregation_attribute' on aggregation layer, if user runs merging
tools with aggregation layer chosen
The things that we validate are:
1. 'map_title' keyword must exist on each impact layer
2. 'exposure_title' keyword must exist on each impact layer
3. 'postprocessing_report' keyword must exist on each impact layer
4. 'hazard_title' keyword must exist on each impact layer. Hazard title
from first impact layer must be the same with second impact layer
to indicate that both are generated from the same hazard layer.
5. 'aggregation attribute' must exist when user wants to run merging
tools with aggregation layer chosen.
"""
required_attribute = [
'map_title', 'exposure_title', 'hazard_title',
'postprocessing_report'
]
# Fetch for first impact layer
for attribute in required_attribute:
try:
# noinspection PyTypeChecker
self.first_impact[attribute] = self.keyword_io.read_keywords(
self.first_impact['layer'], attribute)
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords found for first impact layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword %s not found for first layer.' %
attribute))
# Fetch for second impact layer
for attribute in required_attribute:
try:
# noinspection PyTypeChecker
self.second_impact[attribute] = self.keyword_io.read_keywords(
self.second_impact['layer'], attribute)
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords found for second impact layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword %s not found for second layer.' %
attribute))
# Validate that two impact layers are obtained from the same hazard.
# Indicated by the same 'hazard_title' (to be fixed later by using
# more reliable method)
if (self.first_impact['hazard_title'] !=
self.second_impact['hazard_title']):
raise InvalidLayerError(
self.tr('First impact layer and second impact layer do not '
'use the same hazard layer.'))
# Fetch 'aggregation_attribute'
# If the chosen aggregation layer not Entire Area, it should have
# aggregation attribute keywords
if not self.entire_area_mode:
try:
# noinspection PyTypeChecker
self.aggregation['aggregation_attribute'] = \
self.keyword_io.read_keywords(
self.aggregation['layer'], 'aggregation attribute')
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords exist in aggregation layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword aggregation attribute not found for '
'aggregation layer.'))
def load_layer(full_layer_uri_string, name=None, provider=None):
"""Helper to load and return a single QGIS layer based on our layer URI.
:param provider: The provider name to use if known to open the layer.
Default to None, we will try to guess it, but it's much better if you
can provide it.
:type provider:
:param name: The name of the layer. If not provided, it will be computed
based on the URI.
:type name: basestring
:param full_layer_uri_string: Layer URI, with provider type.
:type full_layer_uri_string: str
:returns: tuple containing layer and its layer_purpose.
:rtype: (QgsMapLayer, str)
"""
if provider:
# Cool !
layer_path = full_layer_uri_string
else:
# Let's check if the driver is included in the path
layer_path, provider = decode_full_layer_uri(full_layer_uri_string)
if not provider:
# Let's try to check if it's file based and look for a extension
if '|' in layer_path:
clean_uri = layer_path.split('|')[0]
else:
clean_uri = layer_path
is_file_based = os.path.exists(clean_uri)
if is_file_based:
# Extract basename and absolute path
file_name = os.path.split(layer_path)[
-1] # If path was absolute
extension = os.path.splitext(file_name)[1]
if extension in OGR_EXTENSIONS:
provider = 'ogr'
elif extension in GDAL_EXTENSIONS:
provider = 'gdal'
else:
provider = None
if not provider:
layer = load_layer_without_provider(layer_path)
else:
layer = load_layer_with_provider(layer_path, provider)
if not layer or not layer.isValid():
message = 'Layer "%s" is not valid' % layer_path
LOGGER.debug(message)
raise InvalidLayerError(message)
# Define the name
if not name:
source = layer.source()
if '|' in source:
clean_uri = source.split('|')[0]
else:
clean_uri = source
is_file_based = os.path.exists(clean_uri)
if is_file_based:
# Extract basename and absolute path
file_name = os.path.split(layer_path)[-1] # If path was absolute
name = os.path.splitext(file_name)[0]
else:
# Might be a DB, take the DB name
source = QgsDataSourceUri(source)
name = source.table()
if not name:
name = 'default'
if qgis_version() >= 21800:
layer.setName(name)
else:
layer.setLayerName(name)
# update the layer keywords
monkey_patch_keywords(layer)
layer_purpose = layer.keywords.get('layer_purpose')
return layer, layer_purpose
def set_contour_properties(contour_file_path):
"""Set the X, Y, RGB, ROMAN attributes of the contour layer.
:param contour_file_path: Path of the contour layer.
:type contour_file_path: str
:raise: InvalidLayerError if anything is amiss with the layer.
"""
LOGGER.debug('Set_contour_properties requested for %s.' %
contour_file_path)
layer = QgsVectorLayer(contour_file_path, 'mmi-contours', "ogr")
if not layer.isValid():
raise InvalidLayerError(contour_file_path)
layer.startEditing()
# Now loop through the db adding selected features to mem layer
request = QgsFeatureRequest()
for feature in layer.getFeatures(request):
if not feature.isValid():
LOGGER.debug('Skipping feature')
continue
# Work out x and y
line = feature.geometry().asPolyline()
y = line[0].y()
x_max = line[0].x()
x_min = x_max
for point in line:
if point.y() < y:
y = point.y()
x = point.x()
if x < x_min:
x_min = x
if x > x_max:
x_max = x
x = x_min + ((x_max - x_min) / 2)
# Get length
length = feature.geometry().length()
mmi_value = float(feature[contour_mmi_field['field_name']])
# We only want labels on the whole number contours
if mmi_value != round(mmi_value):
roman = ''
else:
roman = romanise(mmi_value)
# RGB from http://en.wikipedia.org/wiki/Mercalli_intensity_scale
rgb = mmi_colour(mmi_value)
# Now update the feature
feature_id = feature.id()
layer.changeAttributeValue(
feature_id, field_index_from_definition(layer, contour_x_field), x)
layer.changeAttributeValue(
feature_id, field_index_from_definition(layer, contour_y_field), y)
layer.changeAttributeValue(
feature_id, field_index_from_definition(layer,
contour_colour_field), rgb)
layer.changeAttributeValue(
feature_id,
field_index_from_definition(layer, contour_roman_field), roman)
layer.changeAttributeValue(
feature_id,
field_index_from_definition(layer, contour_halign_field), 'Center')
layer.changeAttributeValue(
feature_id,
field_index_from_definition(layer, contour_valign_field), 'HALF')
layer.changeAttributeValue(
feature_id,
field_index_from_definition(layer, contour_length_field), length)
layer.commitChanges()
