def test_to_from_json(self): """Test FlatTable from_dict method""" json_string = self.flat_table.to_json() flat_table = FlatTable() flat_table.from_json(json_string) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20)
def test_from_json(self): """Test FlatTable from_json method""" json_string = ( '{"data": [["residential", "low", 50], ' '["residential", "medium", 30], ["secondary", "low", 40], ' '["primary", "high", 10], ["primary", "medium", 20]], ' '"groups": ["road_type", "hazard"]}') flat_table = FlatTable() flat_table.from_json(json_string) expected_groups = ["road_type", "hazard"] for i in range(len(flat_table.groups)): self.assertEquals(expected_groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20)
def impact_table(self): """Return data as dictionary""" # prepare area calculator object area_calc = QgsDistanceArea() area_calc.setSourceCrs(self.impact_layer.crs()) area_calc.setEllipsoid('WGS84') area_calc.setEllipsoidalMode(True) impacted_table = FlatTable('landcover', 'hazard', 'zone') for f in self.impact_layer.getFeatures(): area = area_calc.measure(f.geometry()) / 1e4 zone = f[self.zone_field] if self.zone_field is not None else None impacted_table.add_value(area, landcover=f[self.land_cover_field], hazard=f[self.target_field], zone=zone) return impacted_table.to_dict()
def impact_table(self): """Return data as dictionary""" # prepare area calculator object area_calc = QgsDistanceArea() area_calc.setSourceCrs(self.impact_layer.crs()) area_calc.setEllipsoid('WGS84') area_calc.setEllipsoidalMode(True) impacted_table = FlatTable('landcover', 'hazard', 'zone') for f in self.impact_layer.getFeatures(): area = area_calc.measure(f.geometry()) / 1e4 zone = f[self.zone_field] if self.zone_field is not None else None impacted_table.add_value( area, landcover=f[self.land_cover_field], hazard=f[self.target_field], zone=zone) return impacted_table.to_dict()
def setUp(self): """ This will generate a table like this: Affected columns : high and medium road_type | hazard | Totals | | Total Percent | high medium low | Total affected affected residential | 0 30 50 | 80 30 37.5 primary | 10 20 0 | 30 30 100 secondary | 0 0 40 | 40 0 0 Total | 10 50 90 | 150 60 40 """ self.affected_columns = ['medium', 'high'] self.flat_table = FlatTable("road_type", "hazard") self.flat_table.add_value(10, road_type="primary", hazard="high") self.flat_table.add_value(20, road_type="primary", hazard="medium") self.flat_table.add_value(30, road_type="residential", hazard="medium") self.flat_table.add_value(40, road_type="secondary", hazard="low") self.flat_table.add_value(50, road_type="residential", hazard="low")
def test_from_dict(self): """Test FlatTable from_dict method""" groups = ["road_type", "hazard"] data = [ ["residential", "low", 50], ["residential", "medium", 30], ["secondary", "low", 40], ["primary", "high", 10], ["primary", "medium", 20] ] flat_table = FlatTable() flat_table.from_dict(groups, data) for i in range(len(flat_table.groups)): self.assertEquals(groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20) groups = [u'landcover', u'hazard', u'zone'] data = [ [u'Forest', u'high', None, 5172.100048073517], [u'Population', u'high', None, 20689.8283632199], [u'Forest', u'low', None, 5171.381989317935], [u'Population', u'medium', None, 10347.048486941067], [u'Meadow', u'high', None, 5172.81413353821], [u'Population', u'low', None, 10342.763978632318], [u'Meadow', u'medium', None, 5173.5242434723095] ] flat_table = FlatTable() flat_table.from_dict(groups, data) for i in range(len(flat_table.groups)): self.assertEquals(groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[ ('Forest', 'high', None)], 5172.100048073517)
def create_absolute_values_structure(layer, fields): """Helper function to create the structure for absolute values. :param layer: The vector layer. :type layer: QgsVectorLayer :param fields: List of name field on which we want to aggregate. :type fields: list :return: The data structure. :rtype: dict """ # Let's create a structure like : # key is the index of the field : (flat table, definition name) source_fields = layer.keywords['inasafe_fields'] absolute_fields = [field['key'] for field in count_fields] summaries = {} for field in source_fields: if field in absolute_fields: field_name = source_fields[field] index = layer.fields().lookupField(field_name) flat_table = FlatTable(*fields) summaries[index] = (flat_table, field) return summaries
def test_run(self): function = TsunamiRasterLandcoverFunction.instance() hazard_path = standard_data_path('hazard', 'tsunami_wgs84.tif') exposure_path = standard_data_path('exposure', 'landcover.shp') # noinspection PyCallingNonCallable hazard_layer = QgsRasterLayer(hazard_path, 'Tsunami') # noinspection PyCallingNonCallable exposure_layer = QgsVectorLayer(exposure_path, 'Land Cover', 'ogr') self.assertTrue(hazard_layer.isValid()) self.assertTrue(exposure_layer.isValid()) rect_extent = [106.5, -6.5, 107, -6] function.hazard = hazard_layer function.exposure = exposure_layer function.requested_extent = rect_extent function.run() impact = function.impact impact = safe_to_qgis_layer(impact) expected = { 'data': [[u'Population', u'Dry Zone', None, 793.6916054134609], [u'Water', u'Low Hazard Zone', None, 16.298813953855912], [ u'Population', u'Very High Hazard Zone', None, 12.45623642166847 ], [u'Water', u'Very High Hazard Zone', None, 0.08036139883589728], [u'Water', u'Medium Hazard Zone', None, 12.1033540507973], [u'Population', u'Low Hazard Zone', None, 28.866862427357326], [u'Water', u'Dry Zone', None, 164.67113858186028], [u'Meadow', u'Dry Zone', None, 249.95443689559693], [u'Population', u'Medium Hazard Zone', None, 30.69211822286981], [u'Water', u'High Hazard Zone', None, 5.835228232982915], [u'Population', u'High Hazard Zone', None, 29.72789895440279], [u'Forest', u'Dry Zone', None, 99.489344261353]], 'groups': ('landcover', 'hazard', 'zone') } ordered_columns = function.impact.impact_data.get('ordered columns') affected_columns = function.impact.impact_data.get('affected columns') expected = FlatTable().from_dict( groups=expected['groups'], data=expected['data'], ) expected = PivotTable(expected, row_field='landcover', column_field='hazard', columns=ordered_columns, affected_columns=affected_columns) self.assertEqual(impact.dataProvider().featureCount(), 72) table = function.impact.impact_data['impact table'] table = FlatTable().from_dict( groups=table['groups'], data=table['data'], ) table = PivotTable(table, row_field='landcover', column_field='hazard', columns=ordered_columns, affected_columns=affected_columns) for index, value in enumerate(expected.total_rows): self.assertAlmostEqual(value, table.total_rows[index]) for index, value in enumerate(expected.total_columns): self.assertAlmostEqual(value, table.total_columns[index]) for index, value in enumerate(expected.total_rows_affected): self.assertAlmostEqual(value, table.total_rows_affected[index]) for index, value in enumerate(expected.rows): self.assertAlmostEqual(value, table.rows[index]) for index, value in enumerate(expected.columns): self.assertAlmostEqual(value, table.columns[index]) for index, value in enumerate(expected.affected_columns): self.assertAlmostEqual(value, table.affected_columns[index]) # This is a list of list so we unpack both for index, value in enumerate(expected.data): for value_index, value_value in enumerate(value): self.assertAlmostEqual(value_value, table.data[index][value_index]) self.assertAlmostEqual(expected.total_affected, table.total_affected)
def aggregation_summary(aggregate_hazard, aggregation, callback=None): """Compute the summary from the aggregate hazard to the analysis layer. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | aggr_id | aggr_name | Output layer : | aggr_id | aggr_name | count of affected features per exposure type :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param aggregation: The aggregation vector layer where to write statistics. :type aggregation: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new aggregation layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_2_aggregation_steps['output_layer_name'] processing_step = summary_2_aggregation_steps['step_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = aggregation.keywords['inasafe_fields'] target_compulsory_fields = [ aggregation_id_field, aggregation_name_field, ] check_inputs(target_compulsory_fields, target_fields) # Missing exposure_count_field source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, affected_field, ] check_inputs(source_compulsory_fields, source_fields) pattern = exposure_count_field['key'] pattern = pattern.replace('%s', '') unique_exposure = read_dynamic_inasafe_field(source_fields, exposure_count_field) absolute_values = create_absolute_values_structure(aggregate_hazard, ['aggregation_id']) flat_table = FlatTable('aggregation_id', 'exposure_class') aggregation_index = source_fields[aggregation_id_field['key']] # We want to loop over affected features only. request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) expression = '\"%s\" = \'%s\'' % (affected_field['field_name'], tr('True')) request.setFilterExpression(expression) for area in aggregate_hazard.getFeatures(request): for key, name_field in source_fields.iteritems(): if key.endswith(pattern): aggregation_id = area[aggregation_index] exposure_class = key.replace(pattern, '') value = area[name_field] flat_table.add_value(value, aggregation_id=aggregation_id, exposure_class=exposure_class) # We summarize every absolute values. for field, field_definition in absolute_values.iteritems(): value = area[field] if not value or isinstance(value, QPyNullVariant): value = 0 field_definition[0].add_value( value, aggregation_id=area[aggregation_index], ) shift = aggregation.fields().count() aggregation.startEditing() add_fields(aggregation, absolute_values, [total_affected_field], unique_exposure, affected_exposure_count_field) aggregation_index = target_fields[aggregation_id_field['key']] request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregation.getFeatures(request): aggregation_value = area[aggregation_index] total = 0 for i, val in enumerate(unique_exposure): sum = flat_table.get_value(aggregation_id=aggregation_value, exposure_class=val) total += sum aggregation.changeAttributeValue(area.id(), shift + i, sum) aggregation.changeAttributeValue(area.id(), shift + len(unique_exposure), total) for i, field in enumerate(absolute_values.itervalues()): value = field[0].get_value(aggregation_id=aggregation_value, ) target_index = shift + len(unique_exposure) + 1 + i aggregation.changeAttributeValue(area.id(), target_index, value) aggregation.commitChanges() aggregation.keywords['title'] = layer_purpose_aggregation_summary['name'] if qgis_version() >= 21800: aggregation.setName(aggregation.keywords['title']) else: aggregation.setLayerName(aggregation.keywords['title']) aggregation.keywords['layer_purpose'] = ( layer_purpose_aggregation_summary['key']) check_layer(aggregation) return aggregation
def analysis_summary(aggregate_hazard, analysis): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | analysis_name | Output layer : | analysis_name | count_hazard_class | affected_count | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param analysis: The target vector layer where to write statistics. :type analysis: QgsVectorLayer :return: The new target layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = analysis.keywords['inasafe_fields'] target_compulsory_fields = [ analysis_name_field, ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure(aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fields().lookupField(hazard_class) unique_hazard = list(aggregate_hazard.uniqueValues(hazard_class_index)) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] hazard = hazard_keywords['hazard'] classification = hazard_keywords['classification'] exposure_keywords = aggregate_hazard.keywords['exposure_keywords'] exposure = exposure_keywords['exposure'] total = source_fields[total_field['key']] flat_table = FlatTable('hazard_class') # First loop over the aggregate_hazard layer request = QgsFeatureRequest() request.setSubsetOfAttributes([hazard_class, total], aggregate_hazard.fields()) request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] value = area[total] if (value == '' or value is None or isnan(value) or (hasattr(value, 'isNull') and value.isNull())): # For isnan, see ticket #3812 value = 0 if (hazard_value == '' or hazard_value is None or (hasattr(hazard_value, 'isNull') and hazard_value.isNull())): hazard_value = 'NULL' flat_table.add_value(value, hazard_class=hazard_value) # We summarize every absolute values. for field, field_definition in list(absolute_values.items()): value = area[field] if (value == '' or value is None or (hasattr(value, 'isNull') and value.isNull())): value = 0 field_definition[0].add_value(value, all='all') analysis.startEditing() shift = analysis.fields().count() counts = [ total_affected_field, total_not_affected_field, total_exposed_field, total_not_exposed_field, total_field ] dynamic_structure = [ [hazard_count_field, unique_hazard], ] add_fields(analysis, absolute_values, counts, dynamic_structure) affected_sum = 0 not_affected_sum = 0 not_exposed_sum = 0 # Summarization summary_values = {} for key, summary_rule in list(summary_rules.items()): input_field = summary_rule['input_field'] case_field = summary_rule['case_field'] if aggregate_hazard.fields().lookupField(input_field['field_name']) \ == -1: continue if aggregate_hazard.fields().lookupField(case_field['field_name']) \ == -1: continue summary_value = 0 for area in aggregate_hazard.getFeatures(): case_value = area[case_field['field_name']] if case_value in summary_rule['case_values']: summary_value += area[input_field['field_name']] summary_values[key] = summary_value for area in analysis.getFeatures(request): total = 0 for i, val in enumerate(unique_hazard): if (val == '' or val is None or (hasattr(val, 'isNull') and val.isNull())): val = 'NULL' sum = flat_table.get_value(hazard_class=val) total += sum analysis.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( exposure=exposure, hazard=hazard, classification=classification, hazard_class=val) if affected == not_exposed_class['key']: not_exposed_sum += sum elif affected: affected_sum += sum else: not_affected_sum += sum # Total Affected field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard), affected_sum) # Total Not affected field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 1, not_affected_sum) # Total Exposed field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 2, total - not_exposed_sum) # Total Not exposed field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 3, not_exposed_sum) # Total field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 4, total) # Any absolute postprocessors for i, field in enumerate(absolute_values.values()): value = field[0].get_value(all='all') analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 5 + i, value) # Summarizer of custom attributes for key, summary_value in list(summary_values.items()): summary_field = summary_rules[key]['summary_field'] field = create_field_from_definition(summary_field) analysis.addAttribute(field) field_index = analysis.fields().lookupField(field.name()) # noinspection PyTypeChecker analysis.keywords['inasafe_fields'][summary_field['key']] = ( summary_field['field_name']) analysis.changeAttributeValue(area.id(), field_index, summary_value) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is not # enough. ET 13/02/17 # total_computed = ( # affected_sum + not_affected_sum + not_exposed_sum) # if not -1 < (total_computed - total) < 1: # raise ComputationError analysis.commitChanges() analysis.keywords['title'] = layer_purpose_analysis_impacted['name'] if qgis_version() >= 21600: analysis.setName(analysis.keywords['title']) else: analysis.setLayerName(analysis.keywords['title']) analysis.keywords['layer_purpose'] = layer_purpose_analysis_impacted['key'] check_layer(analysis) return analysis
def aggregate_hazard_summary(impact, aggregate_hazard, callback=None): """Compute the summary from the source layer to the aggregate_hazard layer. Source layer : |exp_id|exp_class|haz_id|haz_class|aggr_id|aggr_name|affected|extra*| Target layer : | aggr_id | aggr_name | haz_id | haz_class | extra* | Output layer : |aggr_id| aggr_name|haz_id|haz_class|affected|extra*|count ber exposure*| :param impact: The layer to aggregate vector layer. :type impact: QgsVectorLayer :param aggregate_hazard: The aggregate_hazard vector layer where to write statistics. :type aggregate_hazard: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new aggregate_hazard layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_1_aggregate_hazard_steps['output_layer_name'] processing_step = summary_1_aggregate_hazard_steps['step_name'] source_fields = impact.keywords['inasafe_fields'] target_fields = aggregate_hazard.keywords['inasafe_fields'] target_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ exposure_id_field, exposure_class_field, aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field ] check_inputs(source_compulsory_fields, source_fields) aggregation_id = target_fields[aggregation_id_field['key']] hazard_id = target_fields[hazard_id_field['key']] hazard_class = target_fields[hazard_class_field['key']] exposure_class = source_fields[exposure_class_field['key']] exposure_class_index = impact.fieldNameIndex(exposure_class) unique_exposure = impact.uniqueValues(exposure_class_index) fields = ['aggregation_id', 'hazard_id'] absolute_values = create_absolute_values_structure(impact, fields) # We need to know what kind of exposure we are going to count. # the size, or the number of features or population. field_index = report_on_field(impact) aggregate_hazard.startEditing() shift = aggregate_hazard.fields().count() add_fields( aggregate_hazard, absolute_values, [affected_field, total_field], unique_exposure, exposure_count_field ) flat_table = FlatTable('aggregation_id', 'hazard_id', 'exposure_class') request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) LOGGER.debug('Computing the aggregate hazard summary.') for feature in impact.getFeatures(request): # Field_index can be equal to 0. if field_index is not None: value = feature[field_index] else: value = 1 aggregation_value = feature[aggregation_id] hazard_value = feature[hazard_id] if not hazard_value or isinstance(hazard_value, QPyNullVariant): hazard_value = not_exposed_class['key'] exposure_value = feature[exposure_class] if not exposure_value or isinstance(exposure_value, QPyNullVariant): exposure_value = 'NULL' flat_table.add_value( value, aggregation_id=aggregation_value, hazard_id=hazard_value, exposure_class=exposure_value ) # We summarize every absolute values. for field, field_definition in absolute_values.iteritems(): value = feature[field] if not value or isinstance(value, QPyNullVariant): value = 0 field_definition[0].add_value( value, aggregation_id=aggregation_value, hazard_id=hazard_value ) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] classification = hazard_keywords['classification'] for area in aggregate_hazard.getFeatures(request): aggregation_value = area[aggregation_id] feature_hazard_id = area[hazard_id] if not feature_hazard_id or isinstance( feature_hazard_id, QPyNullVariant): feature_hazard_id = not_exposed_class['key'] feature_hazard_value = area[hazard_class] total = 0 for i, val in enumerate(unique_exposure): sum = flat_table.get_value( aggregation_id=aggregation_value, hazard_id=feature_hazard_id, exposure_class=val ) total += sum aggregate_hazard.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( classification=classification, hazard_class=feature_hazard_value) affected = tr(unicode(affected)) aggregate_hazard.changeAttributeValue( area.id(), shift + len(unique_exposure), affected) aggregate_hazard.changeAttributeValue( area.id(), shift + len(unique_exposure) + 1, total) for i, field in enumerate(absolute_values.itervalues()): value = field[0].get_value( aggregation_id=aggregation_value, hazard_id=feature_hazard_id ) aggregate_hazard.changeAttributeValue( area.id(), shift + len(unique_exposure) + 2 + i, value) aggregate_hazard.commitChanges() aggregate_hazard.keywords['title'] = ( layer_purpose_aggregate_hazard_impacted['name']) if qgis_version() >= 21800: aggregate_hazard.setName(aggregate_hazard.keywords['title']) else: aggregate_hazard.setLayerName(aggregate_hazard.keywords['title']) aggregate_hazard.keywords['layer_purpose'] = ( layer_purpose_aggregate_hazard_impacted['key']) check_layer(aggregate_hazard) return aggregate_hazard
def exposure_summary_table( aggregate_hazard, exposure_summary=None, callback=None): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | exposure_count | Output layer : | exp_type | count_hazard_class | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param exposure_summary: The layer impact layer. :type exposure_summary: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new tabular table, without geometry. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_4_exposure_summary_table_steps[ 'output_layer_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, affected_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure( aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fields().lookupField(hazard_class) unique_hazard = aggregate_hazard.uniqueValues(hazard_class_index) unique_exposure = read_dynamic_inasafe_field( source_fields, exposure_count_field) flat_table = FlatTable('hazard_class', 'exposure_class') request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] for exposure in unique_exposure: key_name = exposure_count_field['key'] % exposure field_name = source_fields[key_name] exposure_count = area[field_name] if not exposure_count: exposure_count = 0 flat_table.add_value( exposure_count, hazard_class=hazard_value, exposure_class=exposure ) # We summarize every absolute values. for field, field_definition in list(absolute_values.items()): value = area[field] if not value: value = 0 field_definition[0].add_value( value, all='all' ) tabular = create_memory_layer(output_layer_name, QgsWkbTypes.NullGeometry) tabular.startEditing() field = create_field_from_definition(exposure_type_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][exposure_type_field['key']] = ( exposure_type_field['field_name']) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] hazard = hazard_keywords['hazard'] classification = hazard_keywords['classification'] exposure_keywords = aggregate_hazard.keywords['exposure_keywords'] exposure = exposure_keywords['exposure'] hazard_affected = {} for hazard_class in unique_hazard: if (hazard_class == '' or hazard_class is None or (hasattr(hazard_class, 'isNull') and hazard_class.isNull())): hazard_class = 'NULL' field = create_field_from_definition(hazard_count_field, hazard_class) tabular.addAttribute(field) key = hazard_count_field['key'] % hazard_class value = hazard_count_field['field_name'] % hazard_class tabular.keywords['inasafe_fields'][key] = value hazard_affected[hazard_class] = post_processor_affected_function( exposure=exposure, hazard=hazard, classification=classification, hazard_class=hazard_class ) field = create_field_from_definition(total_affected_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_affected_field['key']] = ( total_affected_field['field_name']) # essentially have the same value as NULL_hazard_count # but with this, make sure that it exists in layer so it can be used for # reporting, and can be referenced to fields.py to take the label. field = create_field_from_definition(total_not_affected_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_not_affected_field['key']] = ( total_not_affected_field['field_name']) field = create_field_from_definition(total_not_exposed_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_not_exposed_field['key']] = ( total_not_exposed_field['field_name']) field = create_field_from_definition(total_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_field['key']] = ( total_field['field_name']) summarization_dicts = {} if exposure_summary: summarization_dicts = summarize_result(exposure_summary) sorted_keys = sorted(summarization_dicts.keys()) for key in sorted_keys: summary_field = summary_rules[key]['summary_field'] field = create_field_from_definition(summary_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][ summary_field['key']] = ( summary_field['field_name']) # Only add absolute value if there is no summarization / no exposure # classification if not summarization_dicts: # For each absolute values for absolute_field in list(absolute_values.keys()): field_definition = definition(absolute_values[absolute_field][1]) field = create_field_from_definition(field_definition) tabular.addAttribute(field) key = field_definition['key'] value = field_definition['field_name'] tabular.keywords['inasafe_fields'][key] = value for exposure_type in unique_exposure: feature = QgsFeature() attributes = [exposure_type] total_affected = 0 total_not_affected = 0 total_not_exposed = 0 total = 0 for hazard_class in unique_hazard: if hazard_class == '' or hazard_class is None: hazard_class = 'NULL' value = flat_table.get_value( hazard_class=hazard_class, exposure_class=exposure_type ) attributes.append(value) if hazard_affected[hazard_class] == not_exposed_class['key']: total_not_exposed += value elif hazard_affected[hazard_class]: total_affected += value else: total_not_affected += value total += value attributes.append(total_affected) attributes.append(total_not_affected) attributes.append(total_not_exposed) attributes.append(total) if summarization_dicts: for key in sorted_keys: attributes.append(summarization_dicts[key].get( exposure_type, 0)) else: for i, field in enumerate(absolute_values.values()): value = field[0].get_value( all='all' ) attributes.append(value) feature.setAttributes(attributes) tabular.addFeature(feature) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is # not enough. ET 13/02/17 # total_computed = ( # total_affected + total_not_affected + total_not_exposed) # if not -1 < (total_computed - total) < 1: # raise ComputationError tabular.commitChanges() tabular.keywords['title'] = layer_purpose_exposure_summary_table['name'] if qgis_version() >= 21800: tabular.setName(tabular.keywords['title']) else: tabular.setLayerName(tabular.keywords['title']) tabular.keywords['layer_purpose'] = layer_purpose_exposure_summary_table[ 'key'] check_layer(tabular, has_geometry=False) return tabular
def format_impact_summary(self): """The impact summary as per category :returns: The impact summary. :rtype: safe.message.Message """ flat_table = FlatTable().from_dict( groups=self.impact_table['groups'], data=self.impact_table['data'], ) LOGGER.debug(self.impact_table['groups']) LOGGER.debug(self.impact_table['data']) LOGGER.debug(flat_table.groups) LOGGER.debug(flat_table.data) pivot_table = PivotTable(flat_table, row_field='landcover', column_field='hazard', columns=self.ordered_columns, affected_columns=self.affected_columns) report = {'impacted': pivot_table} # breakdown by zones if self.zone_field is not None: report['impacted_zones'] = {} for zone in flat_table.group_values('zone'): table = PivotTable(flat_table, row_field="landcover", column_field='hazard', columns=self.ordered_columns, affected_columns=self.affected_columns, filter_field="zone", filter_value=zone) report['impacted_zones'][zone] = table message = m.Message(style_class='container') affected_text = tr('Affected Area (ha)') show_affected = True if len(self.affected_columns) else False if show_affected: msg = tr( '* Percentage of affected area compared to the total area for ' 'the land cover type.') self.notes['fields'].append(msg) table = format_pivot_table(report['impacted'], header_text=affected_text, total_columns=True, total_affected=show_affected, total_percent_affected=show_affected, bar_chart=False) message.add(table) if 'impacted_zones' in report: message.add( m.Heading(tr('Analysis Results by Aggregation Area'), **INFO_STYLE)) for zone, table in report['impacted_zones'].items(): message.add(m.Heading(zone.lower().title(), **SUB_INFO_STYLE)) m_table = format_pivot_table( table, header_text=affected_text, total_columns=True, total_affected=show_affected, total_percent_affected=show_affected, bar_chart=False) message.add(m_table) return message.to_html(suppress_newlines=True)
def analysis_summary(aggregate_hazard, analysis): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | analysis_name | Output layer : | analysis_name | count_hazard_class | affected_count | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param analysis: The target vector layer where to write statistics. :type analysis: QgsVectorLayer :return: The new target layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = analysis.keywords['inasafe_fields'] target_compulsory_fields = [ analysis_name_field, ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure( aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fields().lookupField(hazard_class) unique_hazard = list(aggregate_hazard.uniqueValues(hazard_class_index)) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] hazard = hazard_keywords['hazard'] classification = hazard_keywords['classification'] exposure_keywords = aggregate_hazard.keywords['exposure_keywords'] exposure = exposure_keywords['exposure'] total = source_fields[total_field['key']] flat_table = FlatTable('hazard_class') # First loop over the aggregate_hazard layer request = QgsFeatureRequest() request.setSubsetOfAttributes( [hazard_class, total], aggregate_hazard.fields()) request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] value = area[total] if (value == '' or value is None or isnan(value) or (hasattr(value, 'isNull') and value.isNull())): # For isnan, see ticket #3812 value = 0 if (hazard_value == '' or hazard_value is None or (hasattr(hazard_value, 'isNull') and hazard_value.isNull())): hazard_value = 'NULL' flat_table.add_value( value, hazard_class=hazard_value ) # We summarize every absolute values. for field, field_definition in list(absolute_values.items()): value = area[field] if (value == '' or value is None or (hasattr(value, 'isNull') and value.isNull())): value = 0 field_definition[0].add_value( value, all='all' ) analysis.startEditing() shift = analysis.fields().count() counts = [ total_affected_field, total_not_affected_field, total_exposed_field, total_not_exposed_field, total_field] dynamic_structure = [ [hazard_count_field, unique_hazard], ] add_fields( analysis, absolute_values, counts, dynamic_structure) affected_sum = 0 not_affected_sum = 0 not_exposed_sum = 0 # Summarization summary_values = {} for key, summary_rule in list(summary_rules.items()): input_field = summary_rule['input_field'] case_field = summary_rule['case_field'] if aggregate_hazard.fields().lookupField(input_field['field_name']) \ == -1: continue if aggregate_hazard.fields().lookupField(case_field['field_name']) \ == -1: continue summary_value = 0 for area in aggregate_hazard.getFeatures(): case_value = area[case_field['field_name']] if case_value in summary_rule['case_values']: summary_value += area[input_field['field_name']] summary_values[key] = summary_value for area in analysis.getFeatures(request): total = 0 for i, val in enumerate(unique_hazard): if (val == '' or val is None or (hasattr(val, 'isNull') and val.isNull())): val = 'NULL' sum = flat_table.get_value(hazard_class=val) total += sum analysis.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( exposure=exposure, hazard=hazard, classification=classification, hazard_class=val) if affected == not_exposed_class['key']: not_exposed_sum += sum elif affected: affected_sum += sum else: not_affected_sum += sum # Total Affected field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard), affected_sum) # Total Not affected field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 1, not_affected_sum) # Total Exposed field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 2, total - not_exposed_sum) # Total Not exposed field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 3, not_exposed_sum) # Total field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 4, total) # Any absolute postprocessors for i, field in enumerate(absolute_values.values()): value = field[0].get_value( all='all' ) analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 5 + i, value) # Summarizer of custom attributes for key, summary_value in list(summary_values.items()): summary_field = summary_rules[key]['summary_field'] field = create_field_from_definition(summary_field) analysis.addAttribute(field) field_index = analysis.fields().lookupField(field.name()) # noinspection PyTypeChecker analysis.keywords['inasafe_fields'][summary_field['key']] = ( summary_field['field_name']) analysis.changeAttributeValue( area.id(), field_index, summary_value) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is not # enough. ET 13/02/17 # total_computed = ( # affected_sum + not_affected_sum + not_exposed_sum) # if not -1 < (total_computed - total) < 1: # raise ComputationError analysis.commitChanges() analysis.keywords['title'] = layer_purpose_analysis_impacted['name'] if qgis_version() >= 21600: analysis.setName(analysis.keywords['title']) else: analysis.setLayerName(analysis.keywords['title']) analysis.keywords['layer_purpose'] = layer_purpose_analysis_impacted['key'] check_layer(analysis) return analysis
def aggregate_hazard_summary(impact, aggregate_hazard): """Compute the summary from the source layer to the aggregate_hazard layer. Source layer : |exp_id|exp_class|haz_id|haz_class|aggr_id|aggr_name|affected|extra*| Target layer : | aggr_id | aggr_name | haz_id | haz_class | extra* | Output layer : |aggr_id| aggr_name|haz_id|haz_class|affected|extra*|count ber exposure*| :param impact: The layer to aggregate vector layer. :type impact: QgsVectorLayer :param aggregate_hazard: The aggregate_hazard vector layer where to write statistics. :type aggregate_hazard: QgsVectorLayer :return: The new aggregate_hazard layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ source_fields = impact.keywords['inasafe_fields'] target_fields = aggregate_hazard.keywords['inasafe_fields'] target_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ exposure_id_field, exposure_class_field, aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field ] check_inputs(source_compulsory_fields, source_fields) aggregation_id = target_fields[aggregation_id_field['key']] hazard_id = target_fields[hazard_id_field['key']] hazard_class = target_fields[hazard_class_field['key']] exposure_class = source_fields[exposure_class_field['key']] exposure_class_index = impact.fields().lookupField(exposure_class) unique_exposure = list(impact.uniqueValues(exposure_class_index)) fields = ['aggregation_id', 'hazard_id'] absolute_values = create_absolute_values_structure(impact, fields) # We need to know what kind of exposure we are going to count. # the size, or the number of features or population. field_index = report_on_field(impact) aggregate_hazard.startEditing() shift = aggregate_hazard.fields().count() dynamic_structure = [ [exposure_count_field, unique_exposure], ] add_fields( aggregate_hazard, absolute_values, [affected_field, total_field], dynamic_structure, ) flat_table = FlatTable('aggregation_id', 'hazard_id', 'exposure_class') request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) LOGGER.debug('Computing the aggregate hazard summary.') for feature in impact.getFeatures(request): # Field_index can be equal to 0. if field_index is not None: value = feature[field_index] else: value = 1 aggregation_value = feature[aggregation_id] hazard_value = feature[hazard_id] if (hazard_value is None or hazard_value == '' or (hasattr(hazard_value, 'isNull') and hazard_value.isNull())): hazard_value = not_exposed_class['key'] exposure_value = feature[exposure_class] if (exposure_value is None or exposure_value == '' or (hasattr(exposure_value, 'isNull') and exposure_value.isNull())): exposure_value = 'NULL' flat_table.add_value(value, aggregation_id=aggregation_value, hazard_id=hazard_value, exposure_class=exposure_value) # We summarize every absolute values. for field, field_definition in list(absolute_values.items()): value = feature[field] if (value == '' or value is None or (hasattr(value, 'isNull') and value.isNull())): value = 0 field_definition[0].add_value(value, aggregation_id=aggregation_value, hazard_id=hazard_value) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] hazard = hazard_keywords['hazard'] classification = hazard_keywords['classification'] exposure_keywords = impact.keywords['exposure_keywords'] exposure = exposure_keywords['exposure'] for area in aggregate_hazard.getFeatures(request): aggregation_value = area[aggregation_id] feature_hazard_id = area[hazard_id] if (feature_hazard_id == '' or feature_hazard_id is None or (hasattr(feature_hazard_id, 'isNull') and feature_hazard_id.isNull())): feature_hazard_id = not_exposed_class['key'] feature_hazard_value = area[hazard_class] total = 0 for i, val in enumerate(unique_exposure): sum = flat_table.get_value(aggregation_id=aggregation_value, hazard_id=feature_hazard_id, exposure_class=val) total += sum aggregate_hazard.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( exposure=exposure, hazard=hazard, classification=classification, hazard_class=feature_hazard_value) affected = tr(str(affected)) aggregate_hazard.changeAttributeValue(area.id(), shift + len(unique_exposure), affected) aggregate_hazard.changeAttributeValue(area.id(), shift + len(unique_exposure) + 1, total) for i, field in enumerate(absolute_values.values()): value = field[0].get_value(aggregation_id=aggregation_value, hazard_id=feature_hazard_id) aggregate_hazard.changeAttributeValue( area.id(), shift + len(unique_exposure) + 2 + i, value) aggregate_hazard.commitChanges() aggregate_hazard.keywords['title'] = ( layer_purpose_aggregate_hazard_impacted['name']) if qgis_version() >= 21800: aggregate_hazard.setName(aggregate_hazard.keywords['title']) else: aggregate_hazard.setLayerName(aggregate_hazard.keywords['title']) aggregate_hazard.keywords['layer_purpose'] = ( layer_purpose_aggregate_hazard_impacted['key']) aggregate_hazard.keywords['exposure_keywords'] = impact.keywords.copy() check_layer(aggregate_hazard) return aggregate_hazard
class PivotTableTest(unittest.TestCase): """Tests for reading and writing of raster and vector data.""" def setUp(self): """ This will generate a table like this: Affected columns : high and medium road_type | hazard | Totals | | Total Percent | high medium low | Total affected affected residential | 0 30 50 | 80 30 37.5 primary | 10 20 0 | 30 30 100 secondary | 0 0 40 | 40 0 0 Total | 10 50 90 | 150 60 40 """ self.affected_columns = ['medium', 'high'] self.flat_table = FlatTable("road_type", "hazard") self.flat_table.add_value(10, road_type="primary", hazard="high") self.flat_table.add_value(20, road_type="primary", hazard="medium") self.flat_table.add_value(30, road_type="residential", hazard="medium") self.flat_table.add_value(40, road_type="secondary", hazard="low") self.flat_table.add_value(50, road_type="residential", hazard="low") def test_basic(self): pivot_table = PivotTable( self.flat_table, row_field="road_type", column_field="hazard") self.assertEqual( pivot_table.rows, ['residential', 'primary', 'secondary']) self.assertEqual(pivot_table.columns, ['high', 'medium', 'low']) self.assertEqual( pivot_table.data, [[0, 30, 50], [10, 20, 0], [0, 0, 40]]) self.assertEqual(pivot_table.total, 150) self.assertEqual(pivot_table.total_rows, [80, 30, 40]) self.assertEqual(pivot_table.total_columns, [10, 50, 90]) self.assertEqual(pivot_table.total_affected, 0) self.assertEqual(pivot_table.total_rows_affected, [0, 0, 0]) def test_one_column(self): pivot_table = PivotTable(self.flat_table, row_field='road_type') self.assertEqual( pivot_table.rows, ['residential', 'primary', 'secondary']) self.assertEqual(pivot_table.columns, ['']) self.assertEqual(pivot_table.data, [[80], [30], [40]]) self.assertEqual(pivot_table.total, 150) self.assertEqual(pivot_table.total_rows, [80, 30, 40]) self.assertEqual(pivot_table.total_columns, [150]) self.assertEqual(pivot_table.total_affected, 0) self.assertEqual(pivot_table.total_rows_affected, [0, 0, 0]) def test_one_row(self): pivot_table = PivotTable(self.flat_table, column_field='hazard') self.assertEqual(pivot_table.rows, ['']) self.assertEqual(pivot_table.columns, ['high', 'medium', 'low']) self.assertEqual(pivot_table.data, [[10, 50, 90]]) self.assertEqual(pivot_table.total, 150) self.assertEqual(pivot_table.total_rows, [150]) self.assertEqual(pivot_table.total_columns, [10, 50, 90]) pivot_table = PivotTable( self.flat_table, column_field='hazard', affected_columns=self.affected_columns) self.assertEqual(pivot_table.total_rows_affected, [60]) self.assertEqual(pivot_table.total_affected, 60) self.assertEqual(pivot_table.total_percent_affected, 40) self.assertEqual(pivot_table.total_percent_rows_affected, [40]) def test_affected(self): pivot_table = PivotTable( self.flat_table, row_field='road_type', column_field='hazard', affected_columns=self.affected_columns) self.assertEqual(pivot_table.total_affected, 60) self.assertEqual(pivot_table.total_rows_affected, [30, 30, 0]) self.assertEqual(pivot_table.affected_columns, ['medium', 'high']) self.assertEqual(pivot_table.total_percent_affected, 40) self.assertEqual( pivot_table.total_percent_rows_affected, [37.5, 100.0, 0]) def test_filter(self): pivot_table = PivotTable( self.flat_table, column_field='hazard', filter_field='road_type', filter_value='primary') self.assertEqual(pivot_table.rows, ['']) self.assertEqual(pivot_table.columns, ['high', 'medium', 'low']) self.assertEqual(pivot_table.data, [[10, 20, 0]]) self.assertEqual(pivot_table.total, 30) self.assertEqual(pivot_table.total_rows, [30]) self.assertEqual(pivot_table.total_columns, [10, 20, 0]) pivot_table = PivotTable( self.flat_table, column_field='hazard', filter_field='road_type', filter_value='primary', affected_columns=self.affected_columns) self.assertEqual(pivot_table.total_affected, 30) self.assertEqual(pivot_table.total_rows_affected, [30]) def test_to_dict(self): """Test for to_dict the FlatTable""" flat_table_dict = self.flat_table.to_dict() self.assertEquals(flat_table_dict['groups'], self.flat_table.groups) self.assertEquals( len(flat_table_dict['data']), len(self.flat_table.data)) def test_to_json(self): """Test for JSON-fy the FlatTable""" json_string = self.flat_table.to_json() flat_table_dict = json.loads(json_string) self.assertEquals( tuple(flat_table_dict['groups']), self.flat_table.groups) self.assertEquals( len(flat_table_dict['data']), len(self.flat_table.data)) def test_from_json(self): """Test FlatTable from_json method""" json_string = ( '{"data": [["residential", "low", 50], ' '["residential", "medium", 30], ["secondary", "low", 40], ' '["primary", "high", 10], ["primary", "medium", 20]], ' '"groups": ["road_type", "hazard"]}') flat_table = FlatTable() flat_table.from_json(json_string) expected_groups = ["road_type", "hazard"] for i in range(len(flat_table.groups)): self.assertEquals(expected_groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20) def test_from_dict(self): """Test FlatTable from_dict method""" groups = ["road_type", "hazard"] data = [ ["residential", "low", 50], ["residential", "medium", 30], ["secondary", "low", 40], ["primary", "high", 10], ["primary", "medium", 20] ] flat_table = FlatTable() flat_table.from_dict(groups, data) for i in range(len(flat_table.groups)): self.assertEquals(groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20) groups = [u'landcover', u'hazard', u'zone'] data = [ [u'Forest', u'high', None, 5172.100048073517], [u'Population', u'high', None, 20689.8283632199], [u'Forest', u'low', None, 5171.381989317935], [u'Population', u'medium', None, 10347.048486941067], [u'Meadow', u'high', None, 5172.81413353821], [u'Population', u'low', None, 10342.763978632318], [u'Meadow', u'medium', None, 5173.5242434723095] ] flat_table = FlatTable() flat_table.from_dict(groups, data) for i in range(len(flat_table.groups)): self.assertEquals(groups[i], flat_table.groups[i]) self.assertEquals(flat_table.data[ ('Forest', 'high', None)], 5172.100048073517) def test_to_from_json(self): """Test FlatTable from_dict method""" json_string = self.flat_table.to_json() flat_table = FlatTable() flat_table.from_json(json_string) self.assertEquals(flat_table.data[('residential', 'low')], 50) self.assertEquals(flat_table.data[('residential', 'medium')], 30) self.assertEquals(flat_table.data[('secondary', 'low')], 40) self.assertEquals(flat_table.data[('primary', 'high')], 10) self.assertEquals(flat_table.data[('primary', 'medium')], 20)
def exposure_summary_table(aggregate_hazard, exposure_summary=None, callback=None): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | exposure_count | Output layer : | exp_type | count_hazard_class | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param exposure_summary: The layer impact layer. :type exposure_summary: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new tabular table, without geometry. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_4_exposure_summary_table_steps[ 'output_layer_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, affected_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure(aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fields().lookupField(hazard_class) unique_hazard = aggregate_hazard.uniqueValues(hazard_class_index) unique_exposure = read_dynamic_inasafe_field(source_fields, exposure_count_field) flat_table = FlatTable('hazard_class', 'exposure_class') request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] for exposure in unique_exposure: key_name = exposure_count_field['key'] % exposure field_name = source_fields[key_name] exposure_count = area[field_name] if not exposure_count: exposure_count = 0 flat_table.add_value(exposure_count, hazard_class=hazard_value, exposure_class=exposure) # We summarize every absolute values. for field, field_definition in list(absolute_values.items()): value = area[field] if not value: value = 0 field_definition[0].add_value(value, all='all') tabular = create_memory_layer(output_layer_name, QgsWkbTypes.NullGeometry) tabular.startEditing() field = create_field_from_definition(exposure_type_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][exposure_type_field['key']] = ( exposure_type_field['field_name']) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] hazard = hazard_keywords['hazard'] classification = hazard_keywords['classification'] exposure_keywords = aggregate_hazard.keywords['exposure_keywords'] exposure = exposure_keywords['exposure'] hazard_affected = {} for hazard_class in unique_hazard: if (hazard_class == '' or hazard_class is None or (hasattr(hazard_class, 'isNull') and hazard_class.isNull())): hazard_class = 'NULL' field = create_field_from_definition(hazard_count_field, hazard_class) tabular.addAttribute(field) key = hazard_count_field['key'] % hazard_class value = hazard_count_field['field_name'] % hazard_class tabular.keywords['inasafe_fields'][key] = value hazard_affected[hazard_class] = post_processor_affected_function( exposure=exposure, hazard=hazard, classification=classification, hazard_class=hazard_class) field = create_field_from_definition(total_affected_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_affected_field['key']] = ( total_affected_field['field_name']) # essentially have the same value as NULL_hazard_count # but with this, make sure that it exists in layer so it can be used for # reporting, and can be referenced to fields.py to take the label. field = create_field_from_definition(total_not_affected_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_not_affected_field['key']] = ( total_not_affected_field['field_name']) field = create_field_from_definition(total_not_exposed_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_not_exposed_field['key']] = ( total_not_exposed_field['field_name']) field = create_field_from_definition(total_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][total_field['key']] = ( total_field['field_name']) summarization_dicts = {} if exposure_summary: summarization_dicts = summarize_result(exposure_summary) sorted_keys = sorted(summarization_dicts.keys()) for key in sorted_keys: summary_field = summary_rules[key]['summary_field'] field = create_field_from_definition(summary_field) tabular.addAttribute(field) tabular.keywords['inasafe_fields'][summary_field['key']] = ( summary_field['field_name']) # Only add absolute value if there is no summarization / no exposure # classification if not summarization_dicts: # For each absolute values for absolute_field in list(absolute_values.keys()): field_definition = definition(absolute_values[absolute_field][1]) field = create_field_from_definition(field_definition) tabular.addAttribute(field) key = field_definition['key'] value = field_definition['field_name'] tabular.keywords['inasafe_fields'][key] = value for exposure_type in unique_exposure: feature = QgsFeature() attributes = [exposure_type] total_affected = 0 total_not_affected = 0 total_not_exposed = 0 total = 0 for hazard_class in unique_hazard: if hazard_class == '' or hazard_class is None: hazard_class = 'NULL' value = flat_table.get_value(hazard_class=hazard_class, exposure_class=exposure_type) attributes.append(value) if hazard_affected[hazard_class] == not_exposed_class['key']: total_not_exposed += value elif hazard_affected[hazard_class]: total_affected += value else: total_not_affected += value total += value attributes.append(total_affected) attributes.append(total_not_affected) attributes.append(total_not_exposed) attributes.append(total) if summarization_dicts: for key in sorted_keys: attributes.append(summarization_dicts[key].get( exposure_type, 0)) else: for i, field in enumerate(absolute_values.values()): value = field[0].get_value(all='all') attributes.append(value) feature.setAttributes(attributes) tabular.addFeature(feature) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is # not enough. ET 13/02/17 # total_computed = ( # total_affected + total_not_affected + total_not_exposed) # if not -1 < (total_computed - total) < 1: # raise ComputationError tabular.commitChanges() tabular.keywords['title'] = layer_purpose_exposure_summary_table['name'] if qgis_version() >= 21800: tabular.setName(tabular.keywords['title']) else: tabular.setLayerName(tabular.keywords['title']) tabular.keywords['layer_purpose'] = layer_purpose_exposure_summary_table[ 'key'] check_layer(tabular, has_geometry=False) return tabular
def aggregation_summary(aggregate_hazard, aggregation, callback=None): """Compute the summary from the aggregate hazard to the analysis layer. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | aggr_id | aggr_name | Output layer : | aggr_id | aggr_name | count of affected features per exposure type :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param aggregation: The aggregation vector layer where to write statistics. :type aggregation: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new aggregation layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_2_aggregation_steps['output_layer_name'] processing_step = summary_2_aggregation_steps['step_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = aggregation.keywords['inasafe_fields'] target_compulsory_fields = [ aggregation_id_field, aggregation_name_field, ] check_inputs(target_compulsory_fields, target_fields) # Missing exposure_count_field source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, affected_field, ] check_inputs(source_compulsory_fields, source_fields) pattern = exposure_count_field['key'] pattern = pattern.replace('%s', '') unique_exposure = read_dynamic_inasafe_field( source_fields, exposure_count_field) absolute_values = create_absolute_values_structure( aggregate_hazard, ['aggregation_id']) flat_table = FlatTable('aggregation_id', 'exposure_class') aggregation_index = source_fields[aggregation_id_field['key']] # We want to loop over affected features only. request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) expression = '\"%s\" = \'%s\'' % ( affected_field['field_name'], tr('True')) request.setFilterExpression(expression) for area in aggregate_hazard.getFeatures(request): for key, name_field in source_fields.iteritems(): if key.endswith(pattern): aggregation_id = area[aggregation_index] exposure_class = key.replace(pattern, '') value = area[name_field] flat_table.add_value( value, aggregation_id=aggregation_id, exposure_class=exposure_class ) # We summarize every absolute values. for field, field_definition in absolute_values.iteritems(): value = area[field] if not value or isinstance(value, QPyNullVariant): value = 0 field_definition[0].add_value( value, aggregation_id=area[aggregation_index], ) shift = aggregation.fields().count() aggregation.startEditing() add_fields( aggregation, absolute_values, [total_affected_field], unique_exposure, affected_exposure_count_field) aggregation_index = target_fields[aggregation_id_field['key']] request = QgsFeatureRequest() request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregation.getFeatures(request): aggregation_value = area[aggregation_index] total = 0 for i, val in enumerate(unique_exposure): sum = flat_table.get_value( aggregation_id=aggregation_value, exposure_class=val ) total += sum aggregation.changeAttributeValue(area.id(), shift + i, sum) aggregation.changeAttributeValue( area.id(), shift + len(unique_exposure), total) for i, field in enumerate(absolute_values.itervalues()): value = field[0].get_value( aggregation_id=aggregation_value, ) target_index = shift + len(unique_exposure) + 1 + i aggregation.changeAttributeValue( area.id(), target_index, value) aggregation.commitChanges() aggregation.keywords['title'] = layer_purpose_aggregation_summary['name'] if qgis_version() >= 21800: aggregation.setName(aggregation.keywords['title']) else: aggregation.setLayerName(aggregation.keywords['title']) aggregation.keywords['layer_purpose'] = ( layer_purpose_aggregation_summary['key']) check_layer(aggregation) return aggregation
def analysis_summary(aggregate_hazard, analysis, callback=None): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | analysis_id | Output layer : | analysis_id | count_hazard_class | affected_count | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param analysis: The target vector layer where to write statistics. :type analysis: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new target layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_3_analysis_steps['output_layer_name'] processing_step = summary_3_analysis_steps['step_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = analysis.keywords['inasafe_fields'] target_compulsory_fields = [ analysis_id_field, analysis_name_field, ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure(aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fieldNameIndex(hazard_class) unique_hazard = aggregate_hazard.uniqueValues(hazard_class_index) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] classification = hazard_keywords['classification'] total = source_fields[total_field['key']] flat_table = FlatTable('hazard_class') # First loop over the aggregate_hazard layer request = QgsFeatureRequest() request.setSubsetOfAttributes([hazard_class, total], aggregate_hazard.fields()) request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] value = area[total] if not value or isinstance(value, QPyNullVariant) or isnan(value): # For isnan, see ticket #3812 value = 0 if not hazard_value or isinstance(hazard_value, QPyNullVariant): hazard_value = 'NULL' flat_table.add_value(value, hazard_class=hazard_value) # We summarize every absolute values. for field, field_definition in absolute_values.iteritems(): value = area[field] if not value or isinstance(value, QPyNullVariant): value = 0 field_definition[0].add_value(value, all='all') analysis.startEditing() shift = analysis.fields().count() counts = [ total_affected_field, total_not_affected_field, total_not_exposed_field, total_field ] add_fields(analysis, absolute_values, counts, unique_hazard, hazard_count_field) affected_sum = 0 not_affected_sum = 0 not_exposed_sum = 0 for area in analysis.getFeatures(request): total = 0 for i, val in enumerate(unique_hazard): if not val or isinstance(val, QPyNullVariant): val = 'NULL' sum = flat_table.get_value(hazard_class=val) total += sum analysis.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( classification=classification, hazard_class=val) if affected == not_exposed_class['key']: not_exposed_sum += sum elif affected: affected_sum += sum else: not_affected_sum += sum # Affected field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard), affected_sum) # Not affected field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 1, not_affected_sum) # Not exposed field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 2, not_exposed_sum) # Total field analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 3, total) # Any absolute postprocessors for i, field in enumerate(absolute_values.itervalues()): value = field[0].get_value(all='all') analysis.changeAttributeValue(area.id(), shift + len(unique_hazard) + 4 + i, value) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is not # enough. ET 13/02/17 # total_computed = ( # affected_sum + not_affected_sum + not_exposed_sum) # if not -1 < (total_computed - total) < 1: # raise ComputationError analysis.commitChanges() analysis.keywords['title'] = layer_purpose_analysis_impacted['name'] if qgis_version() >= 21600: analysis.setName(analysis.keywords['title']) else: analysis.setLayerName(analysis.keywords['title']) analysis.keywords['layer_purpose'] = layer_purpose_analysis_impacted['key'] check_layer(analysis) return analysis
def analysis_summary(aggregate_hazard, analysis, callback=None): """Compute the summary from the aggregate hazard to analysis. Source layer : | haz_id | haz_class | aggr_id | aggr_name | total_feature | Target layer : | analysis_id | Output layer : | analysis_id | count_hazard_class | affected_count | total | :param aggregate_hazard: The layer to aggregate vector layer. :type aggregate_hazard: QgsVectorLayer :param analysis: The target vector layer where to write statistics. :type analysis: QgsVectorLayer :param callback: A function to all to indicate progress. The function should accept params 'current' (int), 'maximum' (int) and 'step' (str). Defaults to None. :type callback: function :return: The new target layer with summary. :rtype: QgsVectorLayer .. versionadded:: 4.0 """ output_layer_name = summary_3_analysis_steps['output_layer_name'] processing_step = summary_3_analysis_steps['step_name'] source_fields = aggregate_hazard.keywords['inasafe_fields'] target_fields = analysis.keywords['inasafe_fields'] target_compulsory_fields = [ analysis_id_field, analysis_name_field, ] check_inputs(target_compulsory_fields, target_fields) source_compulsory_fields = [ aggregation_id_field, aggregation_name_field, hazard_id_field, hazard_class_field, total_field ] check_inputs(source_compulsory_fields, source_fields) absolute_values = create_absolute_values_structure( aggregate_hazard, ['all']) hazard_class = source_fields[hazard_class_field['key']] hazard_class_index = aggregate_hazard.fieldNameIndex(hazard_class) unique_hazard = aggregate_hazard.uniqueValues(hazard_class_index) hazard_keywords = aggregate_hazard.keywords['hazard_keywords'] classification = hazard_keywords['classification'] total = source_fields[total_field['key']] flat_table = FlatTable('hazard_class') # First loop over the aggregate_hazard layer request = QgsFeatureRequest() request.setSubsetOfAttributes( [hazard_class, total], aggregate_hazard.fields()) request.setFlags(QgsFeatureRequest.NoGeometry) for area in aggregate_hazard.getFeatures(): hazard_value = area[hazard_class_index] value = area[total] if not value or isinstance(value, QPyNullVariant) or isnan(value): # For isnan, see ticket #3812 value = 0 if not hazard_value or isinstance(hazard_value, QPyNullVariant): hazard_value = 'NULL' flat_table.add_value( value, hazard_class=hazard_value ) # We summarize every absolute values. for field, field_definition in absolute_values.iteritems(): value = area[field] if not value or isinstance(value, QPyNullVariant): value = 0 field_definition[0].add_value( value, all='all' ) analysis.startEditing() shift = analysis.fields().count() counts = [ total_affected_field, total_not_affected_field, total_not_exposed_field, total_field] add_fields( analysis, absolute_values, counts, unique_hazard, hazard_count_field) affected_sum = 0 not_affected_sum = 0 not_exposed_sum = 0 for area in analysis.getFeatures(request): total = 0 for i, val in enumerate(unique_hazard): if not val or isinstance(val, QPyNullVariant): val = 'NULL' sum = flat_table.get_value(hazard_class=val) total += sum analysis.changeAttributeValue(area.id(), shift + i, sum) affected = post_processor_affected_function( classification=classification, hazard_class=val) if affected == not_exposed_class['key']: not_exposed_sum += sum elif affected: affected_sum += sum else: not_affected_sum += sum # Affected field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard), affected_sum) # Not affected field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 1, not_affected_sum) # Not exposed field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 2, not_exposed_sum) # Total field analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 3, total) # Any absolute postprocessors for i, field in enumerate(absolute_values.itervalues()): value = field[0].get_value( all='all' ) analysis.changeAttributeValue( area.id(), shift + len(unique_hazard) + 4 + i, value) # Sanity check ± 1 to the result. Disabled for now as it seems ± 1 is not # enough. ET 13/02/17 # total_computed = ( # affected_sum + not_affected_sum + not_exposed_sum) # if not -1 < (total_computed - total) < 1: # raise ComputationError analysis.commitChanges() analysis.keywords['title'] = layer_purpose_analysis_impacted['name'] if qgis_version() >= 21600: analysis.setName(analysis.keywords['title']) else: analysis.setLayerName(analysis.keywords['title']) analysis.keywords['layer_purpose'] = layer_purpose_analysis_impacted['key'] check_layer(analysis) return analysis
def format_impact_summary(self): """The impact summary as per category :returns: The impact summary. :rtype: safe.message.Message """ flat_table = FlatTable().from_dict( groups=self.impact_table['groups'], data=self.impact_table['data'], ) LOGGER.debug(self.impact_table['groups']) LOGGER.debug(self.impact_table['data']) LOGGER.debug(flat_table.groups) LOGGER.debug(flat_table.data) pivot_table = PivotTable( flat_table, row_field='landcover', column_field='hazard', columns=self.ordered_columns, affected_columns=self.affected_columns) report = {'impacted': pivot_table} # breakdown by zones if self.zone_field is not None: report['impacted_zones'] = {} for zone in flat_table.group_values('zone'): table = PivotTable( flat_table, row_field="landcover", column_field='hazard', columns=self.ordered_columns, affected_columns=self.affected_columns, filter_field="zone", filter_value=zone) report['impacted_zones'][zone] = table message = m.Message(style_class='container') affected_text = tr('Affected Area (ha)') show_affected = True if len(self.affected_columns) else False if show_affected: msg = tr( '* Percentage of affected area compared to the total area for ' 'the land cover type.') self.notes['fields'].append(msg) table = format_pivot_table( report['impacted'], header_text=affected_text, total_columns=True, total_affected=show_affected, total_percent_affected=show_affected, bar_chart=False) message.add(table) if 'impacted_zones' in report: message.add(m.Heading( tr('Analysis Results by Aggregation Area'), **INFO_STYLE)) for zone, table in report['impacted_zones'].items(): message.add(m.Heading(zone.lower().title(), **SUB_INFO_STYLE)) m_table = format_pivot_table( table, header_text=affected_text, total_columns=True, total_affected=show_affected, total_percent_affected=show_affected, bar_chart=False) message.add(m_table) return message.to_html(suppress_newlines=True)