def get_output(self, aoi_mode):
"""Returns the results of the post processing as a table.
:param aoi_mode: aoi mode of the aggregator.
:type aoi_mode: bool
:returns: str - a string containing the html in the requested format.
"""
if self.error_message is not None:
message = m.Message(
m.Heading(self.tr('Postprocessing report skipped')),
m.Paragraph(
self.tr(
'Due to a problem while processing the results,'
' the detailed postprocessing report is unavailable:'
' %s') % self.error_message))
return message
else:
try:
if (self.keyword_io.read_keywords(self.aggregator.layer,
'had multipart polygon')):
self._consolidate_multipart_stats()
except KeywordNotFoundError:
pass
return self._generate_tables(aoi_mode)
def getOutput(self):
"""Returns the results of the post processing as a table.
Args:
theSingleTableFlag - bool indicating if result should be rendered
as a single table. Default False.
Returns: str - a string containing the html in the requested format.
"""
# LOGGER.debug(self.postProcessingOutput)
if self.errorMessage is not None:
myMessage = m.Message(
m.Heading(self.tr('Postprocessing report skipped')),
m.Paragraph(
self.tr(
'Due to a problem while processing the results,'
' the detailed postprocessing report is unavailable:'
' %1').arg(self.errorMessage)))
return myMessage
return self._generateTables()
def check_memory_usage(buffered_geo_extent, cell_size):
"""Helper to check if analysis is feasible when extents change.
For simplicity, we will do all our calculations in geocrs.
:param buffered_geo_extent: An extent in the for [xmin, ymin, xmax, ymax]
:type buffered_geo_extent: list
:param cell_size: The size of a cell (assumes in the X direction).
:type cell_size: float
:returns: True if it appears we have enough memory (or we can't compute
it), False if it appears we do not have enough.
:rtype: bool
:raises: A Message containing notes about how much memory is needed
for a single raster and if this is likely to result in an error.
:returns: True if it is supposed that there is sufficient memory,
False if it is supposed that too little memory exists.
:rtype: bool
"""
message = m.Message()
check_heading = m.Heading(tr('Checking available memory'),
**PROGRESS_UPDATE_STYLE)
message.add(check_heading)
width = buffered_geo_extent[2] - buffered_geo_extent[0]
height = buffered_geo_extent[3] - buffered_geo_extent[1]
try:
# noinspection PyAugmentAssignment
width = width / cell_size
# noinspection PyAugmentAssignment
height = height / cell_size
except TypeError:
# Could have been a vector layer for example
reason = tr(
'Computed cellsize was None. Memory check currently only works '
'for raster input layers.')
message.add(reason)
send_message(message)
return True # assume enough mem since we have no vector check logic
bullet_list = m.BulletedList()
bullet = m.Paragraph(m.ImportantText(tr('Width: ')), str(width))
bullet_list.add(bullet)
bullet = m.Paragraph(m.ImportantText(tr('Height: ')), str(height))
bullet_list.add(bullet)
bullet = m.Paragraph(m.ImportantText(tr('Cell Size: ')), str(cell_size))
bullet_list.add(bullet)
message.add(bullet_list)
# Compute mem requirement in MB (assuming numpy uses 8 bytes by per
# cell) see this link:
# http://stackoverflow.com/questions/11784329/
# python-memory-usage-of-numpy-arrays
# Also note that the on-disk requirement of the clipped tifs is about
# half this since the tifs as in single precision,
# whereas numpy arrays are in double precision.
requirement = ((width * height * 8) / 1024 / 1024)
try:
free_memory = get_free_memory()
except ValueError:
error_heading = m.Heading(tr('Memory check error'), **WARNING_STYLE)
error_message = tr('Could not determine free memory')
message.add(error_heading)
message.add(error_message)
send_message(message)
LOGGER.exception(message)
return True # still let the user try to run their analysis
# We work on the assumption that if more than 10% of the available
# memory is occupied by a single layer we could run out of memory
# (depending on the impact function). This is because multiple
# in memory copies of the layer are often made during processing.
warning_limit = 10
usage_indicator = (float(requirement) / float(free_memory)) * 100
counts_message = tr('Memory requirement: about %d mb per raster layer ('
'%d mb available)') % (requirement, free_memory)
usage_message = tr('Memory used / available: %d/%d') % (usage_indicator,
warning_limit)
message.add(counts_message)
message.add(usage_message)
if warning_limit <= usage_indicator:
warning_heading = m.Heading(tr('Potential memory issue'),
**WARNING_STYLE)
warning_message = tr(
'There may not be enough free memory to run this analysis. You can'
' attempt to run the analysis anyway, but note that your computer '
'may become unresponsive during execution, and / or the analysis '
'may fail due to insufficient memory. Proceed at your own risk.')
suggestion_heading = m.Heading(tr('Suggestion'), **INFO_STYLE)
suggestion = tr(
'Try zooming in to a smaller area or using a raster layer with a '
'coarser resolution to speed up execution and reduce memory '
'requirements. You could also try adding more RAM to your '
'computer.')
message.add(warning_heading)
message.add(warning_message)
message.add(suggestion_heading)
message.add(suggestion)
send_message(message)
LOGGER.info(message.to_text())
return False
send_message(message)
LOGGER.info(message.to_text())
return True
def impact_attribution(keywords, inasafe_flag=False):
"""Make a little table for attribution of data sources used in impact.
:param keywords: A keywords dict for an impact layer.
:type keywords: dict
:param inasafe_flag: bool - whether to show a little InaSAFE promotional
text in the attribution output. Defaults to False.
:returns: An html snippet containing attribution information for the impact
layer. If no keywords are present or no appropriate keywords are
present, None is returned.
:rtype: str
"""
if keywords is None:
return None
myJoinWords = ' - %s ' % tr('sourced from')
myHazardDetails = tr('Hazard details')
myHazardTitleKeyword = 'hazard_title'
myHazardSourceKeyword = 'hazard_source'
myExposureDetails = tr('Exposure details')
myExposureTitleKeyword = 'exposure_title'
myExposureSourceKeyword = 'exposure_source'
if myHazardTitleKeyword in keywords:
# We use safe translation infrastructure for this one (rather than Qt)
myHazardTitle = safeTr(keywords[myHazardTitleKeyword])
else:
myHazardTitle = tr('Hazard layer')
if myHazardSourceKeyword in keywords:
# We use safe translation infrastructure for this one (rather than Qt)
myHazardSource = safeTr(keywords[myHazardSourceKeyword])
else:
myHazardSource = tr('an unknown source')
if myExposureTitleKeyword in keywords:
myExposureTitle = keywords[myExposureTitleKeyword]
else:
myExposureTitle = tr('Exposure layer')
if myExposureSourceKeyword in keywords:
myExposureSource = keywords[myExposureSourceKeyword]
else:
myExposureSource = tr('an unknown source')
myReport = m.Message()
myReport.add(m.Heading(myHazardDetails, **INFO_STYLE))
myReport.add(m.Paragraph(myHazardTitle, myJoinWords, myHazardSource))
myReport.add(m.Heading(myExposureDetails, **INFO_STYLE))
myReport.add(m.Paragraph(myExposureTitle, myJoinWords, myExposureSource))
if inasafe_flag:
myReport.add(m.Heading(tr('Software notes'), **INFO_STYLE))
myInaSAFEPhrase = tr(
'This report was created using InaSAFE version %1. Visit '
'http://inasafe.org to get your free copy of this software!'
'InaSAFE has been jointly developed by BNPB, AusAid/AIFDRR & the '
'World Bank').arg(get_version())
myReport.add(m.Paragraph(m.Text(myInaSAFEPhrase)))
return myReport
def impact_attribution(keywords, inasafe_flag=False):
"""Make a little table for attribution of data sources used in impact.
:param keywords: A keywords dict for an impact layer.
:type keywords: dict
:param inasafe_flag: bool - whether to show a little InaSAFE promotional
text in the attribution output. Defaults to False.
:returns: An html snippet containing attribution information for the impact
layer. If no keywords are present or no appropriate keywords are
present, None is returned.
:rtype: safe.messaging.Message
"""
if keywords is None:
return None
join_words = ' - %s ' % tr('sourced from')
hazard_details = tr('Hazard details')
hazard_title_keywords = 'hazard_title'
hazard_source_keywords = 'hazard_source'
exposure_details = tr('Exposure details')
exposure_title_keywords = 'exposure_title'
exposure_source_keyword = 'exposure_source'
if hazard_title_keywords in keywords:
# We use safe translation infrastructure for this one (rather than Qt)
hazard_title = safeTr(keywords[hazard_title_keywords])
else:
hazard_title = tr('Hazard layer')
if hazard_source_keywords in keywords:
# We use safe translation infrastructure for this one (rather than Qt)
hazard_source = safeTr(keywords[hazard_source_keywords])
else:
hazard_source = tr('an unknown source')
if exposure_title_keywords in keywords:
exposure_title = keywords[exposure_title_keywords]
else:
exposure_title = tr('Exposure layer')
if exposure_source_keyword in keywords:
exposure_source = keywords[exposure_source_keyword]
else:
exposure_source = tr('an unknown source')
report = m.Message()
report.add(m.Heading(hazard_details, **INFO_STYLE))
report.add(m.Paragraph(hazard_title, join_words, hazard_source))
report.add(m.Heading(exposure_details, **INFO_STYLE))
report.add(m.Paragraph(exposure_title, join_words, exposure_source))
if inasafe_flag:
report.add(m.Heading(tr('Software notes'), **INFO_STYLE))
# noinspection PyUnresolvedReferences
inasafe_phrase = tr(
'This report was created using InaSAFE version %s. Visit '
'http://inasafe.org to get your free copy of this software!'
'InaSAFE has been jointly developed by BNPB, AusAid/AIFDRR & the '
'World Bank') % (get_version())
report.add(m.Paragraph(m.Text(inasafe_phrase)))
return report
def run(self):
"""Run any post processors requested by the impact function.
"""
try:
requested_postprocessors = self.function_parameters[
'postprocessors']
postprocessors = get_postprocessors(requested_postprocessors,
self.aggregator.aoi_mode)
except (TypeError, KeyError):
# TypeError is for when function_parameters is none
# KeyError is for when ['postprocessors'] is unavailable
postprocessors = {}
LOGGER.debug('Running this postprocessors: ' + str(postprocessors))
feature_names_attribute = self.aggregator.attributes[
self.aggregator.get_default_keyword('AGGR_ATTR_KEY')]
if feature_names_attribute is None:
self.attribute_title = self.tr('Aggregation unit')
else:
self.attribute_title = feature_names_attribute
name_filed_index = self.aggregator.layer.fieldNameIndex(
self.attribute_title)
sum_field_index = self.aggregator.layer.fieldNameIndex(
self._sum_field_name())
user_defined_female_ratio = False
female_ratio_field_index = None
female_ratio = None
user_defined_age_ratios = False
youth_ratio_field_index = None
youth_ratio = None
adult_ratio_field_index = None
adult_ratio = None
elderly_ratio_field_index = None
elderly_ratio = None
if 'Gender' in postprocessors:
# look if we need to look for a variable female ratio in a layer
try:
female_ratio_field = self.aggregator.attributes[
self.aggregator.get_default_keyword(
'FEMALE_RATIO_ATTR_KEY')]
female_ratio_field_index = \
self.aggregator.layer.fieldNameIndex(female_ratio_field)
# something went wrong finding the female ratio field,
# use defaults from below except block
if female_ratio_field_index == -1:
raise KeyError
user_defined_female_ratio = True
except KeyError:
try:
female_ratio = self.keyword_io.read_keywords(
self.aggregator.layer,
self.aggregator.get_default_keyword(
'FEMALE_RATIO_KEY'))
except KeywordNotFoundError:
female_ratio = \
self.aggregator.get_default_keyword('FEMALE_RATIO')
if 'Age' in postprocessors:
# look if we need to look for a variable age ratio in a layer
try:
youth_ratio_field = self.aggregator.attributes[
self.aggregator.get_default_keyword(
'YOUTH_RATIO_ATTR_KEY')]
youth_ratio_field_index = \
self.aggregator.layer.fieldNameIndex(youth_ratio_field)
adult_ratio_field = self.aggregator.attributes[
self.aggregator.get_default_keyword(
'ADULT_RATIO_ATTR_KEY')]
adult_ratio_field_index = \
self.aggregator.layer.fieldNameIndex(adult_ratio_field)
elderly_ratio_field = self.aggregator.attributes[
self.aggregator.get_default_keyword(
'ELDERLY_RATIO_ATTR_KEY')]
elderly_ratio_field_index = \
self.aggregator.layer.fieldNameIndex(elderly_ratio_field)
# something went wrong finding the youth ratio field,
# use defaults from below except block
if (youth_ratio_field_index == -1
or adult_ratio_field_index == -1
or elderly_ratio_field_index == -1):
raise KeyError
user_defined_age_ratios = True
except KeyError:
try:
youth_ratio = self.keyword_io.read_keywords(
self.aggregator.layer,
self.aggregator.get_default_keyword('YOUTH_RATIO_KEY'))
adult_ratio = self.keyword_io.read_keywords(
self.aggregator.layer,
self.aggregator.get_default_keyword('ADULT_RATIO_KEY'))
elderly_ratio = self.keyword_io.read_keywords(
self.aggregator.layer,
self.aggregator.get_default_keyword(
'ELDERLY_RATIO_KEY'))
except KeywordNotFoundError:
youth_ratio = \
self.aggregator.get_default_keyword('YOUTH_RATIO')
adult_ratio = \
self.aggregator.get_default_keyword('ADULT_RATIO')
elderly_ratio = \
self.aggregator.get_default_keyword('ELDERLY_RATIO')
if 'BuildingType' or 'RoadType' in postprocessors:
try:
key_attribute = self.keyword_io.read_keywords(
self.aggregator.exposure_layer, 'key_attribute')
except KeywordNotFoundError:
# use 'type' as default
key_attribute = 'type'
# iterate zone features
request = QgsFeatureRequest()
request.setFlags(QgsFeatureRequest.NoGeometry)
provider = self.aggregator.layer.dataProvider()
# start data retrieval: fetch no geometry and all attributes for each
# feature
polygon_index = 0
for feature in provider.getFeatures(request):
# if a feature has no field called
if name_filed_index == -1:
zone_name = str(feature.id())
else:
zone_name = feature[name_filed_index]
# create dictionary of attributes to pass to postprocessor
general_params = {
'target_field': self.aggregator.target_field,
'function_params': self.function_parameters
}
if self.aggregator.statistics_type == 'class_count':
general_params['impact_classes'] = (
self.aggregator.statistics_classes)
elif self.aggregator.statistics_type == 'sum':
impact_total = feature[sum_field_index]
general_params['impact_total'] = impact_total
try:
general_params['impact_attrs'] = (
self.aggregator.impact_layer_attributes[polygon_index])
except IndexError:
# rasters and attributeless vectors have no attributes
general_params['impact_attrs'] = None
for key, value in postprocessors.iteritems():
parameters = general_params
try:
# look if params are available for this postprocessor
parameters.update(
self.function_parameters['postprocessors'][key]
['params'])
except KeyError:
pass
if key == 'Gender':
if user_defined_female_ratio:
female_ratio = feature[female_ratio_field_index]
if female_ratio is None:
female_ratio = self.aggregator.defaults[
'FEMALE_RATIO']
LOGGER.warning('Data Driven Female ratio '
'incomplete, using defaults for'
' aggregation unit'
' %s' % feature.id)
parameters['female_ratio'] = female_ratio
if key == 'Age':
if user_defined_age_ratios:
youth_ratio = feature[youth_ratio_field_index]
adult_ratio = feature[adult_ratio_field_index]
elderly_ratio = feature[elderly_ratio_field_index]
if (youth_ratio is None or adult_ratio is None
or elderly_ratio is None):
youth_ratio = self.aggregator.defaults[
'YOUTH_RATIO']
adult_ratio = self.aggregator.defaults[
'ADULT_RATIO']
elderly_ratio = self.aggregator.defaults[
'ELDERLY_RATIO']
LOGGER.warning('Data Driven Age ratios '
'incomplete, using defaults for'
' aggregation unit'
' %s' % feature.id)
parameters['youth_ratio'] = youth_ratio
parameters['adult_ratio'] = adult_ratio
parameters['elderly_ratio'] = elderly_ratio
if key == 'BuildingType' or key == 'RoadType':
parameters['key_attribute'] = key_attribute
try:
value.setup(parameters)
value.process()
results = value.results()
value.clear()
# LOGGER.debug(results)
# this can raise a KeyError
self.output[key].append((zone_name, results))
except PostProcessorError as e:
message = m.Message(
m.Heading(self.tr('%s postprocessor problem' % key),
**styles.DETAILS_STYLE),
m.Paragraph(self.tr(str(e))))
self.error_message = message
except KeyError:
self.output[key] = []
self.output[key].append((zone_name, results))
# increment the index
polygon_index += 1