def test_provenance_without_aggregation(self):
"""Test provenance of impact function without aggregation."""
hazard_layer = load_test_vector_layer(
'gisv4', 'hazard', 'classified_vector.geojson')
exposure_layer = load_test_vector_layer(
'gisv4', 'exposure', 'building-points.geojson')
hazard = definition(hazard_layer.keywords['hazard'])
exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
'gdal_version': gdal.__version__,
'host_name': gethostname(),
'map_title': get_map_title(hazard, exposure, hazard_category),
'map_legend_title': exposure['layer_legend_title'],
'inasafe_version': get_version(),
'pyqt_version': PYQT_VERSION_STR,
'qgis_version': QGis.QGIS_VERSION,
'qt_version': QT_VERSION_STR,
'user': getpass.getuser(),
'os': readable_os_version(),
'aggregation_layer': None,
'aggregation_layer_id': None,
'exposure_layer': exposure_layer.source(),
'exposure_layer_id': exposure_layer.id(),
'hazard_layer': hazard_layer.source(),
'hazard_layer_id': hazard_layer.id(),
'analysis_question': get_analysis_question(hazard, exposure),
'aggregation_keywords': None,
'exposure_keywords': deepcopy(exposure_layer.keywords),
'hazard_keywords': deepcopy(hazard_layer.keywords),
}
# Set up impact function
impact_function = ImpactFunction()
impact_function.exposure = exposure_layer
impact_function.hazard = hazard_layer
status, message = impact_function.prepare()
self.assertEqual(PREPARE_SUCCESS, status, message)
status, message = impact_function.run()
self.assertEqual(ANALYSIS_SUCCESS, status, message)
self.maxDiff = None
expected_provenance.update({
'action_checklist': impact_function.action_checklist(),
'analysis_extent': impact_function.analysis_extent.exportToWkt(),
'impact_function_name': impact_function.name,
'impact_function_title': impact_function.title,
'notes': impact_function.notes(),
'requested_extent': impact_function.requested_extent,
'data_store_uri': impact_function.datastore.uri_path,
'start_datetime': impact_function.start_datetime,
'end_datetime': impact_function.end_datetime,
'duration': impact_function.duration
})
self.assertDictEqual(expected_provenance, impact_function.provenance)
def test_provenance_without_aggregation(self):
"""Test provenance of impact function without aggregation."""
hazard_layer = load_test_vector_layer('gisv4', 'hazard',
'classified_vector.geojson')
exposure_layer = load_test_vector_layer('gisv4', 'exposure',
'building-points.geojson')
hazard = definition(hazard_layer.keywords['hazard'])
exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
provenance_gdal_version['provenance_key']:
gdal.__version__,
provenance_host_name['provenance_key']:
gethostname(),
provenance_map_title['provenance_key']:
get_map_title(hazard, exposure, hazard_category),
provenance_map_legend_title['provenance_key']:
exposure['layer_legend_title'],
provenance_user['provenance_key']:
getpass.getuser(),
provenance_os['provenance_key']:
readable_os_version(),
provenance_pyqt_version['provenance_key']:
PYQT_VERSION_STR,
provenance_qgis_version['provenance_key']:
QGis.QGIS_VERSION,
provenance_qt_version['provenance_key']:
QT_VERSION_STR,
provenance_inasafe_version['provenance_key']:
get_version(),
provenance_aggregation_layer['provenance_key']:
None,
provenance_aggregation_layer_id['provenance_key']:
None,
provenance_exposure_layer['provenance_key']:
exposure_layer.source(),
provenance_exposure_layer_id['provenance_key']:
exposure_layer.id(),
provenance_hazard_layer['provenance_key']:
hazard_layer.source(),
provenance_hazard_layer_id['provenance_key']:
hazard_layer.id(),
provenance_analysis_question['provenance_key']:
get_analysis_question(hazard, exposure),
provenance_aggregation_keywords['provenance_key']:
None,
provenance_exposure_keywords['provenance_key']:
deepcopy(exposure_layer.keywords),
provenance_hazard_keywords['provenance_key']:
deepcopy(hazard_layer.keywords),
}
# Set up impact function
impact_function = ImpactFunction()
impact_function.exposure = exposure_layer
impact_function.hazard = hazard_layer
status, message = impact_function.prepare()
self.assertEqual(PREPARE_SUCCESS, status, message)
status, message = impact_function.run()
self.assertEqual(ANALYSIS_SUCCESS, status, message)
self.maxDiff = None
expected_provenance.update({
provenance_action_checklist['provenance_key']:
impact_function.action_checklist(),
provenance_analysis_extent['provenance_key']:
impact_function.analysis_extent.exportToWkt(),
provenance_impact_function_name['provenance_key']:
impact_function.name,
provenance_impact_function_title['provenance_key']:
impact_function.title,
provenance_notes['provenance_key']:
impact_function.notes(),
provenance_requested_extent['provenance_key']:
impact_function.requested_extent,
provenance_data_store_uri['provenance_key']:
impact_function.datastore.uri_path,
provenance_start_datetime['provenance_key']:
impact_function.start_datetime,
provenance_end_datetime['provenance_key']:
impact_function.end_datetime,
provenance_duration['provenance_key']:
impact_function.duration
})
self.assertDictContainsSubset(expected_provenance,
impact_function.provenance)
output_layer_provenance_keys = [
provenance_layer_exposure_summary['provenance_key'],
provenance_layer_aggregate_hazard_impacted['provenance_key'],
provenance_layer_aggregation_summary['provenance_key'],
provenance_layer_analysis_impacted['provenance_key'],
provenance_layer_exposure_summary_table['provenance_key']
]
for key in output_layer_provenance_keys:
self.assertIn(key, impact_function.provenance.keys())
def test_multi_exposure(self):
"""Test we can run a multi exposure analysis."""
hazard_layer = load_test_vector_layer('gisv4', 'hazard',
'classified_vector.geojson')
building_layer = load_test_vector_layer('gisv4', 'exposure',
'building-points.geojson')
population_layer = load_test_vector_layer('gisv4', 'exposure',
'population.geojson')
roads_layer = load_test_vector_layer('gisv4', 'exposure',
'roads.geojson')
aggregation_layer = load_test_vector_layer('gisv4', 'aggregation',
'small_grid.geojson')
impact_function = MultiExposureImpactFunction()
impact_function.hazard = hazard_layer
impact_function.exposures = [
building_layer, population_layer, roads_layer
]
impact_function.aggregation = aggregation_layer
code, message = impact_function.prepare()
self.assertEqual(code, PREPARE_SUCCESS, message)
code, message, exposure = impact_function.run()
self.assertEqual(code, ANALYSIS_SUCCESS, message)
# Test provenance
hazard = definition(hazard_layer.keywords['hazard'])
# exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
provenance_gdal_version['provenance_key']:
gdal.__version__,
provenance_host_name['provenance_key']:
gethostname(),
provenance_user['provenance_key']:
getpass.getuser(),
provenance_os['provenance_key']:
readable_os_version(),
provenance_pyqt_version['provenance_key']:
PYQT_VERSION_STR,
provenance_qgis_version['provenance_key']:
qgis_version(),
provenance_qt_version['provenance_key']:
QT_VERSION_STR,
provenance_inasafe_version['provenance_key']:
get_version(),
provenance_aggregation_layer['provenance_key']:
aggregation_layer.source(),
provenance_aggregation_layer_id['provenance_key']:
aggregation_layer.id(),
# provenance_exposure_layer['provenance_key']:
# exposure_layer.source(),
# provenance_exposure_layer_id['provenance_key']:
# exposure_layer.id(),
provenance_hazard_layer['provenance_key']:
hazard_layer.source(),
provenance_hazard_layer_id['provenance_key']:
hazard_layer.id(),
provenance_aggregation_keywords['provenance_key']:
deepcopy(aggregation_layer.keywords),
# provenance_exposure_keywords['provenance_key']:
# deepcopy(exposure_layer.keywords),
provenance_hazard_keywords['provenance_key']:
deepcopy(hazard_layer.keywords),
}
self.maxDiff = None
expected_provenance.update({
provenance_analysis_extent['provenance_key']:
impact_function.analysis_extent.asWkt(),
provenance_impact_function_name['provenance_key']:
impact_function.name,
# provenance_requested_extent['provenance_key']: impact_function.
# requested_extent,
provenance_data_store_uri['provenance_key']:
impact_function.datastore.uri_path,
provenance_start_datetime['provenance_key']:
impact_function.start_datetime,
provenance_end_datetime['provenance_key']:
impact_function.end_datetime,
provenance_duration['provenance_key']:
impact_function.duration
})
self.assertDictContainsSubset(expected_provenance,
impact_function.provenance)
output_layer_provenance_keys = [
provenance_layer_aggregation_summary['provenance_key'],
provenance_layer_analysis_impacted['provenance_key'],
]
for key in output_layer_provenance_keys:
self.assertIn(key, list(impact_function.provenance.keys()))
# Test serialization/deserialization
output_metadata = impact_function.aggregation_summary.keywords
new_impact_function = MultiExposureImpactFunction. \
load_from_output_metadata(output_metadata)
self.assertEqualImpactFunction(impact_function, new_impact_function)
# Check the analysis layer id equal with the actual layer
old_analysis_layer_id = impact_function.provenance[
provenance_layer_analysis_impacted['provenance_key']]
new_analysis_layer_id = new_impact_function.provenance[
provenance_layer_analysis_impacted['provenance_key']]
self.assertEqual(old_analysis_layer_id, new_analysis_layer_id)
def setup_logger(logger_name, log_file=None, sentry_url=None):
"""Run once when the module is loaded and enable logging.
:param logger_name: The logger name that we want to set up.
:type logger_name: str
:param log_file: Optional full path to a file to write logs to.
:type log_file: str
:param sentry_url: Optional url to sentry api for remote
logging. Defaults to URL defined in safe.definitions.sentry.py
which is the sentry project for InaSAFE desktop.
:type sentry_url: str
Borrowed heavily from this:
http://docs.python.org/howto/logging-cookbook.html
Now to log a message do::
LOGGER.debug('Some debug message')
.. note:: The file logs are written to the inasafe user tmp dir e.g.:
/tmp/inasafe/23-08-2012/timlinux/logs/inasafe.log
"""
logger = logging.getLogger(logger_name)
logging_level = int(os.environ.get('INASAFE_LOGGING_LEVEL', logging.DEBUG))
logger.setLevel(logging_level)
default_handler_level = logging_level
# create formatter that will be added to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# create syslog handler which logs even debug messages
# (ariel): Make this log to /var/log/safe.log instead of
# /var/log/syslog
# (Tim) Ole and I discussed this - we prefer to log into the
# user's temporary working directory.
inasafe_log_path = log_file_path()
if log_file is None:
file_handler = logging.FileHandler(inasafe_log_path)
else:
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(default_handler_level)
file_handler.setFormatter(formatter)
add_logging_handler_once(logger, file_handler)
if 'MUTE_LOGS' not in os.environ:
# create console handler with a higher log level
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(formatter)
add_logging_handler_once(logger, console_handler)
# create a QGIS handler
qgis_handler = QgsLogHandler()
qgis_handler.setFormatter(formatter)
add_logging_handler_once(logger, qgis_handler)
# Sentry handler - this is optional hence the localised import
# If raven is available logging messages will be sent to
# http://sentry.kartoza.com
# We will log exceptions only there. You need to either:
# * Set env var 'INASAFE_SENTRY=1' present (value can be anything)
# before this will be enabled or sentry is enabled in QSettings
qsettings_flag = QSettings().value('inasafe/useSentry', False, type=bool)
environment_flag = 'INASAFE_SENTRY' in os.environ
if environment_flag or qsettings_flag:
if sentry_url is None:
sentry_url = PRODUCTION_SERVER
tags = dict()
tags[provenance_gdal_version['provenance_key']] = gdal.__version__
tags[provenance_os['provenance_key']] = readable_os_version()
qgis_short_version = provenance_qgis_version['provenance_key']
qgis_full_version = qgis_short_version + '_full'
versions = [str(v) for v in qgis_version_detailed()]
tags[qgis_short_version] = '.'.join(versions[0:2])
tags[qgis_full_version] = '.'.join(versions[0:3])
tags[provenance_qt_version['provenance_key']] = QT_VERSION_STR
hostname = os.environ.get('HOSTNAME_SENTRY', socket.gethostname())
sentry_handler = SentryHandler(
dsn=sentry_url,
name=hostname,
release=get_version(),
tags=tags,
)
sentry_handler.setFormatter(formatter)
sentry_handler.setLevel(logging.ERROR)
if add_logging_handler_once(logger, sentry_handler):
logger.debug('Sentry logging enabled in safe')
else:
logger.debug('Sentry logging disabled in safe')
def test_provenance_without_aggregation(self):
"""Test provenance of impact function without aggregation."""
hazard_layer = load_test_vector_layer(
'gisv4', 'hazard', 'classified_vector.geojson')
exposure_layer = load_test_vector_layer(
'gisv4', 'exposure', 'building-points.geojson')
hazard = definition(hazard_layer.keywords['hazard'])
exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
provenance_gdal_version['provenance_key']: gdal.__version__,
provenance_host_name['provenance_key']: gethostname(),
provenance_map_title['provenance_key']: get_map_title(
hazard, exposure, hazard_category),
provenance_map_legend_title['provenance_key']: exposure[
'layer_legend_title'],
provenance_user['provenance_key']: getpass.getuser(),
provenance_os['provenance_key']: readable_os_version(),
provenance_pyqt_version['provenance_key']: PYQT_VERSION_STR,
provenance_qgis_version['provenance_key']: QGis.QGIS_VERSION,
provenance_qt_version['provenance_key']: QT_VERSION_STR,
provenance_inasafe_version['provenance_key']: get_version(),
provenance_aggregation_layer['provenance_key']: None,
provenance_aggregation_layer_id['provenance_key']: None,
provenance_exposure_layer['provenance_key']:
exposure_layer.source(),
provenance_exposure_layer_id['provenance_key']:
exposure_layer.id(),
provenance_hazard_layer['provenance_key']: hazard_layer.source(),
provenance_hazard_layer_id['provenance_key']: hazard_layer.id(),
provenance_analysis_question['provenance_key']:
get_analysis_question(hazard, exposure),
provenance_aggregation_keywords['provenance_key']: None,
provenance_exposure_keywords['provenance_key']:
deepcopy(exposure_layer.keywords),
provenance_hazard_keywords['provenance_key']: deepcopy(
hazard_layer.keywords),
}
# Set up impact function
impact_function = ImpactFunction()
impact_function.exposure = exposure_layer
impact_function.hazard = hazard_layer
status, message = impact_function.prepare()
self.assertEqual(PREPARE_SUCCESS, status, message)
status, message = impact_function.run()
self.assertEqual(ANALYSIS_SUCCESS, status, message)
self.maxDiff = None
expected_provenance.update({
provenance_action_checklist['provenance_key']:
impact_function.action_checklist(),
provenance_analysis_extent['provenance_key']:
impact_function.analysis_extent.exportToWkt(),
provenance_impact_function_name['provenance_key']:
impact_function.name,
provenance_impact_function_title['provenance_key']:
impact_function.title,
provenance_notes['provenance_key']: impact_function.notes(),
provenance_requested_extent['provenance_key']: impact_function.
requested_extent,
provenance_data_store_uri['provenance_key']: impact_function.
datastore.uri_path,
provenance_start_datetime['provenance_key']: impact_function.
start_datetime,
provenance_end_datetime['provenance_key']:
impact_function.end_datetime,
provenance_duration['provenance_key']: impact_function.duration
})
self.assertDictContainsSubset(
expected_provenance, impact_function.provenance)
output_layer_provenance_keys = [
provenance_layer_exposure_summary['provenance_key'],
provenance_layer_aggregate_hazard_impacted['provenance_key'],
provenance_layer_aggregation_summary['provenance_key'],
provenance_layer_analysis_impacted['provenance_key'],
provenance_layer_exposure_summary_table['provenance_key']
]
for key in output_layer_provenance_keys:
self.assertIn(key, impact_function.provenance.keys())
def test_provenance_without_aggregation(self):
"""Test provenance of impact function without aggregation."""
hazard_layer = load_test_vector_layer('gisv4', 'hazard',
'classified_vector.geojson')
exposure_layer = load_test_vector_layer('gisv4', 'exposure',
'building-points.geojson')
hazard = definition(hazard_layer.keywords['hazard'])
exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
'gdal_version': gdal.__version__,
'host_name': gethostname(),
'map_title': get_map_title(hazard, exposure, hazard_category),
'map_legend_title': exposure['layer_legend_title'],
'inasafe_version': get_version(),
'pyqt_version': PYQT_VERSION_STR,
'qgis_version': QGis.QGIS_VERSION,
'qt_version': QT_VERSION_STR,
'user': getpass.getuser(),
'os': readable_os_version(),
'aggregation_layer': None,
'aggregation_layer_id': None,
'exposure_layer': exposure_layer.source(),
'exposure_layer_id': exposure_layer.id(),
'hazard_layer': hazard_layer.source(),
'hazard_layer_id': hazard_layer.id(),
'analysis_question': get_analysis_question(hazard, exposure),
'aggregation_keywords': None,
'exposure_keywords': deepcopy(exposure_layer.keywords),
'hazard_keywords': deepcopy(hazard_layer.keywords),
}
# Set up impact function
impact_function = ImpactFunction()
impact_function.exposure = exposure_layer
impact_function.hazard = hazard_layer
status, message = impact_function.prepare()
self.assertEqual(PREPARE_SUCCESS, status, message)
status, message = impact_function.run()
self.assertEqual(ANALYSIS_SUCCESS, status, message)
self.maxDiff = None
expected_provenance.update({
'action_checklist':
impact_function.action_checklist(),
'analysis_extent':
impact_function.analysis_extent.exportToWkt(),
'impact_function_name':
impact_function.name,
'impact_function_title':
impact_function.title,
'notes':
impact_function.notes(),
'requested_extent':
impact_function.requested_extent,
'data_store_uri':
impact_function.datastore.uri_path,
'start_datetime':
impact_function.start_datetime,
'end_datetime':
impact_function.end_datetime,
'duration':
impact_function.duration
})
self.assertDictEqual(expected_provenance, impact_function.provenance)
def test_multi_exposure(self):
"""Test we can run a multi exposure analysis."""
hazard_layer = load_test_vector_layer(
'gisv4', 'hazard', 'classified_vector.geojson')
building_layer = load_test_vector_layer(
'gisv4', 'exposure', 'building-points.geojson')
population_layer = load_test_vector_layer(
'gisv4', 'exposure', 'population.geojson')
roads_layer = load_test_vector_layer(
'gisv4', 'exposure', 'roads.geojson')
aggregation_layer = load_test_vector_layer(
'gisv4', 'aggregation', 'small_grid.geojson')
impact_function = MultiExposureImpactFunction()
impact_function.hazard = hazard_layer
impact_function.exposures = [
building_layer, population_layer, roads_layer]
impact_function.aggregation = aggregation_layer
code, message = impact_function.prepare()
self.assertEqual(code, PREPARE_SUCCESS, message)
code, message, exposure = impact_function.run()
self.assertEqual(code, ANALYSIS_SUCCESS, message)
# Test provenance
hazard = definition(hazard_layer.keywords['hazard'])
# exposure = definition(exposure_layer.keywords['exposure'])
hazard_category = definition(hazard_layer.keywords['hazard_category'])
expected_provenance = {
provenance_gdal_version['provenance_key']: gdal.__version__,
provenance_host_name['provenance_key']: gethostname(),
provenance_user['provenance_key']: getpass.getuser(),
provenance_os['provenance_key']: readable_os_version(),
provenance_pyqt_version['provenance_key']: PYQT_VERSION_STR,
provenance_qgis_version['provenance_key']: qgis_version(),
provenance_qt_version['provenance_key']: QT_VERSION_STR,
provenance_inasafe_version['provenance_key']: get_version(),
provenance_aggregation_layer['provenance_key']:
aggregation_layer.source(),
provenance_aggregation_layer_id['provenance_key']:
aggregation_layer.id(),
# provenance_exposure_layer['provenance_key']:
# exposure_layer.source(),
# provenance_exposure_layer_id['provenance_key']:
# exposure_layer.id(),
provenance_hazard_layer['provenance_key']: hazard_layer.source(),
provenance_hazard_layer_id['provenance_key']: hazard_layer.id(),
provenance_aggregation_keywords['provenance_key']: deepcopy(
aggregation_layer.keywords),
# provenance_exposure_keywords['provenance_key']:
# deepcopy(exposure_layer.keywords),
provenance_hazard_keywords['provenance_key']: deepcopy(
hazard_layer.keywords),
}
self.maxDiff = None
expected_provenance.update({
provenance_analysis_extent['provenance_key']:
impact_function.analysis_extent.asWkt(),
provenance_impact_function_name['provenance_key']:
impact_function.name,
# provenance_requested_extent['provenance_key']: impact_function.
# requested_extent,
provenance_data_store_uri['provenance_key']: impact_function.
datastore.uri_path,
provenance_start_datetime['provenance_key']: impact_function.
start_datetime,
provenance_end_datetime['provenance_key']:
impact_function.end_datetime,
provenance_duration['provenance_key']: impact_function.duration
})
self.assertDictContainsSubset(
expected_provenance, impact_function.provenance)
output_layer_provenance_keys = [
provenance_layer_aggregation_summary['provenance_key'],
provenance_layer_analysis_impacted['provenance_key'],
]
for key in output_layer_provenance_keys:
self.assertIn(key, list(impact_function.provenance.keys()))
# Test serialization/deserialization
output_metadata = impact_function.aggregation_summary.keywords
new_impact_function = MultiExposureImpactFunction. \
load_from_output_metadata(output_metadata)
self.assertEqualImpactFunction(
impact_function, new_impact_function)
# Check the analysis layer id equal with the actual layer
old_analysis_layer_id = impact_function.provenance[
provenance_layer_analysis_impacted['provenance_key']]
new_analysis_layer_id = new_impact_function.provenance[
provenance_layer_analysis_impacted['provenance_key']]
self.assertEqual(old_analysis_layer_id, new_analysis_layer_id)