def test_metadata(self):
"""Check we can't instantiate with unsupported xml types"""
metadata = ImpactLayerMetadata('random_layer_id')
path = 'gmd:MD_Metadata/gmd:dateStamp/gco:RandomString'
# using unsupported xml types
test_value = 'Random string'
with self.assertRaises(KeyError):
metadata.set('ISO19115_TEST', test_value, path)
def test_metadata(self):
"""Check we can't instantiate with unsupported xml types"""
metadata = ImpactLayerMetadata('random_layer_id')
path = 'gmd:MD_Metadata/gmd:dateStamp/gco:RandomString'
# using unsupported xml types
test_value = 'Random string'
with self.assertRaises(KeyError):
metadata.set('ISO19115_TEST', test_value, path)
def test_xml_to_json_to_xml(self):
generated_metadata = ImpactLayerMetadata(EXISTING_IMPACT_FILE,
xml_uri=EXISTING_IMPACT_XML)
with open(EXISTING_IMPACT_XML) as f:
expected_metadata = f.read()
json_tmp_file = unique_filename(suffix='.json', dir=TEMP_DIR)
generated_metadata.write_to_file(json_tmp_file)
read_tmp_metadata = ImpactLayerMetadata(EXISTING_IMPACT_FILE,
json_uri=json_tmp_file)
self.assertEquals(expected_metadata, read_tmp_metadata.xml)
def test_xml_to_json_to_xml(self):
generated_metadata = ImpactLayerMetadata(
EXISTING_IMPACT_FILE,
xml_uri=EXISTING_IMPACT_XML)
with open(EXISTING_IMPACT_XML) as f:
expected_metadata = f.read()
json_tmp_file = unique_filename(suffix='.json', dir=TEMP_DIR)
generated_metadata.write_to_file(json_tmp_file)
read_tmp_metadata = ImpactLayerMetadata(EXISTING_IMPACT_FILE,
json_uri=json_tmp_file)
self.assertEquals(expected_metadata, read_tmp_metadata.xml)
def test_update_from_dict(self):
"""Test update_from_dict method."""
good_data = {
'start_time': '20140714_060955',
'finish_time': '20140714_061255',
'hazard_layer': 'path/to/hazard/layer',
'exposure_layer': 'path/to/exposure/layer',
'impact_function_id': 'IF_id',
'impact_function_version': '2.1',
'host_name': 'my_computer',
'user': '******',
'qgis_version': '2.4',
'gdal_version': '1.9.1',
'qt_version': '4.5',
'pyqt_version': '5.1',
'os': 'ubuntu 12.04',
'inasafe_version': '2.1',
'exposure_pixel_size': '0.1',
'hazard_pixel_size': '0.2',
'impact_pixel_size': '0.1',
'actual_extent': [0, 1, 2, 2],
'requested_extent': [0, 1, 2, 2],
'actual_extent_crs': 'EPSG: 4326',
'requested_extent_crs': 'EPSG: 4326',
'parameter': {},
}
metadata = ImpactLayerMetadata('random_layer_id')
provenance = Provenance()
provenance.append_step(
'Title 1', 'Description of step 1', '2015-06-25T13:14:24.508974')
provenance.append_step(
'Title 2', 'Description of step 2', '2015-06-25T13:14:24.508980')
provenance.append_if_provenance_step(
'Title 3',
'Description of step 3',
'2015-06-25T13:14:24.508984',
data=good_data
)
keywords = {
'layer_purpose': 'impact_layer',
'layer_geometry': 'raster',
'if_provenance': provenance,
}
metadata.update_from_dict(keywords)
self.assertEqual(metadata.layer_purpose, 'impact_layer')
self.assertEqual(metadata.layer_geometry, 'raster')
self.assertNotEqual(metadata.layer_mode, 'raster')
self.assertEqual(len(metadata.provenance.steps), 3)
self.assertEqual(metadata.provenance.get(2), provenance.get(2))
self.assertEqual(metadata.provenance.get(2).user, 'my_user')
def test_update_from_dict(self):
"""Test update_from_dict method."""
good_data = {
'start_time': '20140714_060955',
'finish_time': '20140714_061255',
'hazard_layer': 'path/to/hazard/layer',
'exposure_layer': 'path/to/exposure/layer',
'impact_function_id': 'IF_id',
'impact_function_version': '2.1',
'host_name': 'my_computer',
'user': '******',
'qgis_version': '2.4',
'gdal_version': '1.9.1',
'qt_version': '4.5',
'pyqt_version': '5.1',
'os': 'ubuntu 12.04',
'inasafe_version': '2.1',
'exposure_pixel_size': '0.1',
'hazard_pixel_size': '0.2',
'impact_pixel_size': '0.1',
'actual_extent': [0, 1, 2, 2],
'requested_extent': [0, 1, 2, 2],
'actual_extent_crs': 'EPSG: 4326',
'requested_extent_crs': 'EPSG: 4326',
'parameter': {},
}
metadata = ImpactLayerMetadata('random_layer_id')
provenance = Provenance()
provenance.append_step(
'Title 1', 'Description of step 1', '2015-06-25T13:14:24.508974')
provenance.append_step(
'Title 2', 'Description of step 2', '2015-06-25T13:14:24.508980')
provenance.append_if_provenance_step(
'Title 3',
'Description of step 3',
'2015-06-25T13:14:24.508984',
data=good_data
)
keywords = {
'layer_purpose': 'impact_layer',
'layer_geometry': 'raster',
'if_provenance': provenance,
}
metadata.update_from_dict(keywords)
self.assertEqual(metadata.layer_purpose, 'impact_layer')
self.assertEqual(metadata.layer_geometry, 'raster')
self.assertNotEqual(metadata.layer_mode, 'raster')
self.assertEqual(len(metadata.provenance.steps), 3)
self.assertEqual(metadata.provenance.get(2), provenance.get(2))
self.assertEqual(metadata.provenance.get(2).user, 'my_user')
def write_iso19115_metadata(layer_uri, keywords):
"""Create metadata object from a layer path and keywords dictionary.
:param layer_uri: Uri to layer.
:type layer_uri: str
:param keywords: Dictionary of keywords.
:type keywords: dict
"""
if 'layer_purpose' in keywords.keys():
if keywords['layer_purpose'] == 'exposure':
metadata = ExposureLayerMetadata(layer_uri)
elif keywords['layer_purpose'] == 'hazard':
metadata = HazardLayerMetadata(layer_uri)
elif keywords['layer_purpose'] == 'aggregation':
metadata = AggregationLayerMetadata(layer_uri)
elif keywords['layer_purpose'] == 'impact':
metadata = ImpactLayerMetadata(layer_uri)
else:
metadata = GenericLayerMetadata(layer_uri)
else:
metadata = GenericLayerMetadata(layer_uri)
metadata.update_from_dict(keywords)
# Always set keyword_version to the latest one.
metadata.update_from_dict({'keyword_version': inasafe_keyword_version})
if metadata.layer_is_file_based:
xml_file_path = os.path.splitext(layer_uri)[0] + '.xml'
metadata.write_to_file(xml_file_path)
else:
metadata.write_to_db()
return metadata
def test_metadata_url(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gmd:URL'
# using QUrl
test_value = QUrl('http://inasafe.org')
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'),
'http://inasafe.org')
# using str should work as it is casted
test_value = 'http://inasafe.org'
metadata.update('ISO19115_TEST', test_value)
# using invalid QUrl (has a space)
test_value = QUrl('http://inasafe.org ')
with self.assertRaises(ValueError):
metadata.set('ISO19115_TEST', test_value, path)
def test_metadata_url(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gmd:URL'
# using QUrl
test_value = QUrl('http://inasafe.org')
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(
metadata.get_xml_value('ISO19115_TEST'), 'http://inasafe.org')
# using str should work as it is casted
test_value = 'http://inasafe.org'
metadata.update('ISO19115_TEST', test_value)
# using invalid QUrl (has a space)
test_value = QUrl('http://inasafe.org ')
with self.assertRaises(ValueError):
metadata.set('ISO19115_TEST', test_value, path)
def generate_test_metadata(self):
# if you change this you need to update IMPACT_TEST_FILE_JSON
metadata = ImpactLayerMetadata('random_layer_id')
path = 'gmd:MD_Metadata/gmd:dateStamp/'
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_STR', test_value, path)
test_value = 1234
metadata.set('ISO19115_INT', test_value, path)
test_value = 1234.5678
metadata.set('ISO19115_FLOAT', test_value, path)
metadata.report = 'My super report'
metadata.summary_data = {'res1': 1234, 'res2': 4321}
metadata.append_provenance_step('Title 1', 'Description of step 1',
'2015-06-25T13:14:24.508974')
metadata.append_provenance_step('Title 2', 'Description of step 2',
'2015-06-25T13:14:24.508980')
metadata.append_provenance_step('Title 3', 'Description of step 3',
'2015-06-25T13:14:24.508984')
return metadata
def test_metadata_str(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(
metadata.get_xml_value('ISO19115_TEST'), 'Random string')
# using int
test_value = 1234
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '1234')
# using float
test_value = 1234.5678
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '1234.5678')
# using invalid QUrl
test_value = QUrl()
with self.assertRaises(TypeError):
metadata.update('ISO19115_TEST', test_value)
def test_metadata_date(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:Date'
# using QDate
test_value = QDate(2015, 6, 7)
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using datetime
test_value = datetime(2015, 6, 7)
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using date
test_value = date(2015, 6, 7)
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using str should fail
test_value = '2015-06-07'
with self.assertRaises(TypeError):
metadata.update('ISO19115_TEST', test_value)
def generate_test_metadata(self):
# if you change this you need to update IMPACT_TEST_FILE_JSON
good_data = {
'start_time': '20140714_060955',
'finish_time': '20140714_061255',
'hazard_layer': 'path/to/hazard/layer',
'exposure_layer': 'path/to/exposure/layer',
'impact_function_id': 'IF_id',
'impact_function_version': '2.1',
'host_name': 'my_computer',
'user': '******',
'qgis_version': '2.4',
'gdal_version': '1.9.1',
'qt_version': '4.5',
'pyqt_version': '5.1',
'os': 'ubuntu 12.04',
'inasafe_version': '2.1',
'exposure_pixel_size': '0.1',
'hazard_pixel_size': '0.2',
'impact_pixel_size': '0.1',
'analysis_extent': [0, 1, 2, 2],
'parameter': {},
}
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_STR', test_value, path)
test_value = 1234
metadata.set('ISO19115_INT', test_value, path)
test_value = 1234.5678
metadata.set('ISO19115_FLOAT', test_value, path)
metadata.report = 'My super report'
metadata.summary_data = {'res1': 1234, 'res2': 4321}
metadata.date = datetime.strptime(
'2016-02-18T12:34:56', '%Y-%m-%dT%H:%M:%S')
metadata.url = QUrl('http://inasafe.org')
metadata.append_provenance_step(
'Title 1', 'Description of step 1', '2015-06-25T13:14:24.508974')
metadata.append_provenance_step(
'Title 2', 'Description of step 2', '2015-06-25T13:14:24.508980')
metadata.append_provenance_step(
'Title 3', 'Description of step 3', '2015-06-25T13:14:24.508984')
metadata.append_if_provenance_step(
'IF Provenance',
'IF Provenance',
'2015-06-25T13:14:24.510000',
good_data
)
return metadata
def generate_test_metadata(self):
# if you change this you need to update IMPACT_TEST_FILE_JSON
metadata = ImpactLayerMetadata('random_layer_id')
path = 'gmd:MD_Metadata/gmd:dateStamp/'
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_STR', test_value, path)
test_value = 1234
metadata.set('ISO19115_INT', test_value, path)
test_value = 1234.5678
metadata.set('ISO19115_FLOAT', test_value, path)
metadata.report = 'My super report'
metadata.summary_data = {'res1': 1234, 'res2': 4321}
metadata.append_provenance_step(
'Title 1', 'Description of step 1', '2015-06-25T13:14:24.508974')
metadata.append_provenance_step(
'Title 2', 'Description of step 2', '2015-06-25T13:14:24.508980')
metadata.append_provenance_step(
'Title 3', 'Description of step 3', '2015-06-25T13:14:24.508984')
return metadata
def test_json_read(self):
metadata = ImpactLayerMetadata(EXISTING_IMPACT_FILE)
with open(EXISTING_IMPACT_JSON) as f:
expected_metadata = f.read()
self.assertEquals(expected_metadata, metadata.json)
def test_xml_read(self):
generated_metadata = ImpactLayerMetadata(
EXISTING_IMPACT_FILE, xml_uri=EXISTING_IMPACT_XML)
# TODO (MB): add more checks
self.assertEquals(generated_metadata.get_xml_value('license'), 'GPLv2')
def test_invalid_json_read(self):
with self.assertRaises(MetadataReadError):
ImpactLayerMetadata(EXISTING_IMPACT_FILE,
json_uri=INVALID_IMPACT_JSON)
def read_iso19115_metadata(layer_uri, keyword=None):
"""Retrieve keywords from a metadata object
:param layer_uri:
:param keyword:
:return:
"""
xml_uri = os.path.splitext(layer_uri)[0] + '.xml'
if not os.path.exists(xml_uri):
xml_uri = None
if not xml_uri and os.path.exists(layer_uri):
message = 'Layer based file but no xml file.\n'
message += 'Layer path: %s.' % layer_uri
raise NoKeywordsFoundError(message)
metadata = GenericLayerMetadata(layer_uri, xml_uri)
if metadata.layer_purpose == 'exposure':
metadata = ExposureLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'hazard':
metadata = HazardLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'aggregation':
metadata = AggregationLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'impact':
metadata = ImpactLayerMetadata(layer_uri, xml_uri)
# dictionary comprehension
keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
if x[1]['value'] is not None
}
if 'keyword_version' not in keywords.keys() and xml_uri:
message = 'No keyword version found. Metadata xml file is invalid.\n'
message += 'Layer uri: %s\n' % layer_uri
message += 'Keywords file: %s\n' % os.path.exists(
os.path.splitext(layer_uri)[0] + '.xml')
message += 'keywords:\n'
for k, v in keywords.iteritems():
message += '%s: %s\n' % (k, v)
raise MetadataReadError(message)
keywords = {}
temp_keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
}
included = [
'aggregation attribute',
'female ratio attribute',
'youth ratio attribute',
'adult ratio attribute',
'elderly ratio attribute',
]
for key in temp_keywords.iterkeys():
if key in included:
keywords[key] = temp_keywords[key]
else:
if temp_keywords[key] is not None:
keywords[key] = temp_keywords[key]
if keyword:
try:
return keywords[keyword]
except KeyError:
message = 'Keyword with key %s is not found' % keyword
message += 'Layer path: %s' % layer_uri
raise KeywordNotFoundError(message)
if isinstance(metadata, ImpactLayerMetadata):
keywords['if_provenance'] = metadata.provenance
return keywords
def test_metadata_str(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'),
'Random string')
# using int
test_value = 1234
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '1234')
# using float
test_value = 1234.5678
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '1234.5678')
# using invalid QUrl
test_value = QUrl()
with self.assertRaises(TypeError):
metadata.update('ISO19115_TEST', test_value)
def test_incomplete_json_read(self):
ImpactLayerMetadata(EXISTING_IMPACT_FILE,
json_uri=INCOMPLETE_IMPACT_JSON)
def test_metadata_date(self):
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:Date'
# using QDate
test_value = QDate(2015, 6, 7)
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using datetime
test_value = datetime(2015, 6, 7)
metadata.update('ISO19115_TEST', test_value)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using date
test_value = date(2015, 6, 7)
metadata.set('ISO19115_TEST', test_value, path)
self.assertEqual(metadata.get_xml_value('ISO19115_TEST'), '2015-06-07')
# using str should fail
test_value = 'String'
with self.assertRaises(TypeError):
metadata.update('ISO19115_TEST', test_value)
def generate_test_metadata(self):
# if you change this you need to update IMPACT_TEST_FILE_JSON
good_data = {
'start_time': '20140714_060955',
'finish_time': '20140714_061255',
'hazard_layer': 'path/to/hazard/layer',
'exposure_layer': 'path/to/exposure/layer',
'impact_function_id': 'IF_id',
'impact_function_version': '2.1',
'host_name': 'my_computer',
'user': '******',
'qgis_version': '2.4',
'gdal_version': '1.9.1',
'qt_version': '4.5',
'pyqt_version': '5.1',
'os': 'ubuntu 12.04',
'inasafe_version': '2.1',
'exposure_pixel_size': '0.1',
'hazard_pixel_size': '0.2',
'impact_pixel_size': '0.1',
'actual_extent': [0, 1, 2, 2],
'requested_extent': [0, 1, 2, 2],
'actual_extent_crs': 'EPSG: 4326',
'requested_extent_crs': 'EPSG: 4326',
'parameter': {},
}
metadata = ImpactLayerMetadata('random_layer_id')
path = TEST_XML_BASEPATH + 'gco:CharacterString'
# using str
test_value = 'Random string'
metadata.set('ISO19115_STR', test_value, path)
test_value = 1234
metadata.set('ISO19115_INT', test_value, path)
test_value = 1234.5678
metadata.set('ISO19115_FLOAT', test_value, path)
metadata.report = 'My super report'
metadata.summary_data = {'res1': 1234, 'res2': 4321}
metadata.date = datetime.strptime('2016-02-18T12:34:56',
'%Y-%m-%dT%H:%M:%S')
metadata.url = QUrl('http://inasafe.org')
metadata.append_provenance_step('Title 1', 'Description of step 1',
'2015-06-25T13:14:24.508974')
metadata.append_provenance_step('Title 2', 'Description of step 2',
'2015-06-25T13:14:24.508980')
metadata.append_provenance_step('Title 3', 'Description of step 3',
'2015-06-25T13:14:24.508984')
metadata.append_if_provenance_step('IF Provenance', 'IF Provenance',
'2015-06-25T13:14:24.510000',
good_data)
return metadata
def test_xml_read(self):
generated_metadata = ImpactLayerMetadata(EXISTING_IMPACT_FILE,
xml_uri=EXISTING_IMPACT_XML)
# TODO (MB): add more checks
self.assertEquals(generated_metadata.get_xml_value('license'), 'GPLv2')
