def read_keywords_file(cls, filename, keyword=None):
"""Read keywords from a keywords file and return as dictionary
This serves as a wrapper function that should be provided by Keyword
IO. Use this if you are sure that the filename is a keyword file.
:param filename: The filename of the keyword, typically with .xml or
.keywords extension. If not, will raise exceptions
:type filename: str
:param keyword: If set, will extract only the specified keyword
from the keywords dict.
:type keyword: str
:returns: A dict if keyword is omitted, otherwise the value for the
given key if it is present.
:rtype: dict, str
:raises: KeywordNotFoundError, InvalidParameterError
"""
# Try to read from ISO metadata first.
_, ext = os.path.splitext(filename)
dictionary = {}
if ext == '.xml':
try:
dictionary = read_iso19115_metadata(filename)
except (MetadataReadError, NoKeywordsFoundError):
pass
elif ext == '.keywords':
try:
dictionary = read_file_keywords(filename)
# update to xml based metadata
write_read_iso_19115_metadata(filename, dictionary)
except (HashNotFoundError,
Exception,
OperationalError,
NoKeywordsFoundError,
KeywordNotFoundError,
InvalidParameterError,
UnsupportedProviderError):
raise
else:
raise InvalidParameterError(
'Keywords file have .xml or .keywords extension')
# if no keyword was supplied, just return the dict
if keyword is None:
return dictionary
if keyword not in dictionary:
message = tr('No value was found in file %s for keyword %s' % (
filename, keyword))
raise KeywordNotFoundError(message)
return dictionary[keyword]
def read_iso19115_metadata(layer_uri, keyword=None):
"""Retrieve keywords from a metadata object
:param layer_uri: Uri to layer.
:type layer_uri: basestring
:param keyword: The key of keyword that want to be read. If None, return
all keywords in dictionary.
:returns: Dictionary of keywords or value of key as string.
:rtype: dict, basestring
"""
xml_uri = os.path.splitext(layer_uri)[0] + '.xml'
if not os.path.exists(xml_uri):
xml_uri = None
if not xml_uri and os.path.exists(layer_uri):
message = 'Layer based file but no xml file.\n'
message += 'Layer path: %s.' % layer_uri
raise NoKeywordsFoundError(message)
metadata = GenericLayerMetadata(layer_uri, xml_uri)
if metadata.layer_purpose in METADATA_CLASSES:
metadata = METADATA_CLASSES[metadata.layer_purpose](layer_uri, xml_uri)
# dictionary comprehension
keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
if x[1]['value'] is not None
}
if 'keyword_version' not in keywords.keys() and xml_uri:
message = 'No keyword version found. Metadata xml file is invalid.\n'
message += 'Layer uri: %s\n' % layer_uri
message += 'Keywords file: %s\n' % os.path.exists(
os.path.splitext(layer_uri)[0] + '.xml')
message += 'keywords:\n'
for k, v in keywords.iteritems():
message += '%s: %s\n' % (k, v)
raise MetadataReadError(message)
# Get dictionary keywords that has value != None
keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
if x[1]['value'] is not None
}
if keyword:
try:
return keywords[keyword]
except KeyError:
message = 'Keyword with key %s is not found. ' % keyword
message += 'Layer path: %s' % layer_uri
raise KeywordNotFoundError(message)
if isinstance(metadata, OutputLayerMetadata):
keywords['if_provenance'] = metadata.provenance
return keywords
def read_file_keywords(layer_path, keyword=None):
"""Get metadata from the keywords file associated with a local
file in the file system.
.. note:: Requires a str representing a file path instance
as parameter As opposed to read_keywords_from_layer which
takes a inasafe file object as parameter.
.. seealso:: read_keywords_from_layer
:param: layer_path: a string representing a path to a layer
(e.g. '/tmp/foo.shp', '/tmp/foo.tif')
:type layer_path: str
:param keyword: optional - the metadata keyword to retrieve e.g. 'title'
:type keyword: str
:return: A string containing the retrieved value for the keyword if
the keyword argument is specified, otherwise the
complete keywords dictionary is returned.
:raises: KeywordNotFoundError, NoKeywordsFoundError, InvalidParameterError
Note:
* KeywordNotFoundError - occurs when the keyword is not recognised.
* NoKeywordsFoundError - occurs when no keyword file exists.
* InvalidParameterError - occurs when the layer does not exist.
"""
# check the source layer path is valid
if not os.path.isfile(layer_path):
message = tr('Cannot get keywords from a non-existent file. File '
'%s does not exist.' % layer_path)
raise InvalidParameterError(message)
# check there really is a keywords file for this layer
# priority for iso path first
keyword_file_path = os.path.splitext(layer_path)[0]
keyword_file_path += '.keywords'
keyword_iso_path = os.path.splitext(layer_path)[0]
keyword_iso_path += '.xml'
if not os.path.isfile(keyword_file_path)\
and not os.path.isfile(keyword_iso_path):
message = tr('No keywords file found for %s' % keyword_file_path)
raise NoKeywordsFoundError(message)
elif os.path.isfile(keyword_file_path) \
and not os.path.isfile(keyword_iso_path):
# switch to .keywords file if iso xml file didn't exist
keyword_iso_path = keyword_file_path
# now get the requested keyword using the inasafe library
try:
dictionary = read_keywords(keyword_iso_path)
except Exception, e:
message = tr('Keyword retrieval failed for %s (%s) \n %s' %
(keyword_file_path, keyword, str(e)))
raise KeywordNotFoundError(message)
def get_keywords(self, key=None):
"""Return a copy of the keywords dictionary
Args:
* key (optional): If specified value will be returned for key only
"""
if key is None:
return self.keywords.copy()
else:
if key in self.keywords:
return self.keywords[key]
else:
msg = ('Keyword %s does not exist in %s: Options are '
'%s' % (key, self.get_name(), self.keywords.keys()))
raise KeywordNotFoundError(msg)
def keyword(self, key):
"""Return value of key from keywords.
It will raise KeywordNotFoundError if the key is not found.
:param key: The key of a keyword.
:type key: str
:returns: The value of the key in keywords dictionary.
:rtype: str, dict, int, float
"""
try:
return self.keywords[key]
except KeyError as e:
raise KeywordNotFoundError(e)
def keyword(self, key):
"""Return value of key from keywords.
It will raise KeywordNotFoundError if the key is not found.
:param key: The key of a keyword.
:type key: str
:returns: The value of the key in keywords dictionary.
:rtype: str, dict, int, float
"""
try:
return self.keywords[key]
except KeyError:
message = tr(
'Keyword "%s" is not found in layer %s, please add it to your '
'layer' % (key, self.name))
raise KeywordNotFoundError(message,
layer_name=self.layer.name,
keyword=key)
def set_layer(self, layer, keywords=None):
"""Set layer and update UI accordingly.
:param layer: A vector layer that has been already patched with
metadata.
:type layer: QgsVectorLayer
:param keywords: Custom keyword for the layer.
:type keywords: dict, None
"""
self.layer = layer
if keywords is not None:
self.metadata = keywords
else:
# Check if it has keywords
if not hasattr(layer, 'keywords'):
message = 'Layer {layer_name} does not have keywords.'.format(
layer_name=layer.name())
raise KeywordNotFoundError(message)
self.metadata = layer.keywords
self.populate_parameter()
def populate_tabs(self):
"""Populating tabs based on layer metadata."""
self.delete_tabs()
layer_purpose = self.metadata.get('layer_purpose')
if not layer_purpose:
message = tr(
'Key layer_purpose is not found in the layer {layer_name}'
).format(layer_name=self.layer.name())
raise KeywordNotFoundError(message)
if layer_purpose == layer_purpose_exposure['key']:
layer_subcategory = self.metadata.get('exposure')
elif layer_purpose == layer_purpose_hazard['key']:
layer_subcategory = self.metadata.get('hazard')
else:
layer_subcategory = None
field_groups = get_field_groups(layer_purpose, layer_subcategory)
for field_group in field_groups:
tab = FieldMappingTab(field_group, self, self.iface)
tab.set_layer(self.layer, self.metadata)
self.addTab(tab, field_group['name'])
self.tabs.append(tab)
def read_keyword_from_uri(self, uri, keyword=None):
"""Get metadata from the keywords file associated with a URI.
This is used for layers that are non local layer (e.g. postgresql
connection) and so we need to retrieve the keywords from the sqlite
keywords db.
A hash will be constructed from the supplied uri and a lookup made
in a local SQLITE database for the keywords. If there is an existing
record it will be returned, if not and error will be thrown.
If the record is a dictionary, it means that it was inserted into the
DB in a pre 2.2 version which had no ISO metadata. In this case, we use
that dictionary to update the entry to the new ISO based metadata
.. seealso:: write_keywords_for_uri, delete_keywords_for_uri
:param uri: A layer uri. e.g. ```dbname=\'osm\' host=localhost
port=5432 user=\'foo\' password=\'bar\' sslmode=disable
key=\'id\' srid=4326```
:type uri: str
:param keyword: The metadata keyword to retrieve. If none,
all keywords are returned.
:type keyword: str
:returns: A string containing the retrieved value for the keyword if
the keyword argument is specified, otherwise the
complete keywords dictionary is returned.
:raises: KeywordNotFoundError if the keyword is not found.
"""
hash_value = self.hash_for_datasource(uri)
try:
self.open_connection()
except OperationalError:
raise
try:
cursor = self.get_cursor()
# now see if we have any data for our hash
sql = (
'select dict from keyword where hash = \'%s\';' % hash_value)
cursor.execute(sql)
data = cursor.fetchone()
# unpickle it to get our dict back
if data is None:
raise HashNotFoundError('No hash found for %s' % hash_value)
data = data[0] # first field
# get the ISO XML out of the DB
metadata = pickle.loads(str(data))
# the uri already had a KW entry in the DB using the old KW system
# we use that dictionary to update the entry to the new ISO based
# metadata system
if isinstance(metadata, dict):
metadata = self.write_keywords_for_uri(uri, metadata)
root = ElementTree.fromstring(metadata)
keyword_element = root.find(ISO_METADATA_KEYWORD_TAG)
dict_str = keyword_element.text
picked_dict = json.loads(dict_str)
if keyword is None:
return picked_dict
if keyword in picked_dict:
return picked_dict[keyword]
else:
raise KeywordNotFoundError('Keyword "%s" not found in %s' % (
keyword, picked_dict))
except sqlite.Error, e:
LOGGER.debug("Error %s:" % e.args[0])
dictionary = read_keywords(keyword_file_path)
except Exception, e:
message = tr('Keyword retrieval failed for %s (%s) \n %s' %
(keyword_file_path, keyword, str(e)))
raise KeywordNotFoundError(message)
# if no keyword was supplied, just return the dict
if keyword is None:
if 'keyword_version' in dictionary.keys():
dictionary['keyword_version'] = get_string(
dictionary['keyword_version'])
return dictionary
if keyword not in dictionary:
message = tr('No value was found in file %s for keyword %s' %
(keyword_file_path, keyword))
raise KeywordNotFoundError(message)
try:
value = dictionary[keyword]
except:
raise
if 'keyword_version' == keyword:
value = get_string(value)
return value
def replace_accentuated_characters(message):
"""Normalize unicode data in Python to remove umlauts, accents etc.
:param message: The string where to delete accentuated characters.
:type message: str, unicode
def read_iso19115_metadata(layer_uri, keyword=None):
"""Retrieve keywords from a metadata object
:param layer_uri:
:param keyword:
:return:
"""
xml_uri = os.path.splitext(layer_uri)[0] + '.xml'
if not os.path.exists(xml_uri):
xml_uri = None
if not xml_uri and os.path.exists(layer_uri):
message = 'Layer based file but no xml file.\n'
message += 'Layer path: %s.' % layer_uri
raise NoKeywordsFoundError(message)
metadata = GenericLayerMetadata(layer_uri, xml_uri)
if metadata.layer_purpose == 'exposure':
metadata = ExposureLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'hazard':
metadata = HazardLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'aggregation':
metadata = AggregationLayerMetadata(layer_uri, xml_uri)
elif metadata.layer_purpose == 'impact':
metadata = ImpactLayerMetadata(layer_uri, xml_uri)
# dictionary comprehension
keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
if x[1]['value'] is not None
}
if 'keyword_version' not in keywords.keys() and xml_uri:
message = 'No keyword version found. Metadata xml file is invalid.\n'
message += 'Layer uri: %s\n' % layer_uri
message += 'Keywords file: %s\n' % os.path.exists(
os.path.splitext(layer_uri)[0] + '.xml')
message += 'keywords:\n'
for k, v in keywords.iteritems():
message += '%s: %s\n' % (k, v)
raise MetadataReadError(message)
keywords = {}
temp_keywords = {
x[0]: x[1]['value']
for x in metadata.dict['properties'].iteritems()
}
included = [
'aggregation attribute',
'female ratio attribute',
'youth ratio attribute',
'adult ratio attribute',
'elderly ratio attribute',
]
for key in temp_keywords.iterkeys():
if key in included:
keywords[key] = temp_keywords[key]
else:
if temp_keywords[key] is not None:
keywords[key] = temp_keywords[key]
if keyword:
try:
return keywords[keyword]
except KeyError:
message = 'Keyword with key %s is not found' % keyword
message += 'Layer path: %s' % layer_uri
raise KeywordNotFoundError(message)
if isinstance(metadata, ImpactLayerMetadata):
keywords['if_provenance'] = metadata.provenance
return keywords
def validate_all_layers(self):
"""Validate all layers based on the keywords.
When we do the validation, we also fetch the information we need:
1. 'map_title' from each impact layer
2. 'exposure_title' from each impact layer
3. 'postprocessing_report' from each impact layer
4. 'aggregation_attribute' on aggregation layer, if user runs merging
tools with aggregation layer chosen
The things that we validate are:
1. 'map_title' keyword must exist on each impact layer
2. 'exposure_title' keyword must exist on each impact layer
3. 'postprocessing_report' keyword must exist on each impact layer
4. 'hazard_title' keyword must exist on each impact layer. Hazard title
from first impact layer must be the same with second impact layer
to indicate that both are generated from the same hazard layer.
5. 'aggregation attribute' must exist when user wants to run merging
tools with aggregation layer chosen.
"""
required_attribute = [
'map_title', 'exposure_title', 'hazard_title',
'postprocessing_report'
]
# Fetch for first impact layer
for attribute in required_attribute:
try:
# noinspection PyTypeChecker
self.first_impact[attribute] = self.keyword_io.read_keywords(
self.first_impact['layer'], attribute)
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords found for first impact layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword %s not found for first layer.' %
attribute))
# Fetch for second impact layer
for attribute in required_attribute:
try:
# noinspection PyTypeChecker
self.second_impact[attribute] = self.keyword_io.read_keywords(
self.second_impact['layer'], attribute)
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords found for second impact layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword %s not found for second layer.' %
attribute))
# Validate that two impact layers are obtained from the same hazard.
# Indicated by the same 'hazard_title' (to be fixed later by using
# more reliable method)
if (self.first_impact['hazard_title'] !=
self.second_impact['hazard_title']):
raise InvalidLayerError(
self.tr('First impact layer and second impact layer do not '
'use the same hazard layer.'))
# Fetch 'aggregation_attribute'
# If the chosen aggregation layer not Entire Area, it should have
# aggregation attribute keywords
if not self.entire_area_mode:
try:
# noinspection PyTypeChecker
self.aggregation['aggregation_attribute'] = \
self.keyword_io.read_keywords(
self.aggregation['layer'], 'aggregation attribute')
except NoKeywordsFoundError:
raise NoKeywordsFoundError(
self.tr('No keywords exist in aggregation layer.'))
except KeywordNotFoundError:
raise KeywordNotFoundError(
self.tr('Keyword aggregation attribute not found for '
'aggregation layer.'))
def read_iso19115_metadata(layer_uri, keyword=None, version_35=False):
"""Retrieve keywords from a metadata object
:param layer_uri: Uri to layer.
:type layer_uri: basestring
:param keyword: The key of keyword that want to be read. If None, return
all keywords in dictionary.
:type keyword: basestring
:returns: Dictionary of keywords or value of key as string.
:rtype: dict, basestring
"""
xml_uri = os.path.splitext(layer_uri)[0] + '.xml'
# Remove the prefix for local file. For example csv.
file_prefix = 'file:'
if xml_uri.startswith(file_prefix):
xml_uri = xml_uri[len(file_prefix):]
if not os.path.exists(xml_uri):
xml_uri = None
if not xml_uri and os.path.exists(layer_uri):
message = 'Layer based file but no xml file.\n'
message += 'Layer path: %s.' % layer_uri
raise NoKeywordsFoundError(message)
if version_35:
metadata = GenericLayerMetadata35(layer_uri, xml_uri)
else:
metadata = GenericLayerMetadata(layer_uri, xml_uri)
active_metadata_classes = METADATA_CLASSES
if version_35:
active_metadata_classes = METADATA_CLASSES35
if metadata.layer_purpose in active_metadata_classes:
metadata = active_metadata_classes[
metadata.layer_purpose](layer_uri, xml_uri)
# dictionary comprehension
keywords = {
x[0]: x[1]['value'] for x in list(metadata.dict['properties'].items())
if x[1]['value'] is not None}
if 'keyword_version' not in list(keywords.keys()) and xml_uri:
message = 'No keyword version found. Metadata xml file is invalid.\n'
message += 'Layer uri: %s\n' % layer_uri
message += 'Keywords file: %s\n' % os.path.exists(
os.path.splitext(layer_uri)[0] + '.xml')
message += 'keywords:\n'
for k, v in list(keywords.items()):
message += '%s: %s\n' % (k, v)
raise MetadataReadError(message)
# Get dictionary keywords that has value != None
keywords = {
x[0]: x[1]['value'] for x in list(metadata.dict['properties'].items())
if x[1]['value'] is not None}
if keyword:
try:
return keywords[keyword]
except KeyError:
message = 'Keyword with key %s is not found. ' % keyword
message += 'Layer path: %s' % layer_uri
raise KeywordNotFoundError(message)
return keywords