def read_keywords_file(cls, filename, keyword=None): """Read keywords from a keywords file and return as dictionary This serves as a wrapper function that should be provided by Keyword IO. Use this if you are sure that the filename is a keyword file. :param filename: The filename of the keyword, typically with .xml or .keywords extension. If not, will raise exceptions :type filename: str :param keyword: If set, will extract only the specified keyword from the keywords dict. :type keyword: str :returns: A dict if keyword is omitted, otherwise the value for the given key if it is present. :rtype: dict, str :raises: KeywordNotFoundError, InvalidParameterError """ # Try to read from ISO metadata first. _, ext = os.path.splitext(filename) dictionary = {} if ext == '.xml': try: dictionary = read_iso19115_metadata(filename) except (MetadataReadError, NoKeywordsFoundError): pass elif ext == '.keywords': try: dictionary = read_file_keywords(filename) # update to xml based metadata write_read_iso_19115_metadata(filename, dictionary) except (HashNotFoundError, Exception, OperationalError, NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError, UnsupportedProviderError): raise else: raise InvalidParameterError( 'Keywords file have .xml or .keywords extension') # if no keyword was supplied, just return the dict if keyword is None: return dictionary if keyword not in dictionary: message = tr('No value was found in file %s for keyword %s' % ( filename, keyword)) raise KeywordNotFoundError(message) return dictionary[keyword]
def read_iso19115_metadata(layer_uri, keyword=None): """Retrieve keywords from a metadata object :param layer_uri: Uri to layer. :type layer_uri: basestring :param keyword: The key of keyword that want to be read. If None, return all keywords in dictionary. :returns: Dictionary of keywords or value of key as string. :rtype: dict, basestring """ xml_uri = os.path.splitext(layer_uri)[0] + '.xml' if not os.path.exists(xml_uri): xml_uri = None if not xml_uri and os.path.exists(layer_uri): message = 'Layer based file but no xml file.\n' message += 'Layer path: %s.' % layer_uri raise NoKeywordsFoundError(message) metadata = GenericLayerMetadata(layer_uri, xml_uri) if metadata.layer_purpose in METADATA_CLASSES: metadata = METADATA_CLASSES[metadata.layer_purpose](layer_uri, xml_uri) # dictionary comprehension keywords = { x[0]: x[1]['value'] for x in metadata.dict['properties'].iteritems() if x[1]['value'] is not None } if 'keyword_version' not in keywords.keys() and xml_uri: message = 'No keyword version found. Metadata xml file is invalid.\n' message += 'Layer uri: %s\n' % layer_uri message += 'Keywords file: %s\n' % os.path.exists( os.path.splitext(layer_uri)[0] + '.xml') message += 'keywords:\n' for k, v in keywords.iteritems(): message += '%s: %s\n' % (k, v) raise MetadataReadError(message) # Get dictionary keywords that has value != None keywords = { x[0]: x[1]['value'] for x in metadata.dict['properties'].iteritems() if x[1]['value'] is not None } if keyword: try: return keywords[keyword] except KeyError: message = 'Keyword with key %s is not found. ' % keyword message += 'Layer path: %s' % layer_uri raise KeywordNotFoundError(message) if isinstance(metadata, OutputLayerMetadata): keywords['if_provenance'] = metadata.provenance return keywords
def read_file_keywords(layer_path, keyword=None): """Get metadata from the keywords file associated with a local file in the file system. .. note:: Requires a str representing a file path instance as parameter As opposed to read_keywords_from_layer which takes a inasafe file object as parameter. .. seealso:: read_keywords_from_layer :param: layer_path: a string representing a path to a layer (e.g. '/tmp/foo.shp', '/tmp/foo.tif') :type layer_path: str :param keyword: optional - the metadata keyword to retrieve e.g. 'title' :type keyword: str :return: A string containing the retrieved value for the keyword if the keyword argument is specified, otherwise the complete keywords dictionary is returned. :raises: KeywordNotFoundError, NoKeywordsFoundError, InvalidParameterError Note: * KeywordNotFoundError - occurs when the keyword is not recognised. * NoKeywordsFoundError - occurs when no keyword file exists. * InvalidParameterError - occurs when the layer does not exist. """ # check the source layer path is valid if not os.path.isfile(layer_path): message = tr('Cannot get keywords from a non-existent file. File ' '%s does not exist.' % layer_path) raise InvalidParameterError(message) # check there really is a keywords file for this layer # priority for iso path first keyword_file_path = os.path.splitext(layer_path)[0] keyword_file_path += '.keywords' keyword_iso_path = os.path.splitext(layer_path)[0] keyword_iso_path += '.xml' if not os.path.isfile(keyword_file_path)\ and not os.path.isfile(keyword_iso_path): message = tr('No keywords file found for %s' % keyword_file_path) raise NoKeywordsFoundError(message) elif os.path.isfile(keyword_file_path) \ and not os.path.isfile(keyword_iso_path): # switch to .keywords file if iso xml file didn't exist keyword_iso_path = keyword_file_path # now get the requested keyword using the inasafe library try: dictionary = read_keywords(keyword_iso_path) except Exception, e: message = tr('Keyword retrieval failed for %s (%s) \n %s' % (keyword_file_path, keyword, str(e))) raise KeywordNotFoundError(message)
def get_keywords(self, key=None): """Return a copy of the keywords dictionary Args: * key (optional): If specified value will be returned for key only """ if key is None: return self.keywords.copy() else: if key in self.keywords: return self.keywords[key] else: msg = ('Keyword %s does not exist in %s: Options are ' '%s' % (key, self.get_name(), self.keywords.keys())) raise KeywordNotFoundError(msg)
def keyword(self, key): """Return value of key from keywords. It will raise KeywordNotFoundError if the key is not found. :param key: The key of a keyword. :type key: str :returns: The value of the key in keywords dictionary. :rtype: str, dict, int, float """ try: return self.keywords[key] except KeyError as e: raise KeywordNotFoundError(e)
def keyword(self, key): """Return value of key from keywords. It will raise KeywordNotFoundError if the key is not found. :param key: The key of a keyword. :type key: str :returns: The value of the key in keywords dictionary. :rtype: str, dict, int, float """ try: return self.keywords[key] except KeyError: message = tr( 'Keyword "%s" is not found in layer %s, please add it to your ' 'layer' % (key, self.name)) raise KeywordNotFoundError(message, layer_name=self.layer.name, keyword=key)
def set_layer(self, layer, keywords=None): """Set layer and update UI accordingly. :param layer: A vector layer that has been already patched with metadata. :type layer: QgsVectorLayer :param keywords: Custom keyword for the layer. :type keywords: dict, None """ self.layer = layer if keywords is not None: self.metadata = keywords else: # Check if it has keywords if not hasattr(layer, 'keywords'): message = 'Layer {layer_name} does not have keywords.'.format( layer_name=layer.name()) raise KeywordNotFoundError(message) self.metadata = layer.keywords self.populate_parameter()
def populate_tabs(self): """Populating tabs based on layer metadata.""" self.delete_tabs() layer_purpose = self.metadata.get('layer_purpose') if not layer_purpose: message = tr( 'Key layer_purpose is not found in the layer {layer_name}' ).format(layer_name=self.layer.name()) raise KeywordNotFoundError(message) if layer_purpose == layer_purpose_exposure['key']: layer_subcategory = self.metadata.get('exposure') elif layer_purpose == layer_purpose_hazard['key']: layer_subcategory = self.metadata.get('hazard') else: layer_subcategory = None field_groups = get_field_groups(layer_purpose, layer_subcategory) for field_group in field_groups: tab = FieldMappingTab(field_group, self, self.iface) tab.set_layer(self.layer, self.metadata) self.addTab(tab, field_group['name']) self.tabs.append(tab)
def read_keyword_from_uri(self, uri, keyword=None): """Get metadata from the keywords file associated with a URI. This is used for layers that are non local layer (e.g. postgresql connection) and so we need to retrieve the keywords from the sqlite keywords db. A hash will be constructed from the supplied uri and a lookup made in a local SQLITE database for the keywords. If there is an existing record it will be returned, if not and error will be thrown. If the record is a dictionary, it means that it was inserted into the DB in a pre 2.2 version which had no ISO metadata. In this case, we use that dictionary to update the entry to the new ISO based metadata .. seealso:: write_keywords_for_uri, delete_keywords_for_uri :param uri: A layer uri. e.g. ```dbname=\'osm\' host=localhost port=5432 user=\'foo\' password=\'bar\' sslmode=disable key=\'id\' srid=4326``` :type uri: str :param keyword: The metadata keyword to retrieve. If none, all keywords are returned. :type keyword: str :returns: A string containing the retrieved value for the keyword if the keyword argument is specified, otherwise the complete keywords dictionary is returned. :raises: KeywordNotFoundError if the keyword is not found. """ hash_value = self.hash_for_datasource(uri) try: self.open_connection() except OperationalError: raise try: cursor = self.get_cursor() # now see if we have any data for our hash sql = ( 'select dict from keyword where hash = \'%s\';' % hash_value) cursor.execute(sql) data = cursor.fetchone() # unpickle it to get our dict back if data is None: raise HashNotFoundError('No hash found for %s' % hash_value) data = data[0] # first field # get the ISO XML out of the DB metadata = pickle.loads(str(data)) # the uri already had a KW entry in the DB using the old KW system # we use that dictionary to update the entry to the new ISO based # metadata system if isinstance(metadata, dict): metadata = self.write_keywords_for_uri(uri, metadata) root = ElementTree.fromstring(metadata) keyword_element = root.find(ISO_METADATA_KEYWORD_TAG) dict_str = keyword_element.text picked_dict = json.loads(dict_str) if keyword is None: return picked_dict if keyword in picked_dict: return picked_dict[keyword] else: raise KeywordNotFoundError('Keyword "%s" not found in %s' % ( keyword, picked_dict)) except sqlite.Error, e: LOGGER.debug("Error %s:" % e.args[0])
dictionary = read_keywords(keyword_file_path) except Exception, e: message = tr('Keyword retrieval failed for %s (%s) \n %s' % (keyword_file_path, keyword, str(e))) raise KeywordNotFoundError(message) # if no keyword was supplied, just return the dict if keyword is None: if 'keyword_version' in dictionary.keys(): dictionary['keyword_version'] = get_string( dictionary['keyword_version']) return dictionary if keyword not in dictionary: message = tr('No value was found in file %s for keyword %s' % (keyword_file_path, keyword)) raise KeywordNotFoundError(message) try: value = dictionary[keyword] except: raise if 'keyword_version' == keyword: value = get_string(value) return value def replace_accentuated_characters(message): """Normalize unicode data in Python to remove umlauts, accents etc. :param message: The string where to delete accentuated characters. :type message: str, unicode
def read_iso19115_metadata(layer_uri, keyword=None): """Retrieve keywords from a metadata object :param layer_uri: :param keyword: :return: """ xml_uri = os.path.splitext(layer_uri)[0] + '.xml' if not os.path.exists(xml_uri): xml_uri = None if not xml_uri and os.path.exists(layer_uri): message = 'Layer based file but no xml file.\n' message += 'Layer path: %s.' % layer_uri raise NoKeywordsFoundError(message) metadata = GenericLayerMetadata(layer_uri, xml_uri) if metadata.layer_purpose == 'exposure': metadata = ExposureLayerMetadata(layer_uri, xml_uri) elif metadata.layer_purpose == 'hazard': metadata = HazardLayerMetadata(layer_uri, xml_uri) elif metadata.layer_purpose == 'aggregation': metadata = AggregationLayerMetadata(layer_uri, xml_uri) elif metadata.layer_purpose == 'impact': metadata = ImpactLayerMetadata(layer_uri, xml_uri) # dictionary comprehension keywords = { x[0]: x[1]['value'] for x in metadata.dict['properties'].iteritems() if x[1]['value'] is not None } if 'keyword_version' not in keywords.keys() and xml_uri: message = 'No keyword version found. Metadata xml file is invalid.\n' message += 'Layer uri: %s\n' % layer_uri message += 'Keywords file: %s\n' % os.path.exists( os.path.splitext(layer_uri)[0] + '.xml') message += 'keywords:\n' for k, v in keywords.iteritems(): message += '%s: %s\n' % (k, v) raise MetadataReadError(message) keywords = {} temp_keywords = { x[0]: x[1]['value'] for x in metadata.dict['properties'].iteritems() } included = [ 'aggregation attribute', 'female ratio attribute', 'youth ratio attribute', 'adult ratio attribute', 'elderly ratio attribute', ] for key in temp_keywords.iterkeys(): if key in included: keywords[key] = temp_keywords[key] else: if temp_keywords[key] is not None: keywords[key] = temp_keywords[key] if keyword: try: return keywords[keyword] except KeyError: message = 'Keyword with key %s is not found' % keyword message += 'Layer path: %s' % layer_uri raise KeywordNotFoundError(message) if isinstance(metadata, ImpactLayerMetadata): keywords['if_provenance'] = metadata.provenance return keywords
def validate_all_layers(self): """Validate all layers based on the keywords. When we do the validation, we also fetch the information we need: 1. 'map_title' from each impact layer 2. 'exposure_title' from each impact layer 3. 'postprocessing_report' from each impact layer 4. 'aggregation_attribute' on aggregation layer, if user runs merging tools with aggregation layer chosen The things that we validate are: 1. 'map_title' keyword must exist on each impact layer 2. 'exposure_title' keyword must exist on each impact layer 3. 'postprocessing_report' keyword must exist on each impact layer 4. 'hazard_title' keyword must exist on each impact layer. Hazard title from first impact layer must be the same with second impact layer to indicate that both are generated from the same hazard layer. 5. 'aggregation attribute' must exist when user wants to run merging tools with aggregation layer chosen. """ required_attribute = [ 'map_title', 'exposure_title', 'hazard_title', 'postprocessing_report' ] # Fetch for first impact layer for attribute in required_attribute: try: # noinspection PyTypeChecker self.first_impact[attribute] = self.keyword_io.read_keywords( self.first_impact['layer'], attribute) except NoKeywordsFoundError: raise NoKeywordsFoundError( self.tr('No keywords found for first impact layer.')) except KeywordNotFoundError: raise KeywordNotFoundError( self.tr('Keyword %s not found for first layer.' % attribute)) # Fetch for second impact layer for attribute in required_attribute: try: # noinspection PyTypeChecker self.second_impact[attribute] = self.keyword_io.read_keywords( self.second_impact['layer'], attribute) except NoKeywordsFoundError: raise NoKeywordsFoundError( self.tr('No keywords found for second impact layer.')) except KeywordNotFoundError: raise KeywordNotFoundError( self.tr('Keyword %s not found for second layer.' % attribute)) # Validate that two impact layers are obtained from the same hazard. # Indicated by the same 'hazard_title' (to be fixed later by using # more reliable method) if (self.first_impact['hazard_title'] != self.second_impact['hazard_title']): raise InvalidLayerError( self.tr('First impact layer and second impact layer do not ' 'use the same hazard layer.')) # Fetch 'aggregation_attribute' # If the chosen aggregation layer not Entire Area, it should have # aggregation attribute keywords if not self.entire_area_mode: try: # noinspection PyTypeChecker self.aggregation['aggregation_attribute'] = \ self.keyword_io.read_keywords( self.aggregation['layer'], 'aggregation attribute') except NoKeywordsFoundError: raise NoKeywordsFoundError( self.tr('No keywords exist in aggregation layer.')) except KeywordNotFoundError: raise KeywordNotFoundError( self.tr('Keyword aggregation attribute not found for ' 'aggregation layer.'))
def read_iso19115_metadata(layer_uri, keyword=None, version_35=False): """Retrieve keywords from a metadata object :param layer_uri: Uri to layer. :type layer_uri: basestring :param keyword: The key of keyword that want to be read. If None, return all keywords in dictionary. :type keyword: basestring :returns: Dictionary of keywords or value of key as string. :rtype: dict, basestring """ xml_uri = os.path.splitext(layer_uri)[0] + '.xml' # Remove the prefix for local file. For example csv. file_prefix = 'file:' if xml_uri.startswith(file_prefix): xml_uri = xml_uri[len(file_prefix):] if not os.path.exists(xml_uri): xml_uri = None if not xml_uri and os.path.exists(layer_uri): message = 'Layer based file but no xml file.\n' message += 'Layer path: %s.' % layer_uri raise NoKeywordsFoundError(message) if version_35: metadata = GenericLayerMetadata35(layer_uri, xml_uri) else: metadata = GenericLayerMetadata(layer_uri, xml_uri) active_metadata_classes = METADATA_CLASSES if version_35: active_metadata_classes = METADATA_CLASSES35 if metadata.layer_purpose in active_metadata_classes: metadata = active_metadata_classes[ metadata.layer_purpose](layer_uri, xml_uri) # dictionary comprehension keywords = { x[0]: x[1]['value'] for x in list(metadata.dict['properties'].items()) if x[1]['value'] is not None} if 'keyword_version' not in list(keywords.keys()) and xml_uri: message = 'No keyword version found. Metadata xml file is invalid.\n' message += 'Layer uri: %s\n' % layer_uri message += 'Keywords file: %s\n' % os.path.exists( os.path.splitext(layer_uri)[0] + '.xml') message += 'keywords:\n' for k, v in list(keywords.items()): message += '%s: %s\n' % (k, v) raise MetadataReadError(message) # Get dictionary keywords that has value != None keywords = { x[0]: x[1]['value'] for x in list(metadata.dict['properties'].items()) if x[1]['value'] is not None} if keyword: try: return keywords[keyword] except KeyError: message = 'Keyword with key %s is not found. ' % keyword message += 'Layer path: %s' % layer_uri raise KeywordNotFoundError(message) return keywords