def test_write_keyword_in_iso_metadata(self): keyword_file = test_data_path('other', 'expected_multilayer.keywords') with open(keyword_file) as f: keywords = f.read() basename, _ = os.path.splitext(keyword_file) xml_file = basename + '.xml' if os.path.isfile(xml_file): os.remove(xml_file) # there should be no xml file now self.assertFalse( os.path.isfile(xml_file), 'File %s should not exist' % xml_file) xml_file = write_keyword_in_iso_metadata(keyword_file) tree = ElementTree.parse(xml_file) root = tree.getroot() keyword_tag = root.find(ISO_METADATA_KEYWORD_TAG) self.assertIn(keywords, keyword_tag.text) # there should be an xml file now self.assertTrue( os.path.isfile(xml_file), 'File %s should exist' % xml_file) # lets update the file xml_file = write_keyword_in_iso_metadata(keyword_file) tree = ElementTree.parse(xml_file) keyword_tag = tree.getroot().find(ISO_METADATA_KEYWORD_TAG) self.assertIn(keywords, keyword_tag.text) os.remove(xml_file)
def read_keywords(keyword_filename, sublayer=None, all_blocks=False): """Read keywords dictionary from file :param keyword_filename: Name of keywords file. Extension expected to be .keywords. The format of one line is expected to be either string: string or string :type keyword_filename: str :param sublayer: Optional sublayer applicable only to multilayer formats such as sqlite or netcdf which can potentially hold more than one layer. The string should map to the layer group as per the example below. If the keywords file contains sublayer definitions but no sublayer was defined, the first layer group will be returned. :type sublayer: str :param all_blocks: Optional, defaults to False. If True will return a dict of dicts, where the top level dict entries each represent a sublayer, and the values of that dict will be dicts of keyword entries. :type all_blocks: bool :returns: keywords: Dictionary of keyword, value pairs A keyword layer with sublayers may look like this: [osm_buildings] datatype: osm category: exposure subcategory: building purpose: dki title: buildings_osm_4326 [osm_flood] datatype: flood category: hazard subcategory: building title: flood_osm_4326 Whereas a simple keywords file would look like this datatype: flood category: hazard subcategory: building title: flood_osm_4326 If filename does not exist, an empty dictionary is returned Blank lines are ignored Surrounding whitespace is removed from values, but keys are unmodified If there are no ':', then the keyword is treated as a key with no value """ # Input checks basename, ext = os.path.splitext(keyword_filename) msg = ('Unknown extension for file %s. ' 'Expected %s.keywords' % (keyword_filename, basename)) verify(ext == '.keywords', msg) metadata = False keywords_file = os.path.isfile(keyword_filename) try: metadata = read_iso_metadata(keyword_filename) except (IOError, ReadMetadataError): if keywords_file: # if there is a keyword file generate an xml file also write_keyword_in_iso_metadata(keyword_filename) metadata = read_iso_metadata(keyword_filename) # we have no valid xml metadata nor a keyword file if not metadata and not keywords_file: return {} if metadata: lines = metadata['keywords'] else: # Read all entries with open(keyword_filename, 'r') as fid: lines = fid.readlines() blocks = {} keywords = {} current_block = None first_keywords = None for line in lines: # Remove trailing (but not preceeding!) whitespace # FIXME: Can be removed altogether text = line.rstrip() # Ignore blank lines if text == '': continue # Check if it is an ini style group header block_flag = re.search(r'^\[.*]$', text, re.M | re.I) if block_flag: # Write the old block if it exists - must have a current # block to prevent orphans if len(keywords) > 0 and current_block is not None: blocks[current_block] = keywords if first_keywords is None and len(keywords) > 0: first_keywords = keywords # Now set up for a new block current_block = text[1:-1] # Reset the keywords each time we encounter a new block # until we know we are on the desired one keywords = {} continue if ':' not in text: key = text.strip() val = None else: # Get splitting point idx = text.find(':') # Take key as everything up to the first ':' key = text[:idx] # Take value as everything after the first ':' textval = text[idx + 1:].strip() try: # Take care of python structures like # booleans, None, lists, dicts etc val = literal_eval(textval) except (ValueError, SyntaxError): if 'OrderedDict(' == textval[:12]: try: val = OrderedDict( literal_eval(textval[12:-1])) except (ValueError, SyntaxError, TypeError): val = textval else: val = textval # Add entry to dictionary keywords[key] = val # Write our any unfinalised block data if len(keywords) > 0 and current_block is not None: blocks[current_block] = keywords if first_keywords is None: first_keywords = keywords # Ok we have generated a structure that looks like this: # blocks = {{ 'foo' : { 'a': 'b', 'c': 'd'}, # { 'bar' : { 'd': 'e', 'f': 'g'}} # where foo and bar are sublayers and their dicts are the sublayer keywords if all_blocks: return blocks if sublayer is not None: if sublayer in blocks: return blocks[sublayer] else: return first_keywords
def read_keywords(keyword_filename, sublayer=None, all_blocks=False): """Read keywords dictionary from file :param keyword_filename: Name of keywords file. Extension expected to be .keywords. The format of one line is expected to be either string: string or string :type keyword_filename: str :param sublayer: Optional sublayer applicable only to multilayer formats such as sqlite or netcdf which can potentially hold more than one layer. The string should map to the layer group as per the example below. If the keywords file contains sublayer definitions but no sublayer was defined, the first layer group will be returned. :type sublayer: str :param all_blocks: Optional, defaults to False. If True will return a dict of dicts, where the top level dict entries each represent a sublayer, and the values of that dict will be dicts of keyword entries. :type all_blocks: bool :returns: keywords: Dictionary of keyword, value pairs A keyword layer with sublayers may look like this: [osm_buildings] datatype: osm category: exposure subcategory: building purpose: dki title: buildings_osm_4326 [osm_flood] datatype: flood category: hazard subcategory: building title: flood_osm_4326 Whereas a simple keywords file would look like this datatype: flood category: hazard subcategory: building title: flood_osm_4326 If filename does not exist, an empty dictionary is returned Blank lines are ignored Surrounding whitespace is removed from values, but keys are unmodified If there are no ':', then the keyword is treated as a key with no value """ # Input checks basename, ext = os.path.splitext(keyword_filename) msg = ('Unknown extension for file %s. ' 'Expected %s.keywords' % (keyword_filename, basename)) verify(ext == '.keywords', msg) metadata = False keywords_file = os.path.isfile(keyword_filename) try: metadata = read_iso_metadata(keyword_filename) except (IOError, ReadMetadataError): if keywords_file: # if there is a keyword file generate an xml file also write_keyword_in_iso_metadata(keyword_filename) metadata = read_iso_metadata(keyword_filename) # we have no valid xml metadata nor a keyword file if not metadata and not keywords_file: return {} if metadata: lines = metadata['keywords'] else: # Read all entries with open(keyword_filename, 'r') as fid: lines = fid.readlines() blocks = {} keywords = {} current_block = None first_keywords = None for line in lines: # Remove trailing (but not preceeding!) whitespace # FIXME: Can be removed altogether text = line.rstrip() # Ignore blank lines if text == '': continue # Check if it is an ini style group header block_flag = re.search(r'^\[.*]$', text, re.M | re.I) if block_flag: # Write the old block if it exists - must have a current # block to prevent orphans if len(keywords) > 0 and current_block is not None: blocks[current_block] = keywords if first_keywords is None and len(keywords) > 0: first_keywords = keywords # Now set up for a new block current_block = text[1:-1] # Reset the keywords each time we encounter a new block # until we know we are on the desired one keywords = {} continue if ':' not in text: key = text.strip() val = None else: # Get splitting point idx = text.find(':') # Take key as everything up to the first ':' key = text[:idx] # Take value as everything after the first ':' textval = text[idx + 1:].strip() try: # Take care of python structures like # booleans, None, lists, dicts etc val = literal_eval(textval) except (ValueError, SyntaxError): if 'OrderedDict(' == textval[:12]: try: val = OrderedDict(literal_eval(textval[12:-1])) except (ValueError, SyntaxError, TypeError): val = textval else: val = textval # Add entry to dictionary keywords[key] = val # Write our any unfinalised block data if len(keywords) > 0 and current_block is not None: blocks[current_block] = keywords if first_keywords is None: first_keywords = keywords # Ok we have generated a structure that looks like this: # blocks = {{ 'foo' : { 'a': 'b', 'c': 'd'}, # { 'bar' : { 'd': 'e', 'f': 'g'}} # where foo and bar are sublayers and their dicts are the sublayer keywords if all_blocks: return blocks if sublayer is not None: if sublayer in blocks: return blocks[sublayer] else: return first_keywords
def write_keywords(keywords, filename, sublayer=None): """Write keywords dictonary to file :param keywords: Dictionary of keyword, value pairs :type keywords: dict :param filename: Name of keywords file. Extension expected to be .keywords :type filename: str :param sublayer: Optional sublayer applicable only to multilayer formats such as sqlite or netcdf which can potentially hold more than one layer. The string should map to the layer group as per the example below. **If the keywords file contains sublayer definitions but no sublayer was defined, keywords file content will be removed and replaced with only the keywords provided here.** :type sublayer: str A keyword file with sublayers may look like this: [osm_buildings] datatype: osm category: exposure subcategory: building purpose: dki title: buildings_osm_4326 [osm_flood] datatype: flood category: hazard subcategory: building title: flood_osm_4326 Keys must be strings not containing the ":" character Values can be anything that can be converted to a string (using Python's str function) Surrounding whitespace is removed from values, but keys are unmodified The reason being that keys must always be valid for the dictionary they came from. For values we have decided to be flexible and treat entries like 'unit:m' the same as 'unit: m', or indeed 'unit: m '. Otherwise, unintentional whitespace in values would lead to surprising errors in the application. """ # Input checks basename, ext = os.path.splitext(filename) msg = ('Unknown extension for file %s. ' 'Expected %s.keywords' % (filename, basename)) verify(ext == '.keywords', msg) # First read any keywords out of the file so that we can retain # keywords for other sublayers existing_keywords = read_keywords(filename, all_blocks=True) first_value = None if len(existing_keywords) > 0: first_value = existing_keywords[existing_keywords.keys()[0]] multilayer_flag = type(first_value) == dict handle = file(filename, 'w') if multilayer_flag: if sublayer is not None and sublayer != '': # replace existing keywords / add new for this layer existing_keywords[sublayer] = keywords for key, value in existing_keywords.iteritems(): handle.write(_keywords_to_string(value, sublayer=key)) handle.write('\n') else: # It is currently a multilayer but we will replace it with # a single keyword block since the user passed no sublayer handle.write(_keywords_to_string(keywords)) else: # currently a simple layer so replace it with our content handle.write(_keywords_to_string(keywords, sublayer=sublayer)) handle.close() write_keyword_in_iso_metadata(filename)