def test_write_keyword_in_iso_metadata(self):
keyword_file = test_data_path('other', 'expected_multilayer.keywords')
with open(keyword_file) as f:
keywords = f.read()
basename, _ = os.path.splitext(keyword_file)
xml_file = basename + '.xml'
if os.path.isfile(xml_file):
os.remove(xml_file)
# there should be no xml file now
self.assertFalse(
os.path.isfile(xml_file), 'File %s should not exist' % xml_file)
xml_file = write_keyword_in_iso_metadata(keyword_file)
tree = ElementTree.parse(xml_file)
root = tree.getroot()
keyword_tag = root.find(ISO_METADATA_KEYWORD_TAG)
self.assertIn(keywords, keyword_tag.text)
# there should be an xml file now
self.assertTrue(
os.path.isfile(xml_file), 'File %s should exist' % xml_file)
# lets update the file
xml_file = write_keyword_in_iso_metadata(keyword_file)
tree = ElementTree.parse(xml_file)
keyword_tag = tree.getroot().find(ISO_METADATA_KEYWORD_TAG)
self.assertIn(keywords, keyword_tag.text)
os.remove(xml_file)
def read_keywords(keyword_filename, sublayer=None, all_blocks=False):
"""Read keywords dictionary from file
:param keyword_filename: Name of keywords file. Extension expected to be
.keywords. The format of one line is expected to be either
string: string or string
:type keyword_filename: str
:param sublayer: Optional sublayer applicable only to multilayer formats
such as sqlite or netcdf which can potentially hold more than
one layer. The string should map to the layer group as per the
example below. If the keywords file contains sublayer definitions
but no sublayer was defined, the first layer group will be
returned.
:type sublayer: str
:param all_blocks: Optional, defaults to False. If True will return
a dict of dicts, where the top level dict entries each represent
a sublayer, and the values of that dict will be dicts of keyword
entries.
:type all_blocks: bool
:returns: keywords: Dictionary of keyword, value pairs
A keyword layer with sublayers may look like this:
[osm_buildings]
datatype: osm
category: exposure
subcategory: building
purpose: dki
title: buildings_osm_4326
[osm_flood]
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
Whereas a simple keywords file would look like this
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
If filename does not exist, an empty dictionary is returned
Blank lines are ignored
Surrounding whitespace is removed from values, but keys are unmodified
If there are no ':', then the keyword is treated as a key with no value
"""
# Input checks
basename, ext = os.path.splitext(keyword_filename)
msg = ('Unknown extension for file %s. '
'Expected %s.keywords' % (keyword_filename, basename))
verify(ext == '.keywords', msg)
metadata = False
keywords_file = os.path.isfile(keyword_filename)
try:
metadata = read_iso_metadata(keyword_filename)
except (IOError, ReadMetadataError):
if keywords_file:
# if there is a keyword file generate an xml file also
write_keyword_in_iso_metadata(keyword_filename)
metadata = read_iso_metadata(keyword_filename)
# we have no valid xml metadata nor a keyword file
if not metadata and not keywords_file:
return {}
if metadata:
lines = metadata['keywords']
else:
# Read all entries
with open(keyword_filename, 'r') as fid:
lines = fid.readlines()
blocks = {}
keywords = {}
current_block = None
first_keywords = None
for line in lines:
# Remove trailing (but not preceeding!) whitespace
# FIXME: Can be removed altogether
text = line.rstrip()
# Ignore blank lines
if text == '':
continue
# Check if it is an ini style group header
block_flag = re.search(r'^\[.*]$', text, re.M | re.I)
if block_flag:
# Write the old block if it exists - must have a current
# block to prevent orphans
if len(keywords) > 0 and current_block is not None:
blocks[current_block] = keywords
if first_keywords is None and len(keywords) > 0:
first_keywords = keywords
# Now set up for a new block
current_block = text[1:-1]
# Reset the keywords each time we encounter a new block
# until we know we are on the desired one
keywords = {}
continue
if ':' not in text:
key = text.strip()
val = None
else:
# Get splitting point
idx = text.find(':')
# Take key as everything up to the first ':'
key = text[:idx]
# Take value as everything after the first ':'
textval = text[idx + 1:].strip()
try:
# Take care of python structures like
# booleans, None, lists, dicts etc
val = literal_eval(textval)
except (ValueError, SyntaxError):
if 'OrderedDict(' == textval[:12]:
try:
val = OrderedDict(
literal_eval(textval[12:-1]))
except (ValueError, SyntaxError, TypeError):
val = textval
else:
val = textval
# Add entry to dictionary
keywords[key] = val
# Write our any unfinalised block data
if len(keywords) > 0 and current_block is not None:
blocks[current_block] = keywords
if first_keywords is None:
first_keywords = keywords
# Ok we have generated a structure that looks like this:
# blocks = {{ 'foo' : { 'a': 'b', 'c': 'd'},
# { 'bar' : { 'd': 'e', 'f': 'g'}}
# where foo and bar are sublayers and their dicts are the sublayer keywords
if all_blocks:
return blocks
if sublayer is not None:
if sublayer in blocks:
return blocks[sublayer]
else:
return first_keywords
def read_keywords(keyword_filename, sublayer=None, all_blocks=False):
"""Read keywords dictionary from file
:param keyword_filename: Name of keywords file. Extension expected to be
.keywords. The format of one line is expected to be either
string: string or string
:type keyword_filename: str
:param sublayer: Optional sublayer applicable only to multilayer formats
such as sqlite or netcdf which can potentially hold more than
one layer. The string should map to the layer group as per the
example below. If the keywords file contains sublayer definitions
but no sublayer was defined, the first layer group will be
returned.
:type sublayer: str
:param all_blocks: Optional, defaults to False. If True will return
a dict of dicts, where the top level dict entries each represent
a sublayer, and the values of that dict will be dicts of keyword
entries.
:type all_blocks: bool
:returns: keywords: Dictionary of keyword, value pairs
A keyword layer with sublayers may look like this:
[osm_buildings]
datatype: osm
category: exposure
subcategory: building
purpose: dki
title: buildings_osm_4326
[osm_flood]
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
Whereas a simple keywords file would look like this
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
If filename does not exist, an empty dictionary is returned
Blank lines are ignored
Surrounding whitespace is removed from values, but keys are unmodified
If there are no ':', then the keyword is treated as a key with no value
"""
# Input checks
basename, ext = os.path.splitext(keyword_filename)
msg = ('Unknown extension for file %s. '
'Expected %s.keywords' % (keyword_filename, basename))
verify(ext == '.keywords', msg)
metadata = False
keywords_file = os.path.isfile(keyword_filename)
try:
metadata = read_iso_metadata(keyword_filename)
except (IOError, ReadMetadataError):
if keywords_file:
# if there is a keyword file generate an xml file also
write_keyword_in_iso_metadata(keyword_filename)
metadata = read_iso_metadata(keyword_filename)
# we have no valid xml metadata nor a keyword file
if not metadata and not keywords_file:
return {}
if metadata:
lines = metadata['keywords']
else:
# Read all entries
with open(keyword_filename, 'r') as fid:
lines = fid.readlines()
blocks = {}
keywords = {}
current_block = None
first_keywords = None
for line in lines:
# Remove trailing (but not preceeding!) whitespace
# FIXME: Can be removed altogether
text = line.rstrip()
# Ignore blank lines
if text == '':
continue
# Check if it is an ini style group header
block_flag = re.search(r'^\[.*]$', text, re.M | re.I)
if block_flag:
# Write the old block if it exists - must have a current
# block to prevent orphans
if len(keywords) > 0 and current_block is not None:
blocks[current_block] = keywords
if first_keywords is None and len(keywords) > 0:
first_keywords = keywords
# Now set up for a new block
current_block = text[1:-1]
# Reset the keywords each time we encounter a new block
# until we know we are on the desired one
keywords = {}
continue
if ':' not in text:
key = text.strip()
val = None
else:
# Get splitting point
idx = text.find(':')
# Take key as everything up to the first ':'
key = text[:idx]
# Take value as everything after the first ':'
textval = text[idx + 1:].strip()
try:
# Take care of python structures like
# booleans, None, lists, dicts etc
val = literal_eval(textval)
except (ValueError, SyntaxError):
if 'OrderedDict(' == textval[:12]:
try:
val = OrderedDict(literal_eval(textval[12:-1]))
except (ValueError, SyntaxError, TypeError):
val = textval
else:
val = textval
# Add entry to dictionary
keywords[key] = val
# Write our any unfinalised block data
if len(keywords) > 0 and current_block is not None:
blocks[current_block] = keywords
if first_keywords is None:
first_keywords = keywords
# Ok we have generated a structure that looks like this:
# blocks = {{ 'foo' : { 'a': 'b', 'c': 'd'},
# { 'bar' : { 'd': 'e', 'f': 'g'}}
# where foo and bar are sublayers and their dicts are the sublayer keywords
if all_blocks:
return blocks
if sublayer is not None:
if sublayer in blocks:
return blocks[sublayer]
else:
return first_keywords
def write_keywords(keywords, filename, sublayer=None):
"""Write keywords dictonary to file
:param keywords: Dictionary of keyword, value pairs
:type keywords: dict
:param filename: Name of keywords file. Extension expected to be .keywords
:type filename: str
:param sublayer: Optional sublayer applicable only to multilayer formats
such as sqlite or netcdf which can potentially hold more than
one layer. The string should map to the layer group as per the
example below. **If the keywords file contains sublayer
definitions but no sublayer was defined, keywords file content
will be removed and replaced with only the keywords provided
here.**
:type sublayer: str
A keyword file with sublayers may look like this:
[osm_buildings]
datatype: osm
category: exposure
subcategory: building
purpose: dki
title: buildings_osm_4326
[osm_flood]
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
Keys must be strings not containing the ":" character
Values can be anything that can be converted to a string (using
Python's str function)
Surrounding whitespace is removed from values, but keys are unmodified
The reason being that keys must always be valid for the dictionary they
came from. For values we have decided to be flexible and treat entries like
'unit:m' the same as 'unit: m', or indeed 'unit: m '.
Otherwise, unintentional whitespace in values would lead to surprising
errors in the application.
"""
# Input checks
basename, ext = os.path.splitext(filename)
msg = ('Unknown extension for file %s. '
'Expected %s.keywords' % (filename, basename))
verify(ext == '.keywords', msg)
# First read any keywords out of the file so that we can retain
# keywords for other sublayers
existing_keywords = read_keywords(filename, all_blocks=True)
first_value = None
if len(existing_keywords) > 0:
first_value = existing_keywords[existing_keywords.keys()[0]]
multilayer_flag = type(first_value) == dict
handle = file(filename, 'w')
if multilayer_flag:
if sublayer is not None and sublayer != '':
# replace existing keywords / add new for this layer
existing_keywords[sublayer] = keywords
for key, value in existing_keywords.iteritems():
handle.write(_keywords_to_string(value, sublayer=key))
handle.write('\n')
else:
# It is currently a multilayer but we will replace it with
# a single keyword block since the user passed no sublayer
handle.write(_keywords_to_string(keywords))
else:
# currently a simple layer so replace it with our content
handle.write(_keywords_to_string(keywords, sublayer=sublayer))
handle.close()
write_keyword_in_iso_metadata(filename)
def write_keywords(keywords, filename, sublayer=None):
"""Write keywords dictonary to file
:param keywords: Dictionary of keyword, value pairs
:type keywords: dict
:param filename: Name of keywords file. Extension expected to be .keywords
:type filename: str
:param sublayer: Optional sublayer applicable only to multilayer formats
such as sqlite or netcdf which can potentially hold more than
one layer. The string should map to the layer group as per the
example below. **If the keywords file contains sublayer
definitions but no sublayer was defined, keywords file content
will be removed and replaced with only the keywords provided
here.**
:type sublayer: str
A keyword file with sublayers may look like this:
[osm_buildings]
datatype: osm
category: exposure
subcategory: building
purpose: dki
title: buildings_osm_4326
[osm_flood]
datatype: flood
category: hazard
subcategory: building
title: flood_osm_4326
Keys must be strings not containing the ":" character
Values can be anything that can be converted to a string (using
Python's str function)
Surrounding whitespace is removed from values, but keys are unmodified
The reason being that keys must always be valid for the dictionary they
came from. For values we have decided to be flexible and treat entries like
'unit:m' the same as 'unit: m', or indeed 'unit: m '.
Otherwise, unintentional whitespace in values would lead to surprising
errors in the application.
"""
# Input checks
basename, ext = os.path.splitext(filename)
msg = ('Unknown extension for file %s. '
'Expected %s.keywords' % (filename, basename))
verify(ext == '.keywords', msg)
# First read any keywords out of the file so that we can retain
# keywords for other sublayers
existing_keywords = read_keywords(filename, all_blocks=True)
first_value = None
if len(existing_keywords) > 0:
first_value = existing_keywords[existing_keywords.keys()[0]]
multilayer_flag = type(first_value) == dict
handle = file(filename, 'w')
if multilayer_flag:
if sublayer is not None and sublayer != '':
# replace existing keywords / add new for this layer
existing_keywords[sublayer] = keywords
for key, value in existing_keywords.iteritems():
handle.write(_keywords_to_string(value, sublayer=key))
handle.write('\n')
else:
# It is currently a multilayer but we will replace it with
# a single keyword block since the user passed no sublayer
handle.write(_keywords_to_string(keywords))
else:
# currently a simple layer so replace it with our content
handle.write(_keywords_to_string(keywords, sublayer=sublayer))
handle.close()
write_keyword_in_iso_metadata(filename)
