def test_cached_expanded_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
deref_schema = v.get_expanded_schema(schema_name)
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
# get the schame again
deref_schema = v.get_expanded_schema(schema_name)
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_cached_expanded_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
deref_schema = v.get_expanded_schema(schema_name)
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
# get the schame again
deref_schema = v.get_expanded_schema(schema_name)
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_deref():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
print(json.dumps(jsn_schema, indent=4))
print(jsn_schema["properties"]["filter"])
assert (list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
deref_schema = v.get_expanded_schema(schema_name)
print(json.dumps(deref_schema, indent=4))
print(deref_schema["properties"]["filter"])
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_deref():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
print(json.dumps(jsn_schema, indent=4))
print(jsn_schema["properties"]["filter"])
assert(list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
deref_schema = v.get_expanded_schema(schema_name)
print(json.dumps(deref_schema, indent=4))
print(deref_schema["properties"]["filter"])
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
class PrettyPrinter(object):
def __init__(self,
indent=4,
spacer=" ",
quote='"',
newlinechar="\n",
end_comment=False,
align_values=False,
separate_complex_types=False,
**kwargs):
"""
Option use "\t" for spacer with an indent of 1
"""
assert (quote == "'" or quote == '"')
self.indent = indent
self.spacer = spacer * self.indent
self.quoter = Quoter(quote)
self.newlinechar = newlinechar
self.end_comment = end_comment
self.end = u"END"
self.validator = Validator()
self.align_values = align_values
self.separate_complex_types = separate_complex_types
def __is_metadata(self, key):
"""
Check to see if the property is hidden metadata
e.g. "__type__", "__comments__", "__position__"
"""
if key.startswith("__") and key.endswith("__"):
return True
else:
return False
def compute_aligned_max_indent(self, max_key_length):
"""
Computes the indentation as a multiple of self.indent for aligning
values at the same column based on the maximum key length.
Example:
key value1
longkey value2
longestkey value3 <-- column at 12, indent of 4, determined by "longestkey"
"""
indent = max(1, self.indent)
return int((int(max_key_length / indent) + 1) * indent)
def compute_max_key_length(self, composite):
"""
Computes the maximum length of all keys (non-recursive) in the passed
composite.
"""
length = 0
for attr, value in composite.items():
attr_length = len(attr)
if (not self.__is_metadata(attr)
and attr not in ("metadata", "validation", "values",
"connectionoptions")
and not self.is_hidden_container(attr, value)
and not attr == "pattern" and not attr == "projection"
and not attr == "points" and not attr == "config"
and not self.is_composite(value)):
length = max(length, attr_length)
return length
def separate_complex(self, composite, level):
if not self.separate_complex_types:
return
for key in list(composite.keys()):
if self.is_complex_type(composite, key, level):
utils.dict_move_to_end(composite, key)
def whitespace(self, level, indent):
return self.spacer * (level + indent)
def add_start_line(self, key, level):
return self.whitespace(level, 1) + key.upper()
def add_end_line(self, level, indent, key):
end_line = self.whitespace(level, indent) + self.end
if self.end_comment:
end_line = "{} # {}".format(end_line, key.upper())
return end_line
def __format_line(self, spacer, key, value, aligned_max_indent=0):
if ((aligned_max_indent is None) or (aligned_max_indent == 0)):
aligned_max_indent = len(key) + 1
indent = " " * (aligned_max_indent - len(key))
tmpl = u"{spacer}{key}{indent}{value}"
d = {"spacer": spacer, "key": key, "value": value, "indent": indent}
return tmpl.format(**d)
def process_key_dict(self, key, d, level):
"""
Process key value dicts e.g. METADATA "key" "value"
"""
# add any composite level comments
comments = d.get("__comments__", {})
lines = []
self._add_type_comment(level, comments, lines)
lines += [self.add_start_line(key, level)]
lines += self.process_dict(d, level, comments)
lines.append(self.add_end_line(level, 1, key))
return lines
def process_dict(self, d, level, comments):
"""
Process keys and values within a block
"""
lines = []
aligned_max_indent = 0
if (self.align_values):
max_key_length = self.compute_max_key_length(
d) + 2 # add length of quotes
aligned_max_indent = self.compute_aligned_max_indent(
max_key_length)
for k, v in d.items():
if not self.__is_metadata(k):
qk = self.quoter.add_quotes(k)
qv = self.quoter.add_quotes(v)
line = self.__format_line(self.whitespace(level, 2), qk, qv,
aligned_max_indent)
line += self.process_attribute_comment(comments, k)
lines.append(line)
return lines
def process_config_dict(self, key, d, level):
"""
Process the CONFIG block
"""
lines = []
for k, v in d.items():
k = "CONFIG {}".format(self.quoter.add_quotes(k.upper()))
v = self.quoter.add_quotes(v)
lines.append(self.__format_line(self.whitespace(level, 1), k, v))
return lines
def process_repeated_list(self, key, lst, level, aligned_max_indent=1):
"""
Process blocks of repeated keys e.g. FORMATOPTION
"""
lines = []
for v in lst:
k = key.upper()
v = self.quoter.add_quotes(v)
lines.append(
self.__format_line(self.whitespace(level, 1), k, v,
aligned_max_indent))
return lines
def process_projection(self, key, lst, level):
lines = [self.add_start_line(key, level)]
if self.quoter.is_string(lst):
val = self.quoter.add_quotes(lst)
# the value has been manually set to a single string projection
lines.append(u"{}{}".format(self.whitespace(level, 2), val))
elif len(lst) == 1 and lst[0].upper() == "AUTO":
lines.append(u"{}{}".format(self.whitespace(level, 2), "AUTO"))
else:
for v in lst:
v = self.quoter.add_quotes(v)
lines.append(u"{}{}".format(self.whitespace(level, 2), v))
lines.append(self.add_end_line(level, 1, key))
return lines
def format_pair_list(self, key, pair_list, level):
"""
Process lists of pairs (e.g. PATTERN block)
"""
lines = [self.add_start_line(key, level)]
list_spacer = self.spacer * (level + 2)
pairs = ["{}{} {}".format(list_spacer, p[0], p[1]) for p in pair_list]
lines += pairs
lines.append(self.add_end_line(level, 1, key))
return lines
def format_repeated_pair_list(self, key, root_list, level):
"""
Process (possibly) repeated lists of pairs e.g. POINTs blocks
"""
lines = []
def depth(L):
return isinstance(L, (tuple, list)) and max(map(depth, L)) + 1
if depth(root_list) == 2:
# single set of points only
root_list = [root_list]
for pair_list in root_list:
lines += self.format_pair_list(key, pair_list, level)
return lines
def is_composite(self, val):
if isinstance(val, dict) and "__type__" in val:
return True
else:
return False
def is_complex_type(self, composite, key, level):
# symbol needs special treatment
if key == "symbol" and level > 0:
return False
return key in COMPLEX_TYPES or self.is_composite(
key) or self.is_hidden_container(key, composite[key])
def is_hidden_container(self, key, val):
"""
The key is not one of the Mapfile keywords, and its
values are a list
"""
if key in OBJECT_LIST_KEYS and isinstance(val, list):
return True
else:
return False
def pprint(self, composites):
"""
Print out a nicely indented Mapfile
"""
# if only a single composite is used then cast to list
# and allow for multiple root composites
if composites and not isinstance(composites, list):
composites = [composites]
lines = []
for composite in composites:
type_ = composite["__type__"]
if type_ in ("metadata", "validation", "connectionoptions"):
# types are being parsed directly, and not as an attr of a parent
lines += self.process_key_dict(type_, composite, level=0)
else:
lines += self._format(composite)
result = str(self.newlinechar.join(lines))
return result
def get_attribute_properties(self, type_, attr):
jsn_schema = self.validator.get_expanded_schema(type_)
props = jsn_schema["properties"]
# check if a value needs to be quoted or not, by referring to the JSON schema
try:
attr_props = props[attr]
except KeyError as ex:
log.error("The key '{}' was not found in the JSON schema for '{}'".
format(attr, type_))
log.error(ex)
return {}
return attr_props
def is_expression(self, option):
return "description" in option and (option["description"]
== "expression")
def check_options_list(self, options_list, value):
for option in options_list:
if "enum" in option and value.lower() in option["enum"]:
if value.lower() == "end":
# in GEOTRANSFORM "end" is an attribute value
return self.quoter.add_quotes(value)
else:
return value.upper()
elif self.is_expression(option):
if value.endswith("'i") or value.endswith('"i'):
return value
if self.quoter.in_slashes(value):
return value
else:
return self.quoter.add_quotes(value)
def format_value(self, attr, attr_props, value):
"""
TODO - refactor and add more specific tests (particularly for expressions)
"""
if isinstance(value, bool):
return str(value).upper()
if any(i in ["enum"] for i in attr_props):
if isinstance(value, dict) and not value:
raise ValueError(
"The property {} has an empty dictionary as a value".
format(attr))
if not isinstance(value, numbers.Number):
if attr == "compop":
return self.quoter.add_quotes(value)
else:
return value.upper(
) # value is from a set list, no need for quote
else:
return value
if "type" in attr_props and attr_props[
"type"] == "string": # and "enum" not in attr_props
# check schemas for expressions and handle accordingly
if self.is_expression(attr_props) and self.quoter.in_slashes(
value):
return value
elif self.is_expression(attr_props) and (value.endswith("'i")
or value.endswith('"i')):
# for case insensitive regex
return value
else:
return self.quoter.add_quotes(value)
# expressions can be one of a string or an expression in brackets
if any(i in ["oneOf", "anyOf"]
for i in attr_props): # and check that type string is in list
if "oneOf" in attr_props:
options_list = attr_props["oneOf"]
else:
options_list = attr_props["anyOf"]
if self.quoter.is_string(value):
if self.quoter.in_parenthesis(value):
pass
elif attr == "expression" and self.quoter.in_braces(value):
# don't add quotes to list expressions such as {val1, val2}
pass
elif attr != "text" and self.quoter.in_brackets(value):
# TEXT expressions are often "[field1]-[field2]" so need to leave quotes for these
pass
elif value.startswith("NOT ") and self.quoter.in_parenthesis(
value[4:]):
value = "NOT {}".format(value[4:])
else:
value = self.check_options_list(options_list, value)
if isinstance(value, list):
new_values = []
for v in value:
if not isinstance(v, numbers.Number) and attr not in [
"offset", "polaroffset"
]:
# don't add quotes to list of attributes for offset / polaroffset
v = self.quoter.add_quotes(v)
new_values.append(v)
value = " ".join(list(map(str, new_values)))
else:
value = self.quoter.escape_quotes(value)
return value
def process_attribute(self,
type_,
attr,
value,
level,
aligned_max_indent=1):
"""
Process one of the main composite types (see the type_ value)
"""
attr_props = self.get_attribute_properties(type_, attr)
value = self.format_value(attr, attr_props, value)
line = self.__format_line(self.whitespace(level, 1), attr.upper(),
value, aligned_max_indent)
return line
def format_comment(self, spacer, value):
return "{}{}".format(spacer, value)
def process_composite_comment(self, level, comments, key):
"""
Process comments for composites such as MAP, LAYER etc.
"""
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = self.whitespace(level, 0)
if isinstance(value, list):
comments = [self.format_comment(spacer, v) for v in value]
comment = self.newlinechar.join(comments)
else:
comment = self.format_comment(spacer, value)
return comment
def process_attribute_comment(self, comments, key):
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = " "
# for multiple comments associated with an attribute
# simply join them together as a single string
if isinstance(value, list):
value = " ".join(value)
comment = self.format_comment(spacer, value)
return comment
def _add_type_comment(self, level, comments, lines):
comment = self.process_composite_comment(level, comments, '__type__')
if comment:
lines.append(str(comment))
def _format(self, composite, level=0):
lines = []
type_ = None
# get any comments associated with the composite
comments = composite.get("__comments__", {})
if isinstance(composite, dict) and '__type__' in composite:
type_ = composite['__type__']
assert type_ in COMPOSITE_NAMES.union(SINGLETON_COMPOSITE_NAMES)
is_hidden = False
self._add_type_comment(level, comments, lines)
s = self.whitespace(level, 0) + type_.upper()
lines.append(s)
aligned_max_indent = 0
if self.align_values:
max_key_length = self.compute_max_key_length(composite)
aligned_max_indent = self.compute_aligned_max_indent(
max_key_length)
self.separate_complex(composite, level)
for attr, value in composite.items():
if self.__is_metadata(attr):
# skip hidden attributes
continue
elif self.is_hidden_container(attr, value):
# now recursively print all the items in the container
for v in value:
lines += self._format(v, level + 1)
elif attr == "pattern":
lines += self.format_pair_list(attr, value, level)
elif attr in ("metadata", "validation", "values",
"connectionoptions"):
# metadata and values are also composites
# but will be processed here
lines += self.process_key_dict(attr, value, level)
elif attr == "projection":
lines += self.process_projection(attr, value, level)
elif attr in REPEATED_KEYS:
lines += self.process_repeated_list(attr, value, level,
aligned_max_indent)
elif attr == "points":
lines += self.format_repeated_pair_list(attr, value, level)
elif attr == "config":
lines += self.process_config_dict(attr, value, level)
elif self.is_composite(value):
lines += self._format(value, level +
1) # recursively add the child class
else:
# standard key value pair
if not type_:
raise UnboundLocalError(
"The Mapfile object is missing a __type__ attribute")
line = self.process_attribute(type_, attr, value, level,
aligned_max_indent)
line += self.process_attribute_comment(comments, attr)
lines.append(line)
if not is_hidden:
# close the container block with an END
lines.append(self.add_end_line(level, 0, type_))
return lines
class PrettyPrinter(object):
def __init__(self, indent=4, spacer=" ", quote='"', newlinechar="\n"):
"""
Option use "\t" for spacer with an indent of 1
"""
assert (quote == "'" or quote == '"')
self.indent = indent
self.spacer = spacer * self.indent
self.newlinechar = newlinechar
self.quoter = Quoter(quote)
self.end = u"END"
self.validator = Validator()
def whitespace(self, level, indent):
return self.spacer * (level + indent)
def singular(self, s):
if s == 'points':
return s
elif s.endswith('es'):
return s[:-2]
return s[:-1]
def add_start_line(self, key, level):
return self.whitespace(level, 1) + key.upper()
def add_end_line(self, level, indent):
return self.whitespace(level, indent) + self.end
def __format_line(self, spacer, key, value):
tmpl = u"{spacer}{key} {value}"
d = {"spacer": spacer, "key": key, "value": value}
return tmpl.format(**d)
def process_key_dict(self, key, d, level):
"""
Process key value dicts e.g. METADATA "key" "value"
"""
# add any composite level comments
comments = d.get("__comments__", {})
lines = []
self._add_type_comment(level, comments, lines)
lines += [self.add_start_line(key, level)]
lines += self.process_dict(d, level, comments)
lines.append(self.add_end_line(level, 1))
return lines
def process_dict(self, d, level, comments):
"""
Process keys and values within a block
"""
lines = []
for k, v in d.items():
if k in ("__comments__", "__position__"):
pass
elif k != "__type__":
qk = self.quoter.add_quotes(k)
qv = self.quoter.add_quotes(v)
line = self.__format_line(self.whitespace(level, 2), qk, qv)
line += self.process_attribute_comment(comments, k)
lines.append(line)
return lines
def process_config_dict(self, key, d, level):
"""
Process the CONFIG block
"""
lines = []
for k, v in d.items():
k = "CONFIG {}".format(self.quoter.add_quotes(k.upper()))
v = self.quoter.add_quotes(v)
lines.append(self.__format_line(self.whitespace(level, 1), k, v))
return lines
def process_repeated_list(self, key, lst, level):
"""
Process blocks of repeated keys e.g. FORMATOPTION
"""
lines = []
for v in lst:
k = key.upper()
v = self.quoter.add_quotes(v)
lines.append(self.__format_line(self.whitespace(level, 1), k, v))
return lines
def process_projection(self, key, lst, level):
lines = [self.add_start_line(key, level)]
if len(lst) == 1 and lst[0].upper() == "AUTO":
lines.append(u"{}{}".format(self.whitespace(level, 2), "AUTO"))
else:
for v in lst:
v = self.quoter.add_quotes(v)
lines.append(u"{}{}".format(self.whitespace(level, 2), v))
lines.append(self.add_end_line(level, 1))
return lines
def format_pair_list(self, key, pair_list, level):
"""
Process lists of pairs (e.g. PATTERN block)
"""
lines = [self.add_start_line(key, level)]
list_spacer = self.spacer * (level + 2)
pairs = ["{}{} {}".format(list_spacer, p[0], p[1]) for p in pair_list]
lines += pairs
lines.append(self.add_end_line(level, 1))
return lines
def format_repeated_pair_list(self, key, root_list, level):
"""
Process (possibly) repeated lists of pairs e.g. POINTs blocks
"""
lines = []
def depth(L):
return isinstance(L, (tuple, list)) and max(map(depth, L)) + 1
if depth(root_list) == 2:
# single set of points only
root_list = [root_list]
for pair_list in root_list:
lines += self.format_pair_list(key, pair_list, level)
return lines
def is_composite(self, val):
if isinstance(val, dict) and "__type__" in val:
return True
else:
return False
def is_hidden_container(self, key, val):
"""
The key is not one of the Mapfile keywords, and its
values are a list
"""
if key in ("layers", "classes", "styles", "symbols", "labels",
"outputformats", "features", "scaletokens",
"composites") and isinstance(val, list):
return True
else:
return False
def pprint(self, composites):
"""
Print out a nicely indented Mapfile
"""
# if only a single composite is used then cast to list
# and allow for multiple root composites
if composites and not isinstance(composites, list):
composites = [composites]
lines = []
for composite in composites:
type_ = composite["__type__"]
if type_ in ("metadata", "validation"):
# types are being parsed directly, and not as an attr of a parent
lines += self.process_key_dict(type_, composite, level=0)
else:
lines += self._format(composite)
result = str(self.newlinechar.join(lines))
return result
def get_attribute_properties(self, type_, attr):
jsn_schema = self.validator.get_expanded_schema(type_)
props = jsn_schema["properties"]
# check if a value needs to be quoted or not, by referring to the JSON schema
try:
attr_props = props[attr]
except KeyError as ex:
log.error("The key '{}' was not found in the JSON schema for '{}'".
format(attr, type_))
log.error(ex)
return {}
return attr_props
def is_expression(self, option):
return "description" in option and (option["description"]
== "expression")
def check_options_list(self, options_list, value):
for option in options_list:
if "enum" in option and value.lower() in option["enum"]:
if value.lower() == "end":
# in GEOTRANSFORM "end" is an attribute value
return self.quoter.add_quotes(value)
else:
return value.upper()
elif self.is_expression(option):
if value.endswith("'i") or value.endswith('"i'):
return value
if self.quoter.in_slashes(value):
return value
else:
return self.quoter.add_quotes(value)
def format_value(self, attr, attr_props, value):
"""
TODO - refactor and add more specific tests (particularly for expressions)
"""
if isinstance(value, bool):
return str(value).upper()
if any(i in ["enum"] for i in attr_props):
if not isinstance(value, numbers.Number):
return value.upper(
) # value is from a set list, no need for quote
else:
return value
if "type" in attr_props and attr_props[
"type"] == "string": # and "enum" not in attr_props
# check schemas for expressions and handle accordingly
if self.is_expression(attr_props) and self.quoter.in_slashes(
value):
return value
elif self.is_expression(attr_props) and (value.endswith("'i")
or value.endswith('"i')):
# for case insensitive regex
return value
else:
return self.quoter.add_quotes(value)
# expressions can be one of a string or an expression in brackets
if any(i in ["oneOf", "anyOf"]
for i in attr_props): # and check that type string is in list
if "oneOf" in attr_props:
options_list = attr_props["oneOf"]
else:
options_list = attr_props["anyOf"]
if self.quoter.is_string(value):
if self.quoter.in_parenthesis(value):
pass
elif attr == "expression" and self.quoter.in_braces(value):
# don't add quotes to list expressions such as {val1, val2}
pass
elif attr != "text" and self.quoter.in_brackets(value):
# TEXT expressions are often "[field1]-[field2]" so need to leave quotes for these
pass
elif value.startswith("NOT ") and self.quoter.in_parenthesis(
value[4:]):
value = "NOT {}".format(value[4:])
else:
value = self.check_options_list(options_list, value)
if isinstance(value, list):
new_values = []
for v in value:
if not isinstance(v, numbers.Number):
v = self.quoter.add_quotes(v)
new_values.append(v)
value = " ".join(list(map(str, new_values)))
else:
value = self.quoter.escape_quotes(value)
return value
def process_attribute(self, type_, attr, value, level):
"""
Process one of the main composite types (see the type_ value)
"""
attr_props = self.get_attribute_properties(type_, attr)
value = self.format_value(attr, attr_props, value)
line = self.__format_line(self.whitespace(level, 1), attr.upper(),
value)
return line
def format_comment(self, spacer, value):
return "{}{}".format(spacer, value)
def process_composite_comment(self, level, comments, key):
"""
Process comments for composites such as MAP, LAYER etc.
"""
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = self.whitespace(level, 0)
if isinstance(value, list):
comments = [self.format_comment(spacer, v) for v in value]
comment = self.newlinechar.join(comments)
else:
comment = self.format_comment(spacer, value)
return comment
def process_attribute_comment(self, comments, key):
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = " "
# for multiple comments associated with an attribute
# simply join them together as a single string
if isinstance(value, list):
value = " ".join(value)
comment = self.format_comment(spacer, value)
return comment
def _add_type_comment(self, level, comments, lines):
comment = self.process_composite_comment(level, comments, '__type__')
if comment:
lines.append(str(comment))
def _format(self, composite, level=0):
lines = []
# get any comments associated with the composite
comments = composite.get("__comments__", {})
if isinstance(composite, dict) and '__type__' in composite:
type_ = composite['__type__']
assert type_ in COMPOSITE_NAMES.union(SINGLETON_COMPOSITE_NAMES)
is_hidden = False
self._add_type_comment(level, comments, lines)
s = self.whitespace(level, 0) + type_.upper()
lines.append(s)
for attr, value in composite.items():
if attr in ("__type__", "__comments__", "__position__"):
# skip hidden attributes
continue
elif self.is_hidden_container(attr, value):
# now recursively print all the items in the container
for v in value:
lines += self._format(v, level + 1)
elif attr == "pattern":
lines += self.format_pair_list(attr, value, level)
elif attr in ("metadata", "validation", "values"):
# metadata and values are also composites
# but will be processed here
lines += self.process_key_dict(attr, value, level)
elif attr == "projection":
lines += self.process_projection(attr, value, level)
elif attr in ("processing", "formatoption", "include"):
lines += self.process_repeated_list(attr, value, level)
elif attr == "points":
lines += self.format_repeated_pair_list(attr, value, level)
elif attr == "config":
lines += self.process_config_dict(attr, value, level)
elif self.is_composite(value):
lines += self._format(value, level +
1) # recursively add the child class
else:
# standard key value pair
line = self.process_attribute(type_, attr, value, level)
line += self.process_attribute_comment(comments, attr)
lines.append(line)
if not is_hidden:
# close the container block with an END
lines.append(self.add_end_line(level, 0))
return lines