def test_double_error():
s = """MAP
NAME "sample"
STATUS ON
SIZE 600 400
SYMBOLSET "../etc/symbols.txt"
EXTENT -180 -90 180
UNITS DD
SHAPEPATH "../data"
IMAGECOLOR 255 255 256
FONTSET "../etc/fonts.txt"
WEB
IMAGEPATH "/ms4w/tmp/ms_tmp/"
IMAGEURL "/ms_tmp/"
END
LAYER
NAME "global-raster"
TYPE RASTER
STATUS DEFAULT
DATA "bluemarble.gif"
END
END"""
d = mappyfile.loads(s, include_position=True)
# print(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, add_comments=True)
# print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert(len(errors) == 2)
print(mappyfile.dumps(d))
def test_keyword_versioning():
properties = {
"type": "object",
"properties": {
"__type__": {
"enum": ["label"]
},
"align": {
"oneOf": [{
"type": "string",
"enum": ["left", "center", "right"],
"additionalProperties": False
}, {
"type": "string",
"pattern": "^\\[(.*?)\\]$",
"description": "attribute"
}],
"metadata": {
"minVersion": 5.4
}
}
}
}
v = Validator()
assert "align" in properties["properties"].keys()
properties = v.get_versioned_properties(properties, 5.2)
print(json.dumps(properties, indent=4))
assert "align" not in properties["properties"].keys()
def test_double_error():
s = """MAP
NAME "sample"
STATUS ON
SIZE 600 400
SYMBOLSET "../etc/symbols.txt"
EXTENT -180 -90 180
UNITS DD
SHAPEPATH "../data"
IMAGECOLOR 255 255 256
FONTSET "../etc/fonts.txt"
WEB
IMAGEPATH "/ms4w/tmp/ms_tmp/"
IMAGEURL "/ms_tmp/"
END
LAYER
NAME "global-raster"
TYPE RASTER
STATUS DEFAULT
DATA "bluemarble.gif"
END
END"""
d = mappyfile.loads(s, include_position=True)
# print(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, add_comments=True)
# print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert (len(errors) == 2)
print(mappyfile.dumps(d))
def test_all_maps():
sample_dir = os.path.join(os.path.dirname(__file__), "sample_maps")
p = Parser(expand_includes=False)
m = MapfileToDict(include_position=True)
v = Validator()
failing_maps = []
for fn in os.listdir(sample_dir):
print(fn)
try:
ast = p.parse_file(os.path.join(sample_dir, fn))
d = m.transform(ast)
errors = v.validate(d)
try:
assert (len(errors) == 0)
except AssertionError as ex:
logging.warning("Validation errors in %s ", fn)
logging.warning(errors)
except (BaseException, UnexpectedToken) as ex:
logging.warning("Cannot process %s ", fn)
logging.error(ex)
failing_maps.append(fn)
logging.warning(failing_maps)
def test_object_versioning():
"""
Exclude whole objects if they were added in a
later version of MapServer
"""
s = """MAP
NAME "sample"
LAYER
TYPE POLYGON
COMPOSITE
COMPOP "lighten"
OPACITY 50
COMPFILTER "blur(10)"
END
END
END"""
d = mappyfile.loads(s, include_position=False)
v = Validator()
errors = v.validate(d, add_comments=True, version=6.0)
assert len(errors) == 1
d = mappyfile.loads(s, include_position=False)
errors = v.validate(d, add_comments=True, version=7.0)
assert len(errors) == 0
def test_all_maps():
sample_dir = os.path.join(os.path.dirname(__file__), "sample_maps")
p = Parser(expand_includes=False)
m = MapfileToDict(include_position=True)
v = Validator()
failing_maps = []
for fn in os.listdir(sample_dir):
print(fn)
try:
ast = p.parse_file(os.path.join(sample_dir, fn))
d = m.transform(ast)
errors = v.validate(d)
try:
assert(len(errors) == 0)
except AssertionError as ex:
logging.warning("Validation errors in %s ", fn)
logging.warning(errors)
except (BaseException, UnexpectedToken) as ex:
logging.warning("Cannot process %s ", fn)
logging.error(ex)
failing_maps.append(fn)
logging.warning(failing_maps)
def test_line_position_mutlilines():
s = """MAP
NAME "sample"
LAYER
NAME "test"
STATUS DEFAULT
DATA "SELECT GEOM
FROM
TABLE"
TYPE LINEX
END
END"""
p = Parser()
ast = p.parse(s)
print(ast)
d = mappyfile.loads(s, include_position=True)
v = Validator()
errors = v.validate(d, add_comments=True)
print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert (len(errors) == 1)
err = errors[0]
assert (err["line"] == 9)
assert (err["column"] == 9)
print(mappyfile.dumps(d))
def validate(d, trace_o_incl=None, version=None):
"""
Validate a mappyfile dictionary by using the Mapfile schema.
An optional version number can be used to specify a specific
a Mapfile is valid for a specific MapServer version.
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
trace_o_incl: list
A trace of the origin of lines for include files, use to find the original location of an error
version: float
The MapServer version number used to validate the Mapfile
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d, trace_o_incl, version=version)
def validate(d):
"""
Validate a mappyfile dictionary by using the Mapfile schema
d: dict
A Python dictionary based on the the mappyfile schema
"""
v = Validator()
return v.validate(d)
def main(msautotest_fld, create_new_copy=True):
msautotest_copy = os.path.join(os.path.dirname(msautotest_fld),
"msautotest_mappyfile")
if create_new_copy:
create_copy(msautotest_fld, msautotest_copy)
parser = Parser()
transformer = MapfileToDict()
pp = PrettyPrinter()
# these two maps aren't in utf8
# see https://github.com/mapserver/mapserver/pull/5460
# ignore_list = ["wms_inspire_scenario1.map","wms_inspire_scenario2.map"]
# transparent_layer.map has an extra END, see https://github.com/mapserver/mapserver/pull/5468
# polyline_no_clip.map needs symbol names in quotes, and SYMBOL is ambiguous
ignore_list = [
"polyline_no_clip.map", "poly-label-multiline-pos-auto.map",
"poly-label-pos-auto.map", "embed_sb_rgba.map",
"embed_sb_rgba_offset.map"
] # has attributes all on the same line
mapfiles = glob.glob(msautotest_fld + '/**/*.map')
mapfiles = [f for f in mapfiles if os.path.basename(f) not in ignore_list]
# target_map = "polyline_no_clip.map"
# mapfiles = [f for f in mapfiles if os.path.basename(f) in (target_map)]
v = Validator()
for fn in mapfiles:
d = parse_mapfile(parser, transformer, pp, fn)
errors = v.validate(d, add_messages=True)
if errors:
logging.warning("{} failed validation".format(fn))
output_file = fn.replace(msautotest_fld, msautotest_copy)
try:
mappyfile.utils.write(d, output_file)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
print(json.dumps(d, indent=4))
d = parse_mapfile(parser, transformer, pp, output_file)
errors = v.validate(d, add_messages=True)
if errors:
logging.warning("{} failed validation".format(fn))
def main(msautotest_fld, create_new_copy=True):
msautotest_copy = os.path.join(os.path.dirname(msautotest_fld), "msautotest_mappyfile")
if create_new_copy:
create_copy(msautotest_fld, msautotest_copy)
parser = Parser()
transformer = MapfileToDict()
pp = PrettyPrinter()
# these two maps aren't in utf8
# see https://github.com/mapserver/mapserver/pull/5460
# ignore_list = ["wms_inspire_scenario1.map","wms_inspire_scenario2.map"]
# transparent_layer.map has an extra END, see https://github.com/mapserver/mapserver/pull/5468
# polyline_no_clip.map needs symbol names in quotes, and SYMBOL is ambiguous
ignore_list = ["polyline_no_clip.map",
"poly-label-multiline-pos-auto.map", "poly-label-pos-auto.map",
"embed_sb_rgba.map", "embed_sb_rgba_offset.map"] # has attributes all on the same line
mapfiles = glob.glob(msautotest_fld + '/**/*.map')
mapfiles = [f for f in mapfiles if os.path.basename(f) not in ignore_list]
# target_map = "polyline_no_clip.map"
# mapfiles = [f for f in mapfiles if os.path.basename(f) in (target_map)]
v = Validator()
for fn in mapfiles:
d = parse_mapfile(parser, transformer, pp, fn)
errors = v.validate(d, add_comments=True)
if errors:
logging.warning("{} failed validation".format(fn))
output_file = fn.replace(msautotest_fld, msautotest_copy)
try:
mappyfile.utils.write(d, output_file)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
print(json.dumps(d, indent=4))
d = parse_mapfile(parser, transformer, pp, output_file)
errors = v.validate(d, add_comments=True)
if errors:
logging.warning("{} failed validation".format(fn))
def test_cached_expanded_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
deref_schema = v.get_expanded_schema(schema_name)
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
# get the schame again
deref_schema = v.get_expanded_schema(schema_name)
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_cached_expanded_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
deref_schema = v.get_expanded_schema(schema_name)
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
# get the schame again
deref_schema = v.get_expanded_schema(schema_name)
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def __init__(self, indent=4, spacer=" ", quote='"', newlinechar="\n"):
"""
Option use "\t" for spacer with an indent of 1
"""
assert (quote == "'" or quote == '"')
self.indent = indent
self.spacer = spacer * self.indent
self.newlinechar = newlinechar
self.quoter = Quoter(quote)
self.end = u"END"
self.validator = Validator()
def test_cached_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
assert (list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
# get the schame again
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
assert (list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
def test_cached_schema():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
assert(list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
# get the schame again
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
assert(list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
def test_deref():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
print(json.dumps(jsn_schema, indent=4))
print(jsn_schema["properties"]["filter"])
assert (list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
deref_schema = v.get_expanded_schema(schema_name)
print(json.dumps(deref_schema, indent=4))
print(deref_schema["properties"]["filter"])
assert (list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_deref():
"""
Check that the full schema properties have been expanded
"""
v = Validator()
schema_name = "cluster"
validator = v.get_schema_validator(schema_name)
jsn_schema = validator.schema
print(json.dumps(jsn_schema, indent=4))
print(jsn_schema["properties"]["filter"])
assert(list(jsn_schema["properties"]["filter"].keys())[0] == "$ref")
deref_schema = v.get_expanded_schema(schema_name)
print(json.dumps(deref_schema, indent=4))
print(deref_schema["properties"]["filter"])
assert(list(deref_schema["properties"]["filter"].keys())[0] == "anyOf")
def test_extra_property_validation():
"""
Check root errors are handled correctly
"""
s = """
MAP
LAYER
TYPE POLYGON
END
END
"""
d = to_dict(s)
d["__unwanted__"] = "error"
v = Validator()
errors = v.validate(d, add_comments=True)
print(errors)
assert (len(errors) == 1)
def test_root_position():
"""
Check the root objects position is found correctly
"""
s = """
MAP
METADATA
"wms_title" "Toronto Landsat 5 TM"
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
assert d["__position__"]["line"] == 2
errors = v.validate(d, add_comments=True)
assert len(errors) == 1
def test_extra_property_validation():
"""
Check root errors are handled correctly
"""
s = """
MAP
LAYER
TYPE POLYGON
END
END
"""
d = to_dict(s)
d["unwanted"] = "error"
v = Validator()
errors = v.validate(d, add_comments=True)
print(errors)
assert(len(errors) == 1)
def test_cluster_validation_fail():
s = u"""
MAP
LAYER
CLUSTER
MAXDISTANCE 50
REGION "ELLIPSEZ"
END
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
errors = v.validate(d, add_comments=True)
print(mappyfile.dumps(d))
assert len(errors) == 1
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert (len(errors) == 0)
return s
def validate(d):
"""
Validate a mappyfile dictionary by using the Mapfile schema
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d)
def test_version_warnings():
s = """MAP
NAME "sample"
LAYER
NAME "test"
TYPE LINE
CLASS
#MADEUP True
COLOR 0 0 0
END
END
END"""
d = mappyfile.loads(s, include_position=False)
v = Validator()
errors = v.validate(d, add_comments=True, version=8.0)
print(errors)
assert len(errors) == 1
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert(len(errors) == 0)
return s
def test_cluster_validation():
s = u"""
MAP
LAYER
TYPE POINT
CLUSTER
MAXDISTANCE 50
REGION "ELLIPSE"
END
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
assert d["__position__"]["line"] == 2
errors = v.validate(d, add_comments=True)
print(mappyfile.dumps(d))
assert len(errors) == 0
def save_full_schema(output_file):
validator = Validator()
# check individual schema files
fld = validator.get_schemas_folder()
jsn_files = glob.glob(fld + '/*.json')
for fn in jsn_files:
check_schema(fn)
# now check the combined schema
jsn = validator.get_versioned_schema()
full_schema = json.dumps(jsn, indent=4, sort_keys=False)
with open(output_file, "w") as f:
f.write(full_schema)
check_schema(output_file)
def schema(ctx, output_file, version=None):
"""
Save the Mapfile schema to a file. Set the version parameter to output a specific version.
Note output-file will be overwritten if it already exists.
Examples:
mappyfile schema C:/Temp/mapfile-schema.json
Example of a specific version:
mappyfile schema C:/Temp/mapfile-schema-7-6.json --version=7.6
"""
validator = Validator()
jsn = validator.get_versioned_schema(version)
with codecs.open(output_file, "w", encoding="utf-8") as f:
f.write(json.dumps(jsn, sort_keys=True, indent=4))
sys.exit(0)
def test_property_versioning():
properties = {
"force": {
"oneOf": [{
"type": "boolean"
}, {
"enum": ["group"],
"metadata": {
"minVersion": 6.2
}
}]
}
}
v = Validator()
assert "enum" in properties["force"]["oneOf"][1].keys()
assert len(properties["force"]["oneOf"]) == 2
properties = v.get_versioned_properties(properties, 6.0)
print(json.dumps(properties, indent=4))
assert len(properties["force"]["oneOf"]) == 1
def output(fn):
"""
Parse, transform, and pretty print
the result
"""
p = Parser(expand_includes=False)
m = MapfileToDict()
v = Validator()
try:
ast = p.parse_file(fn)
# print(ast)
d = m.transform(ast)
logging.debug("Number of layers: {}".format(len(d["layers"])))
errors = v.validate(d)
assert (len(errors) == 0)
except Exception as ex:
logging.exception(ex)
logging.warning("%s could not be successfully parsed", fn)
d = None
raise
if d:
try:
s = mappyfile.utils.dumps(d)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
ast = p.parse(s)
d = m.transform(ast)
errors = v.validate(d)
assert (len(errors) == 0)
def output(fn):
"""
Parse, transform, and pretty print
the result
"""
p = Parser(expand_includes=False, include_comments=True)
m = MapfileToDict(include_position=True, include_comments=True)
v = Validator()
try:
ast = p.parse_file(fn)
# print(ast)
d = m.transform(ast)
logging.debug("Number of layers: {}".format(len(d["layers"])))
errors = v.validate(d, add_comments=True)
assert(len(errors) == 0)
except Exception as ex:
logging.exception(ex)
logging.warning("%s could not be successfully parsed", fn)
d = None
raise
if d:
try:
s = mappyfile.utils.dumps(d)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
ast = p.parse(s)
d = m.transform(ast)
errors = v.validate(d)
assert(len(errors) == 0)
def validate(d, version=None):
"""
Validate a mappyfile dictionary by using the Mapfile schema.
An optional version number can be used to specify a specific
a Mapfile is valid for a specific MapServer version.
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
version: float
The MapServer version number used to validate the Mapfile
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d, version=version)
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
# https://stackoverflow.com/questions/900392/getting-the-caller-function-name-inside-another-function-in-python
logging.info(inspect.stack()[1][3])
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert(len(errors) == 0)
return s
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
# https://stackoverflow.com/questions/900392/getting-the-caller-function-name-inside-another-function-in-python
logging.info(inspect.stack()[1][3])
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert (len(errors) == 0)
return s
def test_multiple_layer_projection():
"""
TODO add validation for this case
"""
s = """
MAP
LAYER
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
END
END
"""
p = Parser()
ast = p.parse(s)
t = MapfileToDict(include_position=True)
d = t.transform(ast)
print(json.dumps(d, indent=4))
assert (len(d["projection"]) == 1)
p = Parser()
m = MapfileToDict()
ast = p.parse(s)
d = m.transform(ast)
print(json.dumps(d, indent=4))
from mappyfile.validator import Validator
v = Validator()
return v.validate(d)
def test_multiple_layer_projection():
"""
TODO add validation for this case
"""
s = """
MAP
LAYER
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
END
END
"""
p = Parser()
ast = p.parse(s)
t = MapfileToDict(include_position=True)
d = t.transform(ast)
print(json.dumps(d, indent=4))
assert(len(d["projection"]) == 1)
p = Parser()
m = MapfileToDict()
ast = p.parse(s)
d = m.transform(ast)
print(json.dumps(d, indent=4))
from mappyfile.validator import Validator
v = Validator()
return v.validate(d)
def create(type, version=None):
"""
Create a new mappyfile object, using MapServer defaults (if any).
Parameters
----------
s: type
The mappyfile type to be stored in the __type__ property
Returns
-------
dict
A Python dictionary representing the Mapfile object in the mappyfile format
"""
# get the schema for this type
v = Validator()
try:
schema = v.get_versioned_schema(version=version, schema_name=type)
except IOError:
raise SyntaxError(
"The mappyfile type '{}' does not exist!".format(type))
d = OrderedDict()
d["__type__"] = type
properties = sorted(schema["properties"].items())
for k, v in properties:
if "default" in v:
d[k] = v["default"]
return d
def test_add_comments():
s = """
MAP
IMAGECOLOR 'FF00FF'
LAYER
EXTENT 0 0 0
TYPE POLYGON
END
END
"""
d = to_dict(s)
v = Validator()
errors = v.validate(d, add_comments=True)
print(len(errors))
print(json.dumps(d, indent=4))
for error in errors:
print(error)
pp = PrettyPrinter(indent=4, quote='"') # expected
res = pp.pprint(d)
print(res)
def __init__(self,
indent=4,
spacer=" ",
quote='"',
newlinechar="\n",
end_comment=False,
align_values=False,
separate_complex_types=False,
**kwargs):
"""
Option use "\t" for spacer with an indent of 1
"""
assert (quote == "'" or quote == '"')
self.indent = indent
self.spacer = spacer * self.indent
self.quoter = Quoter(quote)
self.newlinechar = newlinechar
self.end_comment = end_comment
self.end = u"END"
self.validator = Validator()
self.align_values = align_values
self.separate_complex_types = separate_complex_types
def test_add_comments():
s = """
MAP
IMAGECOLOR 'FF00FF'
LAYER
EXTENT 0 0 0
TYPE POLYGON
END
END
"""
d = to_dict(s)
v = Validator()
errors = v.validate(d, add_comments=True)
print(len(errors))
print(json.dumps(d, indent=4))
for error in errors:
print(error)
pp = PrettyPrinter(indent=4, quote='"') # expected
res = pp.pprint(d)
print(res)
def validate(s):
d = to_dict(s)
v = Validator()
return v.validate(d)
def validate(d):
v = Validator()
return v.validate(d)
class PrettyPrinter(object):
def __init__(self,
indent=4,
spacer=" ",
quote='"',
newlinechar="\n",
end_comment=False,
align_values=False,
separate_complex_types=False,
**kwargs):
"""
Option use "\t" for spacer with an indent of 1
"""
assert (quote == "'" or quote == '"')
self.indent = indent
self.spacer = spacer * self.indent
self.quoter = Quoter(quote)
self.newlinechar = newlinechar
self.end_comment = end_comment
self.end = u"END"
self.validator = Validator()
self.align_values = align_values
self.separate_complex_types = separate_complex_types
def __is_metadata(self, key):
"""
Check to see if the property is hidden metadata
e.g. "__type__", "__comments__", "__position__"
"""
if key.startswith("__") and key.endswith("__"):
return True
else:
return False
def compute_aligned_max_indent(self, max_key_length):
"""
Computes the indentation as a multiple of self.indent for aligning
values at the same column based on the maximum key length.
Example:
key value1
longkey value2
longestkey value3 <-- column at 12, indent of 4, determined by "longestkey"
"""
indent = max(1, self.indent)
return int((int(max_key_length / indent) + 1) * indent)
def compute_max_key_length(self, composite):
"""
Computes the maximum length of all keys (non-recursive) in the passed
composite.
"""
length = 0
for attr, value in composite.items():
attr_length = len(attr)
if (not self.__is_metadata(attr)
and attr not in ("metadata", "validation", "values",
"connectionoptions")
and not self.is_hidden_container(attr, value)
and not attr == "pattern" and not attr == "projection"
and not attr == "points" and not attr == "config"
and not self.is_composite(value)):
length = max(length, attr_length)
return length
def separate_complex(self, composite, level):
if not self.separate_complex_types:
return
for key in list(composite.keys()):
if self.is_complex_type(composite, key, level):
utils.dict_move_to_end(composite, key)
def whitespace(self, level, indent):
return self.spacer * (level + indent)
def add_start_line(self, key, level):
return self.whitespace(level, 1) + key.upper()
def add_end_line(self, level, indent, key):
end_line = self.whitespace(level, indent) + self.end
if self.end_comment:
end_line = "{} # {}".format(end_line, key.upper())
return end_line
def __format_line(self, spacer, key, value, aligned_max_indent=0):
if ((aligned_max_indent is None) or (aligned_max_indent == 0)):
aligned_max_indent = len(key) + 1
indent = " " * (aligned_max_indent - len(key))
tmpl = u"{spacer}{key}{indent}{value}"
d = {"spacer": spacer, "key": key, "value": value, "indent": indent}
return tmpl.format(**d)
def process_key_dict(self, key, d, level):
"""
Process key value dicts e.g. METADATA "key" "value"
"""
# add any composite level comments
comments = d.get("__comments__", {})
lines = []
self._add_type_comment(level, comments, lines)
lines += [self.add_start_line(key, level)]
lines += self.process_dict(d, level, comments)
lines.append(self.add_end_line(level, 1, key))
return lines
def process_dict(self, d, level, comments):
"""
Process keys and values within a block
"""
lines = []
aligned_max_indent = 0
if (self.align_values):
max_key_length = self.compute_max_key_length(
d) + 2 # add length of quotes
aligned_max_indent = self.compute_aligned_max_indent(
max_key_length)
for k, v in d.items():
if not self.__is_metadata(k):
qk = self.quoter.add_quotes(k)
qv = self.quoter.add_quotes(v)
line = self.__format_line(self.whitespace(level, 2), qk, qv,
aligned_max_indent)
line += self.process_attribute_comment(comments, k)
lines.append(line)
return lines
def process_config_dict(self, key, d, level):
"""
Process the CONFIG block
"""
lines = []
for k, v in d.items():
k = "CONFIG {}".format(self.quoter.add_quotes(k.upper()))
v = self.quoter.add_quotes(v)
lines.append(self.__format_line(self.whitespace(level, 1), k, v))
return lines
def process_repeated_list(self, key, lst, level, aligned_max_indent=1):
"""
Process blocks of repeated keys e.g. FORMATOPTION
"""
lines = []
for v in lst:
k = key.upper()
v = self.quoter.add_quotes(v)
lines.append(
self.__format_line(self.whitespace(level, 1), k, v,
aligned_max_indent))
return lines
def process_projection(self, key, lst, level):
lines = [self.add_start_line(key, level)]
if self.quoter.is_string(lst):
val = self.quoter.add_quotes(lst)
# the value has been manually set to a single string projection
lines.append(u"{}{}".format(self.whitespace(level, 2), val))
elif len(lst) == 1 and lst[0].upper() == "AUTO":
lines.append(u"{}{}".format(self.whitespace(level, 2), "AUTO"))
else:
for v in lst:
v = self.quoter.add_quotes(v)
lines.append(u"{}{}".format(self.whitespace(level, 2), v))
lines.append(self.add_end_line(level, 1, key))
return lines
def format_pair_list(self, key, pair_list, level):
"""
Process lists of pairs (e.g. PATTERN block)
"""
lines = [self.add_start_line(key, level)]
list_spacer = self.spacer * (level + 2)
pairs = ["{}{} {}".format(list_spacer, p[0], p[1]) for p in pair_list]
lines += pairs
lines.append(self.add_end_line(level, 1, key))
return lines
def format_repeated_pair_list(self, key, root_list, level):
"""
Process (possibly) repeated lists of pairs e.g. POINTs blocks
"""
lines = []
def depth(L):
return isinstance(L, (tuple, list)) and max(map(depth, L)) + 1
if depth(root_list) == 2:
# single set of points only
root_list = [root_list]
for pair_list in root_list:
lines += self.format_pair_list(key, pair_list, level)
return lines
def is_composite(self, val):
if isinstance(val, dict) and "__type__" in val:
return True
else:
return False
def is_complex_type(self, composite, key, level):
# symbol needs special treatment
if key == "symbol" and level > 0:
return False
return key in COMPLEX_TYPES or self.is_composite(
key) or self.is_hidden_container(key, composite[key])
def is_hidden_container(self, key, val):
"""
The key is not one of the Mapfile keywords, and its
values are a list
"""
if key in OBJECT_LIST_KEYS and isinstance(val, list):
return True
else:
return False
def pprint(self, composites):
"""
Print out a nicely indented Mapfile
"""
# if only a single composite is used then cast to list
# and allow for multiple root composites
if composites and not isinstance(composites, list):
composites = [composites]
lines = []
for composite in composites:
type_ = composite["__type__"]
if type_ in ("metadata", "validation", "connectionoptions"):
# types are being parsed directly, and not as an attr of a parent
lines += self.process_key_dict(type_, composite, level=0)
else:
lines += self._format(composite)
result = str(self.newlinechar.join(lines))
return result
def get_attribute_properties(self, type_, attr):
jsn_schema = self.validator.get_expanded_schema(type_)
props = jsn_schema["properties"]
# check if a value needs to be quoted or not, by referring to the JSON schema
try:
attr_props = props[attr]
except KeyError as ex:
log.error("The key '{}' was not found in the JSON schema for '{}'".
format(attr, type_))
log.error(ex)
return {}
return attr_props
def is_expression(self, option):
return "description" in option and (option["description"]
== "expression")
def check_options_list(self, options_list, value):
for option in options_list:
if "enum" in option and value.lower() in option["enum"]:
if value.lower() == "end":
# in GEOTRANSFORM "end" is an attribute value
return self.quoter.add_quotes(value)
else:
return value.upper()
elif self.is_expression(option):
if value.endswith("'i") or value.endswith('"i'):
return value
if self.quoter.in_slashes(value):
return value
else:
return self.quoter.add_quotes(value)
def format_value(self, attr, attr_props, value):
"""
TODO - refactor and add more specific tests (particularly for expressions)
"""
if isinstance(value, bool):
return str(value).upper()
if any(i in ["enum"] for i in attr_props):
if isinstance(value, dict) and not value:
raise ValueError(
"The property {} has an empty dictionary as a value".
format(attr))
if not isinstance(value, numbers.Number):
if attr == "compop":
return self.quoter.add_quotes(value)
else:
return value.upper(
) # value is from a set list, no need for quote
else:
return value
if "type" in attr_props and attr_props[
"type"] == "string": # and "enum" not in attr_props
# check schemas for expressions and handle accordingly
if self.is_expression(attr_props) and self.quoter.in_slashes(
value):
return value
elif self.is_expression(attr_props) and (value.endswith("'i")
or value.endswith('"i')):
# for case insensitive regex
return value
else:
return self.quoter.add_quotes(value)
# expressions can be one of a string or an expression in brackets
if any(i in ["oneOf", "anyOf"]
for i in attr_props): # and check that type string is in list
if "oneOf" in attr_props:
options_list = attr_props["oneOf"]
else:
options_list = attr_props["anyOf"]
if self.quoter.is_string(value):
if self.quoter.in_parenthesis(value):
pass
elif attr == "expression" and self.quoter.in_braces(value):
# don't add quotes to list expressions such as {val1, val2}
pass
elif attr != "text" and self.quoter.in_brackets(value):
# TEXT expressions are often "[field1]-[field2]" so need to leave quotes for these
pass
elif value.startswith("NOT ") and self.quoter.in_parenthesis(
value[4:]):
value = "NOT {}".format(value[4:])
else:
value = self.check_options_list(options_list, value)
if isinstance(value, list):
new_values = []
for v in value:
if not isinstance(v, numbers.Number) and attr not in [
"offset", "polaroffset"
]:
# don't add quotes to list of attributes for offset / polaroffset
v = self.quoter.add_quotes(v)
new_values.append(v)
value = " ".join(list(map(str, new_values)))
else:
value = self.quoter.escape_quotes(value)
return value
def process_attribute(self,
type_,
attr,
value,
level,
aligned_max_indent=1):
"""
Process one of the main composite types (see the type_ value)
"""
attr_props = self.get_attribute_properties(type_, attr)
value = self.format_value(attr, attr_props, value)
line = self.__format_line(self.whitespace(level, 1), attr.upper(),
value, aligned_max_indent)
return line
def format_comment(self, spacer, value):
return "{}{}".format(spacer, value)
def process_composite_comment(self, level, comments, key):
"""
Process comments for composites such as MAP, LAYER etc.
"""
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = self.whitespace(level, 0)
if isinstance(value, list):
comments = [self.format_comment(spacer, v) for v in value]
comment = self.newlinechar.join(comments)
else:
comment = self.format_comment(spacer, value)
return comment
def process_attribute_comment(self, comments, key):
if key not in comments:
comment = ""
else:
value = comments[key]
spacer = " "
# for multiple comments associated with an attribute
# simply join them together as a single string
if isinstance(value, list):
value = " ".join(value)
comment = self.format_comment(spacer, value)
return comment
def _add_type_comment(self, level, comments, lines):
comment = self.process_composite_comment(level, comments, '__type__')
if comment:
lines.append(str(comment))
def _format(self, composite, level=0):
lines = []
type_ = None
# get any comments associated with the composite
comments = composite.get("__comments__", {})
if isinstance(composite, dict) and '__type__' in composite:
type_ = composite['__type__']
assert type_ in COMPOSITE_NAMES.union(SINGLETON_COMPOSITE_NAMES)
is_hidden = False
self._add_type_comment(level, comments, lines)
s = self.whitespace(level, 0) + type_.upper()
lines.append(s)
aligned_max_indent = 0
if self.align_values:
max_key_length = self.compute_max_key_length(composite)
aligned_max_indent = self.compute_aligned_max_indent(
max_key_length)
self.separate_complex(composite, level)
for attr, value in composite.items():
if self.__is_metadata(attr):
# skip hidden attributes
continue
elif self.is_hidden_container(attr, value):
# now recursively print all the items in the container
for v in value:
lines += self._format(v, level + 1)
elif attr == "pattern":
lines += self.format_pair_list(attr, value, level)
elif attr in ("metadata", "validation", "values",
"connectionoptions"):
# metadata and values are also composites
# but will be processed here
lines += self.process_key_dict(attr, value, level)
elif attr == "projection":
lines += self.process_projection(attr, value, level)
elif attr in REPEATED_KEYS:
lines += self.process_repeated_list(attr, value, level,
aligned_max_indent)
elif attr == "points":
lines += self.format_repeated_pair_list(attr, value, level)
elif attr == "config":
lines += self.process_config_dict(attr, value, level)
elif self.is_composite(value):
lines += self._format(value, level +
1) # recursively add the child class
else:
# standard key value pair
if not type_:
raise UnboundLocalError(
"The Mapfile object is missing a __type__ attribute")
line = self.process_attribute(type_, attr, value, level,
aligned_max_indent)
line += self.process_attribute_comment(comments, attr)
lines.append(line)
if not is_hidden:
# close the container block with an END
lines.append(self.add_end_line(level, 0, type_))
return lines