def test_object_versioning():
"""
Exclude whole objects if they were added in a
later version of MapServer
"""
s = """MAP
NAME "sample"
LAYER
TYPE POLYGON
COMPOSITE
COMPOP "lighten"
OPACITY 50
COMPFILTER "blur(10)"
END
END
END"""
d = mappyfile.loads(s, include_position=False)
v = Validator()
errors = v.validate(d, add_comments=True, version=6.0)
assert len(errors) == 1
d = mappyfile.loads(s, include_position=False)
errors = v.validate(d, add_comments=True, version=7.0)
assert len(errors) == 0
def main(msautotest_fld, create_new_copy=True):
msautotest_copy = os.path.join(os.path.dirname(msautotest_fld),
"msautotest_mappyfile")
if create_new_copy:
create_copy(msautotest_fld, msautotest_copy)
parser = Parser()
transformer = MapfileToDict()
pp = PrettyPrinter()
# these two maps aren't in utf8
# see https://github.com/mapserver/mapserver/pull/5460
# ignore_list = ["wms_inspire_scenario1.map","wms_inspire_scenario2.map"]
# transparent_layer.map has an extra END, see https://github.com/mapserver/mapserver/pull/5468
# polyline_no_clip.map needs symbol names in quotes, and SYMBOL is ambiguous
ignore_list = [
"polyline_no_clip.map", "poly-label-multiline-pos-auto.map",
"poly-label-pos-auto.map", "embed_sb_rgba.map",
"embed_sb_rgba_offset.map"
] # has attributes all on the same line
mapfiles = glob.glob(msautotest_fld + '/**/*.map')
mapfiles = [f for f in mapfiles if os.path.basename(f) not in ignore_list]
# target_map = "polyline_no_clip.map"
# mapfiles = [f for f in mapfiles if os.path.basename(f) in (target_map)]
v = Validator()
for fn in mapfiles:
d = parse_mapfile(parser, transformer, pp, fn)
errors = v.validate(d, add_messages=True)
if errors:
logging.warning("{} failed validation".format(fn))
output_file = fn.replace(msautotest_fld, msautotest_copy)
try:
mappyfile.utils.write(d, output_file)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
print(json.dumps(d, indent=4))
d = parse_mapfile(parser, transformer, pp, output_file)
errors = v.validate(d, add_messages=True)
if errors:
logging.warning("{} failed validation".format(fn))
def main(msautotest_fld, create_new_copy=True):
msautotest_copy = os.path.join(os.path.dirname(msautotest_fld), "msautotest_mappyfile")
if create_new_copy:
create_copy(msautotest_fld, msautotest_copy)
parser = Parser()
transformer = MapfileToDict()
pp = PrettyPrinter()
# these two maps aren't in utf8
# see https://github.com/mapserver/mapserver/pull/5460
# ignore_list = ["wms_inspire_scenario1.map","wms_inspire_scenario2.map"]
# transparent_layer.map has an extra END, see https://github.com/mapserver/mapserver/pull/5468
# polyline_no_clip.map needs symbol names in quotes, and SYMBOL is ambiguous
ignore_list = ["polyline_no_clip.map",
"poly-label-multiline-pos-auto.map", "poly-label-pos-auto.map",
"embed_sb_rgba.map", "embed_sb_rgba_offset.map"] # has attributes all on the same line
mapfiles = glob.glob(msautotest_fld + '/**/*.map')
mapfiles = [f for f in mapfiles if os.path.basename(f) not in ignore_list]
# target_map = "polyline_no_clip.map"
# mapfiles = [f for f in mapfiles if os.path.basename(f) in (target_map)]
v = Validator()
for fn in mapfiles:
d = parse_mapfile(parser, transformer, pp, fn)
errors = v.validate(d, add_comments=True)
if errors:
logging.warning("{} failed validation".format(fn))
output_file = fn.replace(msautotest_fld, msautotest_copy)
try:
mappyfile.utils.write(d, output_file)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
print(json.dumps(d, indent=4))
d = parse_mapfile(parser, transformer, pp, output_file)
errors = v.validate(d, add_comments=True)
if errors:
logging.warning("{} failed validation".format(fn))
def validate(d, trace_o_incl=None, version=None):
"""
Validate a mappyfile dictionary by using the Mapfile schema.
An optional version number can be used to specify a specific
a Mapfile is valid for a specific MapServer version.
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
trace_o_incl: list
A trace of the origin of lines for include files, use to find the original location of an error
version: float
The MapServer version number used to validate the Mapfile
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d, trace_o_incl, version=version)
def test_all_maps():
sample_dir = os.path.join(os.path.dirname(__file__), "sample_maps")
p = Parser(expand_includes=False)
m = MapfileToDict(include_position=True)
v = Validator()
failing_maps = []
for fn in os.listdir(sample_dir):
print(fn)
try:
ast = p.parse_file(os.path.join(sample_dir, fn))
d = m.transform(ast)
errors = v.validate(d)
try:
assert (len(errors) == 0)
except AssertionError as ex:
logging.warning("Validation errors in %s ", fn)
logging.warning(errors)
except (BaseException, UnexpectedToken) as ex:
logging.warning("Cannot process %s ", fn)
logging.error(ex)
failing_maps.append(fn)
logging.warning(failing_maps)
def test_double_error():
s = """MAP
NAME "sample"
STATUS ON
SIZE 600 400
SYMBOLSET "../etc/symbols.txt"
EXTENT -180 -90 180
UNITS DD
SHAPEPATH "../data"
IMAGECOLOR 255 255 256
FONTSET "../etc/fonts.txt"
WEB
IMAGEPATH "/ms4w/tmp/ms_tmp/"
IMAGEURL "/ms_tmp/"
END
LAYER
NAME "global-raster"
TYPE RASTER
STATUS DEFAULT
DATA "bluemarble.gif"
END
END"""
d = mappyfile.loads(s, include_position=True)
# print(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, add_comments=True)
# print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert(len(errors) == 2)
print(mappyfile.dumps(d))
def test_line_position_mutlilines():
s = """MAP
NAME "sample"
LAYER
NAME "test"
STATUS DEFAULT
DATA "SELECT GEOM
FROM
TABLE"
TYPE LINEX
END
END"""
p = Parser()
ast = p.parse(s)
print(ast)
d = mappyfile.loads(s, include_position=True)
v = Validator()
errors = v.validate(d, add_comments=True)
print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert (len(errors) == 1)
err = errors[0]
assert (err["line"] == 9)
assert (err["column"] == 9)
print(mappyfile.dumps(d))
def test_double_error():
s = """MAP
NAME "sample"
STATUS ON
SIZE 600 400
SYMBOLSET "../etc/symbols.txt"
EXTENT -180 -90 180
UNITS DD
SHAPEPATH "../data"
IMAGECOLOR 255 255 256
FONTSET "../etc/fonts.txt"
WEB
IMAGEPATH "/ms4w/tmp/ms_tmp/"
IMAGEURL "/ms_tmp/"
END
LAYER
NAME "global-raster"
TYPE RASTER
STATUS DEFAULT
DATA "bluemarble.gif"
END
END"""
d = mappyfile.loads(s, include_position=True)
# print(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, add_comments=True)
# print(json.dumps(d, indent=4))
# print(errors)
for e in errors:
print(e)
assert (len(errors) == 2)
print(mappyfile.dumps(d))
def test_all_maps():
sample_dir = os.path.join(os.path.dirname(__file__), "sample_maps")
p = Parser(expand_includes=False)
m = MapfileToDict(include_position=True)
v = Validator()
failing_maps = []
for fn in os.listdir(sample_dir):
print(fn)
try:
ast = p.parse_file(os.path.join(sample_dir, fn))
d = m.transform(ast)
errors = v.validate(d)
try:
assert(len(errors) == 0)
except AssertionError as ex:
logging.warning("Validation errors in %s ", fn)
logging.warning(errors)
except (BaseException, UnexpectedToken) as ex:
logging.warning("Cannot process %s ", fn)
logging.error(ex)
failing_maps.append(fn)
logging.warning(failing_maps)
def validate(d):
"""
Validate a mappyfile dictionary by using the Mapfile schema
d: dict
A Python dictionary based on the the mappyfile schema
"""
v = Validator()
return v.validate(d)
def output(fn):
"""
Parse, transform, and pretty print
the result
"""
p = Parser(expand_includes=False)
m = MapfileToDict()
v = Validator()
try:
ast = p.parse_file(fn)
# print(ast)
d = m.transform(ast)
logging.debug("Number of layers: {}".format(len(d["layers"])))
errors = v.validate(d)
assert (len(errors) == 0)
except Exception as ex:
logging.exception(ex)
logging.warning("%s could not be successfully parsed", fn)
d = None
raise
if d:
try:
s = mappyfile.utils.dumps(d)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
ast = p.parse(s)
d = m.transform(ast)
errors = v.validate(d)
assert (len(errors) == 0)
def output(fn):
"""
Parse, transform, and pretty print
the result
"""
p = Parser(expand_includes=False, include_comments=True)
m = MapfileToDict(include_position=True, include_comments=True)
v = Validator()
try:
ast = p.parse_file(fn)
# print(ast)
d = m.transform(ast)
logging.debug("Number of layers: {}".format(len(d["layers"])))
errors = v.validate(d, add_comments=True)
assert(len(errors) == 0)
except Exception as ex:
logging.exception(ex)
logging.warning("%s could not be successfully parsed", fn)
d = None
raise
if d:
try:
s = mappyfile.utils.dumps(d)
except Exception:
logging.warning(json.dumps(d, indent=4))
logging.warning("%s could not be successfully re-written", fn)
raise
# now try reading it again
ast = p.parse(s)
d = m.transform(ast)
errors = v.validate(d)
assert(len(errors) == 0)
def test_cluster_validation_fail():
s = u"""
MAP
LAYER
CLUSTER
MAXDISTANCE 50
REGION "ELLIPSEZ"
END
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
errors = v.validate(d, add_comments=True)
print(mappyfile.dumps(d))
assert len(errors) == 1
def test_root_position():
"""
Check the root objects position is found correctly
"""
s = """
MAP
METADATA
"wms_title" "Toronto Landsat 5 TM"
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
assert d["__position__"]["line"] == 2
errors = v.validate(d, add_comments=True)
assert len(errors) == 1
def test_extra_property_validation():
"""
Check root errors are handled correctly
"""
s = """
MAP
LAYER
TYPE POLYGON
END
END
"""
d = to_dict(s)
d["unwanted"] = "error"
v = Validator()
errors = v.validate(d, add_comments=True)
print(errors)
assert(len(errors) == 1)
def test_extra_property_validation():
"""
Check root errors are handled correctly
"""
s = """
MAP
LAYER
TYPE POLYGON
END
END
"""
d = to_dict(s)
d["__unwanted__"] = "error"
v = Validator()
errors = v.validate(d, add_comments=True)
print(errors)
assert (len(errors) == 1)
def test_version_warnings():
s = """MAP
NAME "sample"
LAYER
NAME "test"
TYPE LINE
CLASS
#MADEUP True
COLOR 0 0 0
END
END
END"""
d = mappyfile.loads(s, include_position=False)
v = Validator()
errors = v.validate(d, add_comments=True, version=8.0)
print(errors)
assert len(errors) == 1
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert(len(errors) == 0)
return s
def validate(d):
"""
Validate a mappyfile dictionary by using the Mapfile schema
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d)
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert (len(errors) == 0)
return s
def test_cluster_validation():
s = u"""
MAP
LAYER
TYPE POINT
CLUSTER
MAXDISTANCE 50
REGION "ELLIPSE"
END
END
END
"""
d = mappyfile.loads(s, include_position=True)
v = Validator()
assert d["__position__"]["line"] == 2
errors = v.validate(d, add_comments=True)
print(mappyfile.dumps(d))
assert len(errors) == 0
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
# https://stackoverflow.com/questions/900392/getting-the-caller-function-name-inside-another-function-in-python
logging.info(inspect.stack()[1][3])
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert(len(errors) == 0)
return s
def output(s, include_position=True, schema_name="map"):
"""
Parse, transform, validate, and pretty print
the result
"""
p = Parser()
m = MapfileToDict(include_position=include_position)
# https://stackoverflow.com/questions/900392/getting-the-caller-function-name-inside-another-function-in-python
logging.info(inspect.stack()[1][3])
ast = p.parse(s)
logging.debug(ast.pretty())
d = m.transform(ast)
logging.debug(json.dumps(d, indent=4))
v = Validator()
errors = v.validate(d, schema_name=schema_name)
logging.error(errors)
pp = PrettyPrinter(indent=0, newlinechar=" ", quote="'")
s = pp.pprint(d)
logging.debug(s)
assert (len(errors) == 0)
return s
def validate(d, version=None):
"""
Validate a mappyfile dictionary by using the Mapfile schema.
An optional version number can be used to specify a specific
a Mapfile is valid for a specific MapServer version.
Parameters
----------
d: dict
A Python dictionary based on the the mappyfile schema
version: float
The MapServer version number used to validate the Mapfile
Returns
-------
list
A list containing validation errors
"""
v = Validator()
return v.validate(d, version=version)
def test_multiple_layer_projection():
"""
TODO add validation for this case
"""
s = """
MAP
LAYER
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
END
END
"""
p = Parser()
ast = p.parse(s)
t = MapfileToDict(include_position=True)
d = t.transform(ast)
print(json.dumps(d, indent=4))
assert (len(d["projection"]) == 1)
p = Parser()
m = MapfileToDict()
ast = p.parse(s)
d = m.transform(ast)
print(json.dumps(d, indent=4))
from mappyfile.validator import Validator
v = Validator()
return v.validate(d)
def test_multiple_layer_projection():
"""
TODO add validation for this case
"""
s = """
MAP
LAYER
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
PROJECTION
"init=epsg:4326"
"init=epsg:4326"
END
END
END
"""
p = Parser()
ast = p.parse(s)
t = MapfileToDict(include_position=True)
d = t.transform(ast)
print(json.dumps(d, indent=4))
assert(len(d["projection"]) == 1)
p = Parser()
m = MapfileToDict()
ast = p.parse(s)
d = m.transform(ast)
print(json.dumps(d, indent=4))
from mappyfile.validator import Validator
v = Validator()
return v.validate(d)
def test_add_comments():
s = """
MAP
IMAGECOLOR 'FF00FF'
LAYER
EXTENT 0 0 0
TYPE POLYGON
END
END
"""
d = to_dict(s)
v = Validator()
errors = v.validate(d, add_comments=True)
print(len(errors))
print(json.dumps(d, indent=4))
for error in errors:
print(error)
pp = PrettyPrinter(indent=4, quote='"') # expected
res = pp.pprint(d)
print(res)
def test_add_comments():
s = """
MAP
IMAGECOLOR 'FF00FF'
LAYER
EXTENT 0 0 0
TYPE POLYGON
END
END
"""
d = to_dict(s)
v = Validator()
errors = v.validate(d, add_comments=True)
print(len(errors))
print(json.dumps(d, indent=4))
for error in errors:
print(error)
pp = PrettyPrinter(indent=4, quote='"') # expected
res = pp.pprint(d)
print(res)
def validate(d):
v = Validator()
return v.validate(d)
def validate(s):
d = to_dict(s)
v = Validator()
return v.validate(d)
def validate(s):
d = to_dict(s)
v = Validator()
return v.validate(d)
def validate(d):
v = Validator()
return v.validate(d)