def test_read_mapchete_input(mapchete_input):
"""Read Mapchete files as input files."""
config = MapcheteConfig(mapchete_input.path)
area = config.area_at_zoom(5)
# testpolygon = box(0.5, 1.5, 3.5, 3.5)
testpolygon = wkt.loads("POLYGON ((3 1.5, 3 1, 2 1, 2 1.5, 0.5 1.5, 0.5 3.5, 2 3.5, 2 4, 3 4, 3 3.5, 3.5 3.5, 3.5 1.5, 3 1.5))")
assert area.equals(testpolygon)
def test_read_mapchete_input():
"""Read Mapchete files as input files."""
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/mapchete_input.mapchete"))
area = config.process_area(5)
testpolygon = "POLYGON ((3 2, 3.5 2, 3.5 1.5, 3 1.5, 3 1, 2 1, 2 4, 3 4, 3 2))"
assert area.equals(loads(testpolygon))
def test_mandatory_params():
"""Check availability of mandatory parameters."""
for param in ["process_file", "input_files", "output"]:
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
del config[param]
config.update(config_dir=SCRIPTDIR)
MapcheteConfig(config)
raise Exception()
except errors.MapcheteConfigError:
pass
# invalid path
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
config.update(config_dir=SCRIPTDIR, process_file="invalid/path.py")
MapcheteConfig(config).process_file
raise Exception()
except errors.MapcheteConfigError:
pass
# no config dir given
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
MapcheteConfig(config).process_file
raise Exception()
except errors.MapcheteConfigError:
pass
def test_read_raster_window():
"""Read array with read_raster_window."""
dummy1 = os.path.join(TESTDATA_DIR, "dummy1.tif")
zoom = 8
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/minmax_zoom.mapchete"))
rasterfile = config.at_zoom(7)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = tile_pyramid.tiles_from_geom(dummy1_bbox, zoom)
width = height = tile_pyramid.tile_size + 2 * pixelbuffer
for tile in tiles:
for band in raster.read_raster_window(dummy1, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in range(4):
band = raster.read_raster_window(dummy1, tile, index).next()
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for resampling in [
"nearest", "bilinear", "cubic", "cubic_spline", "lanczos",
"average", "mode"
]:
raster.read_raster_window(dummy1, tile, resampling=resampling)
def test_read_raster_window(dummy1_tif, minmax_zoom):
"""Read array with read_raster_window."""
zoom = 8
# without reproject
config = MapcheteConfig(minmax_zoom.path)
rasterfile = config.params_at_zoom(zoom)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = list(tile_pyramid.tiles_from_geom(dummy1_bbox, zoom))
# add edge tile
tiles.append(tile_pyramid.tile(8, 0, 0))
for tile in tiles:
width, height = tile.shape
for band in read_raster_window(dummy1_tif, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in range(1, 4):
band = read_raster_window(dummy1_tif, tile, index)
assert isinstance(band, ma.MaskedArray)
assert band.shape == (width, height)
for index in [None, [1, 2, 3]]:
band = read_raster_window(dummy1_tif, tile, index)
assert isinstance(band, ma.MaskedArray)
assert band.ndim == 3
assert band.shape == (3, width, height)
def test_read_raster_window_reproject(dummy1_3857_tif, minmax_zoom):
"""Read array with read_raster_window."""
zoom = 8
# with reproject
config_raw = minmax_zoom.dict
config_raw["input"].update(file1=dummy1_3857_tif)
config = MapcheteConfig(config_raw)
rasterfile = config.params_at_zoom(zoom)["input"]["file1"]
dummy1_bbox = rasterfile.bbox()
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = list(tile_pyramid.tiles_from_geom(dummy1_bbox, zoom))
# target window out of CRS bounds
band = read_raster_window(dummy1_3857_tif, tile_pyramid.tile(12, 0, 0))
assert isinstance(band, ma.MaskedArray)
assert band.mask.all()
# not intersecting tile
tiles.append(tile_pyramid.tile(zoom, 1, 1)) # out of CRS bounds
tiles.append(tile_pyramid.tile(zoom, 16, 1)) # out of file bbox
for tile in tiles:
for band in read_raster_window(dummy1_3857_tif, tile):
assert isinstance(band, ma.MaskedArray)
assert band.shape == tile.shape
bands = read_raster_window(dummy1_3857_tif, tile, [1])
assert isinstance(bands, ma.MaskedArray)
assert bands.shape == tile.shape
# errors
with pytest.raises(IOError):
read_raster_window("nonexisting_path", tile)
def test_bounds_from_input_files():
"""Read bounds from input files."""
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/files_bounds.mapchete"))
test_polygon = Polygon([[3, 2], [4, 2], [4, 1], [3, 1], [2, 1], [2, 4],
[3, 4], [3, 2]])
assert config.process_area(10).equals(test_polygon)
def test_config_zoom11(example_mapchete, dummy2_tif, dummy1_tif):
"""Example configuration at zoom 11."""
config = MapcheteConfig(example_mapchete.path)
zoom11 = config.params_at_zoom(11)
input_files = zoom11["input"]
assert input_files["file1"].path == dummy1_tif
assert input_files["file2"].path == dummy2_tif
assert zoom11["some_integer_parameter"] == 12
assert zoom11["some_float_parameter"] == 5.3
assert zoom11["some_string_parameter"] == "string2"
assert zoom11["some_bool_parameter"] is True
def test_process_tile_read(example_mapchete):
"""Raise ValueError on MapcheteProcess.open()."""
config = MapcheteConfig(example_mapchete.path)
tile = BufferedTilePyramid("mercator").tile(7, 1, 1)
user_process = mapchete.MapcheteProcess(
tile=tile,
params=config.params_at_zoom(tile.zoom),
input=config.get_inputs_for_tile(tile),
)
with pytest.raises(DeprecationWarning):
user_process.read()
def test_config_zoom5():
"""Example configuration at zoom 5."""
config = MapcheteConfig(os.path.join(SCRIPTDIR, "example.mapchete"))
dummy2_abspath = os.path.join(SCRIPTDIR, "testdata/dummy2.tif")
zoom5 = config.at_zoom(5)
input_files = zoom5["input"]
assert input_files["file1"] is None
assert input_files["file2"].path == dummy2_abspath
assert zoom5["some_integer_parameter"] == 12
assert zoom5["some_float_parameter"] == 5.3
assert zoom5["some_string_parameter"] == "string1"
assert zoom5["some_bool_parameter"] is True
def open(some_input, with_cache=False, **kwargs):
"""
Open a Mapchete process.
Parameters
----------
some_input : MapcheteConfig object, config dict, path to mapchete file or path to
TileDirectory
Mapchete process configuration
mode : string
* ``memory``: Generate process output on demand without reading
pre-existing data or writing new data.
* ``readonly``: Just read data without processing new data.
* ``continue``: (default) Don't overwrite existing output.
* ``overwrite``: Overwrite existing output.
zoom : list or integer
process zoom level or a pair of minimum and maximum zoom level
bounds : tuple
left, bottom, right, top process boundaries in output pyramid
single_input_file : string
single input file if supported by process
with_cache : bool
process output data cached in memory
Returns
-------
Mapchete
a Mapchete process object
"""
if isinstance(some_input, str) and not some_input.endswith(".mapchete"):
logger.debug("assuming TileDirectory")
metadata_json = os.path.join(some_input, "metadata.json")
fs = kwargs.get("fs", fs_from_path(metadata_json, **kwargs))
logger.debug("read metadata.json")
metadata = read_output_metadata(metadata_json, fs=fs)
config = dict(process=None,
input=None,
pyramid=metadata["pyramid"].to_dict(),
output=dict(
{
k: v
for k, v in metadata["driver"].items()
if k not in ["delimiters", "mode"]
},
path=some_input,
fs_kwargs=kwargs),
config_dir=os.getcwd(),
zoom_levels=kwargs.get("zoom"))
kwargs.update(mode="readonly")
return Mapchete(MapcheteConfig(config, **kwargs))
else:
return Mapchete(MapcheteConfig(some_input, **kwargs),
with_cache=with_cache)
def test_invalid_output_params(example_mapchete):
"""Check on invalid configuration."""
# missing or invalid params
for param in ["format"]:
config = deepcopy(example_mapchete.dict)
with pytest.raises(errors.MapcheteConfigError):
# invalid
config["output"][param] = "invalid"
MapcheteConfig(config)
with pytest.raises(errors.MapcheteConfigError):
# missing
config["output"].pop(param)
MapcheteConfig(config)
def test_read_input_groups(file_groups):
"""Read input data groups."""
config = MapcheteConfig(file_groups.path)
input_files = config.params_at_zoom(0)["input"]
assert "file1" in input_files["group1"]
assert "file2" in input_files["group1"]
assert "file1" in input_files["group2"]
assert "file2" in input_files["group2"]
assert "nested_group" in input_files
assert "group1" in input_files["nested_group"]
assert "file1" in input_files["nested_group"]["group1"]
assert "file2" in input_files["nested_group"]["group1"]
assert "file1" in input_files["nested_group"]["group2"]
assert "file2" in input_files["nested_group"]["group2"]
def test_invalid_zoom_levels(example_mapchete):
"""Check on invalid zoom configuration."""
# process zooms
# no zoom levels given
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.pop("zoom_levels")
MapcheteConfig(config)
# invalid single zoom level
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.update(zoom_levels=-5)
MapcheteConfig(config)
# invalid zoom level in pair
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.update(zoom_levels=[-5, 0])
MapcheteConfig(config)
# invalid number of zoom levels
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.update(zoom_levels=[0, 5, 7])
MapcheteConfig(config)
# min or max missing
config = deepcopy(example_mapchete.dict)
config.update(zoom_levels=dict(min=0))
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config)
config.update(zoom_levels=dict(max=5))
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config)
# min bigger than max
config = deepcopy(example_mapchete.dict)
config.update(zoom_levels=dict(min=5, max=0))
# init zooms
# invalid single zoom level
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config, zoom=-5)
# invalid zoom level in pair
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config, zoom=[-5, 0])
# invalid number of zoom levels
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config, zoom=[0, 5, 7])
# not a subset
with pytest.raises(errors.MapcheteConfigError):
MapcheteConfig(config, zoom=[0, 20])
def test_read_input_groups():
"""Read input data groups."""
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/file_groups.mapchete"))
input_files = config.at_zoom(0)["input"]
print input_files
assert "file1" in input_files["group1"]
assert "file2" in input_files["group1"]
assert "file1" in input_files["group2"]
assert "file2" in input_files["group2"]
assert "nested_group" in input_files
assert "group1" in input_files["nested_group"]
assert "file1" in input_files["nested_group"]["group1"]
assert "file2" in input_files["nested_group"]["group1"]
assert "file1" in input_files["nested_group"]["group2"]
assert "file2" in input_files["nested_group"]["group2"]
def test_read_vector_window():
"""Read vector data from read_vector_window."""
zoom = 4
config = MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/geojson.mapchete"))
vectorfile = config.at_zoom(zoom)["input"]["file1"]
pixelbuffer = 5
tile_pyramid = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer)
tiles = tile_pyramid.tiles_from_geom(vectorfile.bbox(), zoom)
feature_count = 0
for tile in tiles:
for feature in vector.read_vector_window(vectorfile.path, tile):
assert "properties" in feature
assert shape(feature["geometry"]).is_valid
feature_count += 1
assert feature_count
def test_override_zoom_levels():
"""Override zoom levels when constructing configuration."""
config = MapcheteConfig(os.path.join(SCRIPTDIR,
"testdata/minmax_zoom.mapchete"),
zoom=[1, 4])
for zoom in [1, 2, 3, 4]:
assert zoom in config.zoom_levels
def test_metatiles(example_mapchete):
"""Assert metatile sizes are checked."""
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config["pyramid"].update(metatiling=1)
config["output"].update(metatiling=2)
MapcheteConfig(config)
def __init__(self, input_params):
"""Initialize."""
super(InputData, self).__init__(input_params)
self.path = input_params["path"]
if self.path:
self.process = Mapchete(MapcheteConfig(self.path, mode="readonly"))
self._bbox_cache = {}
def test_process_tile_write(example_mapchete):
"""Raise DeprecationWarning on MapcheteProcess.write()."""
config = MapcheteConfig(example_mapchete.path)
tile = BufferedTilePyramid("mercator").tile(7, 1, 1)
process_tile = mapchete.MapcheteProcess(tile, config)
with pytest.raises(DeprecationWarning):
process_tile.write("data")
def test_process_tile_open(example_mapchete):
"""Raise ValueError on MapcheteProcess.open()."""
config = MapcheteConfig(example_mapchete.path)
tile = BufferedTilePyramid("mercator").tile(7, 1, 1)
process_tile = mapchete.MapcheteProcess(tile, config)
with pytest.raises(ValueError):
process_tile.open("nonexisting_id")
def test_effective_bounds(files_bounds, baselevels):
config = MapcheteConfig(files_bounds.dict)
assert config.effective_bounds == snap_bounds(
bounds=config.bounds,
pyramid=config.process_pyramid,
zoom=min(config.zoom_levels))
config = MapcheteConfig(baselevels.dict, zoom=[5, 7], bounds=(0, 1, 2, 3))
assert config.effective_bounds != config.init_bounds
assert config.effective_bounds == snap_bounds(
bounds=config.init_bounds, pyramid=config.process_pyramid, zoom=5)
with pytest.raises(MapcheteConfigError):
MapcheteConfig(
dict(baselevels.dict,
zoom_levels=dict(min=7, max=7),
baselevels=dict(lower="cubic", max=7)))
def test_invalid_zoom_levels():
"""Check on invalid zoom configuration."""
# no zoom levels given
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
config.update(config_dir=SCRIPTDIR)
del config["process_minzoom"]
del config["process_maxzoom"]
MapcheteConfig(config)
raise Exception()
except errors.MapcheteConfigError:
pass
# invalid single zoom level
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
config.update(config_dir=SCRIPTDIR)
del config["process_minzoom"]
del config["process_maxzoom"]
MapcheteConfig(config, zoom=-5)
raise Exception()
except errors.MapcheteConfigError:
pass
# invalid zoom level in pair
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
config.update(config_dir=SCRIPTDIR)
del config["process_minzoom"]
del config["process_maxzoom"]
MapcheteConfig(config, zoom=[-5, 0])
raise Exception()
except errors.MapcheteConfigError:
pass
# invalid number of zoom levels
try:
with open(os.path.join(SCRIPTDIR, "example.mapchete")) as mc:
config = copy(yaml.load(mc))
config.update(config_dir=SCRIPTDIR)
del config["process_minzoom"]
del config["process_maxzoom"]
MapcheteConfig(config, zoom=[0, 5, 7])
raise Exception()
except errors.MapcheteConfigError:
pass
def test_abstract_input():
"""Read abstract input definitions."""
try:
MapcheteConfig(
os.path.join(SCRIPTDIR, "testdata/abstract_input.mapchete"))
raise Exception
except MapcheteDriverError:
pass
def __init__(self, input_params, **kwargs):
"""Initialize."""
super(InputData, self).__init__(input_params, **kwargs)
self.path = input_params["path"]
self.process = Mapchete(
MapcheteConfig(self.path,
mode="readonly",
bounds=input_params["delimiters"]["bounds"],
zoom=input_params["delimiters"]["zoom"]))
def test_mandatory_params(example_mapchete):
"""Check availability of mandatory parameters."""
for param in ["process", "input", "output"]:
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
del config[param]
MapcheteConfig(config)
# invalid path
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.update(process="invalid/path.py")
MapcheteConfig(config).process
# no config dir given
with pytest.raises(errors.MapcheteConfigError):
config = deepcopy(example_mapchete.dict)
config.pop("config_dir")
MapcheteConfig(config).process
def test_config_modes():
"""Assert process mode is handled correctly."""
# invalid mode
try:
MapcheteConfig(os.path.join(SCRIPTDIR, "example.mapchete"),
mode="invalid")
raise Exception()
except errors.MapcheteConfigError:
pass
def __init__(self, input_params, **kwargs):
"""Initialize."""
super().__init__(input_params, **kwargs)
self.path = input_params["path"]
self.process = Mapchete(
MapcheteConfig(self.path,
mode="readonly",
bounds=(input_params["delimiters"]["bounds"]
if "delimiters" in input_params else None)))
def test_config_errors(example_mapchete):
"""Test various configuration parsing errors."""
config_orig = example_mapchete.dict
# wrong config type
with pytest.raises(MapcheteConfigError):
mapchete.open("not_a_config")
# missing process
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config.pop("process")
MapcheteConfig(config)
# using input and input_files
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config.update(input=None, input_files=None)
mapchete.open(config)
# output configuration not compatible with driver
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config["output"].pop("bands")
MapcheteConfig(config)
# no baselevel params
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config.update(baselevels={})
with mapchete.open(config) as mp:
mp.config.baselevels
# wrong baselevel min or max
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config.update(baselevels={"min": "invalid"})
with mapchete.open(config) as mp:
mp.config.baselevels
# wrong pixelbuffer type
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config["pyramid"].update(pixelbuffer="wrong_type")
mapchete.open(config)
# wrong metatiling type
with pytest.raises(MapcheteConfigError):
config = deepcopy(config_orig)
config["pyramid"].update(metatiling="wrong_type")
mapchete.open(config)
def test_read_baselevels(baselevels):
"""Read baselevels."""
config = MapcheteConfig(baselevels.path)
assert isinstance(config.baselevels, dict)
assert set(config.baselevels["zooms"]) == set([5, 6])
assert config.baselevels["lower"] == "bilinear"
assert config.baselevels["higher"] == "nearest"
# without min
config = deepcopy(baselevels.dict)
del config["baselevels"]["min"]
assert min(MapcheteConfig(config).baselevels["zooms"]) == 3
# without max and resampling
config = deepcopy(baselevels.dict)
del config["baselevels"]["max"]
del config["baselevels"]["lower"]
assert max(MapcheteConfig(config).baselevels["zooms"]) == 7
assert MapcheteConfig(config).baselevels["lower"] == "nearest"
