def execute(self, tile, overwrite=True):
"""
Processes and saves tile.
Returns a tuple of three values:
- Tile ID: zoom, row, col
- status: "exists", "empty", "failed" or "processed"
- error message: optional or None
"""
# Do nothing if tile exists or overwrite is turned off.
if not overwrite and tile.exists():
return tile.id, "exists", None
try:
new_process = imp.load_source(self.process_name + "Process", self.config.process_file)
tile_process = new_process.Process(config=self.config, tile=tile, params=self.config.at_zoom(tile.zoom))
except:
return tile.id, "failed", traceback.print_exc()
# Generate tile using the user defined process.
if not self.config.baselevel or (tile.zoom == self.config.baselevel["zoom"]):
try:
result = tile_process.execute()
except:
return tile.id, "failed", traceback.print_exc()
finally:
tile_process = None
message = None
if result:
if result == "empty":
status = "empty"
else:
status = "custom"
message = result
else:
status = "processed"
return tile.id, status, message
# If baselevel interpolation is activated, generate from neighbor zooms.
else:
if tile.zoom < self.config.baselevel["zoom"]:
# determine tiles from next zoom level
# TODO create option for on demand processing if subtile is not
# available.
# create temporary VRT and create new tile from resampled
# subtiles.
children_paths = [child.path for child in tile.get_children() if child.exists()]
if len(children_paths) == 0:
return tile.id, "empty", None
temp_vrt = NamedTemporaryFile()
build_vrt = "gdalbuildvrt %s %s > /dev/null" % (temp_vrt.name, " ".join(children_paths))
try:
os.system(build_vrt)
assert os.path.isfile(temp_vrt.name)
except:
build_vrt = "gdalbuildvrt %s %s" % (temp_vrt.name, " ".join(children_paths))
os.system(build_vrt)
return tile.id, "failed", "GDAL VRT building"
try:
assert os.path.isfile(temp_vrt.name)
bands = tuple(
read_raster_window(temp_vrt.name, tile, resampling=self.config.baselevel["resampling"])
)
except:
return tile.id, "failed", traceback.print_exc()
try:
write_raster(tile_process, self.output.profile, bands)
return tile.id, "processed", None
except:
return tile.id, "failed", traceback.print_exc()
elif tile.zoom > self.config.baselevel["zoom"]:
# determine tiles from previous zoom level
# check if tiles exist and if not, execute subtiles
parent = tile.get_parent()
if not overwrite and parent.exists():
# LOGGER.info((tile.id, "exists", None))
pass
else:
pass
if not parent.exists():
# TODO create option for on demand processing
return tile.id, "empty", "source tile does not exist"
try:
bands = tuple(read_raster_window(parent.path, tile, resampling=self.config.baselevel["resampling"]))
write_raster(tile_process, self.output.profile, bands)
return tile.id, "processed", None
except:
return tile.id, "failed", traceback.print_exc()
def main():
scriptdir = os.path.dirname(os.path.realpath(__file__))
# YAML configuration
#===================
# Load source process from python file and initialize.
mapchete_file = os.path.join(scriptdir, "example.mapchete")
mapchete = Mapchete(MapcheteConfig(mapchete_file))
dummy1_abspath = os.path.join(scriptdir, "testdata/dummy1.tif")
dummy2_abspath = os.path.join(scriptdir, "testdata/dummy2.tif")
# Validate configuration constructor
## basic run through
try:
config = mapchete.config
print "OK: basic configuraiton constructor run through"
except:
print "FAILED: basic configuraiton constructor run through"
raise
try:
# Check configuration at zoom level 5
zoom5 = config.at_zoom(5)
input_files = zoom5["input_files"]
assert input_files["file1"] == None
assert input_files["file2"] == dummy2_abspath
assert zoom5["some_integer_parameter"] == 12
assert zoom5["some_float_parameter"] == 5.3
assert zoom5["some_string_parameter"] == "string1"
assert zoom5["some_bool_parameter"] == True
# Check configuration at zoom level 11
zoom11 = config.at_zoom(11)
input_files = zoom11["input_files"]
assert input_files["file1"] == dummy1_abspath
assert input_files["file2"] == dummy2_abspath
assert zoom11["some_integer_parameter"] == 12
assert zoom11["some_float_parameter"] == 5.3
assert zoom11["some_string_parameter"] == "string2"
assert zoom11["some_bool_parameter"] == True
except:
print "FAILED: basic configuration parsing"
print input_files
raise
else:
print "OK: basic configuration parsing"
## read zoom level from config file
mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file)).config
try:
assert 5 in config.zoom_levels
print "OK: read zoom level from config file"
except:
print "FAILED: read zoom level from config file"
print mapchete_file
raise
## read min/max zoom levels from config file
mapchete_file = os.path.join(scriptdir, "testdata/minmax_zoom.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file)).config
try:
for zoom in [7, 8, 9, 10]:
assert zoom in config.zoom_levels
print "OK: read min/max zoom levels from config file"
except:
print "FAILED: read min/max zoom levels from config file"
raise
## zoom levels override
mapchete_file = os.path.join(scriptdir, "testdata/minmax_zoom.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file, zoom=[1, 4])).config
try:
for zoom in [1, 2, 3, 4]:
assert zoom in config.zoom_levels
print "OK: zoom levels override"
except:
print "FAILED: zoom levels override"
raise
## read bounds from config file
mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file)).config
try:
test_polygon = Polygon([
[3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5]
])
assert config.process_area(5).equals(test_polygon)
print "OK: read bounds from config file"
except:
print "FAILED: read bounds from config file"
print config.process_area(5), test_polygon
raise
## override bounds
mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete")
config = Mapchete(MapcheteConfig(
mapchete_file,
bounds=[3, 2, 3.5, 1.5]
)).config
try:
test_polygon = Polygon([
[3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5]
])
assert config.process_area(5).equals(test_polygon)
print "OK: override bounds"
except:
print "FAILED: override bounds"
print config.process_area(5)
raise
## read bounds from input files
mapchete_file = os.path.join(scriptdir, "testdata/files_bounds.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file)).config
try:
test_polygon = Polygon(
[[3, 2], [4, 2], [4, 1], [3, 1], [2, 1], [2, 4], [3, 4], [3, 2]]
)
assert config.process_area(10).equals(test_polygon)
print "OK: read bounds from input files"
except:
print "FAILED: read bounds from input files"
print config.process_area(10), test_polygon
raise
## read .mapchete files as input files
mapchete_file = os.path.join(scriptdir, "testdata/mapchete_input.mapchete")
config = Mapchete(MapcheteConfig(mapchete_file)).config
area = config.process_area(5)
testpolygon = "POLYGON ((3 2, 3.5 2, 3.5 1.5, 3 1.5, 3 1, 2 1, 2 4, 3 4, 3 2))"
try:
assert area.equals(loads(testpolygon))
print "OK: read bounding box from .mapchete subfile"
except:
print "FAILED: read bounding box from .mapchete subfile"
raise
mapchete_file = os.path.join(scriptdir, "testdata/gtiff.mapchete")
mapchete_file = os.path.join(scriptdir, "testdata/numpy.mapchete")
mapchete = Mapchete(MapcheteConfig(mapchete_file))
# test io module
testdata_directory = os.path.join(scriptdir, "testdata")
outdata_directory = os.path.join(testdata_directory, "out")
dummy1 = os.path.join(testdata_directory, "dummy1.tif")
# dummy1 = os.path.join(testdata_directory, "sentinel2.tif")
dummy2 = os.path.join(testdata_directory, "dummy2.tif")
zoom = 8
tile_pyramid = TilePyramid("geodetic")
dummy1_bbox = file_bbox(dummy1, tile_pyramid)
tiles = tile_pyramid.tiles_from_geom(dummy1_bbox, zoom)
resampling = "average"
pixelbuffer=5
for tile in tiles:
for band in read_raster_window(
dummy1,
tile,
resampling=resampling,
pixelbuffer=pixelbuffer
):
try:
assert band.shape == (
tile_pyramid.tile_size + 2 * pixelbuffer,
tile_pyramid.tile_size + 2 * pixelbuffer
)
print "OK: read data size"
except:
print "FAILED: read data size"
outname = str(tile.zoom) + str(tile.row) + str(tile.col) + ".tif"
outfile = os.path.join(outdata_directory, outname)