def execute(self, tile, overwrite=True): """ Processes and saves tile. Returns a tuple of three values: - Tile ID: zoom, row, col - status: "exists", "empty", "failed" or "processed" - error message: optional or None """ # Do nothing if tile exists or overwrite is turned off. if not overwrite and tile.exists(): return tile.id, "exists", None try: new_process = imp.load_source(self.process_name + "Process", self.config.process_file) tile_process = new_process.Process(config=self.config, tile=tile, params=self.config.at_zoom(tile.zoom)) except: return tile.id, "failed", traceback.print_exc() # Generate tile using the user defined process. if not self.config.baselevel or (tile.zoom == self.config.baselevel["zoom"]): try: result = tile_process.execute() except: return tile.id, "failed", traceback.print_exc() finally: tile_process = None message = None if result: if result == "empty": status = "empty" else: status = "custom" message = result else: status = "processed" return tile.id, status, message # If baselevel interpolation is activated, generate from neighbor zooms. else: if tile.zoom < self.config.baselevel["zoom"]: # determine tiles from next zoom level # TODO create option for on demand processing if subtile is not # available. # create temporary VRT and create new tile from resampled # subtiles. children_paths = [child.path for child in tile.get_children() if child.exists()] if len(children_paths) == 0: return tile.id, "empty", None temp_vrt = NamedTemporaryFile() build_vrt = "gdalbuildvrt %s %s > /dev/null" % (temp_vrt.name, " ".join(children_paths)) try: os.system(build_vrt) assert os.path.isfile(temp_vrt.name) except: build_vrt = "gdalbuildvrt %s %s" % (temp_vrt.name, " ".join(children_paths)) os.system(build_vrt) return tile.id, "failed", "GDAL VRT building" try: assert os.path.isfile(temp_vrt.name) bands = tuple( read_raster_window(temp_vrt.name, tile, resampling=self.config.baselevel["resampling"]) ) except: return tile.id, "failed", traceback.print_exc() try: write_raster(tile_process, self.output.profile, bands) return tile.id, "processed", None except: return tile.id, "failed", traceback.print_exc() elif tile.zoom > self.config.baselevel["zoom"]: # determine tiles from previous zoom level # check if tiles exist and if not, execute subtiles parent = tile.get_parent() if not overwrite and parent.exists(): # LOGGER.info((tile.id, "exists", None)) pass else: pass if not parent.exists(): # TODO create option for on demand processing return tile.id, "empty", "source tile does not exist" try: bands = tuple(read_raster_window(parent.path, tile, resampling=self.config.baselevel["resampling"])) write_raster(tile_process, self.output.profile, bands) return tile.id, "processed", None except: return tile.id, "failed", traceback.print_exc()
def main(): scriptdir = os.path.dirname(os.path.realpath(__file__)) # YAML configuration #=================== # Load source process from python file and initialize. mapchete_file = os.path.join(scriptdir, "example.mapchete") mapchete = Mapchete(MapcheteConfig(mapchete_file)) dummy1_abspath = os.path.join(scriptdir, "testdata/dummy1.tif") dummy2_abspath = os.path.join(scriptdir, "testdata/dummy2.tif") # Validate configuration constructor ## basic run through try: config = mapchete.config print "OK: basic configuraiton constructor run through" except: print "FAILED: basic configuraiton constructor run through" raise try: # Check configuration at zoom level 5 zoom5 = config.at_zoom(5) input_files = zoom5["input_files"] assert input_files["file1"] == None assert input_files["file2"] == dummy2_abspath assert zoom5["some_integer_parameter"] == 12 assert zoom5["some_float_parameter"] == 5.3 assert zoom5["some_string_parameter"] == "string1" assert zoom5["some_bool_parameter"] == True # Check configuration at zoom level 11 zoom11 = config.at_zoom(11) input_files = zoom11["input_files"] assert input_files["file1"] == dummy1_abspath assert input_files["file2"] == dummy2_abspath assert zoom11["some_integer_parameter"] == 12 assert zoom11["some_float_parameter"] == 5.3 assert zoom11["some_string_parameter"] == "string2" assert zoom11["some_bool_parameter"] == True except: print "FAILED: basic configuration parsing" print input_files raise else: print "OK: basic configuration parsing" ## read zoom level from config file mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete") config = Mapchete(MapcheteConfig(mapchete_file)).config try: assert 5 in config.zoom_levels print "OK: read zoom level from config file" except: print "FAILED: read zoom level from config file" print mapchete_file raise ## read min/max zoom levels from config file mapchete_file = os.path.join(scriptdir, "testdata/minmax_zoom.mapchete") config = Mapchete(MapcheteConfig(mapchete_file)).config try: for zoom in [7, 8, 9, 10]: assert zoom in config.zoom_levels print "OK: read min/max zoom levels from config file" except: print "FAILED: read min/max zoom levels from config file" raise ## zoom levels override mapchete_file = os.path.join(scriptdir, "testdata/minmax_zoom.mapchete") config = Mapchete(MapcheteConfig(mapchete_file, zoom=[1, 4])).config try: for zoom in [1, 2, 3, 4]: assert zoom in config.zoom_levels print "OK: zoom levels override" except: print "FAILED: zoom levels override" raise ## read bounds from config file mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete") config = Mapchete(MapcheteConfig(mapchete_file)).config try: test_polygon = Polygon([ [3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5] ]) assert config.process_area(5).equals(test_polygon) print "OK: read bounds from config file" except: print "FAILED: read bounds from config file" print config.process_area(5), test_polygon raise ## override bounds mapchete_file = os.path.join(scriptdir, "testdata/zoom.mapchete") config = Mapchete(MapcheteConfig( mapchete_file, bounds=[3, 2, 3.5, 1.5] )).config try: test_polygon = Polygon([ [3, 1.5], [3, 2], [3.5, 2], [3.5, 1.5], [3, 1.5] ]) assert config.process_area(5).equals(test_polygon) print "OK: override bounds" except: print "FAILED: override bounds" print config.process_area(5) raise ## read bounds from input files mapchete_file = os.path.join(scriptdir, "testdata/files_bounds.mapchete") config = Mapchete(MapcheteConfig(mapchete_file)).config try: test_polygon = Polygon( [[3, 2], [4, 2], [4, 1], [3, 1], [2, 1], [2, 4], [3, 4], [3, 2]] ) assert config.process_area(10).equals(test_polygon) print "OK: read bounds from input files" except: print "FAILED: read bounds from input files" print config.process_area(10), test_polygon raise ## read .mapchete files as input files mapchete_file = os.path.join(scriptdir, "testdata/mapchete_input.mapchete") config = Mapchete(MapcheteConfig(mapchete_file)).config area = config.process_area(5) testpolygon = "POLYGON ((3 2, 3.5 2, 3.5 1.5, 3 1.5, 3 1, 2 1, 2 4, 3 4, 3 2))" try: assert area.equals(loads(testpolygon)) print "OK: read bounding box from .mapchete subfile" except: print "FAILED: read bounding box from .mapchete subfile" raise mapchete_file = os.path.join(scriptdir, "testdata/gtiff.mapchete") mapchete_file = os.path.join(scriptdir, "testdata/numpy.mapchete") mapchete = Mapchete(MapcheteConfig(mapchete_file)) # test io module testdata_directory = os.path.join(scriptdir, "testdata") outdata_directory = os.path.join(testdata_directory, "out") dummy1 = os.path.join(testdata_directory, "dummy1.tif") # dummy1 = os.path.join(testdata_directory, "sentinel2.tif") dummy2 = os.path.join(testdata_directory, "dummy2.tif") zoom = 8 tile_pyramid = TilePyramid("geodetic") dummy1_bbox = file_bbox(dummy1, tile_pyramid) tiles = tile_pyramid.tiles_from_geom(dummy1_bbox, zoom) resampling = "average" pixelbuffer=5 for tile in tiles: for band in read_raster_window( dummy1, tile, resampling=resampling, pixelbuffer=pixelbuffer ): try: assert band.shape == ( tile_pyramid.tile_size + 2 * pixelbuffer, tile_pyramid.tile_size + 2 * pixelbuffer ) print "OK: read data size" except: print "FAILED: read data size" outname = str(tile.zoom) + str(tile.row) + str(tile.col) + ".tif" outfile = os.path.join(outdata_directory, outname)