def test_input_data(mp_tmpdir, cleantopo_br): """Check GeoTIFF proces output as input data.""" with mapchete.open(cleantopo_br.path) as mp: tp = BufferedTilePyramid("geodetic") # TODO tile with existing but empty data tile = tp.tile(5, 5, 5) output_params = dict( grid="geodetic", format="GeoTIFF", path=mp_tmpdir, pixelbuffer=0, metatiling=1, bands=2, dtype="int16", delimiters=dict(bounds=Bounds(-180.0, -90.0, 180.0, 90.0), effective_bounds=Bounds(-180.439453125, -90.0, 180.439453125, 90.0), zoom=[5], process_bounds=Bounds(-180.0, -90.0, 180.0, 90.0))) output = gtiff.OutputDataWriter(output_params) with output.open(tile, mp) as input_tile: for data in [ input_tile.read(), input_tile.read(1), input_tile.read([1]), # TODO assert valid indexes are passed input_tile.read([1, 2]) ]: assert isinstance(data, ma.masked_array) assert input_tile.is_empty() # open without resampling with output.open(tile, mp) as input_tile: pass
def prepare(self, process_area=None, **kwargs): bounds = snap_bounds( bounds=Bounds(*process_area.intersection( box(*self.output_params["delimiters"] ["effective_bounds"])).bounds), pyramid=self.pyramid, zoom=self.zoom) if process_area else self.output_params[ "delimiters"]["effective_bounds"] height = math.ceil((bounds.top - bounds.bottom) / self.pyramid.pixel_x_size(self.zoom)) width = math.ceil((bounds.right - bounds.left) / self.pyramid.pixel_x_size(self.zoom)) logger.debug("output raster bounds: %s", bounds) logger.debug("output raster shape: %s, %s", height, width) self._profile = dict( GTIFF_DEFAULT_PROFILE, driver="GTiff", transform=Affine(self.pyramid.pixel_x_size(self.zoom), 0, bounds.left, 0, -self.pyramid.pixel_y_size(self.zoom), bounds.top), height=height, width=width, count=self.output_params["bands"], crs=self.pyramid.crs, **{ k: self.output_params.get(k, GTIFF_DEFAULT_PROFILE[k]) for k in GTIFF_DEFAULT_PROFILE.keys() }, bigtiff=self.output_params.get("bigtiff", "NO")) logger.debug("single GTiff profile: %s", self._profile) self.in_memory = (self.in_memory if self.in_memory is False else height * width < IN_MEMORY_THRESHOLD) # set up rasterio if path_exists(self.path): if self.output_params["mode"] != "overwrite": raise MapcheteConfigError( "single GTiff file already exists, use overwrite mode to replace" ) else: logger.debug("remove existing file: %s", self.path) os.remove(self.path) # create output directory if necessary makedirs(os.path.dirname(self.path)) logger.debug("open output file: %s", self.path) self._ctx = ExitStack() # (1) use memfile if output is remote or COG if self.cog or path_is_remote(self.path): if self.in_memory: self._memfile = self._ctx.enter_context(MemoryFile()) self.dst = self._ctx.enter_context( self._memfile.open(**self._profile)) else: # in case output raster is too big, use tempfile on disk self._tempfile = self._ctx.enter_context(NamedTemporaryFile()) self.dst = self._ctx.enter_context( rasterio.open(self._tempfile.name, "w+", **self._profile)) else: self.dst = self._ctx.enter_context( rasterio.open(self.path, "w+", **self._profile))
def prepare(self, process_area=None, **kwargs): bounds = snap_bounds( bounds=Bounds(*process_area.intersection( box(*self.output_params["delimiters"] ["effective_bounds"])).bounds), pyramid=self.pyramid, zoom=self.zoom) if process_area else self.output_params[ "delimiters"]["effective_bounds"] height = math.ceil((bounds.top - bounds.bottom) / self.pyramid.pixel_x_size(self.zoom)) width = math.ceil((bounds.right - bounds.left) / self.pyramid.pixel_x_size(self.zoom)) logger.debug("output raster bounds: %s", bounds) logger.debug("output raster shape: %s, %s", height, width) self._profile = dict( GTIFF_DEFAULT_PROFILE, driver="GTiff", transform=Affine(self.pyramid.pixel_x_size(self.zoom), 0, bounds.left, 0, -self.pyramid.pixel_y_size(self.zoom), bounds.top), height=height, width=width, count=self.output_params["bands"], crs=self.pyramid.crs, **{ k: self.output_params.get(k, GTIFF_DEFAULT_PROFILE[k]) for k in GTIFF_DEFAULT_PROFILE.keys() }) logger.debug("single GTiff profile: %s", self._profile) if height * width > 20000 * 20000: raise ValueError("output GeoTIFF too big") # set up rasterio if path_exists(self.path): if self.output_params["mode"] != "overwrite": raise MapcheteConfigError( "single GTiff file already exists, use overwrite mode to replace" ) else: logger.debug("remove existing file: %s", self.path) os.remove(self.path) logger.debug("open output file: %s", self.path) self.rio_file = rasterio.open(self.path, "w+", **self._profile)
def test_create_mosaic_antimeridian(): """Create mosaic using tiles on opposing antimeridian sides.""" zoom = 5 row = 0 pixelbuffer = 5 tp = BufferedTilePyramid("geodetic", pixelbuffer=pixelbuffer) west = tp.tile(zoom, row, 0) east = tp.tile(zoom, row, tp.matrix_width(zoom) - 1) mosaic = create_mosaic([(west, np.ones(west.shape).astype("uint8")), (east, np.ones(east.shape).astype("uint8") * 2)]) assert isinstance(mosaic, ReferencedRaster) # Huge array gets initialized because the two tiles are on opposing sides of the # projection area. The below test should pass if the tiles are stitched together next # to each other. assert mosaic.data.shape == (1, west.height, west.width * 2 - 2 * pixelbuffer) assert mosaic.data[0][0][0] == 2 assert mosaic.data[0][0][-1] == 1 # If tiles from opposing sides from Antimeridian are mosaicked it will happen that the # output mosaic exceeds the CRS bounds (obviously). In such a case the mosaicking # function shall make sure that the larger part of the output mosaic shall be inside # the CRS bounds. # (1) mosaic crosses Antimeridian in the West, larger part is on Western hemisphere: tiles_ids = [ # Western hemisphere tiles (zoom, row, 0), (zoom, row, 1), # Eastern hemisphere tile (zoom, row, tp.matrix_width(zoom) - 1), ] tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape)) for tile_id in tiles_ids] mosaic = create_mosaic(tiles) control_bounds = Bounds( # Eastern tile has to be shifted -(360 - tp.tile(*tiles_ids[2]).left), tp.tile(*tiles_ids[2]).bottom, tp.tile(*tiles_ids[1]).right, tp.tile(*tiles_ids[1]).top, ) assert mosaic.bounds == control_bounds # (2) mosaic crosses Antimeridian in the West, larger part is on Eastern hemisphere: tiles_ids = [ # Western hemisphere tile (zoom, row, 0), # Eastern hemisphere tiles (zoom, row, tp.matrix_width(zoom) - 1), (zoom, row, tp.matrix_width(zoom) - 2), ] tiles = [(tp.tile(*tile_id), np.ones(tp.tile(*tile_id).shape)) for tile_id in tiles_ids] mosaic = create_mosaic(tiles) control_bounds = Bounds( tp.tile(*tiles_ids[2]).left, tp.tile(*tiles_ids[2]).bottom, # Western tile has to be shifted 360 + tp.tile(*tiles_ids[0]).right, tp.tile(*tiles_ids[0]).top, ) assert mosaic.bounds == control_bounds
def create_mosaic(tiles, nodata=0): """ Create a mosaic from tiles. Tiles must be connected (also possible over Antimeridian), otherwise strange things can happen! Parameters ---------- tiles : iterable an iterable containing tuples of a BufferedTile and an array nodata : integer or float raster nodata value to initialize the mosaic with (default: 0) Returns ------- mosaic : ReferencedRaster """ if isinstance(tiles, GeneratorType): tiles = list(tiles) elif not isinstance(tiles, list): raise TypeError("tiles must be either a list or generator") if not all([isinstance(pair, tuple) for pair in tiles]): raise TypeError("tiles items must be tuples") if not all([ all([isinstance(tile, BufferedTile), isinstance(data, np.ndarray)]) for tile, data in tiles ]): raise TypeError("tuples must be pairs of BufferedTile and array") if len(tiles) == 0: raise ValueError("tiles list is empty") logger.debug("create mosaic from %s tile(s)", len(tiles)) # quick return if there is just one tile if len(tiles) == 1: tile, data = tiles[0] return ReferencedRaster(data=data, affine=tile.affine, bounds=tile.bounds, crs=tile.crs) # assert all tiles have same properties pyramid, resolution, dtype = _get_tiles_properties(tiles) # just handle antimeridian on global pyramid types shift = _shift_required(tiles) logger.debug("shift: %s" % shift) # determine mosaic shape and reference m_left, m_bottom, m_right, m_top = None, None, None, None for tile, data in tiles: num_bands = data.shape[0] if data.ndim > 2 else 1 left, bottom, right, top = tile.bounds if shift: # shift by half of the grid width left += pyramid.x_size / 2 right += pyramid.x_size / 2 # if tile is now shifted outside pyramid bounds, move within if right > pyramid.right: right -= pyramid.x_size left -= pyramid.x_size m_left = min([left, m_left]) if m_left is not None else left m_bottom = min([bottom, m_bottom]) if m_bottom is not None else bottom m_right = max([right, m_right]) if m_right is not None else right m_top = max([top, m_top]) if m_top is not None else top height = int(round((m_top - m_bottom) / resolution)) width = int(round((m_right - m_left) / resolution)) # initialize empty mosaic mosaic = ma.MaskedArray(data=np.full((num_bands, height, width), dtype=dtype, fill_value=nodata), mask=np.ones((num_bands, height, width))) # create Affine affine = Affine(resolution, 0, m_left, 0, -resolution, m_top) # fill mosaic array with tile data for tile, data in tiles: data = prepare_array(data, nodata=nodata, dtype=dtype) t_left, t_bottom, t_right, t_top = tile.bounds if shift: t_left += pyramid.x_size / 2 t_right += pyramid.x_size / 2 # if tile is now shifted outside pyramid bounds, move within if t_right > pyramid.right: t_right -= pyramid.x_size t_left -= pyramid.x_size minrow, maxrow, mincol, maxcol = bounds_to_ranges( out_bounds=(t_left, t_bottom, t_right, t_top), in_affine=affine, in_shape=(height, width)) existing_data = mosaic[:, minrow:maxrow, mincol:maxcol] existing_mask = mosaic.mask[:, minrow:maxrow, mincol:maxcol] mosaic[:, minrow:maxrow, mincol:maxcol] = np.where(data.mask, existing_data, data) mosaic.mask[:, minrow:maxrow, mincol:maxcol] = np.where(data.mask, existing_mask, data.mask) if shift: # shift back output mosaic m_left -= pyramid.x_size / 2 m_right -= pyramid.x_size / 2 # if mosaic crosses Antimeridian, make sure the mosaic output bounds are based on the # hemisphere of the Antimeridian with the larger mosaic intersection if m_left < pyramid.left or m_right > pyramid.right: # mosaic crosses Antimeridian logger.debug("mosaic crosses Antimeridian") left_distance = abs(pyramid.left - m_left) right_distance = abs(pyramid.left - m_right) # per default, the mosaic is placed on the right side of the Antimeridian, so we # only need to move the bounds in case the larger part of the mosaic is on the # left side if left_distance > right_distance: m_left += pyramid.x_size m_right += pyramid.x_size logger.debug(Bounds(m_left, m_bottom, m_right, m_top)) return ReferencedRaster(data=mosaic, affine=Affine(resolution, 0, m_left, 0, -resolution, m_top), bounds=Bounds(m_left, m_bottom, m_right, m_top), crs=tile.crs)
def test_output_data(mp_tmpdir): """Check GeoTIFF as output data.""" output_params = dict( grid="geodetic", format="GeoTIFF", path=mp_tmpdir, pixelbuffer=0, metatiling=1, bands=1, dtype="int16", delimiters=dict(bounds=Bounds(-180.0, -90.0, 180.0, 90.0), effective_bounds=Bounds(-180.439453125, -90.0, 180.439453125, 90.0), zoom=[5], process_bounds=Bounds(-180.0, -90.0, 180.0, 90.0))) output = gtiff.OutputDataWriter(output_params) assert output.path == mp_tmpdir assert output.file_extension == ".tif" tp = BufferedTilePyramid("geodetic") tile = tp.tile(5, 5, 5) # get_path assert output.get_path(tile) == os.path.join( *[mp_tmpdir, "5", "5", "5" + ".tif"]) # prepare_path try: temp_dir = os.path.join(*[mp_tmpdir, "5", "5"]) output.prepare_path(tile) assert os.path.isdir(temp_dir) finally: shutil.rmtree(temp_dir, ignore_errors=True) # profile assert isinstance(output.profile(tile), dict) # write try: data = np.ones((1, ) + tile.shape) * 128 output.write(tile, data) # tiles_exist assert output.tiles_exist(tile) # read data = output.read(tile) assert isinstance(data, np.ndarray) assert not data[0].mask.any() finally: shutil.rmtree(mp_tmpdir, ignore_errors=True) # read empty try: data = output.read(tile) assert isinstance(data, np.ndarray) assert data[0].mask.all() finally: shutil.rmtree(mp_tmpdir, ignore_errors=True) # empty try: empty = output.empty(tile) assert isinstance(empty, ma.MaskedArray) assert not empty.any() finally: shutil.rmtree(mp_tmpdir, ignore_errors=True) # deflate with predictor try: # with pytest.deprecated_call(): output_params.update(compress="deflate", predictor=2) output = gtiff.OutputDataWriter(output_params) assert output.profile(tile)["compress"] == "deflate" assert output.profile(tile)["predictor"] == 2 finally: shutil.rmtree(mp_tmpdir, ignore_errors=True) # using deprecated "compression" property try: with pytest.deprecated_call(): output_params.update(compression="deflate", predictor=2) output = gtiff.OutputDataWriter(output_params) assert output.profile(tile)["compress"] == "deflate" assert output.profile(tile)["predictor"] == 2 finally: shutil.rmtree(mp_tmpdir, ignore_errors=True)