def _get_process_area(self,
area=None,
bounds=None,
area_fallback=None,
bounds_fallback=None,
area_crs=None,
bounds_crs=None):
"""
Determine process area by combining configuration with instantiation arguments.
In the configuration the process area can be provided by using the (1) ``area``
option, (2) ``bounds`` option or (3) a combination of both.
(1) If only ``area`` is provided, output shall be the area geometry
(2) If only ``bounds`` is provided, output shall be box(*self.bounds)
(3) If both are provided, output shall be the intersection between ``area`` and
``bounds``
The area parameter can be provided in multiple variations, see _guess_geometry().
"""
try:
dst_crs = self.process_pyramid.crs
if bounds is None and area is None:
return area_fallback
elif bounds is None:
area, crs = _guess_geometry(area, base_dir=self.config_dir)
# in case vector file has no CRS use manually provided CRS
area_crs = crs or area_crs
return reproject_geometry(area,
src_crs=area_crs or dst_crs,
dst_crs=dst_crs)
elif area is None:
return reproject_geometry(box(*validate_bounds(bounds)),
src_crs=bounds_crs or dst_crs,
dst_crs=dst_crs)
else:
area, crs = _guess_geometry(area, base_dir=self.config_dir)
# in case vector file has no CRS use manually provided CRS
area_crs = crs or area_crs
bounds = validate_bounds(bounds)
# reproject area and bounds to process CRS and return intersection
return reproject_geometry(
area, src_crs=area_crs or dst_crs,
dst_crs=dst_crs).intersection(
reproject_geometry(box(*validate_bounds(bounds)),
src_crs=bounds_crs or dst_crs,
dst_crs=dst_crs), )
except Exception as e:
raise MapcheteConfigError(e)
def bounds_from_opts(wkt_geometry=None,
point=None,
point_crs=None,
zoom=None,
bounds=None,
bounds_crs=None,
raw_conf=None):
"""
Return process bounds depending on given inputs.
Parameters
----------
wkt_geometry : string
WKT geometry used to generate bounds.
point : iterable
x and y coordinates of point whose corresponding process tile bounds shall be
returned.
point_crs : str or CRS
CRS of point (default: process pyramid CRS)
zoom : int
Mandatory zoom level if point is provided.
bounds : iterable
Bounding coordinates to be used
bounds_crs : str or CRS
CRS of bounds (default: process pyramid CRS)
raw_conf : dict
Raw mapchete configuration as dictionary.
Returns
-------
BufferedTilePyramid
"""
if wkt_geometry:
return Bounds(*wkt.loads(wkt_geometry).bounds)
elif point:
x, y = point
tp = raw_conf_process_pyramid(raw_conf)
if point_crs:
reproj = reproject_geometry(Point(x, y),
src_crs=point_crs,
dst_crs=tp.crs)
x = reproj.x
y = reproj.y
zoom_levels = get_zoom_levels(
process_zoom_levels=raw_conf["zoom_levels"], init_zoom_levels=zoom)
return Bounds(*tp.tile_from_xy(x, y, max(zoom_levels)).bounds)
elif bounds:
bounds = validate_bounds(bounds)
if bounds_crs:
tp = raw_conf_process_pyramid(raw_conf)
bounds = Bounds(*reproject_geometry(
box(*bounds), src_crs=bounds_crs, dst_crs=tp.crs).bounds)
return bounds
else:
return
def bbox(self, out_crs=None):
"""
Return data bounding box.
Parameters
----------
out_crs : ``rasterio.crs.CRS``
rasterio CRS object (default: CRS of process pyramid)
Returns
-------
bounding box : geometry
Shapely geometry object
"""
out_crs = self.pyramid.crs if out_crs is None else out_crs
with rasterio.open(self.path) as inp:
inp_crs = inp.crs
out_bbox = bbox = box(*inp.bounds)
# If soucre and target CRSes differ, segmentize and reproject
if inp_crs != out_crs:
# estimate segmentize value (raster pixel size * tile size)
# and get reprojected bounding box
return reproject_geometry(segmentize_geometry(
bbox, inp.transform[0] * self.pyramid.tile_size),
src_crs=inp_crs,
dst_crs=out_crs)
else:
return out_bbox
def nodatamask(self):
"""SAFE file nodata mask as iterable list of geometries."""
return [
reproject_geometry(granule["nodatamask"],
src_crs=CRS.from_epsg(4326),
dst_crs=self.crs)
for granule in self.s2metadata["granules"]
]
def bbox(self, out_crs=None):
"""Return data bounding box."""
out_crs = self.pyramid.crs if out_crs is None else out_crs
inp_crs = CRS().from_epsg(4326)
if inp_crs != out_crs:
return reproject_geometry(self.s2metadata["footprint"],
src_crs=inp_crs,
dst_crs=out_crs)
else:
return self.s2metadata["footprint"]
def test_reproject_geometry():
"""Reproject geometry."""
with fiona.open(os.path.join(TESTDATA_DIR, "landpoly.geojson"),
"r") as src:
for feature in src:
# WGS84 to Spherical Mercator
out_geom = vector.reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(3857))
assert out_geom.is_valid
# WGS84 to LAEA
out_geom = vector.reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(3035))
assert out_geom.is_valid
# WGS84 to WGS84
out_geom = vector.reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(4326))
assert out_geom.is_valid
# WGS84 bounds to Spherical Mercator
big_box = box(-180, -90, 180, 90)
vector.reproject_geometry(big_box,
CRS().from_epsg(4326),
CRS().from_epsg(3857))
# WGS84 bounds to Spherical Mercator raising clip error
try:
vector.reproject_geometry(big_box,
CRS().from_epsg(4326),
CRS().from_epsg(3857),
error_on_clip=True)
raise Exception()
except RuntimeError:
pass
# empty geometry
assert vector.reproject_geometry(Polygon(),
CRS().from_epsg(4326),
CRS().from_epsg(3857)).is_empty
assert vector.reproject_geometry(Polygon(),
CRS().from_epsg(4326),
CRS().from_epsg(4326)).is_empty
def _get_tiles_paths(
self,
tile_directory_zoom=None,
fallback_to_higher_zoom=False,
matching_method="gdal",
matching_precision=8,
matching_max_zoom=None,
):
# determine tile bounds in TileDirectory CRS
td_bounds = reproject_geometry(
self.tile.bbox,
src_crs=self.tile.tp.crs,
dst_crs=self._td_pyramid.crs
).bounds
# find target zoom level
if tile_directory_zoom is None:
zoom = tile_to_zoom_level(
self.tile,
dst_pyramid=self._td_pyramid,
matching_method=matching_method,
precision=matching_precision
)
if matching_max_zoom is not None:
zoom = min([zoom, matching_max_zoom])
else:
zoom = tile_directory_zoom
if fallback_to_higher_zoom:
tiles_paths = []
# check if tiles exist otherwise try higher zoom level
while len(tiles_paths) == 0 and zoom >= 0:
tiles_paths = _get_tiles_paths(
basepath=self._basepath,
ext=self._ext,
pyramid=self._td_pyramid,
bounds=td_bounds,
zoom=zoom
)
logger.debug("%s existing tiles found at zoom %s", len(tiles_paths), zoom)
zoom -= 1
else:
tiles_paths = _get_tiles_paths(
basepath=self._basepath,
ext=self._ext,
pyramid=self._td_pyramid,
bounds=td_bounds,
zoom=zoom
)
logger.debug("%s existing tiles found at zoom %s", len(tiles_paths), zoom)
return tiles_paths
def _get_warped_array(
input_file=None, band_idx=None, dst_bounds=None, dst_shape=None,
dst_affine=None, dst_crs=None, resampling="nearest"
):
"""Extract a numpy array from a raster file."""
LOGGER.debug("read array using rasterio")
with rasterio.open(input_file, "r") as src:
if dst_crs == src.crs:
src_left, src_bottom, src_right, src_top = dst_bounds
else:
# Return empty array if destination bounds don't intersect with
# file bounds.
file_bbox = box(*src.bounds)
tile_bbox = reproject_geometry(
box(*dst_bounds), src_crs=dst_crs, dst_crs=src.crs)
if not file_bbox.intersects(tile_bbox):
LOGGER.debug("file bounding box does not intersect with tile")
return ma.MaskedArray(
data=ma.zeros(dst_shape, dtype=src.profile["dtype"]),
mask=ma.ones(dst_shape), fill_value=src.nodata)
# Reproject tile bounds to source file SRS.
src_left, src_bottom, src_right, src_top = transform_bounds(
dst_crs, src.crs, *dst_bounds, densify_pts=21)
if float('Inf') in (src_left, src_bottom, src_right, src_top):
# Maybe not the best way to deal with it, but if bounding box
# cannot be translated, it is assumed that data is emtpy
LOGGER.debug("tile seems to be outside of input CRS bounds")
return ma.MaskedArray(
data=ma.zeros(dst_shape, dtype=src.profile["dtype"]),
mask=ma.ones(dst_shape), fill_value=src.nodata)
# Read data window.
window = src.window(
src_left, src_bottom, src_right, src_top, boundless=True)
start = time.time()
src_band = src.read(band_idx, window=window, boundless=True)
LOGGER.debug("window read in %ss" % round(time.time() - start, 3))
# Quick fix because None nodata is not allowed.
nodataval = 0 if not src.nodata else src.nodata
# Prepare reprojected array.
dst_band = np.empty(dst_shape, src.dtypes[band_idx-1])
# Run rasterio's reproject().
start = time.time()
reproject(
src_band, dst_band, src_transform=src.window_transform(window),
src_crs=src.crs, src_nodata=nodataval, dst_transform=dst_affine,
dst_crs=dst_crs, dst_nodata=nodataval,
resampling=RESAMPLING_METHODS[resampling])
LOGGER.debug(
"window reprojected in %ss" % round(time.time() - start, 3))
return ma.MaskedArray(dst_band, mask=dst_band == nodataval)
def get_best_zoom_level(input_file, tile_pyramid_type):
"""
Determine the best base zoom level for a raster.
"Best" means the maximum zoom level where no oversampling has to be done.
Parameters
----------
input_file : path to raster file
tile_pyramid_type : ``TilePyramid`` projection (``geodetic`` or
``mercator``)
Returns
-------
zoom : integer
"""
tile_pyramid = TilePyramid(tile_pyramid_type)
with rasterio.open(input_file, "r") as src:
if not src.crs.is_valid:
raise IOError("CRS could not be read from %s" % input_file)
bbox = box(
src.bounds.left, src.bounds.bottom, src.bounds.right,
src.bounds.top)
if src.crs != tile_pyramid.crs:
segmentize = raster_file._get_segmentize_value(
input_file, tile_pyramid)
ogr_bbox = ogr.CreateGeometryFromWkb(bbox.wkb)
ogr_bbox.Segmentize(segmentize)
segmentized_bbox = loads(ogr_bbox.ExportToWkt())
bbox = segmentized_bbox
xmin, ymin, xmax, ymax = reproject_geometry(
bbox, src_crs=src.crs, dst_crs=tile_pyramid.crs).bounds
else:
xmin, ymin, xmax, ymax = bbox.bounds
x_dif = xmax - xmin
y_dif = ymax - ymin
size = float(src.width + src.height)
avg_resolution = (
(x_dif / float(src.width)) * (float(src.width) / size) +
(y_dif / float(src.height)) * (float(src.height) / size)
)
for zoom in range(0, 25):
if tile_pyramid.pixel_x_size(zoom) <= avg_resolution:
return zoom-1
raise ValueError("no fitting zoom level found")
def bbox(self, out_crs=None):
"""
Return data bounding box.
Parameters
----------
out_crs : ``rasterio.crs.CRS``
rasterio CRS object (default: CRS of process pyramid)
Returns
-------
bounding box : geometry
Shapely geometry object
"""
return reproject_geometry(
self.process.config.area_at_zoom(),
src_crs=self.process.config.process_pyramid.crs,
dst_crs=self.pyramid.crs if out_crs is None else out_crs)
def bbox(self, out_crs=None):
"""
Return data bounding box.
Parameters
----------
out_crs : ``rasterio.crs.CRS``
rasterio CRS object (default: CRS of process pyramid)
Returns
-------
bounding box : geometry
Shapely geometry object
"""
out_crs = self.pyramid.crs if out_crs is None else out_crs
with fiona.open(self.path) as inp:
inp_crs = CRS(inp.crs)
bbox = box(*inp.bounds)
# TODO find a way to get a good segmentize value in bbox source CRS
return reproject_geometry(bbox, src_crs=inp_crs, dst_crs=out_crs)
def bbox(self, out_crs=None):
"""
Return data bounding box.
Parameters
----------
out_crs : ``rasterio.crs.CRS``
rasterio CRS object (default: CRS of process pyramid)
Returns
-------
bounding box : geometry
Shapely geometry object
"""
if out_crs is None:
out_crs = self.pyramid.crs
if str(out_crs) not in self._bbox_cache:
with rasterio.open(self.path) as inp:
inp_crs = inp.crs
try:
assert inp_crs.is_valid
except AssertionError:
raise IOError("CRS could not be read from %s" % self.path)
out_bbox = bbox = box(
inp.bounds.left, inp.bounds.bottom, inp.bounds.right,
inp.bounds.top
)
# If soucre and target CRSes differ, segmentize and reproject
if inp_crs != out_crs:
# estimate segmentize value (raster pixel size * tile size)
segmentize = inp.transform[0] * self.pyramid.tile_size
ogr_bbox = ogr.CreateGeometryFromWkb(bbox.wkb)
ogr_bbox.Segmentize(segmentize)
self._bbox_cache[str(out_crs)] = reproject_geometry(
loads(ogr_bbox.ExportToWkt()),
src_crs=inp_crs, dst_crs=out_crs
)
else:
self._bbox_cache[str(out_crs)] = out_bbox
return self._bbox_cache[str(out_crs)]
def bbox(self, out_crs=None):
"""
Return data bounding box.
Parameters
----------
out_crs : ``rasterio.crs.CRS``
rasterio CRS object (default: CRS of process pyramid)
Returns
-------
bounding box : geometry
Shapely geometry object
"""
if out_crs is None:
out_crs = self.pyramid.crs
if str(out_crs) not in self._bbox_cache:
self._bbox_cache[str(out_crs)] = reproject_geometry(
self.process.config.process_area(),
src_crs=self.process.config.crs,
dst_crs=out_crs)
return self._bbox_cache[str(out_crs)]
def get_best_zoom_level(input_file, tile_pyramid_type):
"""
Determine the best base zoom level for a raster.
"Best" means the maximum zoom level where no oversampling has to be done.
Parameters
----------
input_file : path to raster file
tile_pyramid_type : ``TilePyramid`` projection (``geodetic`` or
``mercator``)
Returns
-------
zoom : integer
"""
tile_pyramid = TilePyramid(tile_pyramid_type)
with rasterio.open(input_file, "r") as src:
xmin, ymin, xmax, ymax = reproject_geometry(
segmentize_geometry(
box(src.bounds.left, src.bounds.bottom, src.bounds.right,
src.bounds.top),
get_segmentize_value(input_file, tile_pyramid)),
src_crs=src.crs,
dst_crs=tile_pyramid.crs).bounds
x_dif = xmax - xmin
y_dif = ymax - ymin
size = float(src.width + src.height)
avg_resolution = ((x_dif / float(src.width)) *
(float(src.width) / size) +
(y_dif / float(src.height)) *
(float(src.height) / size))
for zoom in range(0, 40):
if tile_pyramid.pixel_x_size(zoom) <= avg_resolution:
return zoom - 1
def __init__(self, input_params, **kwargs):
"""Initialize."""
self.path = input_params["path"]
self.pyramid = input_params["pyramid"]
self.pixelbuffer = input_params["pixelbuffer"]
self.crs = self.pyramid.crs
self.srid = self.pyramid.srid
with s2reader.open(self.path) as s2dataset:
self.s2metadata = {
"path":
s2dataset.path,
"footprint":
s2dataset.footprint,
"granules": [{
"id":
granule.granule_identifier,
"datastrip_id":
granule.datastrip_identifier,
"srid":
granule.srid,
"footprint":
reproject_geometry(granule.footprint,
src_crs=CRS.from_epsg(4326),
dst_crs=self.crs),
"nodatamask":
granule.nodata_mask,
"cloudmask":
granule.cloudmask,
"band_path": {
index: granule.band_path(_id,
for_gdal=True,
absolute=True)
for index, _id in zip(range(1, 14), BAND_IDS)
}
} for granule in s2dataset.granules]
}
def test_reproject_geometry(landpoly):
"""Reproject geometry."""
with fiona.open(landpoly, "r") as src:
for feature in src:
# WGS84 to Spherical Mercator
out_geom = reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(3857))
assert out_geom.is_valid
# WGS84 to LAEA
out_geom = reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(3035))
assert out_geom.is_valid
# WGS84 to WGS84
out_geom = reproject_geometry(shape(feature["geometry"]),
CRS(src.crs),
CRS().from_epsg(4326))
assert out_geom.is_valid
# WGS84 bounds to Spherical Mercator
big_box = box(-180, -90, 180, 90)
reproject_geometry(big_box, CRS().from_epsg(4326), CRS().from_epsg(3857))
# WGS84 bounds to Spherical Mercator raising clip error
with pytest.raises(RuntimeError):
reproject_geometry(big_box,
CRS().from_epsg(4326),
CRS().from_epsg(3857),
error_on_clip=True)
# empty geometry
assert reproject_geometry(Polygon(),
CRS().from_epsg(4326),
CRS().from_epsg(3857)).is_empty
assert reproject_geometry(Polygon(),
CRS().from_epsg(4326),
CRS().from_epsg(4326)).is_empty
# CRS parameter
big_box = box(-180, -90, 180, 90)
assert reproject_geometry(big_box, 4326, 3857) == reproject_geometry(
big_box, "4326", "3857")
with pytest.raises(TypeError):
reproject_geometry(big_box, 1.0, 1.0)
def _clip_bbox(clip_geometry, dst_crs=None):
with fiona.open(clip_geometry) as src:
return reproject_geometry(box(*src.bounds),
src_crs=src.crs,
dst_crs=dst_crs)
def convert(input_,
output,
zoom=None,
bounds=None,
bounds_crs=None,
area=None,
area_crs=None,
point=None,
point_crs=None,
wkt_geometry=None,
clip_geometry=None,
bidx=None,
output_pyramid=None,
output_metatiling=None,
output_format=None,
output_dtype=None,
output_geometry_type=None,
creation_options=None,
scale_ratio=None,
scale_offset=None,
resampling_method=None,
overviews=False,
overviews_resampling_method=None,
cog=False,
overwrite=False,
logfile=None,
verbose=False,
no_pbar=False,
debug=False,
multi=None,
vrt=False,
idx_out_dir=None):
try:
input_info = _get_input_info(input_)
output_info = _get_output_info(output)
except Exception as e:
raise click.BadArgumentUsage(e)
# collect mapchete configuration
mapchete_config = dict(
process="mapchete.processes.convert",
input=dict(inp=input_, clip=clip_geometry),
pyramid=(dict(grid=output_pyramid,
metatiling=(output_metatiling or
(input_info["pyramid"].get("metatiling", 1)
if input_info["pyramid"] else 1)),
pixelbuffer=(input_info["pyramid"].get("pixelbuffer", 0)
if input_info["pyramid"] else 0))
if output_pyramid else input_info["pyramid"]),
output=dict(
{
k: v
for k, v in input_info["output_params"].items()
if k not in ["delimiters", "bounds", "mode"]
},
path=output,
format=(output_format or output_info["driver"]
or input_info["output_params"]["format"]),
dtype=output_dtype or input_info["output_params"].get("dtype"),
**creation_options,
**dict(overviews=True,
overviews_resampling=overviews_resampling_method)
if overviews else dict(),
),
config_dir=os.getcwd(),
zoom_levels=zoom or input_info["zoom_levels"],
scale_ratio=scale_ratio,
scale_offset=scale_offset,
resampling=resampling_method,
band_indexes=bidx)
# assert all required information is there
if mapchete_config["output"]["format"] is None:
# this happens if input file is e.g. JPEG2000 and output is a tile directory
raise click.BadOptionUsage("output-format", "Output format required.")
if mapchete_config["output"]["format"] == "GTiff":
mapchete_config["output"].update(cog=cog)
output_type = OUTPUT_FORMATS[mapchete_config["output"]
["format"]]["data_type"]
if bidx is not None:
mapchete_config["output"].update(bands=len(bidx))
if mapchete_config["pyramid"] is None:
raise click.BadOptionUsage("output-pyramid",
"Output pyramid required.")
elif mapchete_config["zoom_levels"] is None:
try:
mapchete_config.update(zoom_levels=dict(
min=0,
max=get_best_zoom_level(input_, mapchete_config["pyramid"]
["grid"])))
except:
raise click.BadOptionUsage("zoom", "Zoom levels required.")
elif input_info["input_type"] != output_type:
raise click.BadArgumentUsage(
"Output format type (%s) is incompatible with input format (%s)." %
(output_type, input_info["input_type"]))
if output_metatiling:
mapchete_config["output"].update(metatiling=output_metatiling)
if input_info["output_params"].get("schema") and output_geometry_type:
mapchete_config["output"]["schema"].update(
geometry=output_geometry_type)
# determine process bounds
out_pyramid = BufferedTilePyramid.from_dict(mapchete_config["pyramid"])
inp_bounds = (bounds or reproject_geometry(box(*input_info["bounds"]),
src_crs=input_info["crs"],
dst_crs=out_pyramid.crs).bounds
if input_info["bounds"] else out_pyramid.bounds)
# if clip-geometry is available, intersect determined bounds with clip bounds
if clip_geometry:
clip_intersection = _clip_bbox(clip_geometry,
dst_crs=out_pyramid.crs).intersection(
box(*inp_bounds))
if clip_intersection.is_empty:
click.echo(
"Process area is empty: clip bounds don't intersect with input bounds."
)
return
# add process bounds and output type
mapchete_config.update(
bounds=(clip_intersection.bounds if clip_geometry else inp_bounds),
bounds_crs=bounds_crs,
clip_to_output_dtype=mapchete_config["output"].get("dtype", None))
logger.debug("temporary config generated: %s", pformat(mapchete_config))
utils._process_area(debug=debug,
mapchete_config=mapchete_config,
mode="overwrite" if overwrite else "continue",
zoom=zoom,
wkt_geometry=wkt_geometry,
point=point,
point_crs=point_crs,
bounds=bounds,
bounds_crs=bounds_crs,
area=area,
area_crs=area_crs,
multi=multi or cpu_count(),
verbose_dst=open(os.devnull, 'w')
if debug or not verbose else sys.stdout,
no_pbar=no_pbar,
vrt=vrt,
idx_out_dir=idx_out_dir)
