def _set_attributes(self, output_params):
self.path = output_params["path"]
self.file_extension = ".tif"
self.output_params = dict(
output_params,
nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"])
)
self._bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None
self._fs = fs_from_path(self.path, **self.output_params.get("fs_kwargs", {}))
def open(some_input, with_cache=False, **kwargs):
"""
Open a Mapchete process.
Parameters
----------
some_input : MapcheteConfig object, config dict, path to mapchete file or path to
TileDirectory
Mapchete process configuration
mode : string
* ``memory``: Generate process output on demand without reading
pre-existing data or writing new data.
* ``readonly``: Just read data without processing new data.
* ``continue``: (default) Don't overwrite existing output.
* ``overwrite``: Overwrite existing output.
zoom : list or integer
process zoom level or a pair of minimum and maximum zoom level
bounds : tuple
left, bottom, right, top process boundaries in output pyramid
single_input_file : string
single input file if supported by process
with_cache : bool
process output data cached in memory
Returns
-------
Mapchete
a Mapchete process object
"""
if isinstance(some_input, str) and not some_input.endswith(".mapchete"):
logger.debug("assuming TileDirectory")
metadata_json = os.path.join(some_input, "metadata.json")
fs = kwargs.get("fs", fs_from_path(metadata_json, **kwargs))
logger.debug("read metadata.json")
metadata = read_output_metadata(metadata_json, fs=fs)
config = dict(process=None,
input=None,
pyramid=metadata["pyramid"].to_dict(),
output=dict(
{
k: v
for k, v in metadata["driver"].items()
if k not in ["delimiters", "mode"]
},
path=some_input,
fs_kwargs=kwargs),
config_dir=os.getcwd(),
zoom_levels=kwargs.get("zoom"))
kwargs.update(mode="readonly")
return Mapchete(MapcheteConfig(config, **kwargs))
else:
return Mapchete(MapcheteConfig(some_input, **kwargs),
with_cache=with_cache)
def read(self, output_tile, **kwargs):
"""
Read existing process output.
Parameters
----------
output_tile : ``BufferedTile``
must be member of output ``TilePyramid``
Returns
-------
process output : list
"""
import geobuf
path = self.get_path(output_tile)
try:
with fs_from_path(path).open(path, "rb") as src:
return geobuf.decode(src.read()).get("features", [])
except FileNotFoundError:
return self.empty(output_tile)
def cp(input_,
output,
zoom=None,
area=None,
area_crs=None,
bounds=None,
bounds_crs=None,
wkt_geometry=None,
overwrite=False,
verbose=False,
no_pbar=False,
debug=False,
logfile=None,
multi=None,
username=None,
password=None):
"""Copy TileDirectory."""
if zoom is None: # pragma: no cover
raise click.UsageError("zoom level(s) required")
src_fs = fs_from_path(input_, username=username, password=password)
dst_fs = fs_from_path(output, username=username, password=password)
# open source tile directory
with mapchete.open(input_,
zoom=zoom,
area=area,
area_crs=area_crs,
bounds=bounds,
bounds_crs=bounds_crs,
wkt_geometry=wkt_geometry,
username=username,
password=password,
fs=src_fs) as src_mp:
tp = src_mp.config.output_pyramid
# copy metadata to destination if necessary
src_metadata = os.path.join(input_, "metadata.json")
dst_metadata = os.path.join(output, "metadata.json")
if not dst_fs.exists(dst_metadata):
logger.debug(f"copy {src_metadata} to {dst_metadata}")
_copy(src_fs, src_metadata, dst_fs, dst_metadata)
with mapchete.open(output,
zoom=zoom,
area=area,
area_crs=area_crs,
bounds=bounds,
bounds_crs=bounds_crs,
wkt_geometry=wkt_geometry,
username=username,
password=password,
fs=dst_fs) as dst_mp:
for z in range(min(zoom), max(zoom) + 1):
click.echo(f"copy zoom {z}...")
# materialize all tiles
aoi_geom = src_mp.config.area_at_zoom(z)
tiles = [
t for t in tp.tiles_from_geom(aoi_geom, z)
# this is required to omit tiles touching the config area
if aoi_geom.intersection(t.bbox).area
]
# check which source tiles exist
logger.debug("looking for existing source tiles...")
src_tiles_exist = {
tile: exists
for tile, exists in tiles_exist(
config=src_mp.config, output_tiles=tiles, multi=multi)
}
logger.debug("looking for existing destination tiles...")
# chech which destination tiles exist
dst_tiles_exist = {
tile: exists
for tile, exists in tiles_exist(
config=dst_mp.config, output_tiles=tiles, multi=multi)
}
# copy
for tile in tqdm.tqdm(tiles,
unit="tile",
disable=debug or no_pbar):
src_path = src_mp.config.output_reader.get_path(tile)
# only copy if source tile exists
if src_tiles_exist[tile]:
# skip if destination tile exists and overwrite is deactivated
if dst_tiles_exist[tile] and not overwrite:
logger.debug(f"{tile}: destination tile exists")
continue
# copy from source to target
else:
dst_path = dst_mp.config.output_reader.get_path(
tile)
logger.debug(
f"{tile}: copy {src_path} to {dst_path}")
_copy(src_fs, src_path, dst_fs, dst_path)
else:
logger.debug(
f"{tile}: source tile ({src_path}) does not exist")
def rm_(
input_,
zoom=None,
area=None,
area_crs=None,
bounds=None,
bounds_crs=None,
wkt_geometry=None,
multi=None,
verbose=False,
no_pbar=False,
debug=False,
logfile=None,
force=None,
):
"""Copy TileDirectory."""
if zoom is None: # pragma: no cover
raise click.UsageError("zoom level(s) required")
src_fs = fs_from_path(input_)
# open source tile directory
with mapchete.open(
input_,
zoom=zoom,
area=area,
area_crs=area_crs,
bounds=bounds,
bounds_crs=bounds_crs,
wkt_geometry=wkt_geometry,
fs=src_fs,
mode="readonly"
) as src_mp:
tp = src_mp.config.output_pyramid
tiles = {}
for z in range(min(zoom), max(zoom) + 1):
tiles[z] = []
# check which source tiles exist
logger.debug(f"looking for existing source tiles in zoom {z}...")
for tile, exists in tiles_exist(
config=src_mp.config,
output_tiles=[
t for t in tp.tiles_from_geom(src_mp.config.area_at_zoom(z), z)
# this is required to omit tiles touching the config area
if src_mp.config.area_at_zoom(z).intersection(t.bbox).area
],
multi=multi
):
if exists:
tiles[z].append(tile)
total_tiles = sum([len(v) for v in tiles.values()])
if total_tiles:
if force or click.confirm(
f"Do you want to delete {total_tiles} tiles?", abort=True
):
# remove
rm(
[
src_mp.config.output_reader.get_path(tile)
for zoom_tiles in tiles.values()
for tile in zoom_tiles
],
fs=src_fs
)
else: # pragma: no cover
click.echo("No tiles found to delete.")