def _set_attributes(self, output_params): self.path = output_params["path"] self.file_extension = ".tif" self.output_params = dict( output_params, nodata=output_params.get("nodata", GTIFF_DEFAULT_PROFILE["nodata"]) ) self._bucket = self.path.split("/")[2] if self.path.startswith("s3://") else None self._fs = fs_from_path(self.path, **self.output_params.get("fs_kwargs", {}))
def open(some_input, with_cache=False, **kwargs): """ Open a Mapchete process. Parameters ---------- some_input : MapcheteConfig object, config dict, path to mapchete file or path to TileDirectory Mapchete process configuration mode : string * ``memory``: Generate process output on demand without reading pre-existing data or writing new data. * ``readonly``: Just read data without processing new data. * ``continue``: (default) Don't overwrite existing output. * ``overwrite``: Overwrite existing output. zoom : list or integer process zoom level or a pair of minimum and maximum zoom level bounds : tuple left, bottom, right, top process boundaries in output pyramid single_input_file : string single input file if supported by process with_cache : bool process output data cached in memory Returns ------- Mapchete a Mapchete process object """ if isinstance(some_input, str) and not some_input.endswith(".mapchete"): logger.debug("assuming TileDirectory") metadata_json = os.path.join(some_input, "metadata.json") fs = kwargs.get("fs", fs_from_path(metadata_json, **kwargs)) logger.debug("read metadata.json") metadata = read_output_metadata(metadata_json, fs=fs) config = dict(process=None, input=None, pyramid=metadata["pyramid"].to_dict(), output=dict( { k: v for k, v in metadata["driver"].items() if k not in ["delimiters", "mode"] }, path=some_input, fs_kwargs=kwargs), config_dir=os.getcwd(), zoom_levels=kwargs.get("zoom")) kwargs.update(mode="readonly") return Mapchete(MapcheteConfig(config, **kwargs)) else: return Mapchete(MapcheteConfig(some_input, **kwargs), with_cache=with_cache)
def read(self, output_tile, **kwargs): """ Read existing process output. Parameters ---------- output_tile : ``BufferedTile`` must be member of output ``TilePyramid`` Returns ------- process output : list """ import geobuf path = self.get_path(output_tile) try: with fs_from_path(path).open(path, "rb") as src: return geobuf.decode(src.read()).get("features", []) except FileNotFoundError: return self.empty(output_tile)
def cp(input_, output, zoom=None, area=None, area_crs=None, bounds=None, bounds_crs=None, wkt_geometry=None, overwrite=False, verbose=False, no_pbar=False, debug=False, logfile=None, multi=None, username=None, password=None): """Copy TileDirectory.""" if zoom is None: # pragma: no cover raise click.UsageError("zoom level(s) required") src_fs = fs_from_path(input_, username=username, password=password) dst_fs = fs_from_path(output, username=username, password=password) # open source tile directory with mapchete.open(input_, zoom=zoom, area=area, area_crs=area_crs, bounds=bounds, bounds_crs=bounds_crs, wkt_geometry=wkt_geometry, username=username, password=password, fs=src_fs) as src_mp: tp = src_mp.config.output_pyramid # copy metadata to destination if necessary src_metadata = os.path.join(input_, "metadata.json") dst_metadata = os.path.join(output, "metadata.json") if not dst_fs.exists(dst_metadata): logger.debug(f"copy {src_metadata} to {dst_metadata}") _copy(src_fs, src_metadata, dst_fs, dst_metadata) with mapchete.open(output, zoom=zoom, area=area, area_crs=area_crs, bounds=bounds, bounds_crs=bounds_crs, wkt_geometry=wkt_geometry, username=username, password=password, fs=dst_fs) as dst_mp: for z in range(min(zoom), max(zoom) + 1): click.echo(f"copy zoom {z}...") # materialize all tiles aoi_geom = src_mp.config.area_at_zoom(z) tiles = [ t for t in tp.tiles_from_geom(aoi_geom, z) # this is required to omit tiles touching the config area if aoi_geom.intersection(t.bbox).area ] # check which source tiles exist logger.debug("looking for existing source tiles...") src_tiles_exist = { tile: exists for tile, exists in tiles_exist( config=src_mp.config, output_tiles=tiles, multi=multi) } logger.debug("looking for existing destination tiles...") # chech which destination tiles exist dst_tiles_exist = { tile: exists for tile, exists in tiles_exist( config=dst_mp.config, output_tiles=tiles, multi=multi) } # copy for tile in tqdm.tqdm(tiles, unit="tile", disable=debug or no_pbar): src_path = src_mp.config.output_reader.get_path(tile) # only copy if source tile exists if src_tiles_exist[tile]: # skip if destination tile exists and overwrite is deactivated if dst_tiles_exist[tile] and not overwrite: logger.debug(f"{tile}: destination tile exists") continue # copy from source to target else: dst_path = dst_mp.config.output_reader.get_path( tile) logger.debug( f"{tile}: copy {src_path} to {dst_path}") _copy(src_fs, src_path, dst_fs, dst_path) else: logger.debug( f"{tile}: source tile ({src_path}) does not exist")
def rm_( input_, zoom=None, area=None, area_crs=None, bounds=None, bounds_crs=None, wkt_geometry=None, multi=None, verbose=False, no_pbar=False, debug=False, logfile=None, force=None, ): """Copy TileDirectory.""" if zoom is None: # pragma: no cover raise click.UsageError("zoom level(s) required") src_fs = fs_from_path(input_) # open source tile directory with mapchete.open( input_, zoom=zoom, area=area, area_crs=area_crs, bounds=bounds, bounds_crs=bounds_crs, wkt_geometry=wkt_geometry, fs=src_fs, mode="readonly" ) as src_mp: tp = src_mp.config.output_pyramid tiles = {} for z in range(min(zoom), max(zoom) + 1): tiles[z] = [] # check which source tiles exist logger.debug(f"looking for existing source tiles in zoom {z}...") for tile, exists in tiles_exist( config=src_mp.config, output_tiles=[ t for t in tp.tiles_from_geom(src_mp.config.area_at_zoom(z), z) # this is required to omit tiles touching the config area if src_mp.config.area_at_zoom(z).intersection(t.bbox).area ], multi=multi ): if exists: tiles[z].append(tile) total_tiles = sum([len(v) for v in tiles.values()]) if total_tiles: if force or click.confirm( f"Do you want to delete {total_tiles} tiles?", abort=True ): # remove rm( [ src_mp.config.output_reader.get_path(tile) for zoom_tiles in tiles.values() for tile in zoom_tiles ], fs=src_fs ) else: # pragma: no cover click.echo("No tiles found to delete.")