def raster_convexhull(file_path: str, epsg='EPSG:4326', nodata: int=0) -> dict: """Get footprint from any raster. Args: file_path (str): image file epsg (str): geometry EPSG nodata (int): Custom nodata value See: https://rasterio.readthedocs.io/en/latest/topics/masks.html """ with rasterio_open(file_path) as dataset: # Read raster data, masking nodata values data = dataset.read(1) data[data != nodata] = 1 data[data == nodata] = 0 # Create mask, which 1 represents valid data and 0 nodata mask = data.astype(uint8) geoms = [] res = {'val': []} for geom, val in shapes(mask, mask=mask, transform=dataset.transform): geom = transform_geom(dataset.crs, epsg, geom, precision=6) res['val'].append(val) geoms.append(shape(geom)) if len(geoms) == 1: return geoms[0].convex_hull multi_polygons = MultiPolygon(geoms) return multi_polygons.convex_hull
def merge(cls, list_of_raster, bounds=None): """ Merge several RasterMap instances :param list_of_raster: list of RasterMap class instance(s) :param bounds: bounds of the output RasterMap :return: """ # TODO: use gdal merge method list_of_datasets = [] for raster_map in list_of_raster: list_of_datasets.append(rasterio_open(raster_map.raster_file, 'r')) # Merge using rasterio array, transform = rasterio_merge(list_of_datasets, bounds=bounds) with RasterTempFile() as file: with rasterio_open(file, 'w', driver="GTiff", height=array.shape[1], width=array.shape[2], count=1, dtype=array.dtype, crs=list_of_raster[0].crs, transform=transform) as out_dst: out_dst.write(array.squeeze(), 1) return cls(file, no_data_value=list_of_raster[0].no_data_value)
def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e, n) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: try: w, s, e, n = mercantile.xy_bounds(tile) # inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45 warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, args.ts, args.ts), width=math.ceil((e - w) / transform.a), height=math.ceil((s - n) / transform.e), ) data = warp_vrt.read( out_shape=(len(raster.indexes), args.ts, args.ts), window=warp_vrt.window(w, s, e, n) ) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C except: sys.exit("Error: Unable to tile {} from raster {}.".format(str(tile), raster)) tile_key = (str(tile.x), str(tile.y), str(tile.z)) if not args.label and len(tiles_map[tile_key]) == 1 and is_border(image): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: ret = tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: ret = tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if not ret: sys.exit("Error: Unable to write tile {} from raster {}.".format(str(tile), raster)) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled
def raster_extent(file_path: str, epsg: str='EPSG:4326') -> Polygon: """Get raster extent in arbitrary CRS. Args: file_path (str): Path to image epsg (str): EPSG Code of result crs Returns: dict: geojson-like geometry """ with rasterio_open(file_path) as dataset: _geom = mapping(box(*dataset.bounds)) return shape(transform_geom(dataset.crs, epsg, _geom, precision=6))
def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read(out_shape=(len(raster.indexes), width, height), window=warp_vrt.window(w, s, e, n)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if ( not args.label and len(tiles_map[tile_key]) == 1 and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders) ): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled
def rgb_to_csv(path_to_rgb, out_path, delimiter=","): """ :param path_to_rgb: :param out_path: :param delimiter: :return: """ band_images = [] with rasterio_open(path_to_rgb) as src: for band in range(src.count): band_images.append([src.read(band + 1).ravel()]) rgb = np.concatenate(band_images).transpose() np.savetxt(out_path, rgb, delimiter=delimiter)
def tile_bbox(tile, mercator=False): if isinstance(tile, mercantile.Tile): if mercator: return mercantile.xy_bounds(tile) # EPSG:3857 else: return mercantile.bounds(tile) # EPSG:4326 else: with open(rasterio_open(tile)) as r: if mercator: w, s, e, n = r.bounds w, s = mercantile.xy(w, s) e, n = mercantile.xy(e, n) return w, s, e, n # EPSG:3857 else: return r.bounds # EPSG:4326 assert False, "Unable to open tile"
def tile_image_from_file(path, bands=None): """Return a multiband image numpy array, from an image file path, or None.""" try: if path[-3:] == "png": return np.array( Image.open(os.path.expanduser(path)).convert("RGB")) raster = rasterio_open(os.path.expanduser(path)) except: return None image = None for i in raster.indexes if bands is None else bands: data_band = raster.read(i) data_band = data_band.reshape(data_band.shape[0], data_band.shape[1], 1) # H,W -> H,W,C image = np.concatenate( (image, data_band), axis=2) if image is not None else data_band return image
def tile_image_from_file(path, bands=None): """Return a multiband image numpy array, from an image file path""" path_expanded = os.path.expanduser(path) _, file_extension = os.path.splitext(path_expanded) try: if file_extension in set([".png", ".webp", ".jpeg", ".jpg"]): return np.array(Image.open(path_expanded).convert("RGB")) raster = rasterio_open(path_expanded) assert raster, "Unable to open {}".format(path) except: return None image = None for i in raster.indexes if bands is None else bands: data_band = raster.read(i) data_band = data_band.reshape(data_band.shape[0], data_band.shape[1], 1) # H,W -> H,W,C image = np.concatenate( (image, data_band), axis=2) if image is not None else data_band assert image is not None, "Unable to open {}".format(path) return image
def tile_image_from_file(path, bands=None, force_rgb=False): """Return a multiband image numpy array, from an image file path, or None.""" try: if path[-3:] == "png" and force_rgb: # PIL PNG Color Palette handling return np.array( Image.open(os.path.expanduser(path)).convert("RGB")) elif path[-3:] == "png": return np.array(Image.open(os.path.expanduser(path))) else: raster = rasterio_open(os.path.expanduser(path)) except: return None image = None for i in raster.indexes if bands is None else bands: data_band = raster.read(i) data_band = data_band.reshape(data_band.shape[0], data_band.shape[1], 1) # H,W -> H,W,C image = np.concatenate( (image, data_band), axis=2) if image is not None else data_band assert image is not None, "Unable to open {}".format(path) return image
def main(args): if not args.workers: args.workers = min(os.cpu_count(), len(args.rasters)) if args.label: config = load_config(args.config) check_classes(config) colors = [classe["color"] for classe in config["classes"]] palette = make_palette(*colors) cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None splits_path = os.path.join(os.path.expanduser(args.out), ".splits") tiles_map = {} print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers)) bands = -1 for path in args.rasters: raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) if bands != -1: assert bands == len( raster.indexes), "Coverage must be bands consistent" bands = len(raster.indexes) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiles = list(set(tiles) & set(cover)) if cover else tiles for tile in tiles: tile_key = (str(tile.x), str(tile.y), str(tile.z)) if tile_key not in tiles_map.keys(): tiles_map[tile_key] = [] tiles_map[tile_key].append(path) if args.label: ext = "png" bands = 1 if not args.label: if bands == 1: ext = "png" if bands == 3: ext = "webp" if bands > 3: ext = "tiff" tiles = [] progress = tqdm(total=len(tiles_map), ascii=True, unit="tile") # Begin to tile plain tiles with futures.ThreadPoolExecutor(args.workers) as executor: def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) transform, _, _ = calculate_default_transform( raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e, n) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) # inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45 warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, args.ts, args.ts), width=math.ceil((e - w) / transform.a), height=math.ceil((s - n) / transform.e), ) data = warp_vrt.read(out_shape=(len(raster.indexes), args.ts, args.ts), window=warp_vrt.window(w, s, e, n)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if not args.label and len( tiles_map[tile_key]) == 1 and is_nodata( image, threshold=args.nodata_threshold): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: ret = tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: ret = tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) assert ret, "Unable to write tile {} from raster {}.".format( str(tile), raster) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled for tiled in executor.map(worker, args.rasters): if tiled is not None: tiles.extend(tiled) # Aggregate remaining tiles splits with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((args.ts, args.ts, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape( (args.ts, args.ts, 1)) # H,W -> H,W,C assert image.shape == split.shape image[:, :, :] += split[:, :, :] if not args.label and is_nodata(image, threshold=args.nodata_threshold): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: ret = tile_image_to_file(args.out, tile, image) if args.label: ret = tile_label_to_file(args.out, tile, palette, image) assert ret, "Unable to write tile {} from raster {}.".format( str(tile_key)) progress.update() return tile for tiled in executor.map(worker, tiles_map.keys()): if tiled is not None: tiles.append(tiled) if splits_path and os.path.isdir(splits_path): shutil.rmtree(splits_path) # Delete suffixes dir if any if tiles and not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args): assert not (args.extent and args.splits ), "--splits and --extent are mutually exclusive options." assert not (args.extent and len(args.out) > 1), "--extent option imply a single output." assert not (args.sql and not args.pg), "--sql option imply --pg" assert ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.sql is not None) + int(args.dir is not None) + int(args.raster is not None) + int(args.cover is not None) == 1 ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql" if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: crs = None w, s, e, n = map(float, args.bbox.split(",")) assert isinstance(w, float) and isinstance( s, float), "Invalid bbox parameter." if args.splits: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) and 0 < sum( splits) <= 100, "Invalid split value or incoherent with out paths." assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required." args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format( args.raster, args.zoom), file=sys.stderr, flush=True) with rasterio_open(os.path.expanduser(args.raster)) as r: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) assert isinstance(w, float) and isinstance( s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format( args.geojson, args.zoom), file=sys.stderr, flush=True) with open(os.path.expanduser(args.geojson)) as f: feature_collection = json.load(f) srid = geojson_srid(feature_collection) feature_map = collections.defaultdict(list) for feature in tqdm(feature_collection["features"], ascii=True, unit="feature"): feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature) cover = feature_map.keys() if args.sql: print("RoboSat.pink - cover from {} {} at zoom {}".format( args.sql, args.pg, args.zoom), file=sys.stderr, flush=True) conn = psycopg2.connect(args.pg) assert conn, "Unable to connect to PostgreSQL database." db = conn.cursor() query = """ WITH sql AS ({}), geom AS (SELECT "1" AS geom FROM sql AS t("1")) SELECT '{{"type": "Feature", "geometry": ' || ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6) || '}}' AS features FROM geom """.format(args.sql) db.execute(query) assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result." feature_map = collections.defaultdict(list) for feature in tqdm( db.fetchall(), ascii=True, unit="feature" ): # FIXME: fetchall will not always fit in memory... feature_map = geojson_parse_feature(args.zoom, 4326, feature_map, json.loads(feature[0])) cover = feature_map.keys() if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format( args.bbox, args.zoom), file=sys.stderr, flush=True) if crs: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) assert isinstance(w, float) and isinstance( s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True) cover = [ tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz)) ] _cover = [] extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0) for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: if args.extent: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) _cover.append(tile) if args.extent: extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e)) cover = _cover if args.splits: shuffle(cover) # in-place cover_splits = [ math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1) ] if len(splits) > 1 and sum(map( int, splits)) == 100 and len(cover) > sum(map(int, splits)): cover_splits[0] = len(cover) - sum(map( int, cover_splits[1:])) # no tile waste s = 0 covers = [] for e in cover_splits: covers.append(cover[s:s + e]) s += e else: covers = [cover] if args.extent: if args.out and os.path.dirname(args.out[0]) and not os.path.isdir( os.path.dirname(args.out[0])): os.makedirs(os.path.dirname(args.out[0]), exist_ok=True) extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e) if args.out: with open(args.out[0], "w") as fp: fp.write(extent) else: print(extent) else: for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir( os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def main(args): assert not (args.label and args.format), "Format option not supported for label, output must be kept as png" try: args.bands = list(map(int, args.bands.split(","))) if args.bands else None except: raise ValueError("invalid --args.bands value") if not args.workers: args.workers = min(os.cpu_count(), len(args.rasters)) if args.label: config = load_config(args.config) check_classes(config) colors = [classe["color"] for classe in config["classes"]] palette = make_palette(colors) assert len(args.ts.split(",")) == 2, "--ts expect width,height value (e.g 512,512)" width, height = list(map(int, args.ts.split(","))) cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))] if args.cover else None splits_path = os.path.join(os.path.expanduser(args.out), ".splits") args.out = os.path.expanduser(args.out) if os.path.dirname(os.path.expanduser(args.out)): os.makedirs(args.out, exist_ok=True) log = Logs(os.path.join(args.out, "log"), out=sys.stderr) raster = rasterio_open(os.path.expanduser(args.rasters[0])) args.bands = args.bands if args.bands else raster.indexes raster.close() print( "neo tile {} rasters on bands {}, on CPU with {} workers".format(len(args.rasters), args.bands, args.workers), file=sys.stderr, flush=True, ) skip = [] tiles_map = {} total = 0 for path in args.rasters: raster = rasterio_open(os.path.expanduser(path)) assert set(args.bands).issubset(set(raster.indexes)), "Missing bands in raster {}".format(path) try: w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) except: log.log("WARNING: missing or invalid raster projection, SKIPPING: {}".format(path)) skip.append(path) continue tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiles = list(set(tiles) & set(cover)) if cover else tiles total += len(tiles) for tile in tiles: tile_key = (str(tile.x), str(tile.y), str(tile.z)) if tile_key not in tiles_map.keys(): tiles_map[tile_key] = [] tiles_map[tile_key].append(path) raster.close() assert total, "Nothing left to tile" if len(args.bands) == 1 or args.label: ext = "png" if args.format is None else args.format if len(args.bands) == 3: ext = "webp" if args.format is None else args.format if len(args.bands) > 3: ext = "tiff" if args.format is None else args.format tiles = [] progress = tqdm(desc="Coverage tiling", total=total, ascii=True, unit="tile") with futures.ThreadPoolExecutor(args.workers) as executor: def worker(path): if path in skip: return None raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read( out_shape=(len(args.bands), width, height), indexes=args.bands, window=warp_vrt.window(w, s, e, n) ) if data.dtype == "uint16": # GeoTiff could be 16 bits data = np.uint8(data / 256) elif data.dtype == "uint32": # or 32 bits data = np.uint8(data / (256 * 256)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if ( not args.label and len(tiles_map[tile_key]) == 1 and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders) ): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image, ext=ext) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, args.nodata, image) if len(tiles_map[tile_key]) == 1: tiled.append(mercantile.Tile(x=x, y=y, z=z)) progress.update() raster.close() return tiled for tiled in executor.map(worker, args.rasters): if tiled is not None: tiles.extend(tiled) total = sum([1 for tile_key in tiles_map.keys() if len(tiles_map[tile_key]) > 1]) progress = tqdm(desc="Aggregate splits", total=total, ascii=True, unit="tile") with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((width, height, len(args.bands)), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) if len(split.shape) == 2: split = split.reshape((width, height, 1)) # H,W -> H,W,C assert image.shape == split.shape, "{}, {}".format(image.shape, split.shape) image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: tile_image_to_file(args.out, tile, image) if args.label: tile_label_to_file(args.out, tile, palette, image) progress.update() return tile for tiled in executor.map(worker, tiles_map.keys()): if tiled is not None: tiles.append(tiled) if splits_path and os.path.isdir(splits_path): shutil.rmtree(splits_path) # Delete suffixes dir if any if tiles and not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args): assert not (args.extent and args.splits), "--splits and --extent are mutually exclusive options." assert not (args.extent and len(args.out) > 1), "--extent option imply a single output." assert ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.dir is not None) + int(args.raster is not None) + int(args.cover is not None) == 1 ), "One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson." if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: crs = None w, s, e, n = map(float, args.bbox.split(",")) assert isinstance(w, float) and isinstance(s, float), "Invalid bbox parameter." if args.splits: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) and 0 < sum(splits) <= 100, "Invalid split value or incoherent with out paths." assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required." args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom), file=sys.stderr, flush=True) with rasterio_open(os.path.expanduser(args.raster)) as r: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom), file=sys.stderr, flush=True) with open(os.path.expanduser(args.geojson)) as f: feature_collection = json.load(f) srid = geojson_srid(feature_collection) feature_map = collections.defaultdict(list) for i, feature in enumerate(tqdm(feature_collection["features"], ascii=True, unit="feature")): feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature) cover = feature_map.keys() if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom), file=sys.stderr, flush=True) if crs: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection" cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True) cover = [tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))] _cover = [] extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0) for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: if args.extent: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) _cover.append(tile) if args.extent: extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e)) cover = _cover if args.splits: shuffle(cover) # in-place cover_splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)] if len(splits) > 1 and sum(map(int, splits)) == 100 and len(cover) > sum(map(int, splits)): cover_splits[0] = len(cover) - sum(map(int, cover_splits[1:])) # no tile waste s = 0 covers = [] for e in cover_splits: covers.append(cover[s : s + e]) s += e else: covers = [cover] if args.extent: if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(os.path.dirname(args.out[0])): os.makedirs(os.path.dirname(args.out[0]), exist_ok=True) extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e) if args.out: with open(args.out[0], "w") as fp: fp.write(extent) else: print(extent) else: for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def tile_thread(tile_queue) -> None: while True: try: tile_details = tile_queue.get(timeout=5) except Empty: break cross_section_detector = DwarsprofielDetector() cross_sections_on_tile = cross_section_detector.retrieve_cross_section_with_bounding_box( 28992, tile_details['x_min'], tile_details['y_min'], tile_details['x_max'], tile_details['y_max']) if len(cross_sections_on_tile) == 0: tile_queue.task_done() continue # First try to load the file from disk. try: dataset = open('data/ahn2_geotiff_row_' + str(tile_details['row']) + '_column_' + str(tile_details['column']) + '.tif') cross_section_detector.check_with_ahn_heights( cross_sections_on_tile, dataset, dataset.read(1)) # If the data is not found on the disk, make connection with the AHN server and get the tile. except RasterioIOError: print('Tile not found on disk' + str(tile_details['row']) + '-' + str(tile_details['column'])) ahn_datasource = AhnWebCoverageDatasource() try: ahn_response = ahn_datasource.retrieve_tile_ahn2( 'EPSG:28992', tile_details['x_min'], tile_details['y_min'], tile_details['x_max'], tile_details['y_max'], ) except (ConnectionError, ChunkedEncodingError, AttributeError): print('Ahn connection error, ') print('error on ' + str(tile_details['row']) + ', ' + str(tile_details['column'])) tile_queue.task_done() tile_queue.put(tile_details) continue with MemoryFile(ahn_response) as memfile: with memfile.open() as dataset: tile_array = dataset.read(1) with Env(): profile = dataset.profile with rasterio_open( 'data/ahn2_geotiff_row_' + str(tile_details['row']) + '_column_' + str(tile_details['column']) + '.tif', 'w', **profile) as dst: dst.write(tile_array, 1) cross_section_detector.check_with_ahn_heights( cross_sections_on_tile, dataset, tile_array) tile_queue.task_done() if tile_queue.qsize() % 1000 == 0: print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) print(str(tile_queue.qsize()))
def main(args): if not args.workers: args.workers = min(os.cpu_count(), len(args.rasters)) if args.label: config = load_config(args.config) check_classes(config) colors = [classe["color"] for classe in config["classes"]] palette = make_palette(colors) assert len(args.ts.split( ",")) == 2, "--ts expect width,height value (e.g 512,512)" width, height = list(map(int, args.ts.split(","))) cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover)) ] if args.cover else None splits_path = os.path.join(os.path.expanduser(args.out), ".splits") tiles_map = {} print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers), file=sys.stderr, flush=True) bands = -1 for path in args.rasters: raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) if bands != -1: assert bands == len( raster.indexes), "Coverage must be bands consistent" bands = len(raster.indexes) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiles = list(set(tiles) & set(cover)) if cover else tiles for tile in tiles: tile_key = (str(tile.x), str(tile.y), str(tile.z)) if tile_key not in tiles_map.keys(): tiles_map[tile_key] = [] tiles_map[tile_key].append(path) if args.label: ext = "png" bands = 1 if not args.label: if bands == 1: ext = "png" if bands == 3: ext = "webp" if bands > 3: ext = "tiff" tiles = [] progress = tqdm(total=len(tiles_map), ascii=True, unit="tile") # Begin to tile plain tiles with futures.ThreadPoolExecutor(args.workers) as executor: def worker(path): raster = rasterio_open(path) w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiled = [] for tile in tiles: if cover and tile not in cover: continue w, s, e, n = mercantile.xy_bounds(tile) warp_vrt = WarpedVRT( raster, crs="epsg:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(w, s, e, n, width, height), width=width, height=height, ) data = warp_vrt.read(out_shape=(len(raster.indexes), width, height), window=warp_vrt.window(w, s, e, n)) if data.dtype == "uint16": # GeoTiff could be 16 bits data = np.uint8(data / 256) elif data.dtype == "uint32": # or 32 bits data = np.uint8(data / (256 * 256)) image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C tile_key = (str(tile.x), str(tile.y), str(tile.z)) if (not args.label and len(tiles_map[tile_key]) == 1 and is_nodata( image, args.nodata, args.nodata_threshold, args.keep_borders)): progress.update() continue if len(tiles_map[tile_key]) > 1: out = os.path.join(splits_path, str(tiles_map[tile_key].index(path))) else: out = args.out x, y, z = map(int, tile) if not args.label: tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image) if args.label: tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image) if len(tiles_map[tile_key]) == 1: progress.update() tiled.append(mercantile.Tile(x=x, y=y, z=z)) return tiled for tiled in executor.map(worker, args.rasters): if tiled is not None: tiles.extend(tiled) # Aggregate remaining tiles splits with futures.ThreadPoolExecutor(args.workers) as executor: def worker(tile_key): if len(tiles_map[tile_key]) == 1: return image = np.zeros((width, height, bands), np.uint8) x, y, z = map(int, tile_key) for i in range(len(tiles_map[tile_key])): root = os.path.join(splits_path, str(i)) _, path = tile_from_xyz(root, x, y, z) if not args.label: split = tile_image_from_file(path) if args.label: split = tile_label_from_file(path) split = split.reshape((width, height, 1)) # H,W -> H,W,C assert image.shape == split.shape image[np.where(image == 0)] += split[np.where(image == 0)] if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders): progress.update() return tile = mercantile.Tile(x=x, y=y, z=z) if not args.label: tile_image_to_file(args.out, tile, image) if args.label: tile_label_to_file(args.out, tile, palette, image) progress.update() return tile for tiled in executor.map(worker, tiles_map.keys()): if tiled is not None: tiles.append(tiled) if splits_path and os.path.isdir(splits_path): shutil.rmtree(splits_path) # Delete suffixes dir if any if tiles and not args.no_web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template base_url = args.web_ui_base_url if args.web_ui_base_url else "." web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args): if ( int(args.bbox is not None) + int(args.geojson is not None) + int(args.dir is not None) + int(args.xyz is not None) + int(args.raster is not None) + int(args.cover is not None) != 1 ): sys.exit("ERROR: One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson.") if args.bbox: try: w, s, e, n, crs = args.bbox.split(",") w, s, e, n = map(float, (w, s, e, n)) except: try: crs = None w, s, e, n = map(float, args.bbox.split(",")) except: sys.exit("ERROR: invalid bbox parameter.") if args.splits: try: splits = [int(split) for split in args.splits.split("/")] assert len(splits) == len(args.out) assert sum(splits) == 100 except: sys.exit("ERROR: Invalid split value or incoherent with provided out paths.") if not args.zoom and (args.geojson or args.bbox or args.raster): sys.exit("ERROR: Zoom parameter is required.") args.out = [os.path.expanduser(out) for out in args.out] cover = [] if args.raster: print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom)) with rasterio_open(os.path.expanduser(args.raster)) as r: try: w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.geojson: print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom)) with open(os.path.expanduser(args.geojson)) as f: features = json.load(f) try: for feature in tqdm(features["features"], ascii=True, unit="feature"): cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist())) except: sys.exit("ERROR: invalid or unsupported GeoJSON.") cover = list(set(cover)) # tiles can overlap for multiple features; unique tile ids if args.bbox: print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom)) if crs: try: w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n) except: sys.exit("ERROR: unable to deal with raster projection") cover = [tile for tile in tiles(w, s, e, n, args.zoom)] if args.cover: print("RoboSat.pink - cover from {}".format(args.cover)) cover = [tile for tile in tiles_from_csv(args.cover)] if args.dir: print("RoboSat.pink - cover from {}".format(args.dir)) cover = [tile for tile in tiles_from_dir(args.dir, xyz=False)] if args.xyz: print("RoboSat.pink - cover from {}".format(args.xyz)) cover = [tile for tile in tiles_from_dir(args.xyz, xyz=True)] _cover = [] for tile in tqdm(cover, ascii=True, unit="tile"): if args.zoom and tile.z != args.zoom: w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile)) for t in tiles(w, s, n, e, args.zoom): unique = True for _t in _cover: if _t == t: unique = False if unique: _cover.append(t) else: _cover.append(tile) cover = _cover if args.splits: shuffle(cover) # in-place splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)] s = 0 covers = [] for e in splits: covers.append(cover[s : s + e - 1]) s += e else: covers = [cover] for i, cover in enumerate(covers): if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])): os.makedirs(os.path.dirname(args.out[i]), exist_ok=True) with open(args.out[i], "w") as fp: csv.writer(fp).writerows(cover)
def main(args): if args.type == "label": try: config = load_config(args.config) except: sys.exit("Error: Unable to load DataSet config file") classes = config["classes"]["title"] colors = config["classes"]["colors"] assert len(classes) == len(colors), "classes and colors coincide" assert len(colors) == 2, "only binary models supported right now" try: raster = rasterio_open(args.raster) w, s, e, n = bounds = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds) transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, *bounds) except: sys.exit("Error: Unable to load raster or deal with it's projection") tiles = [ mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom) ] tiles_nodata = [] for tile in tqdm(tiles, desc="Tiling", unit="tile", ascii=True): w, s, e, n = tile_bounds = mercantile.xy_bounds(tile) # Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45 warp_vrt = WarpedVRT( raster, crs="EPSG:3857", resampling=Resampling.bilinear, add_alpha=False, transform=from_bounds(*tile_bounds, args.size, args.size), width=math.ceil((e - w) / transform.a), height=math.ceil((s - n) / transform.e), ) data = warp_vrt.read(out_shape=(len(raster.indexes), args.size, args.size), window=warp_vrt.window(w, s, e, n)) # If no_data is set, remove all tiles with at least one whole border filled only with no_data (on all bands) if type(args.no_data) is not None and ( np.all(data[:, 0, :] == args.no_data) or np.all(data[:, -1, :] == args.no_data) or np.all(data[:, :, 0] == args.no_data) or np.all(data[:, :, -1] == args.no_data)): tiles_nodata.append(tile) continue C, W, H = data.shape os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)), exist_ok=True) path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y)) if args.type == "label": assert C == 1, "Error: Label raster input should be 1 band" ext = "png" img = Image.fromarray(np.squeeze(data, axis=0), mode="P") img.putpalette(make_palette(colors[0], colors[1])) img.save("{}.{}".format(path, ext), optimize=True) elif args.type == "image": assert C == 1 or C == 3, "Error: Image raster input should be either 1 or 3 bands" # GeoTiff could be 16 or 32bits if data.dtype == "uint16": data = np.uint8(data / 256) elif data.dtype == "uint32": data = np.uint8(data / (256 * 256)) if C == 1: ext = "png" Image.fromarray(np.squeeze(data, axis=0), mode="L").save("{}.{}".format(path, ext), optimize=True) elif C == 3: ext = "webp" Image.fromarray(np.moveaxis(data, 0, 2), mode="RGB").save("{}.{}".format(path, ext), optimize=True) if args.web_ui: template = "leaflet.html" if not args.web_ui_template else args.web_ui_template tiles = [tile for tile in tiles if tile not in tiles_nodata] web_ui(args.out, args.web_ui, tiles, tiles, ext, template)