def raster_convexhull(file_path: str, epsg='EPSG:4326', nodata: int=0) -> dict:
"""Get footprint from any raster.
Args:
file_path (str): image file
epsg (str): geometry EPSG
nodata (int): Custom nodata value
See:
https://rasterio.readthedocs.io/en/latest/topics/masks.html
"""
with rasterio_open(file_path) as dataset:
# Read raster data, masking nodata values
data = dataset.read(1)
data[data != nodata] = 1
data[data == nodata] = 0
# Create mask, which 1 represents valid data and 0 nodata
mask = data.astype(uint8)
geoms = []
res = {'val': []}
for geom, val in shapes(mask, mask=mask, transform=dataset.transform):
geom = transform_geom(dataset.crs, epsg, geom, precision=6)
res['val'].append(val)
geoms.append(shape(geom))
if len(geoms) == 1:
return geoms[0].convex_hull
multi_polygons = MultiPolygon(geoms)
return multi_polygons.convex_hull
def merge(cls, list_of_raster, bounds=None):
""" Merge several RasterMap instances
:param list_of_raster: list of RasterMap class instance(s)
:param bounds: bounds of the output RasterMap
:return:
"""
# TODO: use gdal merge method
list_of_datasets = []
for raster_map in list_of_raster:
list_of_datasets.append(rasterio_open(raster_map.raster_file, 'r'))
# Merge using rasterio
array, transform = rasterio_merge(list_of_datasets, bounds=bounds)
with RasterTempFile() as file:
with rasterio_open(file, 'w', driver="GTiff", height=array.shape[1], width=array.shape[2], count=1,
dtype=array.dtype, crs=list_of_raster[0].crs, transform=transform) as out_dst:
out_dst.write(array.squeeze(), 1)
return cls(file, no_data_value=list_of_raster[0].no_data_value)
def worker(path):
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e, n)
tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)]
tiled = []
for tile in tiles:
try:
w, s, e, n = mercantile.xy_bounds(tile)
# inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, args.ts, args.ts),
width=math.ceil((e - w) / transform.a),
height=math.ceil((s - n) / transform.e),
)
data = warp_vrt.read(
out_shape=(len(raster.indexes), args.ts, args.ts), window=warp_vrt.window(w, s, e, n)
)
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
except:
sys.exit("Error: Unable to tile {} from raster {}.".format(str(tile), raster))
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if not args.label and len(tiles_map[tile_key]) == 1 and is_border(image):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path, str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
ret = tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image)
if args.label:
ret = tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image)
if not ret:
sys.exit("Error: Unable to write tile {} from raster {}.".format(str(tile), raster))
if len(tiles_map[tile_key]) == 1:
progress.update()
tiled.append(mercantile.Tile(x=x, y=y, z=z))
return tiled
def raster_extent(file_path: str, epsg: str='EPSG:4326') -> Polygon:
"""Get raster extent in arbitrary CRS.
Args:
file_path (str): Path to image
epsg (str): EPSG Code of result crs
Returns:
dict: geojson-like geometry
"""
with rasterio_open(file_path) as dataset:
_geom = mapping(box(*dataset.bounds))
return shape(transform_geom(dataset.crs, epsg, _geom, precision=6))
def worker(path):
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)]
tiled = []
for tile in tiles:
if cover and tile not in cover:
continue
w, s, e, n = mercantile.xy_bounds(tile)
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, width, height),
width=width,
height=height,
)
data = warp_vrt.read(out_shape=(len(raster.indexes), width, height), window=warp_vrt.window(w, s, e, n))
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if (
not args.label
and len(tiles_map[tile_key]) == 1
and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders)
):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path, str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image)
if args.label:
tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, image)
if len(tiles_map[tile_key]) == 1:
progress.update()
tiled.append(mercantile.Tile(x=x, y=y, z=z))
return tiled
def rgb_to_csv(path_to_rgb, out_path, delimiter=","):
"""
:param path_to_rgb:
:param out_path:
:param delimiter:
:return:
"""
band_images = []
with rasterio_open(path_to_rgb) as src:
for band in range(src.count):
band_images.append([src.read(band + 1).ravel()])
rgb = np.concatenate(band_images).transpose()
np.savetxt(out_path, rgb, delimiter=delimiter)
def tile_bbox(tile, mercator=False):
if isinstance(tile, mercantile.Tile):
if mercator:
return mercantile.xy_bounds(tile) # EPSG:3857
else:
return mercantile.bounds(tile) # EPSG:4326
else:
with open(rasterio_open(tile)) as r:
if mercator:
w, s, e, n = r.bounds
w, s = mercantile.xy(w, s)
e, n = mercantile.xy(e, n)
return w, s, e, n # EPSG:3857
else:
return r.bounds # EPSG:4326
assert False, "Unable to open tile"
def tile_image_from_file(path, bands=None):
"""Return a multiband image numpy array, from an image file path, or None."""
try:
if path[-3:] == "png":
return np.array(
Image.open(os.path.expanduser(path)).convert("RGB"))
raster = rasterio_open(os.path.expanduser(path))
except:
return None
image = None
for i in raster.indexes if bands is None else bands:
data_band = raster.read(i)
data_band = data_band.reshape(data_band.shape[0], data_band.shape[1],
1) # H,W -> H,W,C
image = np.concatenate(
(image, data_band), axis=2) if image is not None else data_band
return image
def tile_image_from_file(path, bands=None):
"""Return a multiband image numpy array, from an image file path"""
path_expanded = os.path.expanduser(path)
_, file_extension = os.path.splitext(path_expanded)
try:
if file_extension in set([".png", ".webp", ".jpeg", ".jpg"]):
return np.array(Image.open(path_expanded).convert("RGB"))
raster = rasterio_open(path_expanded)
assert raster, "Unable to open {}".format(path)
except:
return None
image = None
for i in raster.indexes if bands is None else bands:
data_band = raster.read(i)
data_band = data_band.reshape(data_band.shape[0], data_band.shape[1],
1) # H,W -> H,W,C
image = np.concatenate(
(image, data_band), axis=2) if image is not None else data_band
assert image is not None, "Unable to open {}".format(path)
return image
def tile_image_from_file(path, bands=None, force_rgb=False):
"""Return a multiband image numpy array, from an image file path, or None."""
try:
if path[-3:] == "png" and force_rgb: # PIL PNG Color Palette handling
return np.array(
Image.open(os.path.expanduser(path)).convert("RGB"))
elif path[-3:] == "png":
return np.array(Image.open(os.path.expanduser(path)))
else:
raster = rasterio_open(os.path.expanduser(path))
except:
return None
image = None
for i in raster.indexes if bands is None else bands:
data_band = raster.read(i)
data_band = data_band.reshape(data_band.shape[0], data_band.shape[1],
1) # H,W -> H,W,C
image = np.concatenate(
(image, data_band), axis=2) if image is not None else data_band
assert image is not None, "Unable to open {}".format(path)
return image
def main(args):
if not args.workers:
args.workers = min(os.cpu_count(), len(args.rasters))
if args.label:
config = load_config(args.config)
check_classes(config)
colors = [classe["color"] for classe in config["classes"]]
palette = make_palette(*colors)
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
splits_path = os.path.join(os.path.expanduser(args.out), ".splits")
tiles_map = {}
print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers))
bands = -1
for path in args.rasters:
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
if bands != -1:
assert bands == len(
raster.indexes), "Coverage must be bands consistent"
bands = len(raster.indexes)
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiles = list(set(tiles) & set(cover)) if cover else tiles
for tile in tiles:
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if tile_key not in tiles_map.keys():
tiles_map[tile_key] = []
tiles_map[tile_key].append(path)
if args.label:
ext = "png"
bands = 1
if not args.label:
if bands == 1:
ext = "png"
if bands == 3:
ext = "webp"
if bands > 3:
ext = "tiff"
tiles = []
progress = tqdm(total=len(tiles_map), ascii=True, unit="tile")
# Begin to tile plain tiles
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(path):
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326",
*raster.bounds)
transform, _, _ = calculate_default_transform(
raster.crs, "EPSG:3857", raster.width, raster.height, w, s, e,
n)
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiled = []
for tile in tiles:
if cover and tile not in cover:
continue
w, s, e, n = mercantile.xy_bounds(tile)
# inspired by rio-tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, args.ts, args.ts),
width=math.ceil((e - w) / transform.a),
height=math.ceil((s - n) / transform.e),
)
data = warp_vrt.read(out_shape=(len(raster.indexes), args.ts,
args.ts),
window=warp_vrt.window(w, s, e, n))
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if not args.label and len(
tiles_map[tile_key]) == 1 and is_nodata(
image, threshold=args.nodata_threshold):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path,
str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
ret = tile_image_to_file(out, mercantile.Tile(x=x,
y=y,
z=z), image)
if args.label:
ret = tile_label_to_file(out, mercantile.Tile(x=x,
y=y,
z=z),
palette, image)
assert ret, "Unable to write tile {} from raster {}.".format(
str(tile), raster)
if len(tiles_map[tile_key]) == 1:
progress.update()
tiled.append(mercantile.Tile(x=x, y=y, z=z))
return tiled
for tiled in executor.map(worker, args.rasters):
if tiled is not None:
tiles.extend(tiled)
# Aggregate remaining tiles splits
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile_key):
if len(tiles_map[tile_key]) == 1:
return
image = np.zeros((args.ts, args.ts, bands), np.uint8)
x, y, z = map(int, tile_key)
for i in range(len(tiles_map[tile_key])):
root = os.path.join(splits_path, str(i))
_, path = tile_from_xyz(root, x, y, z)
if not args.label:
split = tile_image_from_file(path)
if args.label:
split = tile_label_from_file(path)
split = split.reshape(
(args.ts, args.ts, 1)) # H,W -> H,W,C
assert image.shape == split.shape
image[:, :, :] += split[:, :, :]
if not args.label and is_nodata(image,
threshold=args.nodata_threshold):
progress.update()
return
tile = mercantile.Tile(x=x, y=y, z=z)
if not args.label:
ret = tile_image_to_file(args.out, tile, image)
if args.label:
ret = tile_label_to_file(args.out, tile, palette, image)
assert ret, "Unable to write tile {} from raster {}.".format(
str(tile_key))
progress.update()
return tile
for tiled in executor.map(worker, tiles_map.keys()):
if tiled is not None:
tiles.append(tiled)
if splits_path and os.path.isdir(splits_path):
shutil.rmtree(splits_path) # Delete suffixes dir if any
if tiles and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args):
assert not (args.extent and args.splits
), "--splits and --extent are mutually exclusive options."
assert not (args.extent and
len(args.out) > 1), "--extent option imply a single output."
assert not (args.sql and not args.pg), "--sql option imply --pg"
assert (
int(args.bbox is not None) + int(args.geojson is not None) +
int(args.sql is not None) + int(args.dir is not None) +
int(args.raster is not None) + int(args.cover is not None) == 1
), "One, and only one, input type must be provided, among: --dir, --bbox, --cover, --raster, --geojson or --sql"
if args.bbox:
try:
w, s, e, n, crs = args.bbox.split(",")
w, s, e, n = map(float, (w, s, e, n))
except:
crs = None
w, s, e, n = map(float, args.bbox.split(","))
assert isinstance(w, float) and isinstance(
s, float), "Invalid bbox parameter."
if args.splits:
splits = [int(split) for split in args.splits.split("/")]
assert len(splits) == len(args.out) and 0 < sum(
splits) <= 100, "Invalid split value or incoherent with out paths."
assert not (not args.zoom and
(args.geojson or args.bbox
or args.raster)), "Zoom parameter is required."
args.out = [os.path.expanduser(out) for out in args.out]
cover = []
if args.raster:
print("RoboSat.pink - cover from {} at zoom {}".format(
args.raster, args.zoom),
file=sys.stderr,
flush=True)
with rasterio_open(os.path.expanduser(args.raster)) as r:
w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
assert isinstance(w, float) and isinstance(
s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.geojson:
print("RoboSat.pink - cover from {} at zoom {}".format(
args.geojson, args.zoom),
file=sys.stderr,
flush=True)
with open(os.path.expanduser(args.geojson)) as f:
feature_collection = json.load(f)
srid = geojson_srid(feature_collection)
feature_map = collections.defaultdict(list)
for feature in tqdm(feature_collection["features"],
ascii=True,
unit="feature"):
feature_map = geojson_parse_feature(args.zoom, srid,
feature_map, feature)
cover = feature_map.keys()
if args.sql:
print("RoboSat.pink - cover from {} {} at zoom {}".format(
args.sql, args.pg, args.zoom),
file=sys.stderr,
flush=True)
conn = psycopg2.connect(args.pg)
assert conn, "Unable to connect to PostgreSQL database."
db = conn.cursor()
query = """
WITH
sql AS ({}),
geom AS (SELECT "1" AS geom FROM sql AS t("1"))
SELECT '{{"type": "Feature", "geometry": '
|| ST_AsGeoJSON((ST_Dump(ST_Transform(ST_Force2D(geom.geom), 4326))).geom, 6)
|| '}}' AS features
FROM geom
""".format(args.sql)
db.execute(query)
assert db.rowcount is not None and db.rowcount != -1, "SQL Query return no result."
feature_map = collections.defaultdict(list)
for feature in tqdm(
db.fetchall(), ascii=True, unit="feature"
): # FIXME: fetchall will not always fit in memory...
feature_map = geojson_parse_feature(args.zoom, 4326, feature_map,
json.loads(feature[0]))
cover = feature_map.keys()
if args.bbox:
print("RoboSat.pink - cover from {} at zoom {}".format(
args.bbox, args.zoom),
file=sys.stderr,
flush=True)
if crs:
w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
assert isinstance(w, float) and isinstance(
s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.cover:
print("RoboSat.pink - cover from {}".format(args.cover),
file=sys.stderr,
flush=True)
cover = [tile for tile in tiles_from_csv(args.cover)]
if args.dir:
print("RoboSat.pink - cover from {}".format(args.dir),
file=sys.stderr,
flush=True)
cover = [
tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))
]
_cover = []
extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
for tile in tqdm(cover, ascii=True, unit="tile"):
if args.zoom and tile.z != args.zoom:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
for t in tiles(w, s, n, e, args.zoom):
unique = True
for _t in _cover:
if _t == t:
unique = False
if unique:
_cover.append(t)
else:
if args.extent:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326",
*xy_bounds(tile))
_cover.append(tile)
if args.extent:
extent_w, extent_s, extent_n, extent_e = (min(extent_w,
w), min(extent_s, s),
max(extent_n,
n), max(extent_e, e))
cover = _cover
if args.splits:
shuffle(cover) # in-place
cover_splits = [
math.floor(len(cover) * split / 100)
for i, split in enumerate(splits, 1)
]
if len(splits) > 1 and sum(map(
int, splits)) == 100 and len(cover) > sum(map(int, splits)):
cover_splits[0] = len(cover) - sum(map(
int, cover_splits[1:])) # no tile waste
s = 0
covers = []
for e in cover_splits:
covers.append(cover[s:s + e])
s += e
else:
covers = [cover]
if args.extent:
if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(
os.path.dirname(args.out[0])):
os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)
extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s,
extent_n, extent_e)
if args.out:
with open(args.out[0], "w") as fp:
fp.write(extent)
else:
print(extent)
else:
for i, cover in enumerate(covers):
if os.path.dirname(args.out[i]) and not os.path.isdir(
os.path.dirname(args.out[i])):
os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)
with open(args.out[i], "w") as fp:
csv.writer(fp).writerows(cover)
def main(args):
assert not (args.label and args.format), "Format option not supported for label, output must be kept as png"
try:
args.bands = list(map(int, args.bands.split(","))) if args.bands else None
except:
raise ValueError("invalid --args.bands value")
if not args.workers:
args.workers = min(os.cpu_count(), len(args.rasters))
if args.label:
config = load_config(args.config)
check_classes(config)
colors = [classe["color"] for classe in config["classes"]]
palette = make_palette(colors)
assert len(args.ts.split(",")) == 2, "--ts expect width,height value (e.g 512,512)"
width, height = list(map(int, args.ts.split(",")))
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))] if args.cover else None
splits_path = os.path.join(os.path.expanduser(args.out), ".splits")
args.out = os.path.expanduser(args.out)
if os.path.dirname(os.path.expanduser(args.out)):
os.makedirs(args.out, exist_ok=True)
log = Logs(os.path.join(args.out, "log"), out=sys.stderr)
raster = rasterio_open(os.path.expanduser(args.rasters[0]))
args.bands = args.bands if args.bands else raster.indexes
raster.close()
print(
"neo tile {} rasters on bands {}, on CPU with {} workers".format(len(args.rasters), args.bands, args.workers),
file=sys.stderr,
flush=True,
)
skip = []
tiles_map = {}
total = 0
for path in args.rasters:
raster = rasterio_open(os.path.expanduser(path))
assert set(args.bands).issubset(set(raster.indexes)), "Missing bands in raster {}".format(path)
try:
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
except:
log.log("WARNING: missing or invalid raster projection, SKIPPING: {}".format(path))
skip.append(path)
continue
tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)]
tiles = list(set(tiles) & set(cover)) if cover else tiles
total += len(tiles)
for tile in tiles:
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if tile_key not in tiles_map.keys():
tiles_map[tile_key] = []
tiles_map[tile_key].append(path)
raster.close()
assert total, "Nothing left to tile"
if len(args.bands) == 1 or args.label:
ext = "png" if args.format is None else args.format
if len(args.bands) == 3:
ext = "webp" if args.format is None else args.format
if len(args.bands) > 3:
ext = "tiff" if args.format is None else args.format
tiles = []
progress = tqdm(desc="Coverage tiling", total=total, ascii=True, unit="tile")
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(path):
if path in skip:
return None
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
tiles = [mercantile.Tile(x=x, y=y, z=z) for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)]
tiled = []
for tile in tiles:
if cover and tile not in cover:
continue
w, s, e, n = mercantile.xy_bounds(tile)
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, width, height),
width=width,
height=height,
)
data = warp_vrt.read(
out_shape=(len(args.bands), width, height), indexes=args.bands, window=warp_vrt.window(w, s, e, n)
)
if data.dtype == "uint16": # GeoTiff could be 16 bits
data = np.uint8(data / 256)
elif data.dtype == "uint32": # or 32 bits
data = np.uint8(data / (256 * 256))
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if (
not args.label
and len(tiles_map[tile_key]) == 1
and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders)
):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path, str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z), image, ext=ext)
if args.label:
tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z), palette, args.nodata, image)
if len(tiles_map[tile_key]) == 1:
tiled.append(mercantile.Tile(x=x, y=y, z=z))
progress.update()
raster.close()
return tiled
for tiled in executor.map(worker, args.rasters):
if tiled is not None:
tiles.extend(tiled)
total = sum([1 for tile_key in tiles_map.keys() if len(tiles_map[tile_key]) > 1])
progress = tqdm(desc="Aggregate splits", total=total, ascii=True, unit="tile")
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile_key):
if len(tiles_map[tile_key]) == 1:
return
image = np.zeros((width, height, len(args.bands)), np.uint8)
x, y, z = map(int, tile_key)
for i in range(len(tiles_map[tile_key])):
root = os.path.join(splits_path, str(i))
_, path = tile_from_xyz(root, x, y, z)
if not args.label:
split = tile_image_from_file(path)
if args.label:
split = tile_label_from_file(path)
if len(split.shape) == 2:
split = split.reshape((width, height, 1)) # H,W -> H,W,C
assert image.shape == split.shape, "{}, {}".format(image.shape, split.shape)
image[np.where(image == 0)] += split[np.where(image == 0)]
if not args.label and is_nodata(image, args.nodata, args.nodata_threshold, args.keep_borders):
progress.update()
return
tile = mercantile.Tile(x=x, y=y, z=z)
if not args.label:
tile_image_to_file(args.out, tile, image)
if args.label:
tile_label_to_file(args.out, tile, palette, image)
progress.update()
return tile
for tiled in executor.map(worker, tiles_map.keys()):
if tiled is not None:
tiles.append(tiled)
if splits_path and os.path.isdir(splits_path):
shutil.rmtree(splits_path) # Delete suffixes dir if any
if tiles and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args):
assert not (args.extent and args.splits), "--splits and --extent are mutually exclusive options."
assert not (args.extent and len(args.out) > 1), "--extent option imply a single output."
assert (
int(args.bbox is not None)
+ int(args.geojson is not None)
+ int(args.dir is not None)
+ int(args.raster is not None)
+ int(args.cover is not None)
== 1
), "One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson."
if args.bbox:
try:
w, s, e, n, crs = args.bbox.split(",")
w, s, e, n = map(float, (w, s, e, n))
except:
crs = None
w, s, e, n = map(float, args.bbox.split(","))
assert isinstance(w, float) and isinstance(s, float), "Invalid bbox parameter."
if args.splits:
splits = [int(split) for split in args.splits.split("/")]
assert len(splits) == len(args.out) and 0 < sum(splits) <= 100, "Invalid split value or incoherent with out paths."
assert not (not args.zoom and (args.geojson or args.bbox or args.raster)), "Zoom parameter is required."
args.out = [os.path.expanduser(out) for out in args.out]
cover = []
if args.raster:
print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom), file=sys.stderr, flush=True)
with rasterio_open(os.path.expanduser(args.raster)) as r:
w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.geojson:
print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom), file=sys.stderr, flush=True)
with open(os.path.expanduser(args.geojson)) as f:
feature_collection = json.load(f)
srid = geojson_srid(feature_collection)
feature_map = collections.defaultdict(list)
for i, feature in enumerate(tqdm(feature_collection["features"], ascii=True, unit="feature")):
feature_map = geojson_parse_feature(args.zoom, srid, feature_map, feature)
cover = feature_map.keys()
if args.bbox:
print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom), file=sys.stderr, flush=True)
if crs:
w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
assert isinstance(w, float) and isinstance(s, float), "Unable to deal with raster projection"
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.cover:
print("RoboSat.pink - cover from {}".format(args.cover), file=sys.stderr, flush=True)
cover = [tile for tile in tiles_from_csv(args.cover)]
if args.dir:
print("RoboSat.pink - cover from {}".format(args.dir), file=sys.stderr, flush=True)
cover = [tile for tile in tiles_from_dir(args.dir, xyz=not (args.no_xyz))]
_cover = []
extent_w, extent_s, extent_n, extent_e = (180.0, 90.0, -180.0, -90.0)
for tile in tqdm(cover, ascii=True, unit="tile"):
if args.zoom and tile.z != args.zoom:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile))
for t in tiles(w, s, n, e, args.zoom):
unique = True
for _t in _cover:
if _t == t:
unique = False
if unique:
_cover.append(t)
else:
if args.extent:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile))
_cover.append(tile)
if args.extent:
extent_w, extent_s, extent_n, extent_e = (min(extent_w, w), min(extent_s, s), max(extent_n, n), max(extent_e, e))
cover = _cover
if args.splits:
shuffle(cover) # in-place
cover_splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)]
if len(splits) > 1 and sum(map(int, splits)) == 100 and len(cover) > sum(map(int, splits)):
cover_splits[0] = len(cover) - sum(map(int, cover_splits[1:])) # no tile waste
s = 0
covers = []
for e in cover_splits:
covers.append(cover[s : s + e])
s += e
else:
covers = [cover]
if args.extent:
if args.out and os.path.dirname(args.out[0]) and not os.path.isdir(os.path.dirname(args.out[0])):
os.makedirs(os.path.dirname(args.out[0]), exist_ok=True)
extent = "{:.8f},{:.8f},{:.8f},{:.8f}".format(extent_w, extent_s, extent_n, extent_e)
if args.out:
with open(args.out[0], "w") as fp:
fp.write(extent)
else:
print(extent)
else:
for i, cover in enumerate(covers):
if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])):
os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)
with open(args.out[i], "w") as fp:
csv.writer(fp).writerows(cover)
def tile_thread(tile_queue) -> None:
while True:
try:
tile_details = tile_queue.get(timeout=5)
except Empty:
break
cross_section_detector = DwarsprofielDetector()
cross_sections_on_tile = cross_section_detector.retrieve_cross_section_with_bounding_box(
28992, tile_details['x_min'], tile_details['y_min'],
tile_details['x_max'], tile_details['y_max'])
if len(cross_sections_on_tile) == 0:
tile_queue.task_done()
continue
# First try to load the file from disk.
try:
dataset = open('data/ahn2_geotiff_row_' +
str(tile_details['row']) + '_column_' +
str(tile_details['column']) + '.tif')
cross_section_detector.check_with_ahn_heights(
cross_sections_on_tile, dataset, dataset.read(1))
# If the data is not found on the disk, make connection with the AHN server and get the tile.
except RasterioIOError:
print('Tile not found on disk' + str(tile_details['row']) +
'-' + str(tile_details['column']))
ahn_datasource = AhnWebCoverageDatasource()
try:
ahn_response = ahn_datasource.retrieve_tile_ahn2(
'EPSG:28992',
tile_details['x_min'],
tile_details['y_min'],
tile_details['x_max'],
tile_details['y_max'],
)
except (ConnectionError, ChunkedEncodingError, AttributeError):
print('Ahn connection error, ')
print('error on ' + str(tile_details['row']) + ', ' +
str(tile_details['column']))
tile_queue.task_done()
tile_queue.put(tile_details)
continue
with MemoryFile(ahn_response) as memfile:
with memfile.open() as dataset:
tile_array = dataset.read(1)
with Env():
profile = dataset.profile
with rasterio_open(
'data/ahn2_geotiff_row_' +
str(tile_details['row']) + '_column_' +
str(tile_details['column']) + '.tif', 'w',
**profile) as dst:
dst.write(tile_array, 1)
cross_section_detector.check_with_ahn_heights(
cross_sections_on_tile, dataset, tile_array)
tile_queue.task_done()
if tile_queue.qsize() % 1000 == 0:
print(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
print(str(tile_queue.qsize()))
def main(args):
if not args.workers:
args.workers = min(os.cpu_count(), len(args.rasters))
if args.label:
config = load_config(args.config)
check_classes(config)
colors = [classe["color"] for classe in config["classes"]]
palette = make_palette(colors)
assert len(args.ts.split(
",")) == 2, "--ts expect width,height value (e.g 512,512)"
width, height = list(map(int, args.ts.split(",")))
cover = [tile for tile in tiles_from_csv(os.path.expanduser(args.cover))
] if args.cover else None
splits_path = os.path.join(os.path.expanduser(args.out), ".splits")
tiles_map = {}
print("RoboSat.pink - tile on CPU, with {} workers".format(args.workers),
file=sys.stderr,
flush=True)
bands = -1
for path in args.rasters:
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326", *raster.bounds)
if bands != -1:
assert bands == len(
raster.indexes), "Coverage must be bands consistent"
bands = len(raster.indexes)
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiles = list(set(tiles) & set(cover)) if cover else tiles
for tile in tiles:
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if tile_key not in tiles_map.keys():
tiles_map[tile_key] = []
tiles_map[tile_key].append(path)
if args.label:
ext = "png"
bands = 1
if not args.label:
if bands == 1:
ext = "png"
if bands == 3:
ext = "webp"
if bands > 3:
ext = "tiff"
tiles = []
progress = tqdm(total=len(tiles_map), ascii=True, unit="tile")
# Begin to tile plain tiles
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(path):
raster = rasterio_open(path)
w, s, e, n = transform_bounds(raster.crs, "EPSG:4326",
*raster.bounds)
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiled = []
for tile in tiles:
if cover and tile not in cover:
continue
w, s, e, n = mercantile.xy_bounds(tile)
warp_vrt = WarpedVRT(
raster,
crs="epsg:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(w, s, e, n, width, height),
width=width,
height=height,
)
data = warp_vrt.read(out_shape=(len(raster.indexes), width,
height),
window=warp_vrt.window(w, s, e, n))
if data.dtype == "uint16": # GeoTiff could be 16 bits
data = np.uint8(data / 256)
elif data.dtype == "uint32": # or 32 bits
data = np.uint8(data / (256 * 256))
image = np.moveaxis(data, 0, 2) # C,H,W -> H,W,C
tile_key = (str(tile.x), str(tile.y), str(tile.z))
if (not args.label
and len(tiles_map[tile_key]) == 1 and is_nodata(
image, args.nodata, args.nodata_threshold,
args.keep_borders)):
progress.update()
continue
if len(tiles_map[tile_key]) > 1:
out = os.path.join(splits_path,
str(tiles_map[tile_key].index(path)))
else:
out = args.out
x, y, z = map(int, tile)
if not args.label:
tile_image_to_file(out, mercantile.Tile(x=x, y=y, z=z),
image)
if args.label:
tile_label_to_file(out, mercantile.Tile(x=x, y=y, z=z),
palette, image)
if len(tiles_map[tile_key]) == 1:
progress.update()
tiled.append(mercantile.Tile(x=x, y=y, z=z))
return tiled
for tiled in executor.map(worker, args.rasters):
if tiled is not None:
tiles.extend(tiled)
# Aggregate remaining tiles splits
with futures.ThreadPoolExecutor(args.workers) as executor:
def worker(tile_key):
if len(tiles_map[tile_key]) == 1:
return
image = np.zeros((width, height, bands), np.uint8)
x, y, z = map(int, tile_key)
for i in range(len(tiles_map[tile_key])):
root = os.path.join(splits_path, str(i))
_, path = tile_from_xyz(root, x, y, z)
if not args.label:
split = tile_image_from_file(path)
if args.label:
split = tile_label_from_file(path)
split = split.reshape((width, height, 1)) # H,W -> H,W,C
assert image.shape == split.shape
image[np.where(image == 0)] += split[np.where(image == 0)]
if not args.label and is_nodata(image, args.nodata,
args.nodata_threshold,
args.keep_borders):
progress.update()
return
tile = mercantile.Tile(x=x, y=y, z=z)
if not args.label:
tile_image_to_file(args.out, tile, image)
if args.label:
tile_label_to_file(args.out, tile, palette, image)
progress.update()
return tile
for tiled in executor.map(worker, tiles_map.keys()):
if tiled is not None:
tiles.append(tiled)
if splits_path and os.path.isdir(splits_path):
shutil.rmtree(splits_path) # Delete suffixes dir if any
if tiles and not args.no_web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
base_url = args.web_ui_base_url if args.web_ui_base_url else "."
web_ui(args.out, base_url, tiles, tiles, ext, template)
def main(args):
if (
int(args.bbox is not None)
+ int(args.geojson is not None)
+ int(args.dir is not None)
+ int(args.xyz is not None)
+ int(args.raster is not None)
+ int(args.cover is not None)
!= 1
):
sys.exit("ERROR: One, and only one, input type must be provided, among: --dir, --bbox, --cover or --geojson.")
if args.bbox:
try:
w, s, e, n, crs = args.bbox.split(",")
w, s, e, n = map(float, (w, s, e, n))
except:
try:
crs = None
w, s, e, n = map(float, args.bbox.split(","))
except:
sys.exit("ERROR: invalid bbox parameter.")
if args.splits:
try:
splits = [int(split) for split in args.splits.split("/")]
assert len(splits) == len(args.out)
assert sum(splits) == 100
except:
sys.exit("ERROR: Invalid split value or incoherent with provided out paths.")
if not args.zoom and (args.geojson or args.bbox or args.raster):
sys.exit("ERROR: Zoom parameter is required.")
args.out = [os.path.expanduser(out) for out in args.out]
cover = []
if args.raster:
print("RoboSat.pink - cover from {} at zoom {}".format(args.raster, args.zoom))
with rasterio_open(os.path.expanduser(args.raster)) as r:
try:
w, s, e, n = transform_bounds(r.crs, "EPSG:4326", *r.bounds)
except:
sys.exit("ERROR: unable to deal with raster projection")
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.geojson:
print("RoboSat.pink - cover from {} at zoom {}".format(args.geojson, args.zoom))
with open(os.path.expanduser(args.geojson)) as f:
features = json.load(f)
try:
for feature in tqdm(features["features"], ascii=True, unit="feature"):
cover.extend(map(tuple, burntiles.burn([feature], args.zoom).tolist()))
except:
sys.exit("ERROR: invalid or unsupported GeoJSON.")
cover = list(set(cover)) # tiles can overlap for multiple features; unique tile ids
if args.bbox:
print("RoboSat.pink - cover from {} at zoom {}".format(args.bbox, args.zoom))
if crs:
try:
w, s, e, n = transform_bounds(crs, "EPSG:4326", w, s, e, n)
except:
sys.exit("ERROR: unable to deal with raster projection")
cover = [tile for tile in tiles(w, s, e, n, args.zoom)]
if args.cover:
print("RoboSat.pink - cover from {}".format(args.cover))
cover = [tile for tile in tiles_from_csv(args.cover)]
if args.dir:
print("RoboSat.pink - cover from {}".format(args.dir))
cover = [tile for tile in tiles_from_dir(args.dir, xyz=False)]
if args.xyz:
print("RoboSat.pink - cover from {}".format(args.xyz))
cover = [tile for tile in tiles_from_dir(args.xyz, xyz=True)]
_cover = []
for tile in tqdm(cover, ascii=True, unit="tile"):
if args.zoom and tile.z != args.zoom:
w, s, n, e = transform_bounds("EPSG:3857", "EPSG:4326", *xy_bounds(tile))
for t in tiles(w, s, n, e, args.zoom):
unique = True
for _t in _cover:
if _t == t:
unique = False
if unique:
_cover.append(t)
else:
_cover.append(tile)
cover = _cover
if args.splits:
shuffle(cover) # in-place
splits = [math.floor(len(cover) * split / 100) for i, split in enumerate(splits, 1)]
s = 0
covers = []
for e in splits:
covers.append(cover[s : s + e - 1])
s += e
else:
covers = [cover]
for i, cover in enumerate(covers):
if os.path.dirname(args.out[i]) and not os.path.isdir(os.path.dirname(args.out[i])):
os.makedirs(os.path.dirname(args.out[i]), exist_ok=True)
with open(args.out[i], "w") as fp:
csv.writer(fp).writerows(cover)
def main(args):
if args.type == "label":
try:
config = load_config(args.config)
except:
sys.exit("Error: Unable to load DataSet config file")
classes = config["classes"]["title"]
colors = config["classes"]["colors"]
assert len(classes) == len(colors), "classes and colors coincide"
assert len(colors) == 2, "only binary models supported right now"
try:
raster = rasterio_open(args.raster)
w, s, e, n = bounds = transform_bounds(raster.crs, "EPSG:4326",
*raster.bounds)
transform, _, _ = calculate_default_transform(raster.crs, "EPSG:3857",
raster.width,
raster.height, *bounds)
except:
sys.exit("Error: Unable to load raster or deal with it's projection")
tiles = [
mercantile.Tile(x=x, y=y, z=z)
for x, y, z in mercantile.tiles(w, s, e, n, args.zoom)
]
tiles_nodata = []
for tile in tqdm(tiles, desc="Tiling", unit="tile", ascii=True):
w, s, e, n = tile_bounds = mercantile.xy_bounds(tile)
# Inspired by Rio-Tiler, cf: https://github.com/mapbox/rio-tiler/pull/45
warp_vrt = WarpedVRT(
raster,
crs="EPSG:3857",
resampling=Resampling.bilinear,
add_alpha=False,
transform=from_bounds(*tile_bounds, args.size, args.size),
width=math.ceil((e - w) / transform.a),
height=math.ceil((s - n) / transform.e),
)
data = warp_vrt.read(out_shape=(len(raster.indexes), args.size,
args.size),
window=warp_vrt.window(w, s, e, n))
# If no_data is set, remove all tiles with at least one whole border filled only with no_data (on all bands)
if type(args.no_data) is not None and (
np.all(data[:, 0, :] == args.no_data)
or np.all(data[:, -1, :] == args.no_data)
or np.all(data[:, :, 0] == args.no_data)
or np.all(data[:, :, -1] == args.no_data)):
tiles_nodata.append(tile)
continue
C, W, H = data.shape
os.makedirs(os.path.join(args.out, str(args.zoom), str(tile.x)),
exist_ok=True)
path = os.path.join(args.out, str(args.zoom), str(tile.x), str(tile.y))
if args.type == "label":
assert C == 1, "Error: Label raster input should be 1 band"
ext = "png"
img = Image.fromarray(np.squeeze(data, axis=0), mode="P")
img.putpalette(make_palette(colors[0], colors[1]))
img.save("{}.{}".format(path, ext), optimize=True)
elif args.type == "image":
assert C == 1 or C == 3, "Error: Image raster input should be either 1 or 3 bands"
# GeoTiff could be 16 or 32bits
if data.dtype == "uint16":
data = np.uint8(data / 256)
elif data.dtype == "uint32":
data = np.uint8(data / (256 * 256))
if C == 1:
ext = "png"
Image.fromarray(np.squeeze(data, axis=0),
mode="L").save("{}.{}".format(path, ext),
optimize=True)
elif C == 3:
ext = "webp"
Image.fromarray(np.moveaxis(data, 0, 2),
mode="RGB").save("{}.{}".format(path, ext),
optimize=True)
if args.web_ui:
template = "leaflet.html" if not args.web_ui_template else args.web_ui_template
tiles = [tile for tile in tiles if tile not in tiles_nodata]
web_ui(args.out, args.web_ui, tiles, tiles, ext, template)
