def convert_to_tiff(
dim_file,
out_folder,
decibel=False,
use_nodata=True,
nodata_value=-9999.0,
):
data_folder = os.path.splitext(dim_file)[0] + ".data/"
name = os.path.splitext(os.path.basename(dim_file))[0].replace("_step_2", "")
vh_path = data_folder + "Gamma0_VH.img"
vv_path = data_folder + "Gamma0_VV.img"
out_paths = [
out_folder + name + "_Gamma0_VH.tif",
out_folder + name + "_Gamma0_VV.tif",
]
if os.path.exists(out_paths[0]) and os.path.exists(out_paths[1]):
print(f"{name} already processed")
return out_paths
vh = raster_to_array(vh_path)
vv = raster_to_array(vv_path)
if use_nodata:
vh = np.ma.masked_equal(vh, 0.0, copy=False)
vv = np.ma.masked_equal(vv, 0.0, copy=False)
vh = np.nan_to_num(vh)
vv = np.nan_to_num(vv)
vh = np.ma.masked_equal(vh.filled(nodata_value), nodata_value)
vv = np.ma.masked_equal(vv.filled(nodata_value), nodata_value)
if decibel:
with np.errstate(divide="ignore", invalid="ignore"):
if use_nodata:
vh = np.ma.multiply(np.ma.log10(np.ma.abs(vh)), 10)
vv = np.ma.multiply(np.ma.log10(np.ma.abs(vv)), 10)
else:
vh = np.multiply(np.log10(np.abs(vh)), 10)
vv = np.multiply(np.log10(np.abs(vv)), 10)
array_to_raster(vh, vh_path, out_paths[0])
array_to_raster(vv, vv_path, out_paths[1])
return out_paths
def pansharpen(pan_path, tar_path, out_path):
target = resample_raster(tar_path, pan_path, resample_alg="bilinear")
aligned = align_rasters(target, master=pan_path)
target = aligned[0]
tar_arr = raster_to_array(target, output_2d=True)
tar_arr = (tar_arr - tar_arr.min()) / (tar_arr.max() - tar_arr.min())
pan_arr = raster_to_array(pan_path, output_2d=True)
_kernel, offsets, weights = create_kernel(
[5, 5], sigma=2, output_2d=True, offsets=True
)
pan = pansharpen_filter(pan_arr, tar_arr, offsets, weights.astype("float32"))
array_to_raster(pan, reference=target, out_path=out_path)
def height_over_terrain(dsm_folder, dtm_folder, out_folder, tmp_folder):
dsm_zipped = glob(dsm_folder + "*.zip")
dtm_zipped = glob(dtm_folder + "*.zip")
completed = 0
for dsm_tile in dsm_zipped:
s = get_tile_from_zipped_url(dsm_tile)
for dtm_tile in dtm_zipped:
t = get_tile_from_zipped_url(dtm_tile)
if s == t:
ZipFile(dsm_tile).extractall(tmp_folder)
ZipFile(dtm_tile).extractall(tmp_folder)
dsm_tiffs = glob(tmp_folder + "DSM_*.tif")
dtm_tiffs = glob(tmp_folder + "DTM_*.tif")
for s_tiff in dsm_tiffs:
s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4]
s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0]
for t_tiff in dtm_tiffs:
t_tiff_tile_base = os.path.basename(t_tiff).split(
"_")[2:4]
t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0]
if s_tiff_tile == t_tiff_tile:
ss = raster_to_array(s_tiff)
tt = raster_to_array(t_tiff)
array_to_raster(
np.abs(np.subtract(ss, tt)),
out_path=out_folder +
f"HOT_1km_{s_tiff_tile}.tif",
reference=s_tiff,
)
for f in glob(tmp_folder + "/*"):
os.remove(f)
completed += 1
print(f"Completed: {completed}/{len(dsm_zipped)}")
# weights_2D,
# )
# panned3 = mad_match(
# raster_to_array(layers[2]),
# raster_to_array(layers[1]),
# offsets_2D,
# weights_2D,
# )
# array_to_raster(panned1, folder + "vv_01.tif", out_path=folder + "01_panned.tif")
# array_to_raster(panned3, folder + "vv_03.tif", out_path=folder + "03_panned.tif")
sar_data = raster_to_array([
folder + "01_panned.tif",
folder + "vv_02.tif",
folder + "03_panned.tif",
])
result = median_collapse(
sar_data,
offsets_3D,
weights_3D,
weighted=True,
nodata_value=nodata_value,
nodata=True,
)
array_to_raster(result,
folder + "vv_01.tif",
out_path=folder + "elips_median_mad.tif")
found_path = building_tile
found = True
if not found:
area_vol_10m = internal_resample_raster(
vrt_file, (10, 10),
resample_alg='average',
out_path=tmp_folder +
f"buildings_volume_{vrt_tile_name}_10m_unscaled.tif")
hot_arr = raster_to_array(area_vol_10m) * 0
vol_10m_path = dst_folder + f"buildings_volume_{vrt_tile_name}_10m.tif"
area_10m_path = dst_folder + f"buildings_area_{vrt_tile_name}_10m.tif"
array_to_raster(hot_arr, reference=area_vol_10m, out_path=vol_10m_path)
array_to_raster(hot_arr,
reference=area_vol_10m,
out_path=area_10m_path)
processed += 1
continue
try:
metadata = internal_raster_to_metadata(vrt_file)
except:
print(f"Error while processing tile: {vrt_file}")
error_files.append(vrt_file)
processed += 1
continue
def extract_patches(
raster_list,
outdir,
tile_size=32,
zones=None,
options=None,
):
"""
Generate patches for machine learning from rasters
"""
base_options = {
"overlaps": True,
"border_check": True,
"merge_output": True,
"force_align": True,
"output_raster_labels": True,
"label_geom": None,
"label_res": 0.2,
"label_mult": 100,
"tolerance": 0.0,
"fill_value": 0,
"zone_layer_id": 0,
"align_with_size": 20,
"prefix": "",
"postfix": "",
}
if options is None:
options = base_options
else:
for key in options:
if key not in base_options:
raise ValueError(f"Invalid option: {key}")
base_options[key] = options[key]
options = base_options
if zones is not None and not is_vector(zones):
raise TypeError(
"Clip geom is invalid. Did you input a valid geometry?")
if not isinstance(raster_list, list):
raster_list = [raster_list]
for raster in raster_list:
if not is_raster(raster):
raise TypeError("raster_list is not a list of rasters.")
if not os.path.isdir(outdir):
raise ValueError(
"Outdir does not exist. Please create before running the function."
)
if not rasters_are_aligned(raster_list, same_extent=True):
if options["force_align"]:
print(
"Rasters we not aligned. Realigning rasters due to force_align=True option."
)
raster_list = align_rasters(raster_list)
else:
raise ValueError("Rasters in raster_list are not aligned.")
offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]]
raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True)
pixel_size = min(raster_metadata["pixel_height"],
raster_metadata["pixel_width"])
if zones is None:
zones = raster_metadata["extent_datasource_path"]
zones_meta = vector_to_metadata(zones)
mem_driver = ogr.GetDriverByName("ESRI Shapefile")
if zones_meta["layer_count"] == 0:
raise ValueError("Vector contains no layers.")
zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]]
if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]:
raise ValueError("clip geom is not Polygon or Multi Polygon.")
zones_ogr = open_vector(zones)
zones_layer = zones_ogr.GetLayer(options["zone_layer_id"])
feature_defn = zones_layer.GetLayerDefn()
fids = vector_get_fids(zones_ogr, options["zone_layer_id"])
progress(0, len(fids) * len(raster_list), "processing fids")
processed_fids = []
processed = 0
labels_processed = False
for idx, raster in enumerate(raster_list):
name = os.path.splitext(os.path.basename(raster))[0]
list_extracted = []
list_masks = []
list_labels = []
for fid in fids:
feature = zones_layer.GetFeature(fid)
geom = feature.GetGeometryRef()
fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp"
fid_ds = mem_driver.CreateDataSource(fid_path)
fid_ds_lyr = fid_ds.CreateLayer(
"fid_layer",
geom_type=ogr.wkbPolygon,
srs=zones_layer_meta["projection_osr"],
)
copied_feature = ogr.Feature(feature_defn)
copied_feature.SetGeometry(geom)
fid_ds_lyr.CreateFeature(copied_feature)
fid_ds.FlushCache()
fid_ds.SyncToDisk()
valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif"
rasterize_vector(
fid_path,
pixel_size,
out_path=valid_path,
extent=fid_path,
)
valid_arr = raster_to_array(valid_path)
if options["label_geom"] is not None and fid not in processed_fids:
if not is_vector(options["label_geom"]):
raise TypeError(
"label geom is invalid. Did you input a valid geometry?"
)
uuid = str(uuid4().int)
label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp"
label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif"
label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif"
intersect_vector(options["label_geom"],
fid_ds,
out_path=label_clip_path)
try:
rasterize_vector(
label_clip_path,
options["label_res"],
out_path=label_ras_path,
extent=valid_path,
)
except Exception:
array_to_raster(
np.zeros(valid_arr.shape, dtype="float32"),
valid_path,
out_path=label_ras_path,
)
resample_raster(
label_ras_path,
pixel_size,
resample_alg="average",
out_path=label_warp_path,
)
labels_arr = (raster_to_array(label_warp_path) *
options["label_mult"]).astype("float32")
if options["output_raster_labels"]:
array_to_raster(
labels_arr,
label_warp_path,
out_path=
f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif",
)
raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif"
try:
clip_raster(
raster,
valid_path,
raster_clip_path,
all_touch=False,
adjust_bbox=False,
)
except Exception as e:
print(
f"Warning: {raster} did not intersect geom with fid: {fid}."
)
print(e)
if options["label_geom"] is not None:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
continue
arr = raster_to_array(raster_clip_path)
if arr.shape[:2] != valid_arr.shape[:2]:
raise Exception(
f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}"
)
arr_offsets = get_overlaps(arr, offsets, tile_size,
options["border_check"])
arr = np.concatenate(arr_offsets)
valid_offsets = np.concatenate(
get_overlaps(valid_arr, offsets, tile_size,
options["border_check"]))
valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) /
(tile_size * tile_size))) <=
options["tolerance"])[:, 0]
arr = arr[valid_mask]
valid_masked = valid_offsets[valid_mask]
if options["label_geom"] is not None and not labels_processed:
labels_masked = np.concatenate(
get_overlaps(labels_arr, offsets, tile_size,
options["border_check"]))[valid_mask]
if options["merge_output"]:
list_extracted.append(arr)
list_masks.append(valid_masked)
if options["label_geom"] is not None and not labels_processed:
list_labels.append(labels_masked)
else:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy",
arr.filled(options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if fid not in processed_fids:
processed_fids.append(fid)
processed += 1
progress(processed,
len(fids) * len(raster_list), "processing fids")
if not options["merge_output"]:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
gdal.Unlink(valid_path)
if options["merge_output"]:
np.save(
f"{outdir}{options['prefix']}{name}{options['postfix']}.npy",
np.ma.concatenate(list_extracted).filled(
options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy",
np.ma.concatenate(list_masks).filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy",
np.ma.concatenate(list_labels).filled(
options["fill_value"]),
)
labels_processed = True
progress(1, 1, "processing fids")
return 1
def process_aligned(
aligned_rasters,
out_path,
folder_tmp,
chunks,
master_raster,
nodata_value,
feather_weights=None,
):
kernel_size = 3
chunk_offset = kernel_size // 2
_kernel, offsets, weights = create_kernel(
(kernel_size, kernel_size, len(aligned_rasters)),
distance_calc=False, # "gaussian"
sigma=1,
spherical=True,
radius_method="ellipsoid",
offsets=True,
edge_weights=True,
normalised=True,
remove_zero_weights=True,
)
arr_aligned = raster_to_array(aligned_rasters)
if feather_weights is not None:
feather_weights_arr = raster_to_array(feather_weights)
if not rasters_are_aligned(aligned_rasters):
raise Exception("Rasters not aligned")
if chunks > 1:
chunks_list = []
print("Chunking rasters")
uids = uuid4()
for chunk in range(chunks):
print(f"Chunk {chunk + 1} of {chunks}")
cut_start = False
cut_end = False
if chunk == 0:
chunk_start = 0
else:
chunk_start = (chunk *
(arr_aligned.shape[0] // chunks)) - chunk_offset
cut_start = True
if chunk == chunks - 1:
chunk_end = arr_aligned.shape[0]
else:
chunk_end = ((chunk + 1) *
(arr_aligned.shape[0] // chunks)) + chunk_offset
cut_end = True
arr_chunk = arr_aligned[chunk_start:chunk_end]
if feather_weights is not None:
weights_chunk = feather_weights_arr[chunk_start:chunk_end]
else:
weights_chunk = np.ones_like(arr_chunk)
print(" Collapsing...")
arr_collapsed = s1_collapse(
arr_chunk,
offsets,
weights,
weights_chunk,
weighted=True,
nodata_value=nodata_value,
nodata=True,
)
offset_start = chunk_offset if cut_start else 0
offset_end = (arr_collapsed.shape[0] -
chunk_offset if cut_end else arr_collapsed.shape[0])
chunk_path = folder_tmp + f"{uids}_chunk_{chunk}.npy"
chunks_list.append(chunk_path)
np.save(chunk_path, arr_collapsed[offset_start:offset_end])
arr_chunk = None
arr_collapsed = None
print("Merging Chunks")
arr_aligned = None
merged = []
for chunk in chunks_list:
merged.append(np.load(chunk))
merged = np.concatenate(merged)
merged = np.ma.masked_array(merged, mask=merged == nodata_value)
merged.fill_value = nodata_value
print("Writing raster.")
array_to_raster(
merged,
master_raster,
out_path=out_path,
)
merged = None
return out_path
if feather_weights is not None:
weights_borders = feather_weights
else:
weights_borders = np.ones_like(arr_aligned)
print("Collapsing rasters")
arr_collapsed = s1_collapse(
arr_aligned,
offsets,
weights,
weights_borders,
weighted=True,
nodata_value=nodata_value,
nodata=True,
)
arr_collapsed = np.ma.masked_array(arr_collapsed,
mask=arr_collapsed == nodata_value)
arr_collapsed.fill_value = nodata_value
arr_aligned = None
print("Writing raster.")
array_to_raster(
arr_collapsed,
master_raster,
out_path=out_path,
)
arr_collapsed = None
return out_path
def volume_over_terrain(
tile_names,
username,
password,
out_folder,
tmp_folder,
):
if not os.path.isdir(tmp_folder):
raise Exception("Error: output directory does not exist.")
error_tiles = []
completed = 0
for tile in tile_names:
base_path_DSM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DSM/"
file_name_DSM = f'DSM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip'
base_path_DTM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DTM/"
file_name_DTM = f'DTM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip'
try:
if not os.path.exists(tmp_folder + file_name_DSM):
get_file(base_path_DSM + file_name_DSM,
tmp_folder + file_name_DSM)
else:
print(f"{file_name_DSM} Already exists.")
if not os.path.exists(tmp_folder + file_name_DTM):
get_file(base_path_DTM + file_name_DTM,
tmp_folder + file_name_DTM)
else:
print(f"{file_name_DTM} Already exists.")
ZipFile(tmp_folder + file_name_DSM).extractall(tmp_folder)
ZipFile(tmp_folder + file_name_DTM).extractall(tmp_folder)
dsm_tiffs = glob(tmp_folder + "DSM_*.tif")
dtm_tiffs = glob(tmp_folder + "DTM_*.tif")
hot_files = []
for s_tiff in dsm_tiffs:
s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4]
s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0]
for t_tiff in dtm_tiffs:
t_tiff_tile_base = os.path.basename(t_tiff).split("_")[2:4]
t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0]
if s_tiff_tile == t_tiff_tile:
ss = raster_to_array(s_tiff)
tt = raster_to_array(t_tiff)
hot_name = out_folder + f"HOT_1km_{s_tiff_tile}.tif"
subtracted = np.subtract(ss, tt)
hot_arr = np.abs((subtracted >= 0) * subtracted)
array_to_raster(
hot_arr,
out_path=hot_name,
reference=s_tiff,
)
hot_files.append(hot_name)
km10_tilename = "_".join(file_name_DSM.split("_")[1:3])
vrt_name = out_folder + f"HOT_10km_{km10_tilename}.vrt"
stack_rasters_vrt(
hot_files,
seperate=False,
out_path=vrt_name,
)
except:
print("Error while processing tile: {tile}")
error_tiles.append(tile)
finally:
for f in glob(tmp_folder + "/*"):
os.remove(f)
completed += 1
print(f"Completed: {completed}/{len(tile_names)} - {tile}")
def super_sample_s2(
B04_link,
B08_link,
B05_link=None,
B06_link=None,
B07_link=None,
B8A_link=None,
out_folder="../raster/",
prefix="",
suffix="",
):
assert (isinstance(B05_link, str) or isinstance(B06_link, str)
or isinstance(B07_link, str) or isinstance(B8A_link, str))
paths = {
"B04": B04_link,
"B05": B05_link,
"B06": B06_link,
"B07": B07_link,
"B08": B08_link,
"B8A": B8A_link,
}
bands = {
"B04":
raster_to_array(B04_link).astype("float32"),
"B05":
raster_to_array(B05_link).astype("float32")
if B05_link is not None else False,
"B06":
raster_to_array(B06_link).astype("float32")
if B06_link is not None else False,
"B07":
raster_to_array(B07_link).astype("float32")
if B07_link is not None else False,
"B08":
raster_to_array(B08_link).astype("float32"),
"B8A":
raster_to_array(B8A_link).astype("float32")
if B8A_link is not None else False,
}
bands_to_pansharpen = []
if bands["B05"] is not False:
bands_to_pansharpen.append("B05")
if bands["B06"] is not False:
bands_to_pansharpen.append("B06")
if bands["B07"] is not False:
bands_to_pansharpen.append("B07")
if bands["B8A"] is not False:
bands_to_pansharpen.append("B8A")
for band_x in bands_to_pansharpen:
if band_x is "B05":
pseudo_band = "B04"
else:
pseudo_band = "B08"
pseudo_path = os.path.join(out_folder,
f"{prefix}{band_x}{suffix}_pseudo.tif")
array_to_raster(
bands[pseudo_band],
reference_raster=paths[pseudo_band],
out_raster=pseudo_path,
)
low_res_10m = raster_to_array(
resample_raster(
paths[band_x],
reference_raster=paths[pseudo_band])).astype("float32")
resampled_path = os.path.join(
out_folder, f"{prefix}{band_x}{suffix}_resampled.tif")
array_to_raster(low_res_10m,
reference_raster=paths[pseudo_band],
out_raster=resampled_path)
low_res_10m = None
pansharpened_path = os.path.join(
out_folder, f"{prefix}{band_x}{suffix}_float.tif")
pansharpen(pseudo_path, resampled_path, pansharpened_path)
os.remove(resampled_path)
os.remove(pseudo_path)
def raster_mask_values(
raster: Union[gdal.Dataset, str, list],
values_to_mask: list,
out_path: Union[list, str, None] = None,
include_original_nodata: bool = True,
dst_nodata: Union[float, int, str, list, None] = "infer",
in_place: bool = False,
overwrite: bool = True,
opened: bool = False,
prefix: str = "",
postfix: str = "_nodata_masked",
creation_options: list = [],
) -> Union[list, gdal.Dataset, str]:
"""Mask a raster with a list of values.
Args:
raster (path | raster | list): The raster(s) to retrieve nodata values from.
values_to_mask (list): The list of values to mask in the raster(s)
**kwargs:
include_original_nodata: (bool): If True, the nodata value of the raster(s) will be
included in the values to mask.
dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata
value is set based on the input datatype. A list of nodata values can be based matching
the amount of input rasters. If multiple nodata values should be set, use raster_mask_values.
out_path (path | list | None): The destination of the changed rasters. If out_paths
are specified, in_place is automatically set to False. The path can be a folder.
in_place (bool): Should the rasters be changed in_place or copied?
prefix (str): Prefix to add the the output if a folder is specified in out_path.
postfix (str): Postfix to add the the output if a folder is specified in out_path.
Returns:
Returns the rasters with nodata removed. If in_place is True a reference to the
changed orignal is returned, otherwise a copied memory raster or the path to the
generated raster is outputted.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(values_to_mask, [list], "values_to_mask")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(include_original_nodata, [bool], "include_original_nodata")
type_check(dst_nodata, [float, int, str, list],
"dst_nodata",
allow_none=True)
type_check(in_place, [bool], "in_place")
type_check(overwrite, [bool], "overwrite")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(opened, [bool], "opened")
type_check(creation_options, [list], "creation_options")
rasters_metadata = []
internal_in_place = in_place if out_path is None else False
internal_dst_nodata = None
for value in values_to_mask:
if not isinstance(value, (int, float)):
raise ValueError("Values in values_to_mask must be ints or floats")
if isinstance(dst_nodata, str) and dst_nodata != "infer":
raise ValueError(f"Invalid dst_nodata value. {dst_nodata}")
if isinstance(dst_nodata, list):
if not isinstance(raster, list) or len(dst_nodata) != len(raster):
raise ValueError(
"If dst_nodata is a list, raster must also be a list of equal length."
)
for value in dst_nodata:
if isinstance(value, (float, int, str, None)):
raise ValueError("Invalid type in dst_nodata list.")
if isinstance(value, str) and value != "infer":
raise ValueError(
"If dst_nodata is a string it must be 'infer'")
raster_list, out_names = ready_io_raster(raster, out_path, overwrite,
prefix, postfix)
output_rasters = []
for index, internal_raster in enumerate(raster_list):
raster_metadata = None
if len(rasters_metadata) == 0:
raster_metadata = raster_to_metadata(internal_raster)
rasters_metadata.append(raster_metadata)
else:
raster_metadata = rasters_metadata[index]
if dst_nodata == "infer":
internal_dst_nodata = gdal_nodata_value_from_type(
raster_metadata["dtype_gdal_raw"])
elif isinstance(dst_nodata, list):
internal_dst_nodata = dst_nodata[index]
else:
internal_dst_nodata = dst_nodata
mask_values = list(values_to_mask)
if include_original_nodata:
if raster_metadata["nodata_value"] is not None:
mask_values.append(raster_metadata["nodata_value"])
arr = raster_to_array(internal_raster, filled=True)
mask = None
for index, mask_value in enumerate(mask_values):
if index == 0:
mask = arr == mask_value
else:
mask = mask | arr == mask_value
arr = np.ma.masked_array(arr,
mask=mask,
fill_value=internal_dst_nodata)
if internal_in_place:
for band in range(raster_metadata["bands"]):
raster_band = internal_raster.GetRasterBand(band + 1)
raster_band.WriteArray(arr[:, :, band])
raster_band = None
else:
out_name = out_names[index]
remove_if_overwrite(out_name, overwrite)
output_rasters.append(
array_to_raster(arr, internal_raster, out_path=out_name))
if isinstance(raster, list):
return output_rasters
return output_rasters[0]
def calc_proximity(
input_rasters,
target_value=1,
out_path=None,
max_dist=1000,
add_border=False,
weighted=False,
invert=False,
return_array=False,
postfix="_proximity",
uuid=False,
overwrite=True,
skip_existing=False,
):
"""
Calculate the proximity of input_raster to values
"""
raster_list, path_list = ready_io_raster(input_rasters,
out_path,
overwrite,
postfix=postfix,
uuid=uuid)
output = []
for index, input_raster in enumerate(raster_list):
out_path = path_list[index]
if skip_existing and os.path.exists(out_path):
output.append(out_path)
continue
in_arr = raster_to_array(input_raster, filled=True)
bin_arr = (in_arr != target_value).astype("uint8")
bin_raster = array_to_raster(bin_arr, reference=input_raster)
in_raster = open_raster(bin_raster)
in_raster_path = bin_raster
if add_border:
border_size = 1
border_raster = add_border_to_raster(
in_raster,
border_size=border_size,
border_value=0,
overwrite=True,
)
in_raster = open_raster(border_raster)
gdal.Unlink(in_raster_path)
in_raster_path = border_raster
src_band = in_raster.GetRasterBand(1)
driver_name = "GTiff" if out_path is None else path_to_driver_raster(
out_path)
if driver_name is None:
raise ValueError(f"Unable to parse filetype from path: {out_path}")
driver = gdal.GetDriverByName(driver_name)
if driver is None:
raise ValueError(
f"Error while creating driver from extension: {out_path}")
mem_path = f"/vsimem/raster_proximity_tmp_{uuid4().int}.tif"
dest_raster = driver.Create(
mem_path,
in_raster.RasterXSize,
in_raster.RasterYSize,
1,
gdal.GetDataTypeByName("Float32"),
)
dest_raster.SetGeoTransform(in_raster.GetGeoTransform())
dest_raster.SetProjection(in_raster.GetProjectionRef())
dst_band = dest_raster.GetRasterBand(1)
gdal.ComputeProximity(
src_band,
dst_band,
[
f"VALUES='1'",
"DISTUNITS=GEO",
f"MAXDIST={max_dist}",
],
)
dst_arr = dst_band.ReadAsArray()
gdal.Unlink(mem_path)
gdal.Unlink(in_raster_path)
dst_arr = np.where(dst_arr > max_dist, max_dist, dst_arr)
if invert:
dst_arr = max_dist - dst_arr
if weighted:
dst_arr = dst_arr / max_dist
if add_border:
dst_arr = dst_arr[border_size:-border_size,
border_size:-border_size]
src_band = None
dst_band = None
in_raster = None
dest_raster = None
if return_array:
output.append(dst_arr)
else:
array_to_raster(dst_arr, reference=input_raster, out_path=out_path)
output.append(out_path)
dst_arr = None
if isinstance(input_rasters, list):
return output
return output[0]
# obt_bandmath(
# [
# folder + "area_residential_clipped.tif",
# folder + "area_slum_clipped.tif",
# folder + "area_industrial_clipped.tif",
# folder + "m2_per_person_smooth.tif",
# ],
# f"((im1b1 * {w1}) + (im2b1 * {w2}) + (im3b1 * {w3})) / im4b1", # afternoon
# folder + "ppl_predictions/population_area_daytime_unscaled.tif",
# ram=32000,
# )
# exit()
target = "nighttime"
unscaled = raster_to_array(
folder + f"ppl_predictions/population_area_{target}_unscaled.tif"
).filled(0)
scaled = (t_pop / unscaled.sum()) * unscaled
import pdb
pdb.set_trace()
array_to_raster(
scaled,
reference=unscaled,
out_path=folder + f"ppl_predictions/population_area_{target}.tif",
)
def norm_rasters(
in_rasters,
out_folder,
method="normalise",
split_bands=False,
min_target=0,
max_target=1,
min_og=-9999,
max_og=-9999,
truncate=True,
prefix="",
postfix="",
overwrite=True,
):
if not isinstance(in_rasters, list):
in_rasters = [in_rasters]
normed_rasters = []
for in_raster in in_rasters:
name = os.path.splitext(os.path.basename(in_raster))[0]
raster = raster_to_array(in_raster)
if method == "normalise":
normed = norm_to_range(raster,
min_target,
max_target,
truncate=False)
elif method == "standardise":
normed = standardise_filter(raster)
elif method == "median_absolute_deviation":
normed = mad_filter(raster)
elif method == "range":
normed = norm_to_range(
raster,
min_target=min_target,
max_target=max_target,
min_og=min_og,
max_og=max_og,
truncate=truncate,
)
elif method == "robust_quantile":
normed = robust_scaler_filter(
raster,
min_q=0.25,
max_q=0.75,
)
elif method == "robust_98":
normed = robust_scaler_filter(raster, min_q=0.02, max_q=0.98)
else:
raise Exception(f"Method {method} not recognised")
if split_bands:
for idx in range(raster.shape[2]):
band = idx + 1
raster_name = prefix + name + f"_B{band}" + postfix + ".tif"
normed_rasters.append(
array_to_raster(
normed[:, :, idx][..., np.newaxis],
reference=in_raster,
out_path=out_folder + raster_name,
overwrite=overwrite,
))
else:
raster_name = prefix + name + postfix + ".tif"
normed_rasters.append(
array_to_raster(
normed,
reference=in_raster,
out_path=out_folder + raster_name,
overwrite=overwrite,
))
if isinstance(in_rasters, list):
return normed_rasters
return normed_rasters[0]
def blocks_to_raster(
blocks: np.ndarray,
reference: Union[str, gdal.Dataset],
out_path: Union[str, None] = None,
offsets: Union[list, tuple, np.ndarray] = [],
border_patches: bool = True,
generate_zero_offset: bool = True,
merge_method: str = "median",
output_array: bool = False,
dtype=None,
verbose: int = 1,
) -> Union[str, np.ndarray]:
"""Recombines a series of blocks to a raster. OBS: Does not work if the patch
extraction was done with clip geom.
Args:
blocks (ndarray): A numpy array with the values to recombine. The shape
should be (blocks, rows, column, channel).
reference (str, raster): A reference raster to help coax the blocks back
into shape.
**kwargs:
out_path (str | None): Where to save the reconstituted raster. If None
are memory raster is returned.
offsets (tuple, list, ndarray): The offsets used in the original. A (0 ,0)
offset is assumed.
border_patches (bool): Do the blocks contain border patches?
generate_zero_offset (bool): if True, an offset is inserted at (0, 0)
if none is present.
merge_method (str): How to handle overlapping pixels. Options are:
median, average, mode, min, max
output_array (bool): If True the output will be a numpy array instead of a
raster.
verbose (int): If 1 will output messages on progress.
Returns:
A reconstituted raster.
"""
type_check(blocks, [str, np.ndarray], "blocks")
type_check(reference, [str, gdal.Dataset], "reference")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(offsets, [list, tuple, np.ndarray], "offsets", allow_none=True)
type_check(border_patches, [bool], "border_patches")
type_check(generate_zero_offset, [bool], "bool")
type_check(merge_method, [str], "merge_method")
type_check(output_array, [bool], "output_array")
type_check(verbose, [int], "verbose")
if isinstance(blocks, str):
try:
blocks = np.load(blocks)
except:
raise ValueError(f"Failed to parse blocks: {blocks}")
if verbose == 1:
print("Reconstituting blocks into target raster.")
metadata = internal_raster_to_metadata(reference)
border_patches_x = False
border_patches_y = False
if blocks.shape[1] != blocks.shape[2]:
raise ValueError(
"The input blocks must be square. Rectangles might supported in the future."
)
size = blocks.shape[1]
if metadata["width"] % size != 0 and border_patches:
border_patches_x = True
if metadata["height"] % size != 0 and border_patches:
border_patches_y = True
# internal offset array. Avoid manipulating the og array.
in_offsets = []
if generate_zero_offset:
if offsets is not None:
if (0, 0) not in offsets:
in_offsets.append((0, 0))
else:
in_offsets.append((0, 0))
for offset in offsets:
if offset != (0, 0):
if not isinstance(offset, (list, tuple)) or len(offset) != 2:
raise ValueError(
f"offset must be a list or tuple of two integers. Recieved: {offset}"
)
in_offsets.append((offset[0], offset[1]))
elif not generate_zero_offset:
in_offsets.append((offset[0], offset[1]))
# Easier to read this way.
has_offsets = False
if generate_zero_offset and len(in_offsets) > 1:
has_offsets = True
if not generate_zero_offset and len(in_offsets) > 0:
has_offsets = True
if has_offsets:
passes = []
previous = 0
largest_x = 0
largest_y = 0
for index, offset in enumerate(in_offsets):
passes.append(
np.ma.masked_all(
(
metadata["height"],
metadata["width"],
blocks.shape[3],
),
dtype=metadata["datatype"] if dtype is None else dtype,
), )
if index == 0:
x_blocks = (
(metadata["width"] - offset[0]) // size) + border_patches_x
y_blocks = ((metadata["height"] - offset[1]) //
size) + border_patches_x
else:
x_blocks = (metadata["width"] - offset[0]) // size
y_blocks = (metadata["height"] - offset[1]) // size
block_size = x_blocks * y_blocks
raster_pass = reconstitute_raster( # pylint: disable=too-many-function-args
blocks[previous:block_size + previous, :, :, :],
metadata["height"],
metadata["width"],
size,
offset,
border_patches,
border_patches_x,
border_patches_y,
)
if raster_pass.shape[1] > largest_x:
largest_x = raster_pass.shape[1]
if raster_pass.shape[0] > largest_y:
largest_y = raster_pass.shape[0]
previous += block_size
passes[index][offset[1]:raster_pass.shape[0] + offset[1],
offset[0]:raster_pass.shape[1] +
offset[0], :, ] = raster_pass
passes[index] = passes[index].filled(np.nan)
if merge_method == "median":
raster = np.nanmedian(passes, axis=0)
elif merge_method == "mean" or merge_method == "average":
raster = np.nanmean(passes, axis=0)
elif merge_method == "min" or merge_method == "minumum":
raster = np.nanmin(passes, axis=0)
elif merge_method == "max" or merge_method == "maximum":
raster = np.nanmax(passes, axis=0)
elif merge_method == "mode" or merge_method == "majority":
for index, _ in enumerate(passes):
passes[index] = np.rint(passes[index]).astype(int)
raster = np.apply_along_axis(lambda x: np.bincount(x).argmax(),
axis=0,
arr=passes)
else:
raise ValueError(f"Unable to parse merge_method: {merge_method}")
else:
raster: np.ndarray = reconstitute_raster(
blocks,
metadata["height"],
metadata["width"],
size,
(0, 0),
border_patches,
border_patches_x,
border_patches_y,
)
if output_array:
return raster
return array_to_raster(raster, reference, out_path=out_path)
def predict_raster(
raster: Union[List[Union[str, gdal.Dataset]], str, gdal.Dataset],
model: str,
out_path: Optional[str] = None,
offsets: Union[List[Tuple[int, int]], List[List[Tuple[int, int]]]] = [],
region: Optional[Union[str, ogr.DataSource]] = None,
device: str = "gpu",
merge_method: str = "median",
mirror: bool = False,
rotate: bool = False,
custom_objects: Dict[str, Any] = {
"Mish": Mish,
"mish": mish,
"tpe": tpe
},
target_raster: Optional[str] = None,
output_size=128,
dtype: str = "same",
batch_size: int = 16,
overwrite: bool = True,
creation_options: List[str] = [],
verbose: int = 1,
) -> str:
"""Runs a raster or list of rasters through a deep learning network (Tensorflow).
Supports tiling and reconstituting the output. Offsets are allowed and will be
bleneded with the merge_method. If the output is a different resolution
than the input. The output will automatically be scaled to match.
Args:
raster (list | path | raster): The raster(s) to convert.
model (path): A path to the tensorflow .h5 model.
**kwargs:
out_path (str | None): Where to save the reconstituted raster. If None
are memory raster is returned.
offsets (tuple, list, ndarray): The offsets used in the original. A (0 ,0)
offset is assumed.
border_patches (bool): Do the blocks contain border patches?
device (str): Either CPU or GPU to use with tensorflow.
merge_method (str): How to handle overlapping pixels. Options are:
median, average, mode, min, max
mirror (bool): Mirror the raster and do predictions as well.
rotate (bool): rotate the raster and do predictions as well.
dtype (str | None): The dtype of the output. If None: Float32, "save"
is the same as the input raster. Otherwise overwrite dtype.
overwrite (bool): Overwrite output files if they exists.
creation_options: Extra creation options for the output raster.
verbose (int): If 1 will output messages on progress.
Returns:
A predicted raster.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(model, [str], "model")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(offsets, [list], "offsets")
type_check(region, [str, ogr.DataSource], allow_none=True)
type_check(device, [str], "device")
type_check(merge_method, [str], "merge_method")
type_check(mirror, [bool], "mirror")
type_check(rotate, [bool], "rotate")
type_check(custom_objects, [dict], "custom_objects")
type_check(dtype, [str], "dtype", allow_none=True)
type_check(batch_size, [int], "batch_size")
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
type_check(verbose, [int], "verbose")
if mirror or rotate:
raise Exception("Mirror and rotate currently disabled.")
import tensorflow as tf
os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true"
if verbose == 1:
print("Loading model.")
if isinstance(model, str):
model_loaded = tf.keras.models.load_model(
model, custom_objects=custom_objects)
else:
model_loaded = model
multi_input = False
if isinstance(model_loaded.input, list) and len(model_loaded.input) > 1:
if not isinstance(raster, list):
raise TypeError("Multi input model must have a list as input.")
if len(offsets) > 0:
for offset in offsets:
if not isinstance(offset, list):
raise TypeError(
"Offsets must be a list of tuples, same length as inputs."
)
for _offset in offset:
if not isinstance(_offset, tuple):
raise TypeError("Offset must be a tuple")
if len(_offset) != 2:
raise ValueError("Offset must be length 2.")
if len(model_loaded.input) != len(offsets):
raise ValueError("Length of offsets must equal model inputs.")
multi_input = True
model_inputs = (model_loaded.input if isinstance(model_loaded.input, list)
else [model_loaded.input])
shape_output = tuple(model_loaded.output.shape)
if shape_output[1] == None or shape_output[2] == None or shape_output[
3] == None:
print(
f"Unable to find output size, using the supplied variable: {output_size}"
)
shape_output = (None, output_size, output_size, 1)
dst_tile_size = shape_output[1]
prediction_arr = []
readied_inputs = []
pixel_factor = 1.0
for index, model_input in enumerate(model_inputs):
if verbose == 1:
print(f"Readying input: {index}")
shape_input = tuple(model_input.shape)
if len(shape_input) != 4 or len(shape_output) != 4:
raise ValueError(
f"Model input not 4d: {shape_input} - {shape_output}")
if shape_input[1] != shape_input[2] or shape_output[1] != shape_output[
2]:
raise ValueError("Model only takes square images.")
src_tile_size = shape_input[1]
pixel_factor = src_tile_size / dst_tile_size
scale_factor = dst_tile_size / src_tile_size
dst_offsets = []
in_offsets: List[Tuple[Number, Number]] = []
if multi_input:
if len(offsets) > 0:
in_offsets = offsets[index]
else:
in_offsets = offsets
for offset in in_offsets:
if not isinstance(offset, tuple):
raise ValueError(
f"Offset must be a tuple of two ints. Recieved: {offset}")
if len(offset) != 2:
raise ValueError(
"Offsets must have two values. Both integers.")
dst_offsets.append((
round(offset[0] * scale_factor),
round(offset[1] * scale_factor),
))
use_raster = raster[index] if isinstance(raster, list) else raster
if region is not None:
use_raster = clip_raster(use_raster,
region,
adjust_bbox=False,
all_touch=False)
# # check raster size here
# use_raster_meta = internal_raster_to_metadata(use_raster)
# import pdb
# pdb.set_trace()
blocks, _ = extract_patches(
use_raster,
size=src_tile_size,
offsets=in_offsets,
generate_border_patches=True,
generate_grid_geom=False,
verbose=verbose,
)
readied_inputs.append(blocks)
first_len = None
for index, readied in enumerate(readied_inputs):
if index == 0:
first_len = readied.shape[0]
else:
if readied.shape[0] != first_len:
import pdb
pdb.set_trace()
raise ValueError(
"Length of inputs do not match. Have you set the offsets in the correct order?"
)
if verbose == 1:
print("Predicting raster.")
start = 0
end = readied_inputs[0].shape[0]
predictions = np.empty(
(end, dst_tile_size, dst_tile_size, shape_output[3]), dtype="float32")
if multi_input is False:
if device == "cpu":
with tf.device("/cpu:0"):
while start < end:
predictions[start:start +
batch_size] = model_loaded.predict_on_batch(
readied_inputs[0][start:start +
batch_size])
start += batch_size
progress(start, end - 1, "Predicting")
else:
while start < end:
predictions[start:start +
batch_size] = model_loaded.predict_on_batch(
readied_inputs[0][start:start + batch_size])
start += batch_size
progress(start, end - 1, "Predicting")
else:
if device == "cpu":
with tf.device("/cpu:0"):
while start < end:
batch = []
for i in range(len(readied_inputs)):
batch.append(readied_inputs[i][start:start +
batch_size])
predictions[start:start +
batch_size] = model_loaded.predict_on_batch(
batch)
start += batch_size
progress(start, end - 1, "Predicting")
else:
while start < end:
batch = []
for i in range(len(readied_inputs)):
batch.append(readied_inputs[i][start:start + batch_size])
predictions[start:start +
batch_size] = model_loaded.predict_on_batch(batch)
start += batch_size
progress(start, end - 1, "Predicting")
print("")
print("Reconstituting Raster.")
rast_meta = None
target_size = None
resampled = None
if target_raster is not None:
resampled = target_raster
elif isinstance(raster, list):
rast_meta = internal_raster_to_metadata(raster[-1])
target_size = (
rast_meta["pixel_width"] * pixel_factor,
rast_meta["pixel_height"] * pixel_factor,
)
resampled = internal_resample_raster(raster[-1],
target_size=target_size,
dtype="float32")
else:
rast_meta = internal_raster_to_metadata(raster)
target_size = (
rast_meta["pixel_width"] * pixel_factor,
rast_meta["pixel_height"] * pixel_factor,
)
resampled = internal_resample_raster(raster,
target_size=target_size,
dtype="float32")
if region is not None:
resampled = clip_raster(resampled, region)
prediction_arr.append(
blocks_to_raster(
predictions,
resampled,
border_patches=True,
offsets=dst_offsets,
merge_method=merge_method,
output_array=True,
dtype="float32",
))
if verbose == 1:
print("Merging rasters.")
if merge_method == "median":
predicted = np.median(prediction_arr, axis=0)
elif merge_method == "mean" or merge_method == "average":
predicted = np.mean(prediction_arr, axis=0)
elif merge_method == "min" or merge_method == "minumum":
predicted = np.min(prediction_arr, axis=0)
elif merge_method == "max" or merge_method == "maximum":
predicted = np.max(prediction_arr, axis=0)
elif merge_method == "mode" or merge_method == "majority":
for index, _ in enumerate(prediction_arr):
prediction_arr[index] = np.rint(prediction_arr[index]).astype(int)
predicted = np.apply_along_axis(lambda x: np.bincount(x).argmax(),
axis=0,
arr=prediction_arr)
else:
raise ValueError(f"Unable to parse merge_method: {merge_method}")
if dtype == "same" or dtype == None:
predicted = array_to_raster(
predicted.astype(rast_meta["datatype"]),
reference=resampled,
)
else:
predicted = array_to_raster(
predicted.astype(dtype),
reference=resampled,
)
if out_path is None:
return predicted
else:
return internal_raster_to_disk(
predicted,
out_path=out_path,
overwrite=overwrite,
creation_options=creation_options,
)
predictions.append(borders)
print("Merging predictions.")
with np.errstate(invalid="ignore"):
pred_readied = np.concatenate(predictions, axis=2).astype("float32")
if method == "mean":
predicted = np.nanmean(pred_readied)
elif method == "olympic" or method == "olympic_1":
sort = np.sort(pred_readied)[1:-1]
predicted = np.nanmean(sort)
elif method == "olympic_2":
sort = np.sort(pred_readied)[2:-2]
predicted = np.nanmean(sort)
elif method == "mad":
predicted = mad_interval_merging(pred_readied)
else:
predicted = np.nanmedian(pred_readied)
if scale_to_sum:
predicted = predicted / np.sum(predicted)
if out_path_variance is not None:
array_to_raster(
np.nanvar(pred_readied),
reference=reference_raster,
out_path=out_path_variance,
)
return array_to_raster(predicted, reference=reference_raster, out_path=out_path)