def assess_quality(safe_folder):
metadata = get_metadata(safe_folder)
scene_classification = raster_to_array(metadata["paths"]["20m"]["SCL"],
filled=True,
output_2d=True)
sun_elevation = metadata["SUN_ELEVATION"]
sun_adjust = ((-0.0012 * (sun_elevation * sun_elevation)) +
(0.2778 * sun_elevation) - 10)
b2 = raster_to_array(metadata["paths"]["20m"]["B02"],
filled=True,
output_2d=True)
b12 = raster_to_array(metadata["paths"]["20m"]["B12"],
filled=True,
output_2d=True)
cld_prop = raster_to_array(metadata["paths"]["QI"]["CLDPRB_20m"],
filled=True,
output_2d=True)
quality = scl_to_quality(scene_classification, b2, b12, cld_prop,
sun_adjust)
quality = smooth_quality(quality)
return quality
def convert_to_tiff(
dim_file,
out_folder,
decibel=False,
use_nodata=True,
nodata_value=-9999.0,
):
data_folder = os.path.splitext(dim_file)[0] + ".data/"
name = os.path.splitext(os.path.basename(dim_file))[0].replace("_step_2", "")
vh_path = data_folder + "Gamma0_VH.img"
vv_path = data_folder + "Gamma0_VV.img"
out_paths = [
out_folder + name + "_Gamma0_VH.tif",
out_folder + name + "_Gamma0_VV.tif",
]
if os.path.exists(out_paths[0]) and os.path.exists(out_paths[1]):
print(f"{name} already processed")
return out_paths
vh = raster_to_array(vh_path)
vv = raster_to_array(vv_path)
if use_nodata:
vh = np.ma.masked_equal(vh, 0.0, copy=False)
vv = np.ma.masked_equal(vv, 0.0, copy=False)
vh = np.nan_to_num(vh)
vv = np.nan_to_num(vv)
vh = np.ma.masked_equal(vh.filled(nodata_value), nodata_value)
vv = np.ma.masked_equal(vv.filled(nodata_value), nodata_value)
if decibel:
with np.errstate(divide="ignore", invalid="ignore"):
if use_nodata:
vh = np.ma.multiply(np.ma.log10(np.ma.abs(vh)), 10)
vv = np.ma.multiply(np.ma.log10(np.ma.abs(vv)), 10)
else:
vh = np.multiply(np.log10(np.abs(vh)), 10)
vv = np.multiply(np.log10(np.abs(vv)), 10)
array_to_raster(vh, vh_path, out_paths[0])
array_to_raster(vv, vv_path, out_paths[1])
return out_paths
def pansharpen(pan_path, tar_path, out_path):
target = resample_raster(tar_path, pan_path, resample_alg="bilinear")
aligned = align_rasters(target, master=pan_path)
target = aligned[0]
tar_arr = raster_to_array(target, output_2d=True)
tar_arr = (tar_arr - tar_arr.min()) / (tar_arr.max() - tar_arr.min())
pan_arr = raster_to_array(pan_path, output_2d=True)
_kernel, offsets, weights = create_kernel(
[5, 5], sigma=2, output_2d=True, offsets=True
)
pan = pansharpen_filter(pan_arr, tar_arr, offsets, weights.astype("float32"))
array_to_raster(pan, reference=target, out_path=out_path)
def height_over_terrain(dsm_folder, dtm_folder, out_folder, tmp_folder):
dsm_zipped = glob(dsm_folder + "*.zip")
dtm_zipped = glob(dtm_folder + "*.zip")
completed = 0
for dsm_tile in dsm_zipped:
s = get_tile_from_zipped_url(dsm_tile)
for dtm_tile in dtm_zipped:
t = get_tile_from_zipped_url(dtm_tile)
if s == t:
ZipFile(dsm_tile).extractall(tmp_folder)
ZipFile(dtm_tile).extractall(tmp_folder)
dsm_tiffs = glob(tmp_folder + "DSM_*.tif")
dtm_tiffs = glob(tmp_folder + "DTM_*.tif")
for s_tiff in dsm_tiffs:
s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4]
s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0]
for t_tiff in dtm_tiffs:
t_tiff_tile_base = os.path.basename(t_tiff).split(
"_")[2:4]
t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0]
if s_tiff_tile == t_tiff_tile:
ss = raster_to_array(s_tiff)
tt = raster_to_array(t_tiff)
array_to_raster(
np.abs(np.subtract(ss, tt)),
out_path=out_folder +
f"HOT_1km_{s_tiff_tile}.tif",
reference=s_tiff,
)
for f in glob(tmp_folder + "/*"):
os.remove(f)
completed += 1
print(f"Completed: {completed}/{len(dsm_zipped)}")
def haralick(in_raster, out_raster, options=None, out_datatype="float", band=None):
"""Performs haralick texture extraction"""
cli = "otbcli_HaralickTextureExtraction"
stats_raster = raster_to_array(in_raster)
stats = {"min": stats_raster.min(), "max": stats_raster.max()}
stats_raster = None
if options is None:
options = {
"texture": "simple",
"channel": 1,
"parameters.nbbin": 64,
"parameters.xrad": 3,
"parameters.yrad": 3,
"parameters.min": stats["min"],
"parameters.max": stats["max"],
}
if out_datatype is None:
out_datatype = "float"
if band is not None:
band = f"&bands={band}"
else:
band = ""
cli_args = [
cli,
"-in",
os.path.abspath(in_raster),
"-out",
f'"{os.path.abspath(out_raster)}?{band}&gdal:co:COMPRESS=DEFLATE&gdal:co:NUM_THREADS=ALL_CPUS&gdal:co:BIGTIFF=YES"',
out_datatype,
]
for key, value in options.items():
cli_args.append("-" + str(key))
cli_args.append(str(value))
cli_string = " ".join(cli_args)
execute_cli_function(cli_string, name="Texture extraction")
return os.path.abspath(out_raster)
def zonal_statistics(
in_vector,
output_vector=None,
in_rasters=[],
prefixes=[],
stats=["mean", "med", "std"],
):
if len(prefixes) != 0:
if len(in_rasters) != len(prefixes):
raise ValueError("Unable to parse prefixes.")
if isinstance(in_rasters, list):
if len(in_rasters) == 0:
raise ValueError("List of rasters (in_rasters) is empty.")
if len(stats) == 0:
raise ValueError("Unable to parse statistics (stats).")
# Translate stats to integers
stats_translated = stats_to_ints(stats)
# Read the raster meta:
raster_metadata = internal_raster_to_metadata(in_rasters[0])
vector = None
if output_vector is None:
vector = open_vector(in_vector, writeable=True)
else:
vector = internal_vector_to_memory(in_vector)
vector_metadata = internal_vector_to_metadata(vector)
vector_layer = vector.GetLayer()
# Check that projections match
if not vector_metadata["projection_osr"].IsSame(
raster_metadata["projection_osr"]):
if output_vector is None:
vector = internal_reproject_vector(in_vector, in_rasters[0])
else:
vector_path = internal_reproject_vector(in_vector, in_rasters[0],
output_vector)
vector = open_vector(vector_path, writeable=True)
vector_metadata = internal_vector_to_metadata(vector)
vector_layer = vector.GetLayer()
vector_projection = vector_metadata["projection_osr"]
raster_projection = raster_metadata["projection"]
# Read raster data in overlap
raster_transform = np.array(raster_metadata["transform"], dtype=np.float32)
raster_size = np.array(raster_metadata["size"], dtype=np.int32)
raster_extent = get_extent(raster_transform, raster_size)
vector_extent = np.array(vector_layer.GetExtent(), dtype=np.float32)
overlap_extent = get_intersection(raster_extent, vector_extent)
if overlap_extent is False:
print("raster_extent: ", raster_extent)
print("vector_extent: ", vector_extent)
raise Exception("Vector and raster do not overlap!")
(
overlap_aligned_extent,
overlap_aligned_rasterized_size,
overlap_aligned_offset,
) = align_extent(raster_transform, overlap_extent, raster_size)
overlap_transform = np.array(
[
overlap_aligned_extent[0],
raster_transform[1],
0,
overlap_aligned_extent[3],
0,
raster_transform[5],
],
dtype=np.float32,
)
overlap_size = overlap_size_calc(overlap_aligned_extent, raster_transform)
# Loop the features
vector_driver = ogr.GetDriverByName("Memory")
vector_feature_count = vector_layer.GetFeatureCount()
vector_layer.StartTransaction()
# Create fields
vector_layer_defn = vector_layer.GetLayerDefn()
vector_field_counts = vector_layer_defn.GetFieldCount()
vector_current_fields = []
# Get current fields
for i in range(vector_field_counts):
vector_current_fields.append(
vector_layer_defn.GetFieldDefn(i).GetName())
# Add fields where missing
for stat in stats:
for i in range(len(in_rasters)):
field_name = f"{prefixes[i]}{stat}"
if field_name not in vector_current_fields:
field_defn = ogr.FieldDefn(field_name, ogr.OFTReal)
vector_layer.CreateField(field_defn)
rasterized_features = []
sizes = np.zeros((vector_feature_count, 4), dtype="float32")
offsets = np.zeros((vector_feature_count, 2), dtype=np.int32)
raster_data = None
for raster_index, raster_value in enumerate(in_rasters):
columns = {}
for stat in stats:
columns[prefixes[raster_index] + stat] = []
fits_in_memory = True
try:
raster_data = raster_to_array(
raster_value,
crop=[
overlap_aligned_offset[0],
overlap_aligned_offset[1],
overlap_aligned_rasterized_size[0],
overlap_aligned_rasterized_size[1],
],
)
except:
fits_in_memory = False
print("Raster does not fit in memory.. Doing IO for each feature.")
for n in range(vector_feature_count):
vector_feature = vector_layer.GetNextFeature()
rasterized_vector = None
if raster_index == 0:
try:
vector_geom = vector_feature.GetGeometryRef()
except:
vector_geom.Buffer(0)
Warning("Invalid geometry at : ", n)
if vector_geom is None:
raise Exception("Invalid geometry. Could not fix.")
feature_extent = vector_geom.GetEnvelope()
# Create temp layer
temp_vector_datasource = vector_driver.CreateDataSource(
f"vector_{n}")
temp_vector_layer = temp_vector_datasource.CreateLayer(
"temp_polygon", vector_projection, ogr.wkbPolygon)
temp_vector_layer.CreateFeature(vector_feature.Clone())
(
feature_aligned_extent,
feature_aligned_rasterized_size,
feature_aligned_offset,
) = align_extent(overlap_transform, feature_extent,
overlap_size)
rasterized_vector = rasterize_vector(
temp_vector_layer,
feature_aligned_extent,
feature_aligned_rasterized_size,
raster_projection,
)
rasterized_features.append(rasterized_vector)
offsets[n] = feature_aligned_offset
sizes[n] = feature_aligned_rasterized_size
if fits_in_memory is True:
cropped_raster = raster_data[offsets[n][1]:offsets[n][1] +
int(sizes[n][1]), # X
offsets[n][0]:offsets[n][0] +
int(sizes[n][0]), # Y
]
else:
cropped_raster = raster_to_array(
raster_value,
crop=[
overlap_aligned_offset[0] + offsets[n][0],
overlap_aligned_offset[1] + offsets[n][1],
int(sizes[n][0]),
int(sizes[n][1]),
],
)
if rasterized_features[n] is None:
for stat in stats:
field_name = f"{prefixes[raster_index]}{stat}"
vector_feature.SetField(field_name, None)
elif cropped_raster is None:
for stat in stats:
field_name = f"{prefixes[raster_index]}{stat}"
vector_feature.SetField(field_name, None)
else:
raster_data_masked = np.ma.masked_array(
cropped_raster,
mask=rasterized_features[n],
dtype="float32").compressed()
zonal_stats = calculate_array_stats(raster_data_masked,
stats_translated)
for index, stat in enumerate(stats):
field_name = f"{prefixes[raster_index]}{stat}"
vector_feature.SetField(field_name,
float(zonal_stats[index]))
vector_layer.SetFeature(vector_feature)
progress(n, vector_feature_count, name=prefixes[raster_index])
vector_layer.ResetReading()
vector_layer.CommitTransaction()
if output_vector is None:
return vector
return output_vector
quantile)
else:
result[x, y] = np.median(
hood_values[np.nonzero(hood_weights)])
return result
folder = "C:/Users/caspe/Desktop/test_area2/"
layers = [
folder + "vv_01.tif",
folder + "vv_02.tif",
folder + "vv_03.tif",
]
sar_data = raster_to_array(layers)
kernel_size = 3
nodata_value = -9999.0
_kernel, offsets_3D, weights_3D = create_kernel(
(kernel_size, kernel_size, sar_data.shape[2]),
distance_calc=False, # "gaussian"
sigma=1,
spherical=True,
radius_method="ellipsoid",
offsets=True,
edge_weights=True,
normalised=True,
remove_zero_weights=True,
)
build_tile_name = "_".join(
os.path.splitext(
os.path.basename(building_tile))[0].split("_")[2:])
if vrt_tile_name == build_tile_name:
found_path = building_tile
found = True
if not found:
area_vol_10m = internal_resample_raster(
vrt_file, (10, 10),
resample_alg='average',
out_path=tmp_folder +
f"buildings_volume_{vrt_tile_name}_10m_unscaled.tif")
hot_arr = raster_to_array(area_vol_10m) * 0
vol_10m_path = dst_folder + f"buildings_volume_{vrt_tile_name}_10m.tif"
area_10m_path = dst_folder + f"buildings_area_{vrt_tile_name}_10m.tif"
array_to_raster(hot_arr, reference=area_vol_10m, out_path=vol_10m_path)
array_to_raster(hot_arr,
reference=area_vol_10m,
out_path=area_10m_path)
processed += 1
continue
try:
metadata = internal_raster_to_metadata(vrt_file)
except:
def extract_patches(
raster_list,
outdir,
tile_size=32,
zones=None,
options=None,
):
"""
Generate patches for machine learning from rasters
"""
base_options = {
"overlaps": True,
"border_check": True,
"merge_output": True,
"force_align": True,
"output_raster_labels": True,
"label_geom": None,
"label_res": 0.2,
"label_mult": 100,
"tolerance": 0.0,
"fill_value": 0,
"zone_layer_id": 0,
"align_with_size": 20,
"prefix": "",
"postfix": "",
}
if options is None:
options = base_options
else:
for key in options:
if key not in base_options:
raise ValueError(f"Invalid option: {key}")
base_options[key] = options[key]
options = base_options
if zones is not None and not is_vector(zones):
raise TypeError(
"Clip geom is invalid. Did you input a valid geometry?")
if not isinstance(raster_list, list):
raster_list = [raster_list]
for raster in raster_list:
if not is_raster(raster):
raise TypeError("raster_list is not a list of rasters.")
if not os.path.isdir(outdir):
raise ValueError(
"Outdir does not exist. Please create before running the function."
)
if not rasters_are_aligned(raster_list, same_extent=True):
if options["force_align"]:
print(
"Rasters we not aligned. Realigning rasters due to force_align=True option."
)
raster_list = align_rasters(raster_list)
else:
raise ValueError("Rasters in raster_list are not aligned.")
offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]]
raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True)
pixel_size = min(raster_metadata["pixel_height"],
raster_metadata["pixel_width"])
if zones is None:
zones = raster_metadata["extent_datasource_path"]
zones_meta = vector_to_metadata(zones)
mem_driver = ogr.GetDriverByName("ESRI Shapefile")
if zones_meta["layer_count"] == 0:
raise ValueError("Vector contains no layers.")
zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]]
if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]:
raise ValueError("clip geom is not Polygon or Multi Polygon.")
zones_ogr = open_vector(zones)
zones_layer = zones_ogr.GetLayer(options["zone_layer_id"])
feature_defn = zones_layer.GetLayerDefn()
fids = vector_get_fids(zones_ogr, options["zone_layer_id"])
progress(0, len(fids) * len(raster_list), "processing fids")
processed_fids = []
processed = 0
labels_processed = False
for idx, raster in enumerate(raster_list):
name = os.path.splitext(os.path.basename(raster))[0]
list_extracted = []
list_masks = []
list_labels = []
for fid in fids:
feature = zones_layer.GetFeature(fid)
geom = feature.GetGeometryRef()
fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp"
fid_ds = mem_driver.CreateDataSource(fid_path)
fid_ds_lyr = fid_ds.CreateLayer(
"fid_layer",
geom_type=ogr.wkbPolygon,
srs=zones_layer_meta["projection_osr"],
)
copied_feature = ogr.Feature(feature_defn)
copied_feature.SetGeometry(geom)
fid_ds_lyr.CreateFeature(copied_feature)
fid_ds.FlushCache()
fid_ds.SyncToDisk()
valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif"
rasterize_vector(
fid_path,
pixel_size,
out_path=valid_path,
extent=fid_path,
)
valid_arr = raster_to_array(valid_path)
if options["label_geom"] is not None and fid not in processed_fids:
if not is_vector(options["label_geom"]):
raise TypeError(
"label geom is invalid. Did you input a valid geometry?"
)
uuid = str(uuid4().int)
label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp"
label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif"
label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif"
intersect_vector(options["label_geom"],
fid_ds,
out_path=label_clip_path)
try:
rasterize_vector(
label_clip_path,
options["label_res"],
out_path=label_ras_path,
extent=valid_path,
)
except Exception:
array_to_raster(
np.zeros(valid_arr.shape, dtype="float32"),
valid_path,
out_path=label_ras_path,
)
resample_raster(
label_ras_path,
pixel_size,
resample_alg="average",
out_path=label_warp_path,
)
labels_arr = (raster_to_array(label_warp_path) *
options["label_mult"]).astype("float32")
if options["output_raster_labels"]:
array_to_raster(
labels_arr,
label_warp_path,
out_path=
f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif",
)
raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif"
try:
clip_raster(
raster,
valid_path,
raster_clip_path,
all_touch=False,
adjust_bbox=False,
)
except Exception as e:
print(
f"Warning: {raster} did not intersect geom with fid: {fid}."
)
print(e)
if options["label_geom"] is not None:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
continue
arr = raster_to_array(raster_clip_path)
if arr.shape[:2] != valid_arr.shape[:2]:
raise Exception(
f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}"
)
arr_offsets = get_overlaps(arr, offsets, tile_size,
options["border_check"])
arr = np.concatenate(arr_offsets)
valid_offsets = np.concatenate(
get_overlaps(valid_arr, offsets, tile_size,
options["border_check"]))
valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) /
(tile_size * tile_size))) <=
options["tolerance"])[:, 0]
arr = arr[valid_mask]
valid_masked = valid_offsets[valid_mask]
if options["label_geom"] is not None and not labels_processed:
labels_masked = np.concatenate(
get_overlaps(labels_arr, offsets, tile_size,
options["border_check"]))[valid_mask]
if options["merge_output"]:
list_extracted.append(arr)
list_masks.append(valid_masked)
if options["label_geom"] is not None and not labels_processed:
list_labels.append(labels_masked)
else:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy",
arr.filled(options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if fid not in processed_fids:
processed_fids.append(fid)
processed += 1
progress(processed,
len(fids) * len(raster_list), "processing fids")
if not options["merge_output"]:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
gdal.Unlink(valid_path)
if options["merge_output"]:
np.save(
f"{outdir}{options['prefix']}{name}{options['postfix']}.npy",
np.ma.concatenate(list_extracted).filled(
options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy",
np.ma.concatenate(list_masks).filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy",
np.ma.concatenate(list_labels).filled(
options["fill_value"]),
)
labels_processed = True
progress(1, 1, "processing fids")
return 1
)
* dA
)
return out
import sys
sys.path.append("../../")
from buteo.raster.io import raster_to_array
from skimage.feature import graycomatrix, greycoprops
folder = "C:/Users/caspe/Desktop/test_area/raster/"
raster = folder + "B12_20m.tif"
arr = raster_to_array(raster)
arr = np.rint((arr / arr.max()) * 256).astype("uint8")
glcm = graycomatrix(
arr[:, :, 0],
[1],
[0, np.pi / 4, np.pi / 2, 3 * np.pi / 4],
256,
symmetric=False,
normed=True,
)
bob = GLCMFeaturesInvariant(glcm)
import pdb
pdb.set_trace()
def volume_over_terrain(
tile_names,
username,
password,
out_folder,
tmp_folder,
):
if not os.path.isdir(tmp_folder):
raise Exception("Error: output directory does not exist.")
error_tiles = []
completed = 0
for tile in tile_names:
base_path_DSM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DSM/"
file_name_DSM = f'DSM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip'
base_path_DTM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DTM/"
file_name_DTM = f'DTM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip'
try:
if not os.path.exists(tmp_folder + file_name_DSM):
get_file(base_path_DSM + file_name_DSM,
tmp_folder + file_name_DSM)
else:
print(f"{file_name_DSM} Already exists.")
if not os.path.exists(tmp_folder + file_name_DTM):
get_file(base_path_DTM + file_name_DTM,
tmp_folder + file_name_DTM)
else:
print(f"{file_name_DTM} Already exists.")
ZipFile(tmp_folder + file_name_DSM).extractall(tmp_folder)
ZipFile(tmp_folder + file_name_DTM).extractall(tmp_folder)
dsm_tiffs = glob(tmp_folder + "DSM_*.tif")
dtm_tiffs = glob(tmp_folder + "DTM_*.tif")
hot_files = []
for s_tiff in dsm_tiffs:
s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4]
s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0]
for t_tiff in dtm_tiffs:
t_tiff_tile_base = os.path.basename(t_tiff).split("_")[2:4]
t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0]
if s_tiff_tile == t_tiff_tile:
ss = raster_to_array(s_tiff)
tt = raster_to_array(t_tiff)
hot_name = out_folder + f"HOT_1km_{s_tiff_tile}.tif"
subtracted = np.subtract(ss, tt)
hot_arr = np.abs((subtracted >= 0) * subtracted)
array_to_raster(
hot_arr,
out_path=hot_name,
reference=s_tiff,
)
hot_files.append(hot_name)
km10_tilename = "_".join(file_name_DSM.split("_")[1:3])
vrt_name = out_folder + f"HOT_10km_{km10_tilename}.vrt"
stack_rasters_vrt(
hot_files,
seperate=False,
out_path=vrt_name,
)
except:
print("Error while processing tile: {tile}")
error_tiles.append(tile)
finally:
for f in glob(tmp_folder + "/*"):
os.remove(f)
completed += 1
print(f"Completed: {completed}/{len(tile_names)} - {tile}")
def norm_rasters(
in_rasters,
out_folder,
method="normalise",
split_bands=False,
min_target=0,
max_target=1,
min_og=-9999,
max_og=-9999,
truncate=True,
prefix="",
postfix="",
overwrite=True,
):
if not isinstance(in_rasters, list):
in_rasters = [in_rasters]
normed_rasters = []
for in_raster in in_rasters:
name = os.path.splitext(os.path.basename(in_raster))[0]
raster = raster_to_array(in_raster)
if method == "normalise":
normed = norm_to_range(raster,
min_target,
max_target,
truncate=False)
elif method == "standardise":
normed = standardise_filter(raster)
elif method == "median_absolute_deviation":
normed = mad_filter(raster)
elif method == "range":
normed = norm_to_range(
raster,
min_target=min_target,
max_target=max_target,
min_og=min_og,
max_og=max_og,
truncate=truncate,
)
elif method == "robust_quantile":
normed = robust_scaler_filter(
raster,
min_q=0.25,
max_q=0.75,
)
elif method == "robust_98":
normed = robust_scaler_filter(raster, min_q=0.02, max_q=0.98)
else:
raise Exception(f"Method {method} not recognised")
if split_bands:
for idx in range(raster.shape[2]):
band = idx + 1
raster_name = prefix + name + f"_B{band}" + postfix + ".tif"
normed_rasters.append(
array_to_raster(
normed[:, :, idx][..., np.newaxis],
reference=in_raster,
out_path=out_folder + raster_name,
overwrite=overwrite,
))
else:
raster_name = prefix + name + postfix + ".tif"
normed_rasters.append(
array_to_raster(
normed,
reference=in_raster,
out_path=out_folder + raster_name,
overwrite=overwrite,
))
if isinstance(in_rasters, list):
return normed_rasters
return normed_rasters[0]
def super_sample_s2(
B04_link,
B08_link,
B05_link=None,
B06_link=None,
B07_link=None,
B8A_link=None,
out_folder="../raster/",
prefix="",
suffix="",
):
assert (isinstance(B05_link, str) or isinstance(B06_link, str)
or isinstance(B07_link, str) or isinstance(B8A_link, str))
paths = {
"B04": B04_link,
"B05": B05_link,
"B06": B06_link,
"B07": B07_link,
"B08": B08_link,
"B8A": B8A_link,
}
bands = {
"B04":
raster_to_array(B04_link).astype("float32"),
"B05":
raster_to_array(B05_link).astype("float32")
if B05_link is not None else False,
"B06":
raster_to_array(B06_link).astype("float32")
if B06_link is not None else False,
"B07":
raster_to_array(B07_link).astype("float32")
if B07_link is not None else False,
"B08":
raster_to_array(B08_link).astype("float32"),
"B8A":
raster_to_array(B8A_link).astype("float32")
if B8A_link is not None else False,
}
bands_to_pansharpen = []
if bands["B05"] is not False:
bands_to_pansharpen.append("B05")
if bands["B06"] is not False:
bands_to_pansharpen.append("B06")
if bands["B07"] is not False:
bands_to_pansharpen.append("B07")
if bands["B8A"] is not False:
bands_to_pansharpen.append("B8A")
for band_x in bands_to_pansharpen:
if band_x is "B05":
pseudo_band = "B04"
else:
pseudo_band = "B08"
pseudo_path = os.path.join(out_folder,
f"{prefix}{band_x}{suffix}_pseudo.tif")
array_to_raster(
bands[pseudo_band],
reference_raster=paths[pseudo_band],
out_raster=pseudo_path,
)
low_res_10m = raster_to_array(
resample_raster(
paths[band_x],
reference_raster=paths[pseudo_band])).astype("float32")
resampled_path = os.path.join(
out_folder, f"{prefix}{band_x}{suffix}_resampled.tif")
array_to_raster(low_res_10m,
reference_raster=paths[pseudo_band],
out_raster=resampled_path)
low_res_10m = None
pansharpened_path = os.path.join(
out_folder, f"{prefix}{band_x}{suffix}_float.tif")
pansharpen(pseudo_path, resampled_path, pansharpened_path)
os.remove(resampled_path)
os.remove(pseudo_path)
def test_extraction(
rasters: Union[list, str, gdal.Dataset],
arrays: Union[list, np.ndarray],
grid: Union[ogr.DataSource, str],
samples: int = 1000, # if 0, all
grid_layer_index: int = 0,
verbose: int = 1,
) -> bool:
"""Validates the output of the patch_extractor. Useful if you need peace of mind.
Set samples to 0 to tests everything.
Args:
rasters (list of rasters | path | raster): The raster(s) used.
arrays (list of arrays | ndarray): The arrays generated.
grid (vector | vector_path): The grid generated.
**kwargs:
samples (int): The amount of patches to randomly test. If 0 all patches will be
tested. This is a long process, so consider only testing everything if absolutely
necessary.
grid_layer_index (int): If the grid is part of a multi-layer vector, specify the
index of the grid.
verbose (int): If 1 will output messages on progress.
Returns:
True if the extraction is valid. Raises an error otherwise.
"""
type_check(rasters, [list, str, gdal.Dataset], "rasters")
type_check(arrays, [list, str, np.ndarray], "arrays")
type_check(grid, [list, str, ogr.DataSource], "grid")
type_check(samples, [int], "samples")
type_check(grid_layer_index, [int], "clip_layer_index")
type_check(verbose, [int], "verbose")
in_rasters = to_raster_list(rasters)
in_arrays = arrays
if verbose == 1:
print("Verifying integrity of output grid..")
# grid_memory = open_vector(internal_vector_to_memory(grid))
grid_memory = open_vector(grid)
grid_metadata = internal_vector_to_metadata(grid)
grid_projection = grid_metadata["projection_osr"]
if grid_layer_index > (grid_metadata["layer_count"] - 1):
raise ValueError(
f"Requested non-existing layer index: {grid_layer_index}")
grid_layer = grid_memory.GetLayer(grid_layer_index)
# Select sample fids
feature_count = grid_metadata["layers"][grid_layer_index]["feature_count"]
test_samples = samples if samples > 0 else feature_count
max_test = min(test_samples, feature_count) - 1
test_fids = np.array(random.sample(range(0, feature_count), max_test),
dtype="uint64")
mem_driver = ogr.GetDriverByName("ESRI Shapefile")
for index, raster in enumerate(in_rasters):
test_rast = open_raster(raster)
test_array = in_arrays[index]
if isinstance(test_array, str):
if not os.path.exists(test_array):
raise ValueError(f"Numpy array does not exist: {test_array}")
try:
test_array = np.load(in_arrays[index])
except:
raise Exception(
f"Attempted to read numpy raster from: {in_arrays[index]}")
base = os.path.basename(raster)
basename = os.path.splitext(base)[0]
if verbose == 1:
print(f"Testing: {basename}")
tested = 0
for test in test_fids:
feature = grid_layer.GetFeature(test)
if feature is None:
raise Exception(f"Feature not found: {test}")
test_ds_path = f"/vsimem/test_mem_grid_{uuid4().int}.gpkg"
test_ds = mem_driver.CreateDataSource(test_ds_path)
test_ds_lyr = test_ds.CreateLayer("test_mem_grid_layer",
geom_type=ogr.wkbPolygon,
srs=grid_projection)
test_ds_lyr.CreateFeature(feature.Clone())
test_ds.SyncToDisk()
clipped = internal_clip_raster(
test_rast,
test_ds_path,
adjust_bbox=False,
crop_to_geom=True,
all_touch=False,
)
if clipped is None:
raise Exception(
"Error while clipping raster. Likely a bad extraction.")
ref_image = raster_to_array(clipped, filled=True)
image_block = test_array[test]
if not np.array_equal(ref_image, image_block):
# from matplotlib import pyplot as plt; plt.imshow(ref_image[:,:,0]); plt.show()
raise Exception(
f"Image {basename} and grid cell did not match..")
if verbose == 1:
progress(tested, len(test_fids) - 1, "Verifying..")
tested += 1
return True
def raster_mask_values(
raster: Union[gdal.Dataset, str, list],
values_to_mask: list,
out_path: Union[list, str, None] = None,
include_original_nodata: bool = True,
dst_nodata: Union[float, int, str, list, None] = "infer",
in_place: bool = False,
overwrite: bool = True,
opened: bool = False,
prefix: str = "",
postfix: str = "_nodata_masked",
creation_options: list = [],
) -> Union[list, gdal.Dataset, str]:
"""Mask a raster with a list of values.
Args:
raster (path | raster | list): The raster(s) to retrieve nodata values from.
values_to_mask (list): The list of values to mask in the raster(s)
**kwargs:
include_original_nodata: (bool): If True, the nodata value of the raster(s) will be
included in the values to mask.
dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata
value is set based on the input datatype. A list of nodata values can be based matching
the amount of input rasters. If multiple nodata values should be set, use raster_mask_values.
out_path (path | list | None): The destination of the changed rasters. If out_paths
are specified, in_place is automatically set to False. The path can be a folder.
in_place (bool): Should the rasters be changed in_place or copied?
prefix (str): Prefix to add the the output if a folder is specified in out_path.
postfix (str): Postfix to add the the output if a folder is specified in out_path.
Returns:
Returns the rasters with nodata removed. If in_place is True a reference to the
changed orignal is returned, otherwise a copied memory raster or the path to the
generated raster is outputted.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(values_to_mask, [list], "values_to_mask")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(include_original_nodata, [bool], "include_original_nodata")
type_check(dst_nodata, [float, int, str, list],
"dst_nodata",
allow_none=True)
type_check(in_place, [bool], "in_place")
type_check(overwrite, [bool], "overwrite")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(opened, [bool], "opened")
type_check(creation_options, [list], "creation_options")
rasters_metadata = []
internal_in_place = in_place if out_path is None else False
internal_dst_nodata = None
for value in values_to_mask:
if not isinstance(value, (int, float)):
raise ValueError("Values in values_to_mask must be ints or floats")
if isinstance(dst_nodata, str) and dst_nodata != "infer":
raise ValueError(f"Invalid dst_nodata value. {dst_nodata}")
if isinstance(dst_nodata, list):
if not isinstance(raster, list) or len(dst_nodata) != len(raster):
raise ValueError(
"If dst_nodata is a list, raster must also be a list of equal length."
)
for value in dst_nodata:
if isinstance(value, (float, int, str, None)):
raise ValueError("Invalid type in dst_nodata list.")
if isinstance(value, str) and value != "infer":
raise ValueError(
"If dst_nodata is a string it must be 'infer'")
raster_list, out_names = ready_io_raster(raster, out_path, overwrite,
prefix, postfix)
output_rasters = []
for index, internal_raster in enumerate(raster_list):
raster_metadata = None
if len(rasters_metadata) == 0:
raster_metadata = raster_to_metadata(internal_raster)
rasters_metadata.append(raster_metadata)
else:
raster_metadata = rasters_metadata[index]
if dst_nodata == "infer":
internal_dst_nodata = gdal_nodata_value_from_type(
raster_metadata["dtype_gdal_raw"])
elif isinstance(dst_nodata, list):
internal_dst_nodata = dst_nodata[index]
else:
internal_dst_nodata = dst_nodata
mask_values = list(values_to_mask)
if include_original_nodata:
if raster_metadata["nodata_value"] is not None:
mask_values.append(raster_metadata["nodata_value"])
arr = raster_to_array(internal_raster, filled=True)
mask = None
for index, mask_value in enumerate(mask_values):
if index == 0:
mask = arr == mask_value
else:
mask = mask | arr == mask_value
arr = np.ma.masked_array(arr,
mask=mask,
fill_value=internal_dst_nodata)
if internal_in_place:
for band in range(raster_metadata["bands"]):
raster_band = internal_raster.GetRasterBand(band + 1)
raster_band.WriteArray(arr[:, :, band])
raster_band = None
else:
out_name = out_names[index]
remove_if_overwrite(out_name, overwrite)
output_rasters.append(
array_to_raster(arr, internal_raster, out_path=out_name))
if isinstance(raster, list):
return output_rasters
return output_rasters[0]
def calc_proximity(
input_rasters,
target_value=1,
out_path=None,
max_dist=1000,
add_border=False,
weighted=False,
invert=False,
return_array=False,
postfix="_proximity",
uuid=False,
overwrite=True,
skip_existing=False,
):
"""
Calculate the proximity of input_raster to values
"""
raster_list, path_list = ready_io_raster(input_rasters,
out_path,
overwrite,
postfix=postfix,
uuid=uuid)
output = []
for index, input_raster in enumerate(raster_list):
out_path = path_list[index]
if skip_existing and os.path.exists(out_path):
output.append(out_path)
continue
in_arr = raster_to_array(input_raster, filled=True)
bin_arr = (in_arr != target_value).astype("uint8")
bin_raster = array_to_raster(bin_arr, reference=input_raster)
in_raster = open_raster(bin_raster)
in_raster_path = bin_raster
if add_border:
border_size = 1
border_raster = add_border_to_raster(
in_raster,
border_size=border_size,
border_value=0,
overwrite=True,
)
in_raster = open_raster(border_raster)
gdal.Unlink(in_raster_path)
in_raster_path = border_raster
src_band = in_raster.GetRasterBand(1)
driver_name = "GTiff" if out_path is None else path_to_driver_raster(
out_path)
if driver_name is None:
raise ValueError(f"Unable to parse filetype from path: {out_path}")
driver = gdal.GetDriverByName(driver_name)
if driver is None:
raise ValueError(
f"Error while creating driver from extension: {out_path}")
mem_path = f"/vsimem/raster_proximity_tmp_{uuid4().int}.tif"
dest_raster = driver.Create(
mem_path,
in_raster.RasterXSize,
in_raster.RasterYSize,
1,
gdal.GetDataTypeByName("Float32"),
)
dest_raster.SetGeoTransform(in_raster.GetGeoTransform())
dest_raster.SetProjection(in_raster.GetProjectionRef())
dst_band = dest_raster.GetRasterBand(1)
gdal.ComputeProximity(
src_band,
dst_band,
[
f"VALUES='1'",
"DISTUNITS=GEO",
f"MAXDIST={max_dist}",
],
)
dst_arr = dst_band.ReadAsArray()
gdal.Unlink(mem_path)
gdal.Unlink(in_raster_path)
dst_arr = np.where(dst_arr > max_dist, max_dist, dst_arr)
if invert:
dst_arr = max_dist - dst_arr
if weighted:
dst_arr = dst_arr / max_dist
if add_border:
dst_arr = dst_arr[border_size:-border_size,
border_size:-border_size]
src_band = None
dst_band = None
in_raster = None
dest_raster = None
if return_array:
output.append(dst_arr)
else:
array_to_raster(dst_arr, reference=input_raster, out_path=out_path)
output.append(out_path)
dst_arr = None
if isinstance(input_rasters, list):
return output
return output[0]
# obt_bandmath( # [folder + "Ghana_classification_float32_v2.tif", folder + "Ghana_float32_v9.tif"], # "((im1b2 + im1b3 + im1b4) == 0) ? 0.0 : (im1b4 / (im1b2 + im1b3 + im1b4)) * im2b1", # folder + "area_slum.tif", # ram=32000, # ) # w = [1.00, 1.00, 1.00] # unweighted # w = [1.25, 1.35, 0.40] # nighttime # w = [0.80, 0.90, 1.30] # daytime # w1 = w[0] # w2 = w[1] # w3 = w[2] t_pop1 = raster_to_array(folder + "area_residential_clipped.tif").filled(0).sum() t_pop2 = raster_to_array(folder + "area_slum_clipped.tif").filled(0).sum() t_pop3 = raster_to_array(folder + "area_industrial_clipped.tif").filled(0).sum() t_pop = t_pop1 + t_pop2 + t_pop3 t_pop = 968159000.0 import pdb pdb.set_trace() # obt_bandmath( # [ # folder + "area_residential_clipped.tif", # folder + "area_slum_clipped.tif", # folder + "area_industrial_clipped.tif",
output_tile_size=32,
output_channels=1,
model_path="",
reference_raster="",
out_path=None,
out_path_variance=None,
offsets=True,
batch_size=32,
method="median",
scale_to_sum=False,
):
print("Loading Model")
model = tf.keras.models.load_model(
model_path, custom_objects={"tpe": tpe, "mse_sumbias": mse_sumbias}
)
reference_arr = raster_to_array(reference_raster)
if offsets == False:
print(
"No offsets provided. Using offsets greatly increases accuracy. Please provide offsets."
)
else:
offsets = []
for val in tile_size:
offsets.append(get_offsets(val))
predictions = []
read_rasters = []
print("Initialising rasters.")
for raster_idx, raster in enumerate(raster_list):
def process_aligned(
aligned_rasters,
out_path,
folder_tmp,
chunks,
master_raster,
nodata_value,
feather_weights=None,
):
kernel_size = 3
chunk_offset = kernel_size // 2
_kernel, offsets, weights = create_kernel(
(kernel_size, kernel_size, len(aligned_rasters)),
distance_calc=False, # "gaussian"
sigma=1,
spherical=True,
radius_method="ellipsoid",
offsets=True,
edge_weights=True,
normalised=True,
remove_zero_weights=True,
)
arr_aligned = raster_to_array(aligned_rasters)
if feather_weights is not None:
feather_weights_arr = raster_to_array(feather_weights)
if not rasters_are_aligned(aligned_rasters):
raise Exception("Rasters not aligned")
if chunks > 1:
chunks_list = []
print("Chunking rasters")
uids = uuid4()
for chunk in range(chunks):
print(f"Chunk {chunk + 1} of {chunks}")
cut_start = False
cut_end = False
if chunk == 0:
chunk_start = 0
else:
chunk_start = (chunk *
(arr_aligned.shape[0] // chunks)) - chunk_offset
cut_start = True
if chunk == chunks - 1:
chunk_end = arr_aligned.shape[0]
else:
chunk_end = ((chunk + 1) *
(arr_aligned.shape[0] // chunks)) + chunk_offset
cut_end = True
arr_chunk = arr_aligned[chunk_start:chunk_end]
if feather_weights is not None:
weights_chunk = feather_weights_arr[chunk_start:chunk_end]
else:
weights_chunk = np.ones_like(arr_chunk)
print(" Collapsing...")
arr_collapsed = s1_collapse(
arr_chunk,
offsets,
weights,
weights_chunk,
weighted=True,
nodata_value=nodata_value,
nodata=True,
)
offset_start = chunk_offset if cut_start else 0
offset_end = (arr_collapsed.shape[0] -
chunk_offset if cut_end else arr_collapsed.shape[0])
chunk_path = folder_tmp + f"{uids}_chunk_{chunk}.npy"
chunks_list.append(chunk_path)
np.save(chunk_path, arr_collapsed[offset_start:offset_end])
arr_chunk = None
arr_collapsed = None
print("Merging Chunks")
arr_aligned = None
merged = []
for chunk in chunks_list:
merged.append(np.load(chunk))
merged = np.concatenate(merged)
merged = np.ma.masked_array(merged, mask=merged == nodata_value)
merged.fill_value = nodata_value
print("Writing raster.")
array_to_raster(
merged,
master_raster,
out_path=out_path,
)
merged = None
return out_path
if feather_weights is not None:
weights_borders = feather_weights
else:
weights_borders = np.ones_like(arr_aligned)
print("Collapsing rasters")
arr_collapsed = s1_collapse(
arr_aligned,
offsets,
weights,
weights_borders,
weighted=True,
nodata_value=nodata_value,
nodata=True,
)
arr_collapsed = np.ma.masked_array(arr_collapsed,
mask=arr_collapsed == nodata_value)
arr_collapsed.fill_value = nodata_value
arr_aligned = None
print("Writing raster.")
array_to_raster(
arr_collapsed,
master_raster,
out_path=out_path,
)
arr_collapsed = None
return out_path
def add_border_to_raster(
input_raster,
out_path=None,
border_size=100,
border_size_unit="px",
border_value=0,
overwrite: bool = True,
creation_options: list = [],
):
in_raster = open_raster(input_raster)
metadata = raster_to_metadata(in_raster)
# Parse the driver
driver_name = "GTiff" if out_path is None else path_to_driver_raster(
out_path)
if driver_name is None:
raise ValueError(f"Unable to parse filetype from path: {out_path}")
driver = gdal.GetDriverByName(driver_name)
if driver is None:
raise ValueError(
f"Error while creating driver from extension: {out_path}")
output_name = None
if out_path is None:
output_name = f"/vsimem/raster_proximity_{uuid4().int}.tif"
else:
output_name = out_path
in_arr = raster_to_array(in_raster)
if border_size_unit == "px":
border_size_y = border_size
border_size_x = border_size
new_shape = (
in_arr.shape[0] + (2 * border_size_y),
in_arr.shape[1] + (2 * border_size_x),
in_arr.shape[2],
)
else:
border_size_y = round(border_size / metadata["pixel_height"])
border_size_x = round(border_size / metadata["pixel_width"])
new_shape = (
in_arr.shape[0] + (2 * border_size_y),
in_arr.shape[1] + (2 * border_size_x),
in_arr.shape[2],
)
new_arr = np.full(new_shape, border_value, dtype=in_arr.dtype)
new_arr[border_size_y:-border_size_y,
border_size_x:-border_size_x, :] = in_arr
if isinstance(in_arr, np.ma.MaskedArray):
mask = np.zeros(new_shape, dtype=bool)
mask[border_size_y:-border_size_y,
border_size_x:-border_size_x, :] = in_arr.mask
new_arr = np.ma.array(new_arr, mask=mask)
new_arr.fill_value = in_arr.fill_value
remove_if_overwrite(out_path, overwrite)
dest_raster = driver.Create(
output_name,
new_shape[1],
new_shape[0],
metadata["band_count"],
numpy_to_gdal_datatype(in_arr.dtype),
default_options(creation_options),
)
og_transform = in_raster.GetGeoTransform()
new_transform = []
for i in og_transform:
new_transform.append(i)
new_transform[0] -= border_size_x * og_transform[1]
new_transform[3] -= border_size_y * og_transform[5]
dest_raster.SetGeoTransform(new_transform)
dest_raster.SetProjection(in_raster.GetProjectionRef())
for band_num in range(1, metadata["band_count"] + 1):
dst_band = dest_raster.GetRasterBand(band_num)
dst_band.WriteArray(new_arr[:, :, band_num - 1])
if metadata["has_nodata"]:
dst_band.SetNoDataValue(metadata["nodata_value"])
return output_name
def extract_patches(
raster: Union[List[Union[str, gdal.Dataset]], str, gdal.Dataset],
out_dir: Optional[str] = None,
prefix: str = "",
postfix: str = "_patches",
size: int = 32,
offsets: Union[list, None] = [],
generate_border_patches: bool = True,
generate_zero_offset: bool = True,
generate_grid_geom: bool = True,
clip_geom: Optional[Union[str, ogr.DataSource, gdal.Dataset]] = None,
clip_layer_index: int = 0,
verify_output=True,
verification_samples=100,
overwrite=True,
epsilon: float = 1e-9,
verbose: int = 1,
) -> tuple:
"""Extracts square tiles from a raster.
Args:
raster (list of rasters | path | raster): The raster(s) to convert.
**kwargs:
out_dir (path | none): Folder to save output. If None, in-memory
arrays and geometries are outputted.
prefix (str): A prefix for all outputs.
postfix (str): A postfix for all outputs.
size (int): The size of the tiles in pixels.
offsets (list of tuples): List of offsets to extract. Example:
offsets=[(16, 16), (16, 0), (0, 16)]. Will offset the initial raster
and extract from there.
generate_border_patches (bool): The tiles often do not align with the
rasters which means borders are trimmed somewhat. If generate_border_patches
is True, an additional tile is added where needed.
generate_zero_offset (bool): if True, an offset is inserted at (0, 0)
if none is present.
generate_grid_geom (bool): Output a geopackage with the grid of tiles.
clip_geom (str, raster, vector): Clip the output to the
intersections with a geometry. Useful if a lot of the target
area is water or similar.
epsilon (float): How much for buffer the arange array function. This
should usually just be left alone.
verbose (int): If 1 will output messages on progress.
Returns:
A tuple with paths to the generated items. (numpy_array, grid_geom)
"""
type_check(raster, [str, list, gdal.Dataset], "raster")
type_check(out_dir, [str], "out_dir", allow_none=True)
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(size, [int], "size")
type_check(offsets, [list], "offsets", allow_none=True)
type_check(generate_grid_geom, [bool], "generate_grid_geom")
type_check(
clip_geom,
[str, ogr.DataSource, gdal.Dataset],
"clip_layer_index",
allow_none=True,
)
type_check(clip_layer_index, [int], "clip_layer_index")
type_check(overwrite, [bool], "overwrite")
type_check(epsilon, [float], "epsilon")
type_check(verbose, [int], "verbose")
in_rasters = to_raster_list(raster)
if out_dir is not None and not os.path.isdir(out_dir):
raise ValueError(f"Output directory does not exists: {out_dir}")
if not rasters_are_aligned(in_rasters):
raise ValueError(
"Input rasters must be aligned. Please use the align function.")
output_geom = None
metadata = internal_raster_to_metadata(in_rasters[0])
if verbose == 1:
print("Generating blocks..")
# internal offset array. Avoid manipulating the og array.
if offsets is None:
offsets = []
in_offsets = []
if generate_zero_offset and (0, 0) not in offsets:
in_offsets.append((0, 0))
for offset in offsets:
if offset != (0, 0):
if not isinstance(offset, (list, tuple)) or len(offset) != 2:
raise ValueError(
f"offset must be a list or tuple of two integers. Recieved: {offset}"
)
in_offsets.append((offset[0], offset[1]))
border_patches_needed_x = True
border_patches_needed_y = True
if clip_geom is not None:
border_patches_needed_x = False
border_patches_needed_y = False
shapes = []
for offset in in_offsets:
block_shape = shape_to_blockshape(metadata["shape"], (size, size),
offset)
if block_shape[0] * size == metadata["width"]:
border_patches_needed_x = False
if block_shape[1] * size == metadata["height"]:
border_patches_needed_y = False
shapes.append(block_shape)
if generate_border_patches:
cut_x = (metadata["width"] - in_offsets[0][0]) - (shapes[0][0] * size)
cut_y = (metadata["height"] - in_offsets[0][1]) - (shapes[0][1] * size)
if border_patches_needed_x and cut_x > 0:
shapes[0][0] += 1
if border_patches_needed_y and cut_y > 0:
shapes[0][1] += 1
# calculate the offsets
all_rows = 0
offset_rows = []
for i in range(len(shapes)):
row = 0
for j in range(len(shapes[i])):
if j == 0:
row = int(shapes[i][j])
else:
row *= int(shapes[i][j])
offset_rows.append(row)
all_rows += row
offset_rows_cumsum = np.cumsum(offset_rows)
if generate_grid_geom is True or clip_geom is not None:
if verbose == 1:
print("Calculating grid cells..")
mask = np.arange(all_rows, dtype="uint64")
ulx, uly, _lrx, _lry = metadata["extent"]
pixel_width = abs(metadata["pixel_width"])
pixel_height = abs(metadata["pixel_height"])
xres = pixel_width * size
yres = pixel_height * size
dx = xres / 2
dy = yres / 2
# Ready clip geom outside of loop.
if clip_geom is not None:
clip_ref = open_vector(
internal_reproject_vector(clip_geom,
metadata["projection_osr"]))
clip_layer = clip_ref.GetLayerByIndex(clip_layer_index)
meta_clip = internal_vector_to_metadata(clip_ref)
# geom_clip = meta_clip["layers"][clip_layer_index]["column_geom"]
clip_extent = meta_clip["extent_ogr"]
# clip_adjust = [
# clip_extent[0] - clip_extent[0] % xres, # x_min
# (clip_extent[1] - clip_extent[1] % xres) + xres, # x_max
# clip_extent[2] - clip_extent[2] % yres, # y_min
# (clip_extent[3] - clip_extent[3] % yres) + yres, # y_max
# ]
coord_grid = np.empty((all_rows, 2), dtype="float64")
# tiled_extent = [None, None, None, None]
row_count = 0
for idx in range(len(in_offsets)):
x_offset = in_offsets[idx][0]
y_offset = in_offsets[idx][1]
x_step = shapes[idx][0]
y_step = shapes[idx][1]
x_min = (ulx + dx) + (x_offset * pixel_width)
x_max = x_min + (x_step * xres)
y_max = (uly - dy) - (y_offset * pixel_height)
y_min = y_max - (y_step * yres)
# if clip_geom is not None:
# if clip_adjust[0] > x_min:
# x_min = clip_adjust[0] + (x_offset * pixel_width)
# if clip_adjust[1] < x_max:
# x_max = clip_adjust[1] + (x_offset * pixel_width)
# if clip_adjust[2] > y_min:
# y_min = clip_adjust[2] - (y_offset * pixel_height)
# if clip_adjust[3] < y_max:
# y_max = clip_adjust[3] - (y_offset * pixel_height)
# if idx == 0:
# tiled_extent[0] = x_min
# tiled_extent[1] = x_max
# tiled_extent[2] = y_min
# tiled_extent[3] = y_max
# else:
# if x_min < tiled_extent[0]:
# tiled_extent[0] = x_min
# if x_max > tiled_extent[1]:
# tiled_extent[1] = x_max
# if y_min < tiled_extent[2]:
# tiled_extent[2] = y_min
# if y_max > tiled_extent[3]:
# tiled_extent[3] = y_max
# y is flipped so: xmin --> xmax, ymax -- ymin to keep same order as numpy array
x_patches = round((x_max - x_min) / xres)
y_patches = round((y_max - y_min) / yres)
xr = np.arange(x_min, x_max, xres)[0:x_step]
if xr.shape[0] < x_patches:
xr = np.arange(x_min, x_max + epsilon, xres)[0:x_step]
elif xr.shape[0] > x_patches:
xr = np.arange(x_min, x_max - epsilon, xres)[0:x_step]
yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step]
if yr.shape[0] < y_patches:
yr = np.arange(y_max, y_min - epsilon, -yres)[0:y_step]
elif yr.shape[0] > y_patches:
yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step]
if generate_border_patches and idx == 0:
if border_patches_needed_x:
xr[-1] = xr[-1] - (
(xr[-1] + dx) - metadata["extent_dict"]["right"])
if border_patches_needed_y:
yr[-1] = yr[-1] - (
(yr[-1] - dy) - metadata["extent_dict"]["bottom"])
oxx, oyy = np.meshgrid(xr, yr)
oxr = oxx.ravel()
oyr = oyy.ravel()
offset_length = oxr.shape[0]
coord_grid[row_count:row_count + offset_length, 0] = oxr
coord_grid[row_count:row_count + offset_length, 1] = oyr
row_count += offset_length
offset_rows_cumsum[idx] = offset_length
offset_rows_cumsum = np.cumsum(offset_rows_cumsum)
coord_grid = coord_grid[:row_count]
# Output geometry
driver = ogr.GetDriverByName("GPKG")
patches_path = f"/vsimem/patches_{uuid4().int}.gpkg"
patches_ds = driver.CreateDataSource(patches_path)
patches_layer = patches_ds.CreateLayer("patches_all",
geom_type=ogr.wkbPolygon,
srs=metadata["projection_osr"])
patches_fdefn = patches_layer.GetLayerDefn()
og_fid = "og_fid"
field_defn = ogr.FieldDefn(og_fid, ogr.OFTInteger)
patches_layer.CreateField(field_defn)
if clip_geom is not None:
clip_feature_count = meta_clip["layers"][clip_layer_index][
"feature_count"]
spatial_index = rtree.index.Index(interleaved=False)
for _ in range(clip_feature_count):
clip_feature = clip_layer.GetNextFeature()
clip_fid = clip_feature.GetFID()
clip_feature_geom = clip_feature.GetGeometryRef()
xmin, xmax, ymin, ymax = clip_feature_geom.GetEnvelope()
spatial_index.insert(clip_fid, (xmin, xmax, ymin, ymax))
fids = 0
mask = []
for tile_id in range(coord_grid.shape[0]):
x, y = coord_grid[tile_id]
if verbose == 1:
progress(tile_id, coord_grid.shape[0], "Patch generation")
x_min = x - dx
x_max = x + dx
y_min = y - dx
y_max = y + dx
tile_intersects = True
grid_geom = None
poly_wkt = None
if clip_geom is not None:
tile_intersects = False
if not ogr_bbox_intersects([x_min, x_max, y_min, y_max],
clip_extent):
continue
intersections = list(
spatial_index.intersection((x_min, x_max, y_min, y_max)))
if len(intersections) == 0:
continue
poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))"
grid_geom = ogr.CreateGeometryFromWkt(poly_wkt)
for fid1 in intersections:
clip_feature = clip_layer.GetFeature(fid1)
clip_geom = clip_feature.GetGeometryRef()
if grid_geom.Intersects(clip_geom):
tile_intersects = True
continue
if tile_intersects:
ft = ogr.Feature(patches_fdefn)
if grid_geom is None:
poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))"
grid_geom = ogr.CreateGeometryFromWkt(poly_wkt)
ft_geom = ogr.CreateGeometryFromWkt(poly_wkt)
ft.SetGeometry(ft_geom)
ft.SetField(og_fid, int(fids))
ft.SetFID(int(fids))
patches_layer.CreateFeature(ft)
ft = None
mask.append(tile_id)
fids += 1
if verbose == 1:
progress(coord_grid.shape[0], coord_grid.shape[0],
"Patch generation")
mask = np.array(mask, dtype=int)
if generate_grid_geom is True:
if out_dir is None:
output_geom = patches_ds
else:
raster_basename = metadata["basename"]
geom_name = f"{prefix}{raster_basename}_geom_{str(size)}{postfix}.gpkg"
output_geom = os.path.join(out_dir, geom_name)
overwrite_required(output_geom, overwrite)
remove_if_overwrite(output_geom, overwrite)
if verbose == 1:
print("Writing output geometry..")
internal_vector_to_disk(patches_ds,
output_geom,
overwrite=overwrite)
if verbose == 1:
print("Writing numpy array..")
output_blocks = []
for raster in in_rasters:
base = None
basename = None
output_block = None
if out_dir is not None:
base = os.path.basename(raster)
basename = os.path.splitext(base)[0]
output_block = os.path.join(out_dir +
f"{prefix}{basename}{postfix}.npy")
metadata = internal_raster_to_metadata(raster)
if generate_grid_geom is True or clip_geom is not None:
output_shape = (row_count, size, size, metadata["band_count"])
else:
output_shape = (all_rows, size, size, metadata["band_count"])
input_datatype = metadata["datatype"]
output_array = np.empty(output_shape, dtype=input_datatype)
# if clip_geom is not None:
# ref = raster_to_array(raster, filled=True, extent=tiled_extent)
# else:
ref = raster_to_array(raster, filled=True)
for k, offset in enumerate(in_offsets):
start = 0
if k > 0:
start = offset_rows_cumsum[k - 1]
blocks = None
if (k == 0 and generate_border_patches
and (border_patches_needed_x or border_patches_needed_y)):
blocks = array_to_blocks(
ref,
(size, size),
offset,
border_patches_needed_x,
border_patches_needed_y,
)
else:
blocks = array_to_blocks(ref, (size, size), offset)
output_array[start:offset_rows_cumsum[k]] = blocks
if generate_grid_geom is False and clip_geom is None:
if out_dir is None:
output_blocks.append(output_array)
else:
output_blocks.append(output_block)
np.save(output_block, output_array)
else:
if out_dir is None:
output_blocks.append(output_array[mask])
else:
output_blocks.append(output_block)
np.save(output_block, output_array[mask])
if verify_output and generate_grid_geom:
test_extraction(
in_rasters,
output_blocks,
output_geom,
samples=verification_samples,
grid_layer_index=0,
verbose=verbose,
)
if len(output_blocks) == 1:
output_blocks = output_blocks[0]
return (output_blocks, output_geom)
