def assess_quality(safe_folder): metadata = get_metadata(safe_folder) scene_classification = raster_to_array(metadata["paths"]["20m"]["SCL"], filled=True, output_2d=True) sun_elevation = metadata["SUN_ELEVATION"] sun_adjust = ((-0.0012 * (sun_elevation * sun_elevation)) + (0.2778 * sun_elevation) - 10) b2 = raster_to_array(metadata["paths"]["20m"]["B02"], filled=True, output_2d=True) b12 = raster_to_array(metadata["paths"]["20m"]["B12"], filled=True, output_2d=True) cld_prop = raster_to_array(metadata["paths"]["QI"]["CLDPRB_20m"], filled=True, output_2d=True) quality = scl_to_quality(scene_classification, b2, b12, cld_prop, sun_adjust) quality = smooth_quality(quality) return quality
def convert_to_tiff( dim_file, out_folder, decibel=False, use_nodata=True, nodata_value=-9999.0, ): data_folder = os.path.splitext(dim_file)[0] + ".data/" name = os.path.splitext(os.path.basename(dim_file))[0].replace("_step_2", "") vh_path = data_folder + "Gamma0_VH.img" vv_path = data_folder + "Gamma0_VV.img" out_paths = [ out_folder + name + "_Gamma0_VH.tif", out_folder + name + "_Gamma0_VV.tif", ] if os.path.exists(out_paths[0]) and os.path.exists(out_paths[1]): print(f"{name} already processed") return out_paths vh = raster_to_array(vh_path) vv = raster_to_array(vv_path) if use_nodata: vh = np.ma.masked_equal(vh, 0.0, copy=False) vv = np.ma.masked_equal(vv, 0.0, copy=False) vh = np.nan_to_num(vh) vv = np.nan_to_num(vv) vh = np.ma.masked_equal(vh.filled(nodata_value), nodata_value) vv = np.ma.masked_equal(vv.filled(nodata_value), nodata_value) if decibel: with np.errstate(divide="ignore", invalid="ignore"): if use_nodata: vh = np.ma.multiply(np.ma.log10(np.ma.abs(vh)), 10) vv = np.ma.multiply(np.ma.log10(np.ma.abs(vv)), 10) else: vh = np.multiply(np.log10(np.abs(vh)), 10) vv = np.multiply(np.log10(np.abs(vv)), 10) array_to_raster(vh, vh_path, out_paths[0]) array_to_raster(vv, vv_path, out_paths[1]) return out_paths
def pansharpen(pan_path, tar_path, out_path): target = resample_raster(tar_path, pan_path, resample_alg="bilinear") aligned = align_rasters(target, master=pan_path) target = aligned[0] tar_arr = raster_to_array(target, output_2d=True) tar_arr = (tar_arr - tar_arr.min()) / (tar_arr.max() - tar_arr.min()) pan_arr = raster_to_array(pan_path, output_2d=True) _kernel, offsets, weights = create_kernel( [5, 5], sigma=2, output_2d=True, offsets=True ) pan = pansharpen_filter(pan_arr, tar_arr, offsets, weights.astype("float32")) array_to_raster(pan, reference=target, out_path=out_path)
def height_over_terrain(dsm_folder, dtm_folder, out_folder, tmp_folder): dsm_zipped = glob(dsm_folder + "*.zip") dtm_zipped = glob(dtm_folder + "*.zip") completed = 0 for dsm_tile in dsm_zipped: s = get_tile_from_zipped_url(dsm_tile) for dtm_tile in dtm_zipped: t = get_tile_from_zipped_url(dtm_tile) if s == t: ZipFile(dsm_tile).extractall(tmp_folder) ZipFile(dtm_tile).extractall(tmp_folder) dsm_tiffs = glob(tmp_folder + "DSM_*.tif") dtm_tiffs = glob(tmp_folder + "DTM_*.tif") for s_tiff in dsm_tiffs: s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4] s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0] for t_tiff in dtm_tiffs: t_tiff_tile_base = os.path.basename(t_tiff).split( "_")[2:4] t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0] if s_tiff_tile == t_tiff_tile: ss = raster_to_array(s_tiff) tt = raster_to_array(t_tiff) array_to_raster( np.abs(np.subtract(ss, tt)), out_path=out_folder + f"HOT_1km_{s_tiff_tile}.tif", reference=s_tiff, ) for f in glob(tmp_folder + "/*"): os.remove(f) completed += 1 print(f"Completed: {completed}/{len(dsm_zipped)}")
def haralick(in_raster, out_raster, options=None, out_datatype="float", band=None): """Performs haralick texture extraction""" cli = "otbcli_HaralickTextureExtraction" stats_raster = raster_to_array(in_raster) stats = {"min": stats_raster.min(), "max": stats_raster.max()} stats_raster = None if options is None: options = { "texture": "simple", "channel": 1, "parameters.nbbin": 64, "parameters.xrad": 3, "parameters.yrad": 3, "parameters.min": stats["min"], "parameters.max": stats["max"], } if out_datatype is None: out_datatype = "float" if band is not None: band = f"&bands={band}" else: band = "" cli_args = [ cli, "-in", os.path.abspath(in_raster), "-out", f'"{os.path.abspath(out_raster)}?{band}&gdal:co:COMPRESS=DEFLATE&gdal:co:NUM_THREADS=ALL_CPUS&gdal:co:BIGTIFF=YES"', out_datatype, ] for key, value in options.items(): cli_args.append("-" + str(key)) cli_args.append(str(value)) cli_string = " ".join(cli_args) execute_cli_function(cli_string, name="Texture extraction") return os.path.abspath(out_raster)
def zonal_statistics( in_vector, output_vector=None, in_rasters=[], prefixes=[], stats=["mean", "med", "std"], ): if len(prefixes) != 0: if len(in_rasters) != len(prefixes): raise ValueError("Unable to parse prefixes.") if isinstance(in_rasters, list): if len(in_rasters) == 0: raise ValueError("List of rasters (in_rasters) is empty.") if len(stats) == 0: raise ValueError("Unable to parse statistics (stats).") # Translate stats to integers stats_translated = stats_to_ints(stats) # Read the raster meta: raster_metadata = internal_raster_to_metadata(in_rasters[0]) vector = None if output_vector is None: vector = open_vector(in_vector, writeable=True) else: vector = internal_vector_to_memory(in_vector) vector_metadata = internal_vector_to_metadata(vector) vector_layer = vector.GetLayer() # Check that projections match if not vector_metadata["projection_osr"].IsSame( raster_metadata["projection_osr"]): if output_vector is None: vector = internal_reproject_vector(in_vector, in_rasters[0]) else: vector_path = internal_reproject_vector(in_vector, in_rasters[0], output_vector) vector = open_vector(vector_path, writeable=True) vector_metadata = internal_vector_to_metadata(vector) vector_layer = vector.GetLayer() vector_projection = vector_metadata["projection_osr"] raster_projection = raster_metadata["projection"] # Read raster data in overlap raster_transform = np.array(raster_metadata["transform"], dtype=np.float32) raster_size = np.array(raster_metadata["size"], dtype=np.int32) raster_extent = get_extent(raster_transform, raster_size) vector_extent = np.array(vector_layer.GetExtent(), dtype=np.float32) overlap_extent = get_intersection(raster_extent, vector_extent) if overlap_extent is False: print("raster_extent: ", raster_extent) print("vector_extent: ", vector_extent) raise Exception("Vector and raster do not overlap!") ( overlap_aligned_extent, overlap_aligned_rasterized_size, overlap_aligned_offset, ) = align_extent(raster_transform, overlap_extent, raster_size) overlap_transform = np.array( [ overlap_aligned_extent[0], raster_transform[1], 0, overlap_aligned_extent[3], 0, raster_transform[5], ], dtype=np.float32, ) overlap_size = overlap_size_calc(overlap_aligned_extent, raster_transform) # Loop the features vector_driver = ogr.GetDriverByName("Memory") vector_feature_count = vector_layer.GetFeatureCount() vector_layer.StartTransaction() # Create fields vector_layer_defn = vector_layer.GetLayerDefn() vector_field_counts = vector_layer_defn.GetFieldCount() vector_current_fields = [] # Get current fields for i in range(vector_field_counts): vector_current_fields.append( vector_layer_defn.GetFieldDefn(i).GetName()) # Add fields where missing for stat in stats: for i in range(len(in_rasters)): field_name = f"{prefixes[i]}{stat}" if field_name not in vector_current_fields: field_defn = ogr.FieldDefn(field_name, ogr.OFTReal) vector_layer.CreateField(field_defn) rasterized_features = [] sizes = np.zeros((vector_feature_count, 4), dtype="float32") offsets = np.zeros((vector_feature_count, 2), dtype=np.int32) raster_data = None for raster_index, raster_value in enumerate(in_rasters): columns = {} for stat in stats: columns[prefixes[raster_index] + stat] = [] fits_in_memory = True try: raster_data = raster_to_array( raster_value, crop=[ overlap_aligned_offset[0], overlap_aligned_offset[1], overlap_aligned_rasterized_size[0], overlap_aligned_rasterized_size[1], ], ) except: fits_in_memory = False print("Raster does not fit in memory.. Doing IO for each feature.") for n in range(vector_feature_count): vector_feature = vector_layer.GetNextFeature() rasterized_vector = None if raster_index == 0: try: vector_geom = vector_feature.GetGeometryRef() except: vector_geom.Buffer(0) Warning("Invalid geometry at : ", n) if vector_geom is None: raise Exception("Invalid geometry. Could not fix.") feature_extent = vector_geom.GetEnvelope() # Create temp layer temp_vector_datasource = vector_driver.CreateDataSource( f"vector_{n}") temp_vector_layer = temp_vector_datasource.CreateLayer( "temp_polygon", vector_projection, ogr.wkbPolygon) temp_vector_layer.CreateFeature(vector_feature.Clone()) ( feature_aligned_extent, feature_aligned_rasterized_size, feature_aligned_offset, ) = align_extent(overlap_transform, feature_extent, overlap_size) rasterized_vector = rasterize_vector( temp_vector_layer, feature_aligned_extent, feature_aligned_rasterized_size, raster_projection, ) rasterized_features.append(rasterized_vector) offsets[n] = feature_aligned_offset sizes[n] = feature_aligned_rasterized_size if fits_in_memory is True: cropped_raster = raster_data[offsets[n][1]:offsets[n][1] + int(sizes[n][1]), # X offsets[n][0]:offsets[n][0] + int(sizes[n][0]), # Y ] else: cropped_raster = raster_to_array( raster_value, crop=[ overlap_aligned_offset[0] + offsets[n][0], overlap_aligned_offset[1] + offsets[n][1], int(sizes[n][0]), int(sizes[n][1]), ], ) if rasterized_features[n] is None: for stat in stats: field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, None) elif cropped_raster is None: for stat in stats: field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, None) else: raster_data_masked = np.ma.masked_array( cropped_raster, mask=rasterized_features[n], dtype="float32").compressed() zonal_stats = calculate_array_stats(raster_data_masked, stats_translated) for index, stat in enumerate(stats): field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, float(zonal_stats[index])) vector_layer.SetFeature(vector_feature) progress(n, vector_feature_count, name=prefixes[raster_index]) vector_layer.ResetReading() vector_layer.CommitTransaction() if output_vector is None: return vector return output_vector
quantile) else: result[x, y] = np.median( hood_values[np.nonzero(hood_weights)]) return result folder = "C:/Users/caspe/Desktop/test_area2/" layers = [ folder + "vv_01.tif", folder + "vv_02.tif", folder + "vv_03.tif", ] sar_data = raster_to_array(layers) kernel_size = 3 nodata_value = -9999.0 _kernel, offsets_3D, weights_3D = create_kernel( (kernel_size, kernel_size, sar_data.shape[2]), distance_calc=False, # "gaussian" sigma=1, spherical=True, radius_method="ellipsoid", offsets=True, edge_weights=True, normalised=True, remove_zero_weights=True, )
build_tile_name = "_".join( os.path.splitext( os.path.basename(building_tile))[0].split("_")[2:]) if vrt_tile_name == build_tile_name: found_path = building_tile found = True if not found: area_vol_10m = internal_resample_raster( vrt_file, (10, 10), resample_alg='average', out_path=tmp_folder + f"buildings_volume_{vrt_tile_name}_10m_unscaled.tif") hot_arr = raster_to_array(area_vol_10m) * 0 vol_10m_path = dst_folder + f"buildings_volume_{vrt_tile_name}_10m.tif" area_10m_path = dst_folder + f"buildings_area_{vrt_tile_name}_10m.tif" array_to_raster(hot_arr, reference=area_vol_10m, out_path=vol_10m_path) array_to_raster(hot_arr, reference=area_vol_10m, out_path=area_10m_path) processed += 1 continue try: metadata = internal_raster_to_metadata(vrt_file) except:
def extract_patches( raster_list, outdir, tile_size=32, zones=None, options=None, ): """ Generate patches for machine learning from rasters """ base_options = { "overlaps": True, "border_check": True, "merge_output": True, "force_align": True, "output_raster_labels": True, "label_geom": None, "label_res": 0.2, "label_mult": 100, "tolerance": 0.0, "fill_value": 0, "zone_layer_id": 0, "align_with_size": 20, "prefix": "", "postfix": "", } if options is None: options = base_options else: for key in options: if key not in base_options: raise ValueError(f"Invalid option: {key}") base_options[key] = options[key] options = base_options if zones is not None and not is_vector(zones): raise TypeError( "Clip geom is invalid. Did you input a valid geometry?") if not isinstance(raster_list, list): raster_list = [raster_list] for raster in raster_list: if not is_raster(raster): raise TypeError("raster_list is not a list of rasters.") if not os.path.isdir(outdir): raise ValueError( "Outdir does not exist. Please create before running the function." ) if not rasters_are_aligned(raster_list, same_extent=True): if options["force_align"]: print( "Rasters we not aligned. Realigning rasters due to force_align=True option." ) raster_list = align_rasters(raster_list) else: raise ValueError("Rasters in raster_list are not aligned.") offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]] raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True) pixel_size = min(raster_metadata["pixel_height"], raster_metadata["pixel_width"]) if zones is None: zones = raster_metadata["extent_datasource_path"] zones_meta = vector_to_metadata(zones) mem_driver = ogr.GetDriverByName("ESRI Shapefile") if zones_meta["layer_count"] == 0: raise ValueError("Vector contains no layers.") zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]] if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]: raise ValueError("clip geom is not Polygon or Multi Polygon.") zones_ogr = open_vector(zones) zones_layer = zones_ogr.GetLayer(options["zone_layer_id"]) feature_defn = zones_layer.GetLayerDefn() fids = vector_get_fids(zones_ogr, options["zone_layer_id"]) progress(0, len(fids) * len(raster_list), "processing fids") processed_fids = [] processed = 0 labels_processed = False for idx, raster in enumerate(raster_list): name = os.path.splitext(os.path.basename(raster))[0] list_extracted = [] list_masks = [] list_labels = [] for fid in fids: feature = zones_layer.GetFeature(fid) geom = feature.GetGeometryRef() fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp" fid_ds = mem_driver.CreateDataSource(fid_path) fid_ds_lyr = fid_ds.CreateLayer( "fid_layer", geom_type=ogr.wkbPolygon, srs=zones_layer_meta["projection_osr"], ) copied_feature = ogr.Feature(feature_defn) copied_feature.SetGeometry(geom) fid_ds_lyr.CreateFeature(copied_feature) fid_ds.FlushCache() fid_ds.SyncToDisk() valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif" rasterize_vector( fid_path, pixel_size, out_path=valid_path, extent=fid_path, ) valid_arr = raster_to_array(valid_path) if options["label_geom"] is not None and fid not in processed_fids: if not is_vector(options["label_geom"]): raise TypeError( "label geom is invalid. Did you input a valid geometry?" ) uuid = str(uuid4().int) label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp" label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif" label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif" intersect_vector(options["label_geom"], fid_ds, out_path=label_clip_path) try: rasterize_vector( label_clip_path, options["label_res"], out_path=label_ras_path, extent=valid_path, ) except Exception: array_to_raster( np.zeros(valid_arr.shape, dtype="float32"), valid_path, out_path=label_ras_path, ) resample_raster( label_ras_path, pixel_size, resample_alg="average", out_path=label_warp_path, ) labels_arr = (raster_to_array(label_warp_path) * options["label_mult"]).astype("float32") if options["output_raster_labels"]: array_to_raster( labels_arr, label_warp_path, out_path= f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif", ) raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif" try: clip_raster( raster, valid_path, raster_clip_path, all_touch=False, adjust_bbox=False, ) except Exception as e: print( f"Warning: {raster} did not intersect geom with fid: {fid}." ) print(e) if options["label_geom"] is not None: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) continue arr = raster_to_array(raster_clip_path) if arr.shape[:2] != valid_arr.shape[:2]: raise Exception( f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}" ) arr_offsets = get_overlaps(arr, offsets, tile_size, options["border_check"]) arr = np.concatenate(arr_offsets) valid_offsets = np.concatenate( get_overlaps(valid_arr, offsets, tile_size, options["border_check"])) valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) / (tile_size * tile_size))) <= options["tolerance"])[:, 0] arr = arr[valid_mask] valid_masked = valid_offsets[valid_mask] if options["label_geom"] is not None and not labels_processed: labels_masked = np.concatenate( get_overlaps(labels_arr, offsets, tile_size, options["border_check"]))[valid_mask] if options["merge_output"]: list_extracted.append(arr) list_masks.append(valid_masked) if options["label_geom"] is not None and not labels_processed: list_labels.append(labels_masked) else: np.save( f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy", arr.filled(options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if fid not in processed_fids: processed_fids.append(fid) processed += 1 progress(processed, len(fids) * len(raster_list), "processing fids") if not options["merge_output"]: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) gdal.Unlink(valid_path) if options["merge_output"]: np.save( f"{outdir}{options['prefix']}{name}{options['postfix']}.npy", np.ma.concatenate(list_extracted).filled( options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy", np.ma.concatenate(list_masks).filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy", np.ma.concatenate(list_labels).filled( options["fill_value"]), ) labels_processed = True progress(1, 1, "processing fids") return 1
) * dA ) return out import sys sys.path.append("../../") from buteo.raster.io import raster_to_array from skimage.feature import graycomatrix, greycoprops folder = "C:/Users/caspe/Desktop/test_area/raster/" raster = folder + "B12_20m.tif" arr = raster_to_array(raster) arr = np.rint((arr / arr.max()) * 256).astype("uint8") glcm = graycomatrix( arr[:, :, 0], [1], [0, np.pi / 4, np.pi / 2, 3 * np.pi / 4], 256, symmetric=False, normed=True, ) bob = GLCMFeaturesInvariant(glcm) import pdb pdb.set_trace()
def volume_over_terrain( tile_names, username, password, out_folder, tmp_folder, ): if not os.path.isdir(tmp_folder): raise Exception("Error: output directory does not exist.") error_tiles = [] completed = 0 for tile in tile_names: base_path_DSM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DSM/" file_name_DSM = f'DSM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip' base_path_DTM = f"ftp://{username}:{password}@ftp.kortforsyningen.dk/dhm_danmarks_hoejdemodel/DTM/" file_name_DTM = f'DTM_{tile.split("_", 1)[1]}_TIF_UTM32-ETRS89.zip' try: if not os.path.exists(tmp_folder + file_name_DSM): get_file(base_path_DSM + file_name_DSM, tmp_folder + file_name_DSM) else: print(f"{file_name_DSM} Already exists.") if not os.path.exists(tmp_folder + file_name_DTM): get_file(base_path_DTM + file_name_DTM, tmp_folder + file_name_DTM) else: print(f"{file_name_DTM} Already exists.") ZipFile(tmp_folder + file_name_DSM).extractall(tmp_folder) ZipFile(tmp_folder + file_name_DTM).extractall(tmp_folder) dsm_tiffs = glob(tmp_folder + "DSM_*.tif") dtm_tiffs = glob(tmp_folder + "DTM_*.tif") hot_files = [] for s_tiff in dsm_tiffs: s_tiff_tile_base = os.path.basename(s_tiff).split("_")[2:4] s_tiff_tile = "_".join(s_tiff_tile_base).split(".")[0] for t_tiff in dtm_tiffs: t_tiff_tile_base = os.path.basename(t_tiff).split("_")[2:4] t_tiff_tile = "_".join(t_tiff_tile_base).split(".")[0] if s_tiff_tile == t_tiff_tile: ss = raster_to_array(s_tiff) tt = raster_to_array(t_tiff) hot_name = out_folder + f"HOT_1km_{s_tiff_tile}.tif" subtracted = np.subtract(ss, tt) hot_arr = np.abs((subtracted >= 0) * subtracted) array_to_raster( hot_arr, out_path=hot_name, reference=s_tiff, ) hot_files.append(hot_name) km10_tilename = "_".join(file_name_DSM.split("_")[1:3]) vrt_name = out_folder + f"HOT_10km_{km10_tilename}.vrt" stack_rasters_vrt( hot_files, seperate=False, out_path=vrt_name, ) except: print("Error while processing tile: {tile}") error_tiles.append(tile) finally: for f in glob(tmp_folder + "/*"): os.remove(f) completed += 1 print(f"Completed: {completed}/{len(tile_names)} - {tile}")
def norm_rasters( in_rasters, out_folder, method="normalise", split_bands=False, min_target=0, max_target=1, min_og=-9999, max_og=-9999, truncate=True, prefix="", postfix="", overwrite=True, ): if not isinstance(in_rasters, list): in_rasters = [in_rasters] normed_rasters = [] for in_raster in in_rasters: name = os.path.splitext(os.path.basename(in_raster))[0] raster = raster_to_array(in_raster) if method == "normalise": normed = norm_to_range(raster, min_target, max_target, truncate=False) elif method == "standardise": normed = standardise_filter(raster) elif method == "median_absolute_deviation": normed = mad_filter(raster) elif method == "range": normed = norm_to_range( raster, min_target=min_target, max_target=max_target, min_og=min_og, max_og=max_og, truncate=truncate, ) elif method == "robust_quantile": normed = robust_scaler_filter( raster, min_q=0.25, max_q=0.75, ) elif method == "robust_98": normed = robust_scaler_filter(raster, min_q=0.02, max_q=0.98) else: raise Exception(f"Method {method} not recognised") if split_bands: for idx in range(raster.shape[2]): band = idx + 1 raster_name = prefix + name + f"_B{band}" + postfix + ".tif" normed_rasters.append( array_to_raster( normed[:, :, idx][..., np.newaxis], reference=in_raster, out_path=out_folder + raster_name, overwrite=overwrite, )) else: raster_name = prefix + name + postfix + ".tif" normed_rasters.append( array_to_raster( normed, reference=in_raster, out_path=out_folder + raster_name, overwrite=overwrite, )) if isinstance(in_rasters, list): return normed_rasters return normed_rasters[0]
def super_sample_s2( B04_link, B08_link, B05_link=None, B06_link=None, B07_link=None, B8A_link=None, out_folder="../raster/", prefix="", suffix="", ): assert (isinstance(B05_link, str) or isinstance(B06_link, str) or isinstance(B07_link, str) or isinstance(B8A_link, str)) paths = { "B04": B04_link, "B05": B05_link, "B06": B06_link, "B07": B07_link, "B08": B08_link, "B8A": B8A_link, } bands = { "B04": raster_to_array(B04_link).astype("float32"), "B05": raster_to_array(B05_link).astype("float32") if B05_link is not None else False, "B06": raster_to_array(B06_link).astype("float32") if B06_link is not None else False, "B07": raster_to_array(B07_link).astype("float32") if B07_link is not None else False, "B08": raster_to_array(B08_link).astype("float32"), "B8A": raster_to_array(B8A_link).astype("float32") if B8A_link is not None else False, } bands_to_pansharpen = [] if bands["B05"] is not False: bands_to_pansharpen.append("B05") if bands["B06"] is not False: bands_to_pansharpen.append("B06") if bands["B07"] is not False: bands_to_pansharpen.append("B07") if bands["B8A"] is not False: bands_to_pansharpen.append("B8A") for band_x in bands_to_pansharpen: if band_x is "B05": pseudo_band = "B04" else: pseudo_band = "B08" pseudo_path = os.path.join(out_folder, f"{prefix}{band_x}{suffix}_pseudo.tif") array_to_raster( bands[pseudo_band], reference_raster=paths[pseudo_band], out_raster=pseudo_path, ) low_res_10m = raster_to_array( resample_raster( paths[band_x], reference_raster=paths[pseudo_band])).astype("float32") resampled_path = os.path.join( out_folder, f"{prefix}{band_x}{suffix}_resampled.tif") array_to_raster(low_res_10m, reference_raster=paths[pseudo_band], out_raster=resampled_path) low_res_10m = None pansharpened_path = os.path.join( out_folder, f"{prefix}{band_x}{suffix}_float.tif") pansharpen(pseudo_path, resampled_path, pansharpened_path) os.remove(resampled_path) os.remove(pseudo_path)
def test_extraction( rasters: Union[list, str, gdal.Dataset], arrays: Union[list, np.ndarray], grid: Union[ogr.DataSource, str], samples: int = 1000, # if 0, all grid_layer_index: int = 0, verbose: int = 1, ) -> bool: """Validates the output of the patch_extractor. Useful if you need peace of mind. Set samples to 0 to tests everything. Args: rasters (list of rasters | path | raster): The raster(s) used. arrays (list of arrays | ndarray): The arrays generated. grid (vector | vector_path): The grid generated. **kwargs: samples (int): The amount of patches to randomly test. If 0 all patches will be tested. This is a long process, so consider only testing everything if absolutely necessary. grid_layer_index (int): If the grid is part of a multi-layer vector, specify the index of the grid. verbose (int): If 1 will output messages on progress. Returns: True if the extraction is valid. Raises an error otherwise. """ type_check(rasters, [list, str, gdal.Dataset], "rasters") type_check(arrays, [list, str, np.ndarray], "arrays") type_check(grid, [list, str, ogr.DataSource], "grid") type_check(samples, [int], "samples") type_check(grid_layer_index, [int], "clip_layer_index") type_check(verbose, [int], "verbose") in_rasters = to_raster_list(rasters) in_arrays = arrays if verbose == 1: print("Verifying integrity of output grid..") # grid_memory = open_vector(internal_vector_to_memory(grid)) grid_memory = open_vector(grid) grid_metadata = internal_vector_to_metadata(grid) grid_projection = grid_metadata["projection_osr"] if grid_layer_index > (grid_metadata["layer_count"] - 1): raise ValueError( f"Requested non-existing layer index: {grid_layer_index}") grid_layer = grid_memory.GetLayer(grid_layer_index) # Select sample fids feature_count = grid_metadata["layers"][grid_layer_index]["feature_count"] test_samples = samples if samples > 0 else feature_count max_test = min(test_samples, feature_count) - 1 test_fids = np.array(random.sample(range(0, feature_count), max_test), dtype="uint64") mem_driver = ogr.GetDriverByName("ESRI Shapefile") for index, raster in enumerate(in_rasters): test_rast = open_raster(raster) test_array = in_arrays[index] if isinstance(test_array, str): if not os.path.exists(test_array): raise ValueError(f"Numpy array does not exist: {test_array}") try: test_array = np.load(in_arrays[index]) except: raise Exception( f"Attempted to read numpy raster from: {in_arrays[index]}") base = os.path.basename(raster) basename = os.path.splitext(base)[0] if verbose == 1: print(f"Testing: {basename}") tested = 0 for test in test_fids: feature = grid_layer.GetFeature(test) if feature is None: raise Exception(f"Feature not found: {test}") test_ds_path = f"/vsimem/test_mem_grid_{uuid4().int}.gpkg" test_ds = mem_driver.CreateDataSource(test_ds_path) test_ds_lyr = test_ds.CreateLayer("test_mem_grid_layer", geom_type=ogr.wkbPolygon, srs=grid_projection) test_ds_lyr.CreateFeature(feature.Clone()) test_ds.SyncToDisk() clipped = internal_clip_raster( test_rast, test_ds_path, adjust_bbox=False, crop_to_geom=True, all_touch=False, ) if clipped is None: raise Exception( "Error while clipping raster. Likely a bad extraction.") ref_image = raster_to_array(clipped, filled=True) image_block = test_array[test] if not np.array_equal(ref_image, image_block): # from matplotlib import pyplot as plt; plt.imshow(ref_image[:,:,0]); plt.show() raise Exception( f"Image {basename} and grid cell did not match..") if verbose == 1: progress(tested, len(test_fids) - 1, "Verifying..") tested += 1 return True
def raster_mask_values( raster: Union[gdal.Dataset, str, list], values_to_mask: list, out_path: Union[list, str, None] = None, include_original_nodata: bool = True, dst_nodata: Union[float, int, str, list, None] = "infer", in_place: bool = False, overwrite: bool = True, opened: bool = False, prefix: str = "", postfix: str = "_nodata_masked", creation_options: list = [], ) -> Union[list, gdal.Dataset, str]: """Mask a raster with a list of values. Args: raster (path | raster | list): The raster(s) to retrieve nodata values from. values_to_mask (list): The list of values to mask in the raster(s) **kwargs: include_original_nodata: (bool): If True, the nodata value of the raster(s) will be included in the values to mask. dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata value is set based on the input datatype. A list of nodata values can be based matching the amount of input rasters. If multiple nodata values should be set, use raster_mask_values. out_path (path | list | None): The destination of the changed rasters. If out_paths are specified, in_place is automatically set to False. The path can be a folder. in_place (bool): Should the rasters be changed in_place or copied? prefix (str): Prefix to add the the output if a folder is specified in out_path. postfix (str): Postfix to add the the output if a folder is specified in out_path. Returns: Returns the rasters with nodata removed. If in_place is True a reference to the changed orignal is returned, otherwise a copied memory raster or the path to the generated raster is outputted. """ type_check(raster, [list, str, gdal.Dataset], "raster") type_check(values_to_mask, [list], "values_to_mask") type_check(out_path, [list, str], "out_path", allow_none=True) type_check(include_original_nodata, [bool], "include_original_nodata") type_check(dst_nodata, [float, int, str, list], "dst_nodata", allow_none=True) type_check(in_place, [bool], "in_place") type_check(overwrite, [bool], "overwrite") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(opened, [bool], "opened") type_check(creation_options, [list], "creation_options") rasters_metadata = [] internal_in_place = in_place if out_path is None else False internal_dst_nodata = None for value in values_to_mask: if not isinstance(value, (int, float)): raise ValueError("Values in values_to_mask must be ints or floats") if isinstance(dst_nodata, str) and dst_nodata != "infer": raise ValueError(f"Invalid dst_nodata value. {dst_nodata}") if isinstance(dst_nodata, list): if not isinstance(raster, list) or len(dst_nodata) != len(raster): raise ValueError( "If dst_nodata is a list, raster must also be a list of equal length." ) for value in dst_nodata: if isinstance(value, (float, int, str, None)): raise ValueError("Invalid type in dst_nodata list.") if isinstance(value, str) and value != "infer": raise ValueError( "If dst_nodata is a string it must be 'infer'") raster_list, out_names = ready_io_raster(raster, out_path, overwrite, prefix, postfix) output_rasters = [] for index, internal_raster in enumerate(raster_list): raster_metadata = None if len(rasters_metadata) == 0: raster_metadata = raster_to_metadata(internal_raster) rasters_metadata.append(raster_metadata) else: raster_metadata = rasters_metadata[index] if dst_nodata == "infer": internal_dst_nodata = gdal_nodata_value_from_type( raster_metadata["dtype_gdal_raw"]) elif isinstance(dst_nodata, list): internal_dst_nodata = dst_nodata[index] else: internal_dst_nodata = dst_nodata mask_values = list(values_to_mask) if include_original_nodata: if raster_metadata["nodata_value"] is not None: mask_values.append(raster_metadata["nodata_value"]) arr = raster_to_array(internal_raster, filled=True) mask = None for index, mask_value in enumerate(mask_values): if index == 0: mask = arr == mask_value else: mask = mask | arr == mask_value arr = np.ma.masked_array(arr, mask=mask, fill_value=internal_dst_nodata) if internal_in_place: for band in range(raster_metadata["bands"]): raster_band = internal_raster.GetRasterBand(band + 1) raster_band.WriteArray(arr[:, :, band]) raster_band = None else: out_name = out_names[index] remove_if_overwrite(out_name, overwrite) output_rasters.append( array_to_raster(arr, internal_raster, out_path=out_name)) if isinstance(raster, list): return output_rasters return output_rasters[0]
def calc_proximity( input_rasters, target_value=1, out_path=None, max_dist=1000, add_border=False, weighted=False, invert=False, return_array=False, postfix="_proximity", uuid=False, overwrite=True, skip_existing=False, ): """ Calculate the proximity of input_raster to values """ raster_list, path_list = ready_io_raster(input_rasters, out_path, overwrite, postfix=postfix, uuid=uuid) output = [] for index, input_raster in enumerate(raster_list): out_path = path_list[index] if skip_existing and os.path.exists(out_path): output.append(out_path) continue in_arr = raster_to_array(input_raster, filled=True) bin_arr = (in_arr != target_value).astype("uint8") bin_raster = array_to_raster(bin_arr, reference=input_raster) in_raster = open_raster(bin_raster) in_raster_path = bin_raster if add_border: border_size = 1 border_raster = add_border_to_raster( in_raster, border_size=border_size, border_value=0, overwrite=True, ) in_raster = open_raster(border_raster) gdal.Unlink(in_raster_path) in_raster_path = border_raster src_band = in_raster.GetRasterBand(1) driver_name = "GTiff" if out_path is None else path_to_driver_raster( out_path) if driver_name is None: raise ValueError(f"Unable to parse filetype from path: {out_path}") driver = gdal.GetDriverByName(driver_name) if driver is None: raise ValueError( f"Error while creating driver from extension: {out_path}") mem_path = f"/vsimem/raster_proximity_tmp_{uuid4().int}.tif" dest_raster = driver.Create( mem_path, in_raster.RasterXSize, in_raster.RasterYSize, 1, gdal.GetDataTypeByName("Float32"), ) dest_raster.SetGeoTransform(in_raster.GetGeoTransform()) dest_raster.SetProjection(in_raster.GetProjectionRef()) dst_band = dest_raster.GetRasterBand(1) gdal.ComputeProximity( src_band, dst_band, [ f"VALUES='1'", "DISTUNITS=GEO", f"MAXDIST={max_dist}", ], ) dst_arr = dst_band.ReadAsArray() gdal.Unlink(mem_path) gdal.Unlink(in_raster_path) dst_arr = np.where(dst_arr > max_dist, max_dist, dst_arr) if invert: dst_arr = max_dist - dst_arr if weighted: dst_arr = dst_arr / max_dist if add_border: dst_arr = dst_arr[border_size:-border_size, border_size:-border_size] src_band = None dst_band = None in_raster = None dest_raster = None if return_array: output.append(dst_arr) else: array_to_raster(dst_arr, reference=input_raster, out_path=out_path) output.append(out_path) dst_arr = None if isinstance(input_rasters, list): return output return output[0]
# obt_bandmath( # [folder + "Ghana_classification_float32_v2.tif", folder + "Ghana_float32_v9.tif"], # "((im1b2 + im1b3 + im1b4) == 0) ? 0.0 : (im1b4 / (im1b2 + im1b3 + im1b4)) * im2b1", # folder + "area_slum.tif", # ram=32000, # ) # w = [1.00, 1.00, 1.00] # unweighted # w = [1.25, 1.35, 0.40] # nighttime # w = [0.80, 0.90, 1.30] # daytime # w1 = w[0] # w2 = w[1] # w3 = w[2] t_pop1 = raster_to_array(folder + "area_residential_clipped.tif").filled(0).sum() t_pop2 = raster_to_array(folder + "area_slum_clipped.tif").filled(0).sum() t_pop3 = raster_to_array(folder + "area_industrial_clipped.tif").filled(0).sum() t_pop = t_pop1 + t_pop2 + t_pop3 t_pop = 968159000.0 import pdb pdb.set_trace() # obt_bandmath( # [ # folder + "area_residential_clipped.tif", # folder + "area_slum_clipped.tif", # folder + "area_industrial_clipped.tif",
output_tile_size=32, output_channels=1, model_path="", reference_raster="", out_path=None, out_path_variance=None, offsets=True, batch_size=32, method="median", scale_to_sum=False, ): print("Loading Model") model = tf.keras.models.load_model( model_path, custom_objects={"tpe": tpe, "mse_sumbias": mse_sumbias} ) reference_arr = raster_to_array(reference_raster) if offsets == False: print( "No offsets provided. Using offsets greatly increases accuracy. Please provide offsets." ) else: offsets = [] for val in tile_size: offsets.append(get_offsets(val)) predictions = [] read_rasters = [] print("Initialising rasters.") for raster_idx, raster in enumerate(raster_list):
def process_aligned( aligned_rasters, out_path, folder_tmp, chunks, master_raster, nodata_value, feather_weights=None, ): kernel_size = 3 chunk_offset = kernel_size // 2 _kernel, offsets, weights = create_kernel( (kernel_size, kernel_size, len(aligned_rasters)), distance_calc=False, # "gaussian" sigma=1, spherical=True, radius_method="ellipsoid", offsets=True, edge_weights=True, normalised=True, remove_zero_weights=True, ) arr_aligned = raster_to_array(aligned_rasters) if feather_weights is not None: feather_weights_arr = raster_to_array(feather_weights) if not rasters_are_aligned(aligned_rasters): raise Exception("Rasters not aligned") if chunks > 1: chunks_list = [] print("Chunking rasters") uids = uuid4() for chunk in range(chunks): print(f"Chunk {chunk + 1} of {chunks}") cut_start = False cut_end = False if chunk == 0: chunk_start = 0 else: chunk_start = (chunk * (arr_aligned.shape[0] // chunks)) - chunk_offset cut_start = True if chunk == chunks - 1: chunk_end = arr_aligned.shape[0] else: chunk_end = ((chunk + 1) * (arr_aligned.shape[0] // chunks)) + chunk_offset cut_end = True arr_chunk = arr_aligned[chunk_start:chunk_end] if feather_weights is not None: weights_chunk = feather_weights_arr[chunk_start:chunk_end] else: weights_chunk = np.ones_like(arr_chunk) print(" Collapsing...") arr_collapsed = s1_collapse( arr_chunk, offsets, weights, weights_chunk, weighted=True, nodata_value=nodata_value, nodata=True, ) offset_start = chunk_offset if cut_start else 0 offset_end = (arr_collapsed.shape[0] - chunk_offset if cut_end else arr_collapsed.shape[0]) chunk_path = folder_tmp + f"{uids}_chunk_{chunk}.npy" chunks_list.append(chunk_path) np.save(chunk_path, arr_collapsed[offset_start:offset_end]) arr_chunk = None arr_collapsed = None print("Merging Chunks") arr_aligned = None merged = [] for chunk in chunks_list: merged.append(np.load(chunk)) merged = np.concatenate(merged) merged = np.ma.masked_array(merged, mask=merged == nodata_value) merged.fill_value = nodata_value print("Writing raster.") array_to_raster( merged, master_raster, out_path=out_path, ) merged = None return out_path if feather_weights is not None: weights_borders = feather_weights else: weights_borders = np.ones_like(arr_aligned) print("Collapsing rasters") arr_collapsed = s1_collapse( arr_aligned, offsets, weights, weights_borders, weighted=True, nodata_value=nodata_value, nodata=True, ) arr_collapsed = np.ma.masked_array(arr_collapsed, mask=arr_collapsed == nodata_value) arr_collapsed.fill_value = nodata_value arr_aligned = None print("Writing raster.") array_to_raster( arr_collapsed, master_raster, out_path=out_path, ) arr_collapsed = None return out_path
def add_border_to_raster( input_raster, out_path=None, border_size=100, border_size_unit="px", border_value=0, overwrite: bool = True, creation_options: list = [], ): in_raster = open_raster(input_raster) metadata = raster_to_metadata(in_raster) # Parse the driver driver_name = "GTiff" if out_path is None else path_to_driver_raster( out_path) if driver_name is None: raise ValueError(f"Unable to parse filetype from path: {out_path}") driver = gdal.GetDriverByName(driver_name) if driver is None: raise ValueError( f"Error while creating driver from extension: {out_path}") output_name = None if out_path is None: output_name = f"/vsimem/raster_proximity_{uuid4().int}.tif" else: output_name = out_path in_arr = raster_to_array(in_raster) if border_size_unit == "px": border_size_y = border_size border_size_x = border_size new_shape = ( in_arr.shape[0] + (2 * border_size_y), in_arr.shape[1] + (2 * border_size_x), in_arr.shape[2], ) else: border_size_y = round(border_size / metadata["pixel_height"]) border_size_x = round(border_size / metadata["pixel_width"]) new_shape = ( in_arr.shape[0] + (2 * border_size_y), in_arr.shape[1] + (2 * border_size_x), in_arr.shape[2], ) new_arr = np.full(new_shape, border_value, dtype=in_arr.dtype) new_arr[border_size_y:-border_size_y, border_size_x:-border_size_x, :] = in_arr if isinstance(in_arr, np.ma.MaskedArray): mask = np.zeros(new_shape, dtype=bool) mask[border_size_y:-border_size_y, border_size_x:-border_size_x, :] = in_arr.mask new_arr = np.ma.array(new_arr, mask=mask) new_arr.fill_value = in_arr.fill_value remove_if_overwrite(out_path, overwrite) dest_raster = driver.Create( output_name, new_shape[1], new_shape[0], metadata["band_count"], numpy_to_gdal_datatype(in_arr.dtype), default_options(creation_options), ) og_transform = in_raster.GetGeoTransform() new_transform = [] for i in og_transform: new_transform.append(i) new_transform[0] -= border_size_x * og_transform[1] new_transform[3] -= border_size_y * og_transform[5] dest_raster.SetGeoTransform(new_transform) dest_raster.SetProjection(in_raster.GetProjectionRef()) for band_num in range(1, metadata["band_count"] + 1): dst_band = dest_raster.GetRasterBand(band_num) dst_band.WriteArray(new_arr[:, :, band_num - 1]) if metadata["has_nodata"]: dst_band.SetNoDataValue(metadata["nodata_value"]) return output_name
def extract_patches( raster: Union[List[Union[str, gdal.Dataset]], str, gdal.Dataset], out_dir: Optional[str] = None, prefix: str = "", postfix: str = "_patches", size: int = 32, offsets: Union[list, None] = [], generate_border_patches: bool = True, generate_zero_offset: bool = True, generate_grid_geom: bool = True, clip_geom: Optional[Union[str, ogr.DataSource, gdal.Dataset]] = None, clip_layer_index: int = 0, verify_output=True, verification_samples=100, overwrite=True, epsilon: float = 1e-9, verbose: int = 1, ) -> tuple: """Extracts square tiles from a raster. Args: raster (list of rasters | path | raster): The raster(s) to convert. **kwargs: out_dir (path | none): Folder to save output. If None, in-memory arrays and geometries are outputted. prefix (str): A prefix for all outputs. postfix (str): A postfix for all outputs. size (int): The size of the tiles in pixels. offsets (list of tuples): List of offsets to extract. Example: offsets=[(16, 16), (16, 0), (0, 16)]. Will offset the initial raster and extract from there. generate_border_patches (bool): The tiles often do not align with the rasters which means borders are trimmed somewhat. If generate_border_patches is True, an additional tile is added where needed. generate_zero_offset (bool): if True, an offset is inserted at (0, 0) if none is present. generate_grid_geom (bool): Output a geopackage with the grid of tiles. clip_geom (str, raster, vector): Clip the output to the intersections with a geometry. Useful if a lot of the target area is water or similar. epsilon (float): How much for buffer the arange array function. This should usually just be left alone. verbose (int): If 1 will output messages on progress. Returns: A tuple with paths to the generated items. (numpy_array, grid_geom) """ type_check(raster, [str, list, gdal.Dataset], "raster") type_check(out_dir, [str], "out_dir", allow_none=True) type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(size, [int], "size") type_check(offsets, [list], "offsets", allow_none=True) type_check(generate_grid_geom, [bool], "generate_grid_geom") type_check( clip_geom, [str, ogr.DataSource, gdal.Dataset], "clip_layer_index", allow_none=True, ) type_check(clip_layer_index, [int], "clip_layer_index") type_check(overwrite, [bool], "overwrite") type_check(epsilon, [float], "epsilon") type_check(verbose, [int], "verbose") in_rasters = to_raster_list(raster) if out_dir is not None and not os.path.isdir(out_dir): raise ValueError(f"Output directory does not exists: {out_dir}") if not rasters_are_aligned(in_rasters): raise ValueError( "Input rasters must be aligned. Please use the align function.") output_geom = None metadata = internal_raster_to_metadata(in_rasters[0]) if verbose == 1: print("Generating blocks..") # internal offset array. Avoid manipulating the og array. if offsets is None: offsets = [] in_offsets = [] if generate_zero_offset and (0, 0) not in offsets: in_offsets.append((0, 0)) for offset in offsets: if offset != (0, 0): if not isinstance(offset, (list, tuple)) or len(offset) != 2: raise ValueError( f"offset must be a list or tuple of two integers. Recieved: {offset}" ) in_offsets.append((offset[0], offset[1])) border_patches_needed_x = True border_patches_needed_y = True if clip_geom is not None: border_patches_needed_x = False border_patches_needed_y = False shapes = [] for offset in in_offsets: block_shape = shape_to_blockshape(metadata["shape"], (size, size), offset) if block_shape[0] * size == metadata["width"]: border_patches_needed_x = False if block_shape[1] * size == metadata["height"]: border_patches_needed_y = False shapes.append(block_shape) if generate_border_patches: cut_x = (metadata["width"] - in_offsets[0][0]) - (shapes[0][0] * size) cut_y = (metadata["height"] - in_offsets[0][1]) - (shapes[0][1] * size) if border_patches_needed_x and cut_x > 0: shapes[0][0] += 1 if border_patches_needed_y and cut_y > 0: shapes[0][1] += 1 # calculate the offsets all_rows = 0 offset_rows = [] for i in range(len(shapes)): row = 0 for j in range(len(shapes[i])): if j == 0: row = int(shapes[i][j]) else: row *= int(shapes[i][j]) offset_rows.append(row) all_rows += row offset_rows_cumsum = np.cumsum(offset_rows) if generate_grid_geom is True or clip_geom is not None: if verbose == 1: print("Calculating grid cells..") mask = np.arange(all_rows, dtype="uint64") ulx, uly, _lrx, _lry = metadata["extent"] pixel_width = abs(metadata["pixel_width"]) pixel_height = abs(metadata["pixel_height"]) xres = pixel_width * size yres = pixel_height * size dx = xres / 2 dy = yres / 2 # Ready clip geom outside of loop. if clip_geom is not None: clip_ref = open_vector( internal_reproject_vector(clip_geom, metadata["projection_osr"])) clip_layer = clip_ref.GetLayerByIndex(clip_layer_index) meta_clip = internal_vector_to_metadata(clip_ref) # geom_clip = meta_clip["layers"][clip_layer_index]["column_geom"] clip_extent = meta_clip["extent_ogr"] # clip_adjust = [ # clip_extent[0] - clip_extent[0] % xres, # x_min # (clip_extent[1] - clip_extent[1] % xres) + xres, # x_max # clip_extent[2] - clip_extent[2] % yres, # y_min # (clip_extent[3] - clip_extent[3] % yres) + yres, # y_max # ] coord_grid = np.empty((all_rows, 2), dtype="float64") # tiled_extent = [None, None, None, None] row_count = 0 for idx in range(len(in_offsets)): x_offset = in_offsets[idx][0] y_offset = in_offsets[idx][1] x_step = shapes[idx][0] y_step = shapes[idx][1] x_min = (ulx + dx) + (x_offset * pixel_width) x_max = x_min + (x_step * xres) y_max = (uly - dy) - (y_offset * pixel_height) y_min = y_max - (y_step * yres) # if clip_geom is not None: # if clip_adjust[0] > x_min: # x_min = clip_adjust[0] + (x_offset * pixel_width) # if clip_adjust[1] < x_max: # x_max = clip_adjust[1] + (x_offset * pixel_width) # if clip_adjust[2] > y_min: # y_min = clip_adjust[2] - (y_offset * pixel_height) # if clip_adjust[3] < y_max: # y_max = clip_adjust[3] - (y_offset * pixel_height) # if idx == 0: # tiled_extent[0] = x_min # tiled_extent[1] = x_max # tiled_extent[2] = y_min # tiled_extent[3] = y_max # else: # if x_min < tiled_extent[0]: # tiled_extent[0] = x_min # if x_max > tiled_extent[1]: # tiled_extent[1] = x_max # if y_min < tiled_extent[2]: # tiled_extent[2] = y_min # if y_max > tiled_extent[3]: # tiled_extent[3] = y_max # y is flipped so: xmin --> xmax, ymax -- ymin to keep same order as numpy array x_patches = round((x_max - x_min) / xres) y_patches = round((y_max - y_min) / yres) xr = np.arange(x_min, x_max, xres)[0:x_step] if xr.shape[0] < x_patches: xr = np.arange(x_min, x_max + epsilon, xres)[0:x_step] elif xr.shape[0] > x_patches: xr = np.arange(x_min, x_max - epsilon, xres)[0:x_step] yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step] if yr.shape[0] < y_patches: yr = np.arange(y_max, y_min - epsilon, -yres)[0:y_step] elif yr.shape[0] > y_patches: yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step] if generate_border_patches and idx == 0: if border_patches_needed_x: xr[-1] = xr[-1] - ( (xr[-1] + dx) - metadata["extent_dict"]["right"]) if border_patches_needed_y: yr[-1] = yr[-1] - ( (yr[-1] - dy) - metadata["extent_dict"]["bottom"]) oxx, oyy = np.meshgrid(xr, yr) oxr = oxx.ravel() oyr = oyy.ravel() offset_length = oxr.shape[0] coord_grid[row_count:row_count + offset_length, 0] = oxr coord_grid[row_count:row_count + offset_length, 1] = oyr row_count += offset_length offset_rows_cumsum[idx] = offset_length offset_rows_cumsum = np.cumsum(offset_rows_cumsum) coord_grid = coord_grid[:row_count] # Output geometry driver = ogr.GetDriverByName("GPKG") patches_path = f"/vsimem/patches_{uuid4().int}.gpkg" patches_ds = driver.CreateDataSource(patches_path) patches_layer = patches_ds.CreateLayer("patches_all", geom_type=ogr.wkbPolygon, srs=metadata["projection_osr"]) patches_fdefn = patches_layer.GetLayerDefn() og_fid = "og_fid" field_defn = ogr.FieldDefn(og_fid, ogr.OFTInteger) patches_layer.CreateField(field_defn) if clip_geom is not None: clip_feature_count = meta_clip["layers"][clip_layer_index][ "feature_count"] spatial_index = rtree.index.Index(interleaved=False) for _ in range(clip_feature_count): clip_feature = clip_layer.GetNextFeature() clip_fid = clip_feature.GetFID() clip_feature_geom = clip_feature.GetGeometryRef() xmin, xmax, ymin, ymax = clip_feature_geom.GetEnvelope() spatial_index.insert(clip_fid, (xmin, xmax, ymin, ymax)) fids = 0 mask = [] for tile_id in range(coord_grid.shape[0]): x, y = coord_grid[tile_id] if verbose == 1: progress(tile_id, coord_grid.shape[0], "Patch generation") x_min = x - dx x_max = x + dx y_min = y - dx y_max = y + dx tile_intersects = True grid_geom = None poly_wkt = None if clip_geom is not None: tile_intersects = False if not ogr_bbox_intersects([x_min, x_max, y_min, y_max], clip_extent): continue intersections = list( spatial_index.intersection((x_min, x_max, y_min, y_max))) if len(intersections) == 0: continue poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))" grid_geom = ogr.CreateGeometryFromWkt(poly_wkt) for fid1 in intersections: clip_feature = clip_layer.GetFeature(fid1) clip_geom = clip_feature.GetGeometryRef() if grid_geom.Intersects(clip_geom): tile_intersects = True continue if tile_intersects: ft = ogr.Feature(patches_fdefn) if grid_geom is None: poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))" grid_geom = ogr.CreateGeometryFromWkt(poly_wkt) ft_geom = ogr.CreateGeometryFromWkt(poly_wkt) ft.SetGeometry(ft_geom) ft.SetField(og_fid, int(fids)) ft.SetFID(int(fids)) patches_layer.CreateFeature(ft) ft = None mask.append(tile_id) fids += 1 if verbose == 1: progress(coord_grid.shape[0], coord_grid.shape[0], "Patch generation") mask = np.array(mask, dtype=int) if generate_grid_geom is True: if out_dir is None: output_geom = patches_ds else: raster_basename = metadata["basename"] geom_name = f"{prefix}{raster_basename}_geom_{str(size)}{postfix}.gpkg" output_geom = os.path.join(out_dir, geom_name) overwrite_required(output_geom, overwrite) remove_if_overwrite(output_geom, overwrite) if verbose == 1: print("Writing output geometry..") internal_vector_to_disk(patches_ds, output_geom, overwrite=overwrite) if verbose == 1: print("Writing numpy array..") output_blocks = [] for raster in in_rasters: base = None basename = None output_block = None if out_dir is not None: base = os.path.basename(raster) basename = os.path.splitext(base)[0] output_block = os.path.join(out_dir + f"{prefix}{basename}{postfix}.npy") metadata = internal_raster_to_metadata(raster) if generate_grid_geom is True or clip_geom is not None: output_shape = (row_count, size, size, metadata["band_count"]) else: output_shape = (all_rows, size, size, metadata["band_count"]) input_datatype = metadata["datatype"] output_array = np.empty(output_shape, dtype=input_datatype) # if clip_geom is not None: # ref = raster_to_array(raster, filled=True, extent=tiled_extent) # else: ref = raster_to_array(raster, filled=True) for k, offset in enumerate(in_offsets): start = 0 if k > 0: start = offset_rows_cumsum[k - 1] blocks = None if (k == 0 and generate_border_patches and (border_patches_needed_x or border_patches_needed_y)): blocks = array_to_blocks( ref, (size, size), offset, border_patches_needed_x, border_patches_needed_y, ) else: blocks = array_to_blocks(ref, (size, size), offset) output_array[start:offset_rows_cumsum[k]] = blocks if generate_grid_geom is False and clip_geom is None: if out_dir is None: output_blocks.append(output_array) else: output_blocks.append(output_block) np.save(output_block, output_array) else: if out_dir is None: output_blocks.append(output_array[mask]) else: output_blocks.append(output_block) np.save(output_block, output_array[mask]) if verify_output and generate_grid_geom: test_extraction( in_rasters, output_blocks, output_geom, samples=verification_samples, grid_layer_index=0, verbose=verbose, ) if len(output_blocks) == 1: output_blocks = output_blocks[0] return (output_blocks, output_geom)