def execute_cli_function(command, name, quiet=False): process = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stdin=subprocess.DEVNULL, stderr=subprocess.STDOUT, universal_newlines=True, ) try: before = time.time() for line in iter(process.stdout.readline, ""): if "FATAL" in line: raise RuntimeError(line) elif "CRITICAL" in line: raise RuntimeError(line) elif "WARNING" in line: continue elif quiet is False: if "INFO" in line: continue try: strip = line.strip() if len(strip) != 0: part = strip.rsplit(":", 1)[1] percent = int(part.split("%")[0]) progress(percent, 100, name) except: # print('runtime error') if len(line.strip()) != 0: raise RuntimeError(line) from None except: print("Critical failure while performing Orfeo-Toolbox action.") pass print(f"{name} completed in {round(time.time() - before, 2)}s.")
def zonal_statistics( in_vector, output_vector=None, in_rasters=[], prefixes=[], stats=["mean", "med", "std"], ): if len(prefixes) != 0: if len(in_rasters) != len(prefixes): raise ValueError("Unable to parse prefixes.") if isinstance(in_rasters, list): if len(in_rasters) == 0: raise ValueError("List of rasters (in_rasters) is empty.") if len(stats) == 0: raise ValueError("Unable to parse statistics (stats).") # Translate stats to integers stats_translated = stats_to_ints(stats) # Read the raster meta: raster_metadata = internal_raster_to_metadata(in_rasters[0]) vector = None if output_vector is None: vector = open_vector(in_vector, writeable=True) else: vector = internal_vector_to_memory(in_vector) vector_metadata = internal_vector_to_metadata(vector) vector_layer = vector.GetLayer() # Check that projections match if not vector_metadata["projection_osr"].IsSame( raster_metadata["projection_osr"]): if output_vector is None: vector = internal_reproject_vector(in_vector, in_rasters[0]) else: vector_path = internal_reproject_vector(in_vector, in_rasters[0], output_vector) vector = open_vector(vector_path, writeable=True) vector_metadata = internal_vector_to_metadata(vector) vector_layer = vector.GetLayer() vector_projection = vector_metadata["projection_osr"] raster_projection = raster_metadata["projection"] # Read raster data in overlap raster_transform = np.array(raster_metadata["transform"], dtype=np.float32) raster_size = np.array(raster_metadata["size"], dtype=np.int32) raster_extent = get_extent(raster_transform, raster_size) vector_extent = np.array(vector_layer.GetExtent(), dtype=np.float32) overlap_extent = get_intersection(raster_extent, vector_extent) if overlap_extent is False: print("raster_extent: ", raster_extent) print("vector_extent: ", vector_extent) raise Exception("Vector and raster do not overlap!") ( overlap_aligned_extent, overlap_aligned_rasterized_size, overlap_aligned_offset, ) = align_extent(raster_transform, overlap_extent, raster_size) overlap_transform = np.array( [ overlap_aligned_extent[0], raster_transform[1], 0, overlap_aligned_extent[3], 0, raster_transform[5], ], dtype=np.float32, ) overlap_size = overlap_size_calc(overlap_aligned_extent, raster_transform) # Loop the features vector_driver = ogr.GetDriverByName("Memory") vector_feature_count = vector_layer.GetFeatureCount() vector_layer.StartTransaction() # Create fields vector_layer_defn = vector_layer.GetLayerDefn() vector_field_counts = vector_layer_defn.GetFieldCount() vector_current_fields = [] # Get current fields for i in range(vector_field_counts): vector_current_fields.append( vector_layer_defn.GetFieldDefn(i).GetName()) # Add fields where missing for stat in stats: for i in range(len(in_rasters)): field_name = f"{prefixes[i]}{stat}" if field_name not in vector_current_fields: field_defn = ogr.FieldDefn(field_name, ogr.OFTReal) vector_layer.CreateField(field_defn) rasterized_features = [] sizes = np.zeros((vector_feature_count, 4), dtype="float32") offsets = np.zeros((vector_feature_count, 2), dtype=np.int32) raster_data = None for raster_index, raster_value in enumerate(in_rasters): columns = {} for stat in stats: columns[prefixes[raster_index] + stat] = [] fits_in_memory = True try: raster_data = raster_to_array( raster_value, crop=[ overlap_aligned_offset[0], overlap_aligned_offset[1], overlap_aligned_rasterized_size[0], overlap_aligned_rasterized_size[1], ], ) except: fits_in_memory = False print("Raster does not fit in memory.. Doing IO for each feature.") for n in range(vector_feature_count): vector_feature = vector_layer.GetNextFeature() rasterized_vector = None if raster_index == 0: try: vector_geom = vector_feature.GetGeometryRef() except: vector_geom.Buffer(0) Warning("Invalid geometry at : ", n) if vector_geom is None: raise Exception("Invalid geometry. Could not fix.") feature_extent = vector_geom.GetEnvelope() # Create temp layer temp_vector_datasource = vector_driver.CreateDataSource( f"vector_{n}") temp_vector_layer = temp_vector_datasource.CreateLayer( "temp_polygon", vector_projection, ogr.wkbPolygon) temp_vector_layer.CreateFeature(vector_feature.Clone()) ( feature_aligned_extent, feature_aligned_rasterized_size, feature_aligned_offset, ) = align_extent(overlap_transform, feature_extent, overlap_size) rasterized_vector = rasterize_vector( temp_vector_layer, feature_aligned_extent, feature_aligned_rasterized_size, raster_projection, ) rasterized_features.append(rasterized_vector) offsets[n] = feature_aligned_offset sizes[n] = feature_aligned_rasterized_size if fits_in_memory is True: cropped_raster = raster_data[offsets[n][1]:offsets[n][1] + int(sizes[n][1]), # X offsets[n][0]:offsets[n][0] + int(sizes[n][0]), # Y ] else: cropped_raster = raster_to_array( raster_value, crop=[ overlap_aligned_offset[0] + offsets[n][0], overlap_aligned_offset[1] + offsets[n][1], int(sizes[n][0]), int(sizes[n][1]), ], ) if rasterized_features[n] is None: for stat in stats: field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, None) elif cropped_raster is None: for stat in stats: field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, None) else: raster_data_masked = np.ma.masked_array( cropped_raster, mask=rasterized_features[n], dtype="float32").compressed() zonal_stats = calculate_array_stats(raster_data_masked, stats_translated) for index, stat in enumerate(stats): field_name = f"{prefixes[raster_index]}{stat}" vector_feature.SetField(field_name, float(zonal_stats[index])) vector_layer.SetFeature(vector_feature) progress(n, vector_feature_count, name=prefixes[raster_index]) vector_layer.ResetReading() vector_layer.CommitTransaction() if output_vector is None: return vector return output_vector
previous_errors = [ '629_65', '618_68', '613_61', '612_65', '609_55', '626_56', '616_56', '626_47', '614_65', '622_57', '629_66', '613_63', '616_74', '628_66', '620_60', '619_71', '634_52', '617_74', '613_47', '608_68', '619_62', '619_58', '616_57', '609_60', '620_69', '610_64', '614_68', '622_51', '613_64', '612_62', '604_68', '633_61', '605_64', '625_44', '613_66', '616_58', '613_59', '619_46', '633_48', '631_45', '618_66', '634_63', '627_57' ] error_files = [] processed = 0 for index, vrt_file in enumerate(vrt_files): progress(processed, len(vrt_files), "Generating") vrt_tile_name = "_".join( os.path.splitext(os.path.basename(vrt_file))[0].split("_")[1:]) short_vrt_tile_name = "_".join(vrt_tile_name.split("_")[1:]) if short_vrt_tile_name not in previous_errors: processed += 1 continue found = False found_path = None for building_tile in building_tiles: if found: continue
def mosaic_s1( vv_or_vv_paths, out_path, folder_tmp, master_raster, nodata_value=-9999.0, chunks=1, feather_borders=True, feather_distance=5000, skip_completed=False, ): if not isinstance(vv_or_vv_paths, list): raise Exception("vv_or_vv_paths must be a list") if len(vv_or_vv_paths) < 2: raise Exception("vv_or_vv_paths must contain more than one file") preprocessed = vv_or_vv_paths clipped = [] for idx, img in enumerate(preprocessed): progress(idx, len(preprocessed), "Clipping Rasters") name = os.path.splitext(os.path.basename(img))[0] + "_clipped.tif" out_name_clip = folder_tmp + name if skip_completed and os.path.exists(out_name_clip): clipped_raster = folder_tmp + name else: reprojected = reproject_raster( img, master_raster, copy_if_already_correct=False, ) if not rasters_intersect(reprojected, master_raster): print("") print( f"{img} does not intersect {master_raster}, continuing\n") progress(idx + 1, len(preprocessed), "Clipping Rasters") gdal.Unlink(reprojected) continue clipped_raster = clip_raster( reprojected, master_raster, out_path=folder_tmp + name, postfix="", adjust_bbox=True, all_touch=False, ) gdal.Unlink(reprojected) clipped.append(clipped_raster) progress(idx + 1, len(preprocessed), "Clipping Rasters") arr_aligned_rasters_feather = None print("Aligning rasters to master") arr_aligned_rasters = align_rasters( clipped, out_path=folder_tmp, master=master_raster, skip_existing=skip_completed, ) if feather_borders: print("Feathering rasters") arr_aligned_rasters_feather = calc_proximity( arr_aligned_rasters, target_value=-9999.0, out_path=folder_tmp, max_dist=feather_distance, invert=False, weighted=True, add_border=True, skip_existing=skip_completed, ) print("Processing") outpath = process_aligned( arr_aligned_rasters, out_path, folder_tmp, chunks, master_raster, nodata_value, feather_weights=arr_aligned_rasters_feather, ) return outpath
def extract_patches( raster_list, outdir, tile_size=32, zones=None, options=None, ): """ Generate patches for machine learning from rasters """ base_options = { "overlaps": True, "border_check": True, "merge_output": True, "force_align": True, "output_raster_labels": True, "label_geom": None, "label_res": 0.2, "label_mult": 100, "tolerance": 0.0, "fill_value": 0, "zone_layer_id": 0, "align_with_size": 20, "prefix": "", "postfix": "", } if options is None: options = base_options else: for key in options: if key not in base_options: raise ValueError(f"Invalid option: {key}") base_options[key] = options[key] options = base_options if zones is not None and not is_vector(zones): raise TypeError( "Clip geom is invalid. Did you input a valid geometry?") if not isinstance(raster_list, list): raster_list = [raster_list] for raster in raster_list: if not is_raster(raster): raise TypeError("raster_list is not a list of rasters.") if not os.path.isdir(outdir): raise ValueError( "Outdir does not exist. Please create before running the function." ) if not rasters_are_aligned(raster_list, same_extent=True): if options["force_align"]: print( "Rasters we not aligned. Realigning rasters due to force_align=True option." ) raster_list = align_rasters(raster_list) else: raise ValueError("Rasters in raster_list are not aligned.") offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]] raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True) pixel_size = min(raster_metadata["pixel_height"], raster_metadata["pixel_width"]) if zones is None: zones = raster_metadata["extent_datasource_path"] zones_meta = vector_to_metadata(zones) mem_driver = ogr.GetDriverByName("ESRI Shapefile") if zones_meta["layer_count"] == 0: raise ValueError("Vector contains no layers.") zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]] if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]: raise ValueError("clip geom is not Polygon or Multi Polygon.") zones_ogr = open_vector(zones) zones_layer = zones_ogr.GetLayer(options["zone_layer_id"]) feature_defn = zones_layer.GetLayerDefn() fids = vector_get_fids(zones_ogr, options["zone_layer_id"]) progress(0, len(fids) * len(raster_list), "processing fids") processed_fids = [] processed = 0 labels_processed = False for idx, raster in enumerate(raster_list): name = os.path.splitext(os.path.basename(raster))[0] list_extracted = [] list_masks = [] list_labels = [] for fid in fids: feature = zones_layer.GetFeature(fid) geom = feature.GetGeometryRef() fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp" fid_ds = mem_driver.CreateDataSource(fid_path) fid_ds_lyr = fid_ds.CreateLayer( "fid_layer", geom_type=ogr.wkbPolygon, srs=zones_layer_meta["projection_osr"], ) copied_feature = ogr.Feature(feature_defn) copied_feature.SetGeometry(geom) fid_ds_lyr.CreateFeature(copied_feature) fid_ds.FlushCache() fid_ds.SyncToDisk() valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif" rasterize_vector( fid_path, pixel_size, out_path=valid_path, extent=fid_path, ) valid_arr = raster_to_array(valid_path) if options["label_geom"] is not None and fid not in processed_fids: if not is_vector(options["label_geom"]): raise TypeError( "label geom is invalid. Did you input a valid geometry?" ) uuid = str(uuid4().int) label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp" label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif" label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif" intersect_vector(options["label_geom"], fid_ds, out_path=label_clip_path) try: rasterize_vector( label_clip_path, options["label_res"], out_path=label_ras_path, extent=valid_path, ) except Exception: array_to_raster( np.zeros(valid_arr.shape, dtype="float32"), valid_path, out_path=label_ras_path, ) resample_raster( label_ras_path, pixel_size, resample_alg="average", out_path=label_warp_path, ) labels_arr = (raster_to_array(label_warp_path) * options["label_mult"]).astype("float32") if options["output_raster_labels"]: array_to_raster( labels_arr, label_warp_path, out_path= f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif", ) raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif" try: clip_raster( raster, valid_path, raster_clip_path, all_touch=False, adjust_bbox=False, ) except Exception as e: print( f"Warning: {raster} did not intersect geom with fid: {fid}." ) print(e) if options["label_geom"] is not None: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) continue arr = raster_to_array(raster_clip_path) if arr.shape[:2] != valid_arr.shape[:2]: raise Exception( f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}" ) arr_offsets = get_overlaps(arr, offsets, tile_size, options["border_check"]) arr = np.concatenate(arr_offsets) valid_offsets = np.concatenate( get_overlaps(valid_arr, offsets, tile_size, options["border_check"])) valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) / (tile_size * tile_size))) <= options["tolerance"])[:, 0] arr = arr[valid_mask] valid_masked = valid_offsets[valid_mask] if options["label_geom"] is not None and not labels_processed: labels_masked = np.concatenate( get_overlaps(labels_arr, offsets, tile_size, options["border_check"]))[valid_mask] if options["merge_output"]: list_extracted.append(arr) list_masks.append(valid_masked) if options["label_geom"] is not None and not labels_processed: list_labels.append(labels_masked) else: np.save( f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy", arr.filled(options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if fid not in processed_fids: processed_fids.append(fid) processed += 1 progress(processed, len(fids) * len(raster_list), "processing fids") if not options["merge_output"]: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) gdal.Unlink(valid_path) if options["merge_output"]: np.save( f"{outdir}{options['prefix']}{name}{options['postfix']}.npy", np.ma.concatenate(list_extracted).filled( options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy", np.ma.concatenate(list_masks).filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy", np.ma.concatenate(list_labels).filled( options["fill_value"]), ) labels_processed = True progress(1, 1, "processing fids") return 1
def extract_patches( raster: Union[List[Union[str, gdal.Dataset]], str, gdal.Dataset], out_dir: Optional[str] = None, prefix: str = "", postfix: str = "_patches", size: int = 32, offsets: Union[list, None] = [], generate_border_patches: bool = True, generate_zero_offset: bool = True, generate_grid_geom: bool = True, clip_geom: Optional[Union[str, ogr.DataSource, gdal.Dataset]] = None, clip_layer_index: int = 0, verify_output=True, verification_samples=100, overwrite=True, epsilon: float = 1e-9, verbose: int = 1, ) -> tuple: """Extracts square tiles from a raster. Args: raster (list of rasters | path | raster): The raster(s) to convert. **kwargs: out_dir (path | none): Folder to save output. If None, in-memory arrays and geometries are outputted. prefix (str): A prefix for all outputs. postfix (str): A postfix for all outputs. size (int): The size of the tiles in pixels. offsets (list of tuples): List of offsets to extract. Example: offsets=[(16, 16), (16, 0), (0, 16)]. Will offset the initial raster and extract from there. generate_border_patches (bool): The tiles often do not align with the rasters which means borders are trimmed somewhat. If generate_border_patches is True, an additional tile is added where needed. generate_zero_offset (bool): if True, an offset is inserted at (0, 0) if none is present. generate_grid_geom (bool): Output a geopackage with the grid of tiles. clip_geom (str, raster, vector): Clip the output to the intersections with a geometry. Useful if a lot of the target area is water or similar. epsilon (float): How much for buffer the arange array function. This should usually just be left alone. verbose (int): If 1 will output messages on progress. Returns: A tuple with paths to the generated items. (numpy_array, grid_geom) """ type_check(raster, [str, list, gdal.Dataset], "raster") type_check(out_dir, [str], "out_dir", allow_none=True) type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(size, [int], "size") type_check(offsets, [list], "offsets", allow_none=True) type_check(generate_grid_geom, [bool], "generate_grid_geom") type_check( clip_geom, [str, ogr.DataSource, gdal.Dataset], "clip_layer_index", allow_none=True, ) type_check(clip_layer_index, [int], "clip_layer_index") type_check(overwrite, [bool], "overwrite") type_check(epsilon, [float], "epsilon") type_check(verbose, [int], "verbose") in_rasters = to_raster_list(raster) if out_dir is not None and not os.path.isdir(out_dir): raise ValueError(f"Output directory does not exists: {out_dir}") if not rasters_are_aligned(in_rasters): raise ValueError( "Input rasters must be aligned. Please use the align function.") output_geom = None metadata = internal_raster_to_metadata(in_rasters[0]) if verbose == 1: print("Generating blocks..") # internal offset array. Avoid manipulating the og array. if offsets is None: offsets = [] in_offsets = [] if generate_zero_offset and (0, 0) not in offsets: in_offsets.append((0, 0)) for offset in offsets: if offset != (0, 0): if not isinstance(offset, (list, tuple)) or len(offset) != 2: raise ValueError( f"offset must be a list or tuple of two integers. Recieved: {offset}" ) in_offsets.append((offset[0], offset[1])) border_patches_needed_x = True border_patches_needed_y = True if clip_geom is not None: border_patches_needed_x = False border_patches_needed_y = False shapes = [] for offset in in_offsets: block_shape = shape_to_blockshape(metadata["shape"], (size, size), offset) if block_shape[0] * size == metadata["width"]: border_patches_needed_x = False if block_shape[1] * size == metadata["height"]: border_patches_needed_y = False shapes.append(block_shape) if generate_border_patches: cut_x = (metadata["width"] - in_offsets[0][0]) - (shapes[0][0] * size) cut_y = (metadata["height"] - in_offsets[0][1]) - (shapes[0][1] * size) if border_patches_needed_x and cut_x > 0: shapes[0][0] += 1 if border_patches_needed_y and cut_y > 0: shapes[0][1] += 1 # calculate the offsets all_rows = 0 offset_rows = [] for i in range(len(shapes)): row = 0 for j in range(len(shapes[i])): if j == 0: row = int(shapes[i][j]) else: row *= int(shapes[i][j]) offset_rows.append(row) all_rows += row offset_rows_cumsum = np.cumsum(offset_rows) if generate_grid_geom is True or clip_geom is not None: if verbose == 1: print("Calculating grid cells..") mask = np.arange(all_rows, dtype="uint64") ulx, uly, _lrx, _lry = metadata["extent"] pixel_width = abs(metadata["pixel_width"]) pixel_height = abs(metadata["pixel_height"]) xres = pixel_width * size yres = pixel_height * size dx = xres / 2 dy = yres / 2 # Ready clip geom outside of loop. if clip_geom is not None: clip_ref = open_vector( internal_reproject_vector(clip_geom, metadata["projection_osr"])) clip_layer = clip_ref.GetLayerByIndex(clip_layer_index) meta_clip = internal_vector_to_metadata(clip_ref) # geom_clip = meta_clip["layers"][clip_layer_index]["column_geom"] clip_extent = meta_clip["extent_ogr"] # clip_adjust = [ # clip_extent[0] - clip_extent[0] % xres, # x_min # (clip_extent[1] - clip_extent[1] % xres) + xres, # x_max # clip_extent[2] - clip_extent[2] % yres, # y_min # (clip_extent[3] - clip_extent[3] % yres) + yres, # y_max # ] coord_grid = np.empty((all_rows, 2), dtype="float64") # tiled_extent = [None, None, None, None] row_count = 0 for idx in range(len(in_offsets)): x_offset = in_offsets[idx][0] y_offset = in_offsets[idx][1] x_step = shapes[idx][0] y_step = shapes[idx][1] x_min = (ulx + dx) + (x_offset * pixel_width) x_max = x_min + (x_step * xres) y_max = (uly - dy) - (y_offset * pixel_height) y_min = y_max - (y_step * yres) # if clip_geom is not None: # if clip_adjust[0] > x_min: # x_min = clip_adjust[0] + (x_offset * pixel_width) # if clip_adjust[1] < x_max: # x_max = clip_adjust[1] + (x_offset * pixel_width) # if clip_adjust[2] > y_min: # y_min = clip_adjust[2] - (y_offset * pixel_height) # if clip_adjust[3] < y_max: # y_max = clip_adjust[3] - (y_offset * pixel_height) # if idx == 0: # tiled_extent[0] = x_min # tiled_extent[1] = x_max # tiled_extent[2] = y_min # tiled_extent[3] = y_max # else: # if x_min < tiled_extent[0]: # tiled_extent[0] = x_min # if x_max > tiled_extent[1]: # tiled_extent[1] = x_max # if y_min < tiled_extent[2]: # tiled_extent[2] = y_min # if y_max > tiled_extent[3]: # tiled_extent[3] = y_max # y is flipped so: xmin --> xmax, ymax -- ymin to keep same order as numpy array x_patches = round((x_max - x_min) / xres) y_patches = round((y_max - y_min) / yres) xr = np.arange(x_min, x_max, xres)[0:x_step] if xr.shape[0] < x_patches: xr = np.arange(x_min, x_max + epsilon, xres)[0:x_step] elif xr.shape[0] > x_patches: xr = np.arange(x_min, x_max - epsilon, xres)[0:x_step] yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step] if yr.shape[0] < y_patches: yr = np.arange(y_max, y_min - epsilon, -yres)[0:y_step] elif yr.shape[0] > y_patches: yr = np.arange(y_max, y_min + epsilon, -yres)[0:y_step] if generate_border_patches and idx == 0: if border_patches_needed_x: xr[-1] = xr[-1] - ( (xr[-1] + dx) - metadata["extent_dict"]["right"]) if border_patches_needed_y: yr[-1] = yr[-1] - ( (yr[-1] - dy) - metadata["extent_dict"]["bottom"]) oxx, oyy = np.meshgrid(xr, yr) oxr = oxx.ravel() oyr = oyy.ravel() offset_length = oxr.shape[0] coord_grid[row_count:row_count + offset_length, 0] = oxr coord_grid[row_count:row_count + offset_length, 1] = oyr row_count += offset_length offset_rows_cumsum[idx] = offset_length offset_rows_cumsum = np.cumsum(offset_rows_cumsum) coord_grid = coord_grid[:row_count] # Output geometry driver = ogr.GetDriverByName("GPKG") patches_path = f"/vsimem/patches_{uuid4().int}.gpkg" patches_ds = driver.CreateDataSource(patches_path) patches_layer = patches_ds.CreateLayer("patches_all", geom_type=ogr.wkbPolygon, srs=metadata["projection_osr"]) patches_fdefn = patches_layer.GetLayerDefn() og_fid = "og_fid" field_defn = ogr.FieldDefn(og_fid, ogr.OFTInteger) patches_layer.CreateField(field_defn) if clip_geom is not None: clip_feature_count = meta_clip["layers"][clip_layer_index][ "feature_count"] spatial_index = rtree.index.Index(interleaved=False) for _ in range(clip_feature_count): clip_feature = clip_layer.GetNextFeature() clip_fid = clip_feature.GetFID() clip_feature_geom = clip_feature.GetGeometryRef() xmin, xmax, ymin, ymax = clip_feature_geom.GetEnvelope() spatial_index.insert(clip_fid, (xmin, xmax, ymin, ymax)) fids = 0 mask = [] for tile_id in range(coord_grid.shape[0]): x, y = coord_grid[tile_id] if verbose == 1: progress(tile_id, coord_grid.shape[0], "Patch generation") x_min = x - dx x_max = x + dx y_min = y - dx y_max = y + dx tile_intersects = True grid_geom = None poly_wkt = None if clip_geom is not None: tile_intersects = False if not ogr_bbox_intersects([x_min, x_max, y_min, y_max], clip_extent): continue intersections = list( spatial_index.intersection((x_min, x_max, y_min, y_max))) if len(intersections) == 0: continue poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))" grid_geom = ogr.CreateGeometryFromWkt(poly_wkt) for fid1 in intersections: clip_feature = clip_layer.GetFeature(fid1) clip_geom = clip_feature.GetGeometryRef() if grid_geom.Intersects(clip_geom): tile_intersects = True continue if tile_intersects: ft = ogr.Feature(patches_fdefn) if grid_geom is None: poly_wkt = f"POLYGON (({x_min} {y_max}, {x_max} {y_max}, {x_max} {y_min}, {x_min} {y_min}, {x_min} {y_max}))" grid_geom = ogr.CreateGeometryFromWkt(poly_wkt) ft_geom = ogr.CreateGeometryFromWkt(poly_wkt) ft.SetGeometry(ft_geom) ft.SetField(og_fid, int(fids)) ft.SetFID(int(fids)) patches_layer.CreateFeature(ft) ft = None mask.append(tile_id) fids += 1 if verbose == 1: progress(coord_grid.shape[0], coord_grid.shape[0], "Patch generation") mask = np.array(mask, dtype=int) if generate_grid_geom is True: if out_dir is None: output_geom = patches_ds else: raster_basename = metadata["basename"] geom_name = f"{prefix}{raster_basename}_geom_{str(size)}{postfix}.gpkg" output_geom = os.path.join(out_dir, geom_name) overwrite_required(output_geom, overwrite) remove_if_overwrite(output_geom, overwrite) if verbose == 1: print("Writing output geometry..") internal_vector_to_disk(patches_ds, output_geom, overwrite=overwrite) if verbose == 1: print("Writing numpy array..") output_blocks = [] for raster in in_rasters: base = None basename = None output_block = None if out_dir is not None: base = os.path.basename(raster) basename = os.path.splitext(base)[0] output_block = os.path.join(out_dir + f"{prefix}{basename}{postfix}.npy") metadata = internal_raster_to_metadata(raster) if generate_grid_geom is True or clip_geom is not None: output_shape = (row_count, size, size, metadata["band_count"]) else: output_shape = (all_rows, size, size, metadata["band_count"]) input_datatype = metadata["datatype"] output_array = np.empty(output_shape, dtype=input_datatype) # if clip_geom is not None: # ref = raster_to_array(raster, filled=True, extent=tiled_extent) # else: ref = raster_to_array(raster, filled=True) for k, offset in enumerate(in_offsets): start = 0 if k > 0: start = offset_rows_cumsum[k - 1] blocks = None if (k == 0 and generate_border_patches and (border_patches_needed_x or border_patches_needed_y)): blocks = array_to_blocks( ref, (size, size), offset, border_patches_needed_x, border_patches_needed_y, ) else: blocks = array_to_blocks(ref, (size, size), offset) output_array[start:offset_rows_cumsum[k]] = blocks if generate_grid_geom is False and clip_geom is None: if out_dir is None: output_blocks.append(output_array) else: output_blocks.append(output_block) np.save(output_block, output_array) else: if out_dir is None: output_blocks.append(output_array[mask]) else: output_blocks.append(output_block) np.save(output_block, output_array[mask]) if verify_output and generate_grid_geom: test_extraction( in_rasters, output_blocks, output_geom, samples=verification_samples, grid_layer_index=0, verbose=verbose, ) if len(output_blocks) == 1: output_blocks = output_blocks[0] return (output_blocks, output_geom)
def internal_vector_add_shapes( vector: Union[str, ogr.DataSource], shapes: list = [ "area", "perimeter", "ipq", "hull", "compactness", "centroid" ], ) -> str: """OBS: Internal. Single output. Adds shape calculations to a vector such as area and perimeter. Can also add compactness measurements. """ type_check(vector, [str, ogr.DataSource], "vector") type_check(shapes, [list], "shapes") datasource = open_vector(vector) out_path = get_vector_path(datasource) metadata = internal_vector_to_metadata(datasource) for index in range(metadata["layer_count"]): vector_current_fields = metadata["layers"][index]["field_names"] vector_layer = datasource.GetLayer(index) vector_layer.StartTransaction() # Add missing fields for attribute in shapes: if attribute == "centroid": if "centroid_x" not in vector_current_fields: field_defn = ogr.FieldDefn("centroid_x", ogr.OFTReal) vector_layer.CreateField(field_defn) if "centroid_y" not in vector_current_fields: field_defn = ogr.FieldDefn("centroid_y", ogr.OFTReal) vector_layer.CreateField(field_defn) elif attribute not in vector_current_fields: field_defn = ogr.FieldDefn(attribute, ogr.OFTReal) vector_layer.CreateField(field_defn) vector_feature_count = vector_layer.GetFeatureCount() for i in range(vector_feature_count): vector_feature = vector_layer.GetNextFeature() try: vector_geom = vector_feature.GetGeometryRef() except: vector_geom.Buffer(0) Warning("Invalid geometry at : ", i) if vector_geom is None: raise Exception("Invalid geometry. Could not fix.") centroid = vector_geom.Centroid() vector_area = vector_geom.GetArea() vector_perimeter = vector_geom.Boundary().Length() if "ipq" or "compact" in shapes: vector_ipq = 0 if vector_perimeter != 0: vector_ipq = (4 * np.pi * vector_area) / vector_perimeter**2 if "centroid" in shapes: vector_feature.SetField("centroid_x", centroid.GetX()) vector_feature.SetField("centroid_y", centroid.GetY()) if "hull" in shapes or "compact" in shapes: vector_hull = vector_geom.ConvexHull() hull_area = vector_hull.GetArea() hull_peri = vector_hull.Boundary().Length() hull_ratio = float(vector_area) / float(hull_area) compactness = np.sqrt(float(hull_ratio) * float(vector_ipq)) if "area" in shapes: vector_feature.SetField("area", vector_area) if "perimeter" in shapes: vector_feature.SetField("perimeter", vector_perimeter) if "ipq" in shapes: vector_feature.SetField("ipq", vector_ipq) if "hull" in shapes: vector_feature.SetField("hull_area", hull_area) vector_feature.SetField("hull_peri", hull_peri) vector_feature.SetField("hull_ratio", hull_ratio) if "compact" in shapes: vector_feature.SetField("compact", compactness) vector_layer.SetFeature(vector_feature) progress(i, vector_feature_count, name="shape") vector_layer.CommitTransaction() return out_path
def test_extraction( rasters: Union[list, str, gdal.Dataset], arrays: Union[list, np.ndarray], grid: Union[ogr.DataSource, str], samples: int = 1000, # if 0, all grid_layer_index: int = 0, verbose: int = 1, ) -> bool: """Validates the output of the patch_extractor. Useful if you need peace of mind. Set samples to 0 to tests everything. Args: rasters (list of rasters | path | raster): The raster(s) used. arrays (list of arrays | ndarray): The arrays generated. grid (vector | vector_path): The grid generated. **kwargs: samples (int): The amount of patches to randomly test. If 0 all patches will be tested. This is a long process, so consider only testing everything if absolutely necessary. grid_layer_index (int): If the grid is part of a multi-layer vector, specify the index of the grid. verbose (int): If 1 will output messages on progress. Returns: True if the extraction is valid. Raises an error otherwise. """ type_check(rasters, [list, str, gdal.Dataset], "rasters") type_check(arrays, [list, str, np.ndarray], "arrays") type_check(grid, [list, str, ogr.DataSource], "grid") type_check(samples, [int], "samples") type_check(grid_layer_index, [int], "clip_layer_index") type_check(verbose, [int], "verbose") in_rasters = to_raster_list(rasters) in_arrays = arrays if verbose == 1: print("Verifying integrity of output grid..") # grid_memory = open_vector(internal_vector_to_memory(grid)) grid_memory = open_vector(grid) grid_metadata = internal_vector_to_metadata(grid) grid_projection = grid_metadata["projection_osr"] if grid_layer_index > (grid_metadata["layer_count"] - 1): raise ValueError( f"Requested non-existing layer index: {grid_layer_index}") grid_layer = grid_memory.GetLayer(grid_layer_index) # Select sample fids feature_count = grid_metadata["layers"][grid_layer_index]["feature_count"] test_samples = samples if samples > 0 else feature_count max_test = min(test_samples, feature_count) - 1 test_fids = np.array(random.sample(range(0, feature_count), max_test), dtype="uint64") mem_driver = ogr.GetDriverByName("ESRI Shapefile") for index, raster in enumerate(in_rasters): test_rast = open_raster(raster) test_array = in_arrays[index] if isinstance(test_array, str): if not os.path.exists(test_array): raise ValueError(f"Numpy array does not exist: {test_array}") try: test_array = np.load(in_arrays[index]) except: raise Exception( f"Attempted to read numpy raster from: {in_arrays[index]}") base = os.path.basename(raster) basename = os.path.splitext(base)[0] if verbose == 1: print(f"Testing: {basename}") tested = 0 for test in test_fids: feature = grid_layer.GetFeature(test) if feature is None: raise Exception(f"Feature not found: {test}") test_ds_path = f"/vsimem/test_mem_grid_{uuid4().int}.gpkg" test_ds = mem_driver.CreateDataSource(test_ds_path) test_ds_lyr = test_ds.CreateLayer("test_mem_grid_layer", geom_type=ogr.wkbPolygon, srs=grid_projection) test_ds_lyr.CreateFeature(feature.Clone()) test_ds.SyncToDisk() clipped = internal_clip_raster( test_rast, test_ds_path, adjust_bbox=False, crop_to_geom=True, all_touch=False, ) if clipped is None: raise Exception( "Error while clipping raster. Likely a bad extraction.") ref_image = raster_to_array(clipped, filled=True) image_block = test_array[test] if not np.array_equal(ref_image, image_block): # from matplotlib import pyplot as plt; plt.imshow(ref_image[:,:,0]); plt.show() raise Exception( f"Image {basename} and grid cell did not match..") if verbose == 1: progress(tested, len(test_fids) - 1, "Verifying..") tested += 1 return True
def predict_raster( raster: Union[List[Union[str, gdal.Dataset]], str, gdal.Dataset], model: str, out_path: Optional[str] = None, offsets: Union[List[Tuple[int, int]], List[List[Tuple[int, int]]]] = [], region: Optional[Union[str, ogr.DataSource]] = None, device: str = "gpu", merge_method: str = "median", mirror: bool = False, rotate: bool = False, custom_objects: Dict[str, Any] = { "Mish": Mish, "mish": mish, "tpe": tpe }, target_raster: Optional[str] = None, output_size=128, dtype: str = "same", batch_size: int = 16, overwrite: bool = True, creation_options: List[str] = [], verbose: int = 1, ) -> str: """Runs a raster or list of rasters through a deep learning network (Tensorflow). Supports tiling and reconstituting the output. Offsets are allowed and will be bleneded with the merge_method. If the output is a different resolution than the input. The output will automatically be scaled to match. Args: raster (list | path | raster): The raster(s) to convert. model (path): A path to the tensorflow .h5 model. **kwargs: out_path (str | None): Where to save the reconstituted raster. If None are memory raster is returned. offsets (tuple, list, ndarray): The offsets used in the original. A (0 ,0) offset is assumed. border_patches (bool): Do the blocks contain border patches? device (str): Either CPU or GPU to use with tensorflow. merge_method (str): How to handle overlapping pixels. Options are: median, average, mode, min, max mirror (bool): Mirror the raster and do predictions as well. rotate (bool): rotate the raster and do predictions as well. dtype (str | None): The dtype of the output. If None: Float32, "save" is the same as the input raster. Otherwise overwrite dtype. overwrite (bool): Overwrite output files if they exists. creation_options: Extra creation options for the output raster. verbose (int): If 1 will output messages on progress. Returns: A predicted raster. """ type_check(raster, [list, str, gdal.Dataset], "raster") type_check(model, [str], "model") type_check(out_path, [str], "out_path", allow_none=True) type_check(offsets, [list], "offsets") type_check(region, [str, ogr.DataSource], allow_none=True) type_check(device, [str], "device") type_check(merge_method, [str], "merge_method") type_check(mirror, [bool], "mirror") type_check(rotate, [bool], "rotate") type_check(custom_objects, [dict], "custom_objects") type_check(dtype, [str], "dtype", allow_none=True) type_check(batch_size, [int], "batch_size") type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") type_check(verbose, [int], "verbose") if mirror or rotate: raise Exception("Mirror and rotate currently disabled.") import tensorflow as tf os.environ["TF_FORCE_GPU_ALLOW_GROWTH"] = "true" if verbose == 1: print("Loading model.") if isinstance(model, str): model_loaded = tf.keras.models.load_model( model, custom_objects=custom_objects) else: model_loaded = model multi_input = False if isinstance(model_loaded.input, list) and len(model_loaded.input) > 1: if not isinstance(raster, list): raise TypeError("Multi input model must have a list as input.") if len(offsets) > 0: for offset in offsets: if not isinstance(offset, list): raise TypeError( "Offsets must be a list of tuples, same length as inputs." ) for _offset in offset: if not isinstance(_offset, tuple): raise TypeError("Offset must be a tuple") if len(_offset) != 2: raise ValueError("Offset must be length 2.") if len(model_loaded.input) != len(offsets): raise ValueError("Length of offsets must equal model inputs.") multi_input = True model_inputs = (model_loaded.input if isinstance(model_loaded.input, list) else [model_loaded.input]) shape_output = tuple(model_loaded.output.shape) if shape_output[1] == None or shape_output[2] == None or shape_output[ 3] == None: print( f"Unable to find output size, using the supplied variable: {output_size}" ) shape_output = (None, output_size, output_size, 1) dst_tile_size = shape_output[1] prediction_arr = [] readied_inputs = [] pixel_factor = 1.0 for index, model_input in enumerate(model_inputs): if verbose == 1: print(f"Readying input: {index}") shape_input = tuple(model_input.shape) if len(shape_input) != 4 or len(shape_output) != 4: raise ValueError( f"Model input not 4d: {shape_input} - {shape_output}") if shape_input[1] != shape_input[2] or shape_output[1] != shape_output[ 2]: raise ValueError("Model only takes square images.") src_tile_size = shape_input[1] pixel_factor = src_tile_size / dst_tile_size scale_factor = dst_tile_size / src_tile_size dst_offsets = [] in_offsets: List[Tuple[Number, Number]] = [] if multi_input: if len(offsets) > 0: in_offsets = offsets[index] else: in_offsets = offsets for offset in in_offsets: if not isinstance(offset, tuple): raise ValueError( f"Offset must be a tuple of two ints. Recieved: {offset}") if len(offset) != 2: raise ValueError( "Offsets must have two values. Both integers.") dst_offsets.append(( round(offset[0] * scale_factor), round(offset[1] * scale_factor), )) use_raster = raster[index] if isinstance(raster, list) else raster if region is not None: use_raster = clip_raster(use_raster, region, adjust_bbox=False, all_touch=False) # # check raster size here # use_raster_meta = internal_raster_to_metadata(use_raster) # import pdb # pdb.set_trace() blocks, _ = extract_patches( use_raster, size=src_tile_size, offsets=in_offsets, generate_border_patches=True, generate_grid_geom=False, verbose=verbose, ) readied_inputs.append(blocks) first_len = None for index, readied in enumerate(readied_inputs): if index == 0: first_len = readied.shape[0] else: if readied.shape[0] != first_len: import pdb pdb.set_trace() raise ValueError( "Length of inputs do not match. Have you set the offsets in the correct order?" ) if verbose == 1: print("Predicting raster.") start = 0 end = readied_inputs[0].shape[0] predictions = np.empty( (end, dst_tile_size, dst_tile_size, shape_output[3]), dtype="float32") if multi_input is False: if device == "cpu": with tf.device("/cpu:0"): while start < end: predictions[start:start + batch_size] = model_loaded.predict_on_batch( readied_inputs[0][start:start + batch_size]) start += batch_size progress(start, end - 1, "Predicting") else: while start < end: predictions[start:start + batch_size] = model_loaded.predict_on_batch( readied_inputs[0][start:start + batch_size]) start += batch_size progress(start, end - 1, "Predicting") else: if device == "cpu": with tf.device("/cpu:0"): while start < end: batch = [] for i in range(len(readied_inputs)): batch.append(readied_inputs[i][start:start + batch_size]) predictions[start:start + batch_size] = model_loaded.predict_on_batch( batch) start += batch_size progress(start, end - 1, "Predicting") else: while start < end: batch = [] for i in range(len(readied_inputs)): batch.append(readied_inputs[i][start:start + batch_size]) predictions[start:start + batch_size] = model_loaded.predict_on_batch(batch) start += batch_size progress(start, end - 1, "Predicting") print("") print("Reconstituting Raster.") rast_meta = None target_size = None resampled = None if target_raster is not None: resampled = target_raster elif isinstance(raster, list): rast_meta = internal_raster_to_metadata(raster[-1]) target_size = ( rast_meta["pixel_width"] * pixel_factor, rast_meta["pixel_height"] * pixel_factor, ) resampled = internal_resample_raster(raster[-1], target_size=target_size, dtype="float32") else: rast_meta = internal_raster_to_metadata(raster) target_size = ( rast_meta["pixel_width"] * pixel_factor, rast_meta["pixel_height"] * pixel_factor, ) resampled = internal_resample_raster(raster, target_size=target_size, dtype="float32") if region is not None: resampled = clip_raster(resampled, region) prediction_arr.append( blocks_to_raster( predictions, resampled, border_patches=True, offsets=dst_offsets, merge_method=merge_method, output_array=True, dtype="float32", )) if verbose == 1: print("Merging rasters.") if merge_method == "median": predicted = np.median(prediction_arr, axis=0) elif merge_method == "mean" or merge_method == "average": predicted = np.mean(prediction_arr, axis=0) elif merge_method == "min" or merge_method == "minumum": predicted = np.min(prediction_arr, axis=0) elif merge_method == "max" or merge_method == "maximum": predicted = np.max(prediction_arr, axis=0) elif merge_method == "mode" or merge_method == "majority": for index, _ in enumerate(prediction_arr): prediction_arr[index] = np.rint(prediction_arr[index]).astype(int) predicted = np.apply_along_axis(lambda x: np.bincount(x).argmax(), axis=0, arr=prediction_arr) else: raise ValueError(f"Unable to parse merge_method: {merge_method}") if dtype == "same" or dtype == None: predicted = array_to_raster( predicted.astype(rast_meta["datatype"]), reference=resampled, ) else: predicted = array_to_raster( predicted.astype(dtype), reference=resampled, ) if out_path is None: return predicted else: return internal_raster_to_disk( predicted, out_path=out_path, overwrite=overwrite, creation_options=creation_options, )
print( "No offsets provided. Using offsets greatly increases accuracy. Please provide offsets." ) else: offsets = [] for val in tile_size: offsets.append(get_offsets(val)) predictions = [] read_rasters = [] print("Initialising rasters.") for raster_idx, raster in enumerate(raster_list): read_rasters.append(raster_to_array(raster).astype("float32")) progress(0, len(offsets[0]) + 3, "Predicting") for offset_idx in range(len(offsets[0])): model_inputs = [] for raster_idx, raster in enumerate(raster_list): array = read_rasters[raster_idx] blocks = array_to_blocks( array, tile_size[raster_idx], offset=offsets[raster_idx][offset_idx] ) model_inputs.append(blocks) prediction_blocks = model.predict(model_inputs, batch_size, verbose=0) prediction = blocks_to_array( prediction_blocks,
def internal_multipart_to_singlepart( vector: Union[str, ogr.DataSource], out_path: Optional[str] = None, copy_attributes: bool = False, overwrite: bool = True, add_index: bool = True, process_layer: int = -1, verbose: int = 1, ) -> str: type_check(vector, [str, ogr.DataSource], "vector") type_check(out_path, [str], "out_path", allow_none=True) type_check(overwrite, [bool], "overwrite") type_check(add_index, [bool], "add_index") type_check(process_layer, [int], "process_layer") type_check(verbose, [int], "verbose") vector_list, path_list = ready_io_vector(vector, out_path, overwrite=overwrite) ref = open_vector(vector_list[0]) out_name = path_list[0] driver = ogr.GetDriverByName(path_to_driver_vector(out_name)) metadata = internal_vector_to_metadata(ref) remove_if_overwrite(out_name, overwrite) destination = driver.CreateDataSource(out_name) for index, layer_meta in enumerate(metadata["layers"]): if process_layer != -1 and index != process_layer: continue if verbose == 1: layer_name = layer_meta["layer_name"] print(f"Splitting layer: {layer_name}") target_unknown = False if layer_meta["geom_type_ogr"] == 4: # MultiPoint target_type = 1 # Point elif layer_meta["geom_type_ogr"] == 5: # MultiLineString target_type = 2 # LineString elif layer_meta["geom_type_ogr"] == 6: # MultiPolygon target_type = 3 # Polygon elif layer_meta["geom_type_ogr"] == 1004: # MultiPoint (z) target_type = 1001 # Point (z) elif layer_meta["geom_type_ogr"] == 1005: # MultiLineString (z) target_type = 1002 # LineString (z) elif layer_meta["geom_type_ogr"] == 1006: # MultiPolygon (z) target_type = 1003 # Polygon (z) elif layer_meta["geom_type_ogr"] == 2004: # MultiPoint (m) target_type = 2001 # Point (m) elif layer_meta["geom_type_ogr"] == 2005: # MultiLineString (m) target_type = 2002 # LineString (m) elif layer_meta["geom_type_ogr"] == 2006: # MultiPolygon (m) target_type = 2003 # Polygon (m) elif layer_meta["geom_type_ogr"] == 3004: # MultiPoint (zm) target_type = 3001 # Point (m) elif layer_meta["geom_type_ogr"] == 3005: # MultiLineString (zm) target_type = 3002 # LineString (m) elif layer_meta["geom_type_ogr"] == 3006: # MultiPolygon (zm) target_type = 3003 # Polygon (m) else: target_unknown = True target_type = layer_meta["geom_type_ogr"] destination_layer = destination.CreateLayer( layer_meta["layer_name"], layer_meta["projection_osr"], target_type ) layer_defn = destination_layer.GetLayerDefn() field_count = layer_meta["field_count"] original_target = ref.GetLayerByIndex(index) feature_count = original_target.GetFeatureCount() if copy_attributes: first_feature = original_target.GetNextFeature() original_target.ResetReading() if verbose == 1: print("Creating attribute fields") for field_id in range(field_count): field_defn = first_feature.GetFieldDefnRef(field_id) fname = field_defn.GetName() ftype = field_defn.GetTypeName() fwidth = field_defn.GetWidth() fprecision = field_defn.GetPrecision() if ftype == "String" or ftype == "Date": fielddefn = ogr.FieldDefn(fname, ogr.OFTString) fielddefn.SetWidth(fwidth) elif ftype == "Real": fielddefn = ogr.FieldDefn(fname, ogr.OFTReal) fielddefn.SetWidth(fwidth) fielddefn.SetPrecision(fprecision) else: fielddefn = ogr.FieldDefn(fname, ogr.OFTInteger) destination_layer.CreateField(fielddefn) for _ in range(feature_count): feature = original_target.GetNextFeature() geom = feature.GetGeometryRef() if target_unknown: out_feat = ogr.Feature(layer_defn) out_feat.SetGeometry(geom) if copy_attributes: for field_id in range(field_count): values = feature.GetField(field_id) out_feat.SetField(field_id, values) destination_layer.CreateFeature(out_feat) for geom_part in geom: out_feat = ogr.Feature(layer_defn) out_feat.SetGeometry(geom_part) if copy_attributes: for field_id in range(field_count): values = feature.GetField(field_id) out_feat.SetField(field_id, values) destination_layer.CreateFeature(out_feat) if verbose == 1: progress(_, feature_count - 1, "Splitting.") if add_index: vector_add_index(destination) return out_name
def raster_to_grid( raster: Union[str, gdal.Dataset], grid: Union[str, ogr.DataSource], out_dir: str, use_field: Optional[str] = None, generate_vrt: bool = True, overwrite: bool = True, process_layer: int = 0, creation_options: list = [], verbose: int = 1, ) -> Union[List[str], Tuple[Optional[List[str]], Optional[str]]]: """Clips a raster to a grid. Generate .vrt. Returns: The filepath for the newly created raster. """ type_check(raster, [str, gdal.Dataset], "raster") type_check(grid, [str, ogr.DataSource], "grid") type_check(out_dir, [str], "out_dir") type_check(overwrite, [bool], "overwrite") type_check(process_layer, [int], "process_layer") type_check(creation_options, [list], "creation_options") type_check(verbose, [int], "verbose") use_grid = open_vector(grid) grid_metadata = internal_vector_to_metadata(use_grid) raster_metadata = raster_to_metadata(raster, create_geometry=True) # Reproject raster if necessary. if not raster_metadata["projection_osr"].IsSame(grid_metadata["projection_osr"]): use_grid = reproject_vector(grid, raster_metadata["projection_osr"]) grid_metadata = internal_vector_to_metadata(use_grid) if not isinstance(grid_metadata, dict): raise Exception("Error while parsing metadata.") # Only use the polygons in the grid that intersect the extent of the raster. use_grid = intersect_vector(use_grid, raster_metadata["extent_datasource"]) ref = open_raster(raster) use_grid = open_vector(use_grid) layer = use_grid.GetLayer(process_layer) feature_count = layer.GetFeatureCount() raster_extent = raster_metadata["extent_ogr"] filetype = path_to_ext(raster) name = raster_metadata["name"] geom_type = grid_metadata["layers"][process_layer]["geom_type_ogr"] if use_field is not None: if use_field not in grid_metadata["layers"][process_layer]["field_names"]: names = grid_metadata["layers"][process_layer]["field_names"] raise ValueError( f"Requested field not found. Fields available are: {names}" ) generated = [] # For the sake of good reporting - lets first establish how many features intersect # the raster. if verbose: print("Finding intersections.") intersections = 0 for _ in range(feature_count): feature = layer.GetNextFeature() geom = feature.GetGeometryRef() if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()): continue intersections += 1 layer.ResetReading() if verbose: print(f"Found {intersections} intersections.") if intersections == 0: print("Warning: Found 0 intersections. Returning empty list.") return ([], None) # TODO: Replace this in gdal. 3.1 driver = ogr.GetDriverByName("Esri Shapefile") clipped = 0 for _ in range(feature_count): feature = layer.GetNextFeature() geom = feature.GetGeometryRef() if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()): continue if verbose == 1: progress(clipped, intersections - 1, "clip_grid") fid = feature.GetFID() test_ds_path = f"/vsimem/grid_{uuid4().int}.shp" test_ds = driver.CreateDataSource(test_ds_path) test_ds_lyr = test_ds.CreateLayer( "mem_layer_grid", geom_type=geom_type, srs=raster_metadata["projection_osr"], ) test_ds_lyr.CreateFeature(feature.Clone()) test_ds.FlushCache() out_name = None if use_field is not None: out_name = f"{out_dir}{feature.GetField(use_field)}{filetype}" else: out_name = f"{out_dir}{name}_{fid}{filetype}" clip_raster( ref, test_ds_path, out_path=out_name, adjust_bbox=True, crop_to_geom=True, all_touch=False, postfix="", prefix="", creation_options=default_options(creation_options), verbose=0, ) generated.append(out_name) clipped += 1 if generate_vrt: vrt_name = f"{out_dir}{name}.vrt" stack_rasters_vrt(generated, vrt_name, seperate=False) return (generated, vrt_name) return generated