def match_projections( rasters, master, out_dir, overwrite=False, dst_nodata="infer", copy_if_already_correct=True, ): target_projection = parse_projection(master) created = [] for raster in rasters: metadata = raster_to_metadata(raster) outname = out_dir + metadata["name"] + ".tif" created.append(outname) if os.path.exists(outname): if not overwrite: continue internal_reproject_raster( raster, target_projection, outname, copy_if_already_correct=copy_if_already_correct, dst_nodata=dst_nodata, ) return created
def raster_get_nodata_value( raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str], ) -> Union[List[Optional[Number]], Optional[Number]]: """Get the nodata value of a raster or a from a list of rasters. Args: raster (path | raster | list): The raster(s) to retrieve nodata values from. Returns: Returns the nodata value from a raster or a list of rasters """ type_check(raster, [list, str, gdal.Dataset], "raster") rasters = get_raster_path(raster, return_list=True) nodata_values = [] for internal_raster in rasters: if not is_raster(internal_raster): raise ValueError(f"Input raster is invalid: {internal_raster}") raster_metadata = raster_to_metadata(internal_raster) if not isinstance(raster_metadata, dict): raise Exception("Metadata is in the wrong format.") raster_nodata = raster_metadata["nodata_value"] nodata_values.append(raster_nodata) if isinstance(raster, list): return nodata_values else: return nodata_values[0]
def raster_has_nodata_value( raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str], ) -> Union[bool, List[bool]]: """Check if a raster or a list of rasters contain nodata values Args: raster (path | raster | list): The raster(s) to check for nodata values. Returns: True if input raster has nodata values. If a list is the input, the output is a list of booleans indicating if the input raster has nodata values. """ type_check(raster, [list, str, gdal.Dataset], "raster") nodata_values = [] rasters = get_raster_path(raster, return_list=True) for internal_raster in rasters: if not is_raster(internal_raster): raise ValueError(f"Input raster is invalid: {internal_raster}") raster_metadata = raster_to_metadata(internal_raster) if not isinstance(raster_metadata, dict): raise Exception("Metadata is in the wrong format.") raster_nodata = raster_metadata["nodata_value"] if raster_nodata is not None: nodata_values.append(True) else: nodata_values.append(False) if isinstance(raster, list): return nodata_values else: return nodata_values[0]
def raster_mask_values( raster: Union[gdal.Dataset, str, list], values_to_mask: list, out_path: Union[list, str, None] = None, include_original_nodata: bool = True, dst_nodata: Union[float, int, str, list, None] = "infer", in_place: bool = False, overwrite: bool = True, opened: bool = False, prefix: str = "", postfix: str = "_nodata_masked", creation_options: list = [], ) -> Union[list, gdal.Dataset, str]: """Mask a raster with a list of values. Args: raster (path | raster | list): The raster(s) to retrieve nodata values from. values_to_mask (list): The list of values to mask in the raster(s) **kwargs: include_original_nodata: (bool): If True, the nodata value of the raster(s) will be included in the values to mask. dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata value is set based on the input datatype. A list of nodata values can be based matching the amount of input rasters. If multiple nodata values should be set, use raster_mask_values. out_path (path | list | None): The destination of the changed rasters. If out_paths are specified, in_place is automatically set to False. The path can be a folder. in_place (bool): Should the rasters be changed in_place or copied? prefix (str): Prefix to add the the output if a folder is specified in out_path. postfix (str): Postfix to add the the output if a folder is specified in out_path. Returns: Returns the rasters with nodata removed. If in_place is True a reference to the changed orignal is returned, otherwise a copied memory raster or the path to the generated raster is outputted. """ type_check(raster, [list, str, gdal.Dataset], "raster") type_check(values_to_mask, [list], "values_to_mask") type_check(out_path, [list, str], "out_path", allow_none=True) type_check(include_original_nodata, [bool], "include_original_nodata") type_check(dst_nodata, [float, int, str, list], "dst_nodata", allow_none=True) type_check(in_place, [bool], "in_place") type_check(overwrite, [bool], "overwrite") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(opened, [bool], "opened") type_check(creation_options, [list], "creation_options") rasters_metadata = [] internal_in_place = in_place if out_path is None else False internal_dst_nodata = None for value in values_to_mask: if not isinstance(value, (int, float)): raise ValueError("Values in values_to_mask must be ints or floats") if isinstance(dst_nodata, str) and dst_nodata != "infer": raise ValueError(f"Invalid dst_nodata value. {dst_nodata}") if isinstance(dst_nodata, list): if not isinstance(raster, list) or len(dst_nodata) != len(raster): raise ValueError( "If dst_nodata is a list, raster must also be a list of equal length." ) for value in dst_nodata: if isinstance(value, (float, int, str, None)): raise ValueError("Invalid type in dst_nodata list.") if isinstance(value, str) and value != "infer": raise ValueError( "If dst_nodata is a string it must be 'infer'") raster_list, out_names = ready_io_raster(raster, out_path, overwrite, prefix, postfix) output_rasters = [] for index, internal_raster in enumerate(raster_list): raster_metadata = None if len(rasters_metadata) == 0: raster_metadata = raster_to_metadata(internal_raster) rasters_metadata.append(raster_metadata) else: raster_metadata = rasters_metadata[index] if dst_nodata == "infer": internal_dst_nodata = gdal_nodata_value_from_type( raster_metadata["dtype_gdal_raw"]) elif isinstance(dst_nodata, list): internal_dst_nodata = dst_nodata[index] else: internal_dst_nodata = dst_nodata mask_values = list(values_to_mask) if include_original_nodata: if raster_metadata["nodata_value"] is not None: mask_values.append(raster_metadata["nodata_value"]) arr = raster_to_array(internal_raster, filled=True) mask = None for index, mask_value in enumerate(mask_values): if index == 0: mask = arr == mask_value else: mask = mask | arr == mask_value arr = np.ma.masked_array(arr, mask=mask, fill_value=internal_dst_nodata) if internal_in_place: for band in range(raster_metadata["bands"]): raster_band = internal_raster.GetRasterBand(band + 1) raster_band.WriteArray(arr[:, :, band]) raster_band = None else: out_name = out_names[index] remove_if_overwrite(out_name, overwrite) output_rasters.append( array_to_raster(arr, internal_raster, out_path=out_name)) if isinstance(raster, list): return output_rasters return output_rasters[0]
def raster_set_nodata( raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str], dst_nodata: Union[float, int, str, list, None], out_path: Union[list, str, None] = None, overwrite: bool = True, in_place: bool = False, prefix: str = "", postfix: str = "_nodata_set", opened: bool = False, creation_options: list = [], ) -> Union[list, gdal.Dataset, str]: """Sets all the nodata from a raster or a list of rasters. Args: raster (path | raster | list): The raster(s) to retrieve nodata values from. dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata value is set based on the input datatype. A list of nodata values can be based matching the amount of input rasters. If multiple nodata values should be set, use raster_mask_values. **kwargs: out_path (path | list | None): The destination of the changed rasters. If out_paths are specified, in_place is automatically set to False. The path can be a folder. in_place (bool): Should the rasters be changed in_place or copied? prefix (str): Prefix to add the the output if a folder is specified in out_path. postfix (str): Postfix to add the the output if a folder is specified in out_path. Returns: Returns the rasters with nodata set. If in_place is True a reference to the changed orignal is returned, otherwise a copied memory raster or the path to the generated raster is outputted. """ type_check(raster, [list, str, gdal.Dataset], "raster") type_check(dst_nodata, [float, int, str, list], "dst_nodata", allow_none=True) type_check(out_path, [list, str], "out_path", allow_none=True) type_check(overwrite, [bool], "overwrite") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(opened, [bool], "opened") type_check(creation_options, [list], "creation_options") rasters, out_names = ready_io_raster(raster, out_path, overwrite, prefix, postfix) rasters_metadata: List[Metadata_raster] = [] internal_dst_nodata = None if isinstance(dst_nodata, str) and dst_nodata != "infer": raise ValueError(f"Invalid dst_nodata value. {dst_nodata}") if isinstance(dst_nodata, list): if not isinstance(raster, list) or len(dst_nodata) != len(raster): raise ValueError( "If dst_nodata is a list, raster must also be a list of equal length." ) for value in dst_nodata: if isinstance(value, (float, int, str, None)): raise ValueError("Invalid type in dst_nodata list.") if isinstance(value, str) and value != "infer": raise ValueError( "If dst_nodata is a string it must be 'infer'") output_rasters = [] for index, internal_raster in enumerate(rasters): raster_metadata = None if len(rasters_metadata) == 0: raster_metadata = raster_to_metadata(internal_raster) if not isinstance(raster_metadata, dict): raise Exception("Metadata is in the wrong format.") rasters_metadata.append(raster_metadata) else: raster_metadata = rasters_metadata[index] if dst_nodata == "infer": internal_dst_nodata = gdal_nodata_value_from_type( raster_metadata["dtype_gdal_raw"]) elif isinstance(dst_nodata, list): internal_dst_nodata = dst_nodata[index] else: internal_dst_nodata = dst_nodata if in_place: for band in range(raster_metadata["bands"]): raster_band = internal_raster.GetRasterBand(band + 1) raster_band.SetNodataValue(internal_dst_nodata) raster_band = None else: if out_path is None: raster_mem = raster_to_memory(internal_raster) raster_mem_ref = raster_to_reference(raster_mem) else: remove_if_overwrite(out_names[index], overwrite) raster_mem = raster_to_disk(internal_raster, out_names[index]) raster_mem_ref = raster_to_reference(raster_mem) for band in range(raster_metadata["bands"]): raster_band = raster_mem_ref.GetRasterBand(band + 1) raster_band.SetNodataValue(internal_dst_nodata) if isinstance(raster, list): return output_rasters return output_rasters[0]
def internal_reproject_raster( raster: Union[str, gdal.Dataset], projection: Union[int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference], out_path: Optional[str] = None, resample_alg: str = "nearest", copy_if_already_correct: bool = True, overwrite: bool = True, creation_options: list = [], dst_nodata: Union[str, int, float] = "infer", prefix: str = "", postfix: str = "_reprojected", ) -> str: """OBS: Internal. Single output. Reproject a raster(s) to a target coordinate reference system. """ type_check(raster, [str, gdal.Dataset], "raster") type_check( projection, [int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference], "projection", ) type_check(out_path, [str], "out_path", allow_none=True) type_check(resample_alg, [str], "resample_alg") type_check(copy_if_already_correct, [bool], "copy_if_already_correct") type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") type_check(dst_nodata, [str, int, float], "dst_nodata") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") raster_list, path_list = ready_io_raster(raster, out_path, overwrite, prefix, postfix) out_name = path_list[0] ref = open_raster(raster_list[0]) metadata = raster_to_metadata(ref) out_creation_options = default_options(creation_options) out_format = path_to_driver_raster(out_name) original_projection = parse_projection(ref) target_projection = parse_projection(projection) if not isinstance(original_projection, osr.SpatialReference): raise Exception("Error while parsing input projection.") if not isinstance(target_projection, osr.SpatialReference): raise Exception("Error while parsing target projection.") if original_projection.IsSame(target_projection): if not copy_if_already_correct: return get_raster_path(ref) src_nodata = metadata["nodata_value"] out_nodata = None if src_nodata is not None: out_nodata = src_nodata else: if dst_nodata == "infer": out_nodata = gdal_nodata_value_from_type( metadata["datatype_gdal_raw"]) elif isinstance(dst_nodata, str): raise TypeError(f"dst_nodata is in a wrong format: {dst_nodata}") else: out_nodata = dst_nodata remove_if_overwrite(out_path, overwrite) reprojected = gdal.Warp( out_name, ref, format=out_format, srcSRS=original_projection, dstSRS=target_projection, resampleAlg=translate_resample_method(resample_alg), creationOptions=out_creation_options, srcNodata=metadata["nodata_value"], dstNodata=out_nodata, multithread=True, ) if reprojected is None: raise Exception(f"Error while reprojecting raster: {raster}") return out_name
def _warp_raster( raster: Union[str, gdal.Dataset], out_path: Optional[str] = None, projection: Optional[Union[int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference]] = None, clip_geom: Optional[Union[str, ogr.DataSource]] = None, target_size: Optional[Union[Tuple[Number], Number]] = None, target_in_pixels: bool = False, resample_alg: str = "nearest", crop_to_geom: bool = True, all_touch: bool = True, adjust_bbox: bool = True, overwrite: bool = True, creation_options: Union[list, None] = None, src_nodata: Union[str, int, float] = "infer", dst_nodata: Union[str, int, float] = "infer", layer_to_clip: int = 0, prefix: str = "", postfix: str = "_resampled", ) -> str: """WARNING: INTERNAL. DO NOT USE.""" raster_list, path_list = ready_io_raster(raster, out_path, overwrite, prefix, postfix) origin = open_raster(raster_list[0]) out_name = path_list[0] raster_metadata = raster_to_metadata(origin, create_geometry=True) # options warp_options = [] if all_touch: warp_options.append("CUTLINE_ALL_TOUCHED=TRUE") else: warp_options.append("CUTLINE_ALL_TOUCHED=FALSE") origin_projection: osr.SpatialReference = raster_metadata["projection_osr"] origin_extent: ogr.Geometry = raster_metadata["extent_geom_latlng"] target_projection = origin_projection if projection is not None: target_projection = parse_projection(projection) if clip_geom is not None: if is_raster(clip_geom): opened_raster = open_raster(clip_geom) clip_metadata_raster = raster_to_metadata(opened_raster, create_geometry=True) clip_ds = clip_metadata_raster["extent_datasource"] clip_metadata = internal_vector_to_metadata(clip_ds, create_geometry=True) elif is_vector(clip_geom): clip_ds = open_vector(clip_geom) clip_metadata = internal_vector_to_metadata(clip_ds, create_geometry=True) else: if file_exists(clip_geom): raise ValueError(f"Unable to parse clip geometry: {clip_geom}") else: raise ValueError(f"Unable to find clip geometry {clip_geom}") if layer_to_clip > (clip_metadata["layer_count"] - 1): raise ValueError("Requested an unable layer_to_clip.") clip_projection = clip_metadata["projection_osr"] clip_extent = clip_metadata["extent_geom_latlng"] # Fast check: Does the extent of the two inputs overlap? if not origin_extent.Intersects(clip_extent): raise Exception("Clipping geometry did not intersect raster.") # Check if projections match, otherwise reproject target geom. if not target_projection.IsSame(clip_projection): clip_metadata["extent"] = reproject_extent( clip_metadata["extent"], clip_projection, target_projection, ) # The extent needs to be reprojected to the target. # this ensures that adjust_bbox works. x_min_og, y_max_og, x_max_og, y_min_og = reproject_extent( raster_metadata["extent"], origin_projection, target_projection, ) output_bounds = (x_min_og, y_min_og, x_max_og, y_max_og ) # gdal_warp format if crop_to_geom: if adjust_bbox: output_bounds = align_bbox( raster_metadata["extent"], clip_metadata["extent"], raster_metadata["pixel_width"], raster_metadata["pixel_height"], warp_format=True, ) else: x_min_og, y_max_og, x_max_og, y_min_og = clip_metadata[ "extent"] output_bounds = ( x_min_og, y_min_og, x_max_og, y_max_og, ) # gdal_warp format if clip_metadata["layer_count"] > 1: clip_ds = vector_to_memory( clip_ds, memory_path=f"clip_geom_{uuid4().int}.gpkg", layer_to_extract=layer_to_clip, ) elif not isinstance(clip_ds, str): clip_ds = vector_to_memory( clip_ds, memory_path=f"clip_geom_{uuid4().int}.gpkg", ) if clip_ds is None: raise ValueError(f"Unable to parse input clip geom: {clip_geom}") x_res, y_res, x_pixels, y_pixels = raster_size_from_list( target_size, target_in_pixels) out_format = path_to_driver_raster(out_name) out_creation_options = default_options(creation_options) # nodata out_nodata = None if src_nodata is not None: out_nodata = raster_metadata["nodata_value"] else: if dst_nodata == "infer": out_nodata = gdal_nodata_value_from_type( raster_metadata["datatype_gdal_raw"]) else: out_nodata = dst_nodata # Removes file if it exists and overwrite is True. remove_if_overwrite(out_path, overwrite) warped = gdal.Warp( out_name, origin, xRes=x_res, yRes=y_res, width=x_pixels, height=y_pixels, cutlineDSName=clip_ds, outputBounds=output_bounds, format=out_format, srcSRS=origin_projection, dstSRS=target_projection, resampleAlg=translate_resample_method(resample_alg), creationOptions=out_creation_options, warpOptions=warp_options, srcNodata=src_nodata, dstNodata=out_nodata, targetAlignedPixels=False, cropToCutline=False, multithread=True, ) if warped is None: raise Exception(f"Error while warping raster: {raster}") return out_name
def backscatter_step1( zip_file, out_path, gpt_path="~/snap/bin/gpt", extent=None, tmp_folder=None, ): graph = "backscatter_step1.xml" # Get absolute location of graph processing tool gpt = find_gpt(gpt_path) out_path_ext = out_path + ".dim" if os.path.exists(out_path_ext): print(f"{out_path_ext} already processed") return out_path_ext xmlfile = os.path.join(os.path.dirname(__file__), f"./graphs/{graph}") snap_graph_step1 = open(xmlfile, "r") snap_graph_step1_str = snap_graph_step1.read() snap_graph_step1.close() if extent is not None: if is_vector(extent): metadata = vector_to_metadata(extent, create_geometry=True) elif is_raster(extent): metadata = raster_to_metadata(extent, create_geometry=True) elif isinstance(extent, str): metadata = raster_to_metadata(extent, create_geometry=True) else: raise ValueError("Extent must be a vector, raster or a path to a raster.") interest_area = metadata["extent_wkt_latlng"] else: interest_area = "POLYGON ((-180.0 -90.0, 180.0 -90.0, 180.0 90.0, -180.0 90.0, -180.0 -90.0))" snap_graph_step1_str = snap_graph_step1_str.replace("${extent}", interest_area) snap_graph_step1_str = snap_graph_step1_str.replace("${inputfile}", zip_file) snap_graph_step1_str = snap_graph_step1_str.replace("${outputfile}", out_path) xmlfile = tmp_folder + os.path.basename(out_path) + "_graph.xml" f = open(xmlfile, "w") f.write(snap_graph_step1_str) f.close() command = [ gpt, os.path.abspath(xmlfile), f"-q {cpu_count()}", ] if platform == "linux" or platform == "linux2": cmd = " ".join(command) else: cmd = f'cmd /c {" ".join(command)}' os.system(cmd) return out_path_ext
def download_s2_tile( scihub_username, scihub_password, onda_username, onda_password, destination, aoi_vector, date_start="20200601", date_end="20210101", clouds=10, producttype="S2MSI2A", tile=None, retry_count=10, retry_wait_min=30, retry_current=0, retry_downloaded=[], api_url="http://apihub.copernicus.eu/apihub", ): print("Downloading Sentinel-2 tiles") try: api = SentinelAPI(scihub_username, scihub_password, api_url, timeout=60) except Exception as e: print(e) raise Exception("Error connecting to SciHub") if is_vector(aoi_vector): geom = internal_vector_to_metadata(aoi_vector, create_geometry=True) elif is_raster(aoi_vector): geom = raster_to_metadata(aoi_vector, create_geometry=True) geom_extent = geom["extent_wkt_latlng"] download_products = OrderedDict() download_ids = [] date = (date_start, date_end) if tile is not None and tile != "": kw = {"raw": f"tileid:{tile} OR filename:*_T{tile}_*"} try: products = api.query( date=date, platformname="Sentinel-2", cloudcoverpercentage=(0, clouds), producttype="S2MSI2A", timeout=60, **kw, ) except Exception as e: print(e) raise Exception("Error connecting to SciHub") else: try: products = api.query( geom_extent, date=date, platformname="Sentinel-2", cloudcoverpercentage=(0, clouds), producttype=producttype, ) except Exception as e: print(e) raise Exception("Error connecting to SciHub") for product in products: dic = products[product] product_tile = dic["title"].split("_")[-2][1:] if (tile is not None and tile != "") and product_tile != tile: continue download_products[product] = dic download_ids.append(product) print(f"Downloading {len(download_products)} tiles") downloaded = [] + retry_downloaded for img_id in download_ids: out_path = destination + download_products[img_id]["filename"] + ".zip" if out_path in downloaded: continue # /footprint url for. download_url = ( f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/$value" ) try: content_size = get_content_size(download_url, auth=(onda_username, onda_password)) except Exception as e: print(f"Failed to get content size for {img_id}") print(e) continue try: if content_size > 0: if os.path.isfile( out_path) and content_size == os.path.getsize( out_path): downloaded.append(out_path) print(f"Skipping {img_id}") else: print(f"Downloading: {img_id}") download( download_url, out_path, auth=HTTPBasicAuth(onda_username, onda_password), verbose=False, skip_if_exists=True, ) downloaded.append(out_path) else: print("Requesting from archive. Not downloaded.") order_url = f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/Ens.Order" order_response = order(order_url, auth=(onda_username, onda_password)) except Exception as e: print(f"Error downloading {img_id}: {e}") if len(downloaded) >= len(download_ids): return downloaded elif retry_current < retry_count: print( f"Retrying {retry_current}/{retry_count}. Sleeping for {retry_wait_min} minutes." ) sleep(retry_wait_min * 60) download_s2_tile( scihub_username, scihub_password, onda_username, onda_password, destination, aoi_vector, date_start=date_start, date_end=date_end, clouds=clouds, producttype=producttype, tile=tile, retry_count=retry_count, retry_wait_min=retry_wait_min, retry_current=retry_current + 1, retry_downloaded=retry_downloaded + downloaded, ) else: return retry_downloaded + downloaded
def internal_resample_raster( raster: Union[str, gdal.Dataset], target_size: Union[tuple, int, float, str, gdal.Dataset], target_in_pixels: bool = False, out_path: Optional[str] = None, resample_alg: str = "nearest", overwrite: bool = True, creation_options: list = [], dtype=None, dst_nodata: Union[str, int, float] = "infer", prefix: str = "", postfix: str = "_resampled", add_uuid: bool = False, ) -> str: """OBS: Internal. Single output. Reprojects a raster given a target projection. Beware if your input is in latitude and longitude, you'll need to specify the target_size in degrees as well. """ type_check(raster, [str, gdal.Dataset], "raster") type_check(target_size, [tuple, int, float, str, gdal.Dataset], "target_size") type_check(target_in_pixels, [bool], "target_in_pixels") type_check(out_path, [list, str], "out_path", allow_none=True) type_check(resample_alg, [str], "resample_alg") type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") type_check(dst_nodata, [str, int, float], "dst_nodata") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") raster_list, path_list = ready_io_raster( raster, out_path, overwrite=overwrite, prefix=prefix, postfix=postfix, uuid=add_uuid, ) ref = open_raster(raster_list[0]) metadata = raster_to_metadata(ref) out_name = path_list[0] x_res, y_res, x_pixels, y_pixels = raster_size_from_list( target_size, target_in_pixels) out_creation_options = default_options(creation_options) out_format = path_to_driver_raster(out_name) src_nodata = metadata["nodata_value"] out_nodata = None if src_nodata is not None: out_nodata = src_nodata else: if dst_nodata == "infer": out_nodata = gdal_nodata_value_from_type( metadata["datatype_gdal_raw"]) elif isinstance(dst_nodata, str): raise TypeError(f"dst_nodata is in a wrong format: {dst_nodata}") else: out_nodata = dst_nodata remove_if_overwrite(out_path, overwrite) resampled = gdal.Warp( out_name, ref, width=x_pixels, height=y_pixels, xRes=x_res, yRes=y_res, format=out_format, outputType=translate_datatypes(dtype), resampleAlg=translate_resample_method(resample_alg), creationOptions=out_creation_options, srcNodata=metadata["nodata_value"], dstNodata=out_nodata, multithread=True, ) if resampled is None: raise Exception(f"Error while resampling raster: {out_name}") return out_name
def shift_raster( raster: Union[gdal.Dataset, str], shift: Union[Number, Tuple[Number, Number], List[Number]], out_path: Optional[str] = None, overwrite: bool = True, creation_options: list = [], ) -> Union[gdal.Dataset, str]: """Shifts a raster in a given direction. Returns: An in-memory raster. If an out_path is given the output is a string containing the path to the newly created raster. """ type_check(raster, [list, str, gdal.Dataset], "raster") type_check(shift, [tuple, list], "shift") type_check(out_path, [list, str], "out_path", allow_none=True) type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") ref = open_raster(raster) metadata = raster_to_metadata(ref) x_shift: float = 0.0 y_shift: float = 0.0 if isinstance(shift, tuple) or isinstance(shift, list): if len(shift) == 1: if is_number(shift[0]): x_shift = float(shift[0]) y_shift = float(shift[0]) else: raise ValueError( "shift is not a number or a list/tuple of numbers.") elif len(shift) == 2: if is_number(shift[0]) and is_number(shift[1]): x_shift = float(shift[0]) y_shift = float(shift[1]) else: raise ValueError("shift is either empty or larger than 2.") elif is_number(shift): x_shift = float(shift) y_shift = float(shift) else: raise ValueError("shift is invalid.") out_name = None out_format = None out_creation_options = [] if out_path is None: raster_name = metadata["basename"] out_name = f"/vsimem/{raster_name}_{uuid4().int}_resampled.tif" out_format = "GTiff" else: out_creation_options = default_options(creation_options) out_name = out_path out_format = path_to_driver_raster(out_path) remove_if_overwrite(out_path, overwrite) driver = gdal.GetDriverByName(out_format) shifted = driver.Create( out_name, # Location of the saved raster, ignored if driver is memory. metadata["width"], # Dataframe width in pixels (e.g. 1920px). metadata["height"], # Dataframe height in pixels (e.g. 1280px). metadata["band_count"], # The number of bands required. metadata["datatype_gdal_raw"], # Datatype of the destination. out_creation_options, ) new_transform = list(metadata["transform"]) new_transform[0] += x_shift new_transform[3] += y_shift shifted.SetGeoTransform(new_transform) shifted.SetProjection(metadata["projection"]) src_nodata = metadata["nodata_value"] for band in range(metadata["band_count"]): origin_raster_band = ref.GetRasterBand(band + 1) target_raster_band = shifted.GetRasterBand(band + 1) target_raster_band.WriteArray(origin_raster_band.ReadAsArray()) target_raster_band.SetNoDataValue(src_nodata) if out_path is not None: shifted = None return out_path else: return shifted
def align_rasters( rasters: List[Union[str, gdal.Dataset]], out_path: Optional[Union[List[str], str]] = None, master: Optional[Union[gdal.Dataset, str]] = None, postfix: str = "_aligned", bounding_box: Union[str, gdal.Dataset, ogr.DataSource, list, tuple] = "intersection", resample_alg: str = "nearest", target_size: Optional[Union[tuple, list, int, float, str, gdal.Dataset]] = None, target_in_pixels: bool = False, projection: Optional[Union[int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference]] = None, overwrite: bool = True, creation_options: list = [], src_nodata: Optional[Union[str, int, float]] = "infer", dst_nodata: Optional[Union[str, int, float]] = "infer", prefix: str = "", ram=8000, skip_existing=False, ) -> List[str]: type_check(rasters, [list], "rasters") type_check(out_path, [list, str], "out_path", allow_none=True) type_check(master, [list, str], "master", allow_none=True) type_check(bounding_box, [str, gdal.Dataset, ogr.DataSource, list, tuple], "bounding_box") type_check(resample_alg, [str], "resample_alg") type_check( target_size, [tuple, list, int, float, str, gdal.Dataset], "target_size", allow_none=True, ) type_check( target_in_pixels, [int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference], "target_in_pixels", allow_none=True, ) type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") type_check(src_nodata, [str, int, float], "src_nodata", allow_none=True) type_check(dst_nodata, [str, int, float], "dst_nodata", allow_none=True) type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") raster_list, path_list = ready_io_raster( rasters, out_path, overwrite=overwrite, prefix=prefix, postfix=postfix, uuid=False, ) x_pixels = None y_pixels = None x_res = None y_res = None target_projection = None target_bounds = None reprojected_rasters: List[str] = [] # Read the metadata for each raster. # Catalogue the used projections, to choose the most common one if necessary. used_projections: List[dict] = [] metadata: List[str] = [] for raster in rasters: meta = raster_to_metadata(raster) metadata.append(meta) used_projections.append(meta["projection"]) # If there is a master layer, copy information from that layer. if master is not None: master_metadata = raster_to_metadata(master) target_projection = master_metadata["projection_osr"] x_min, y_max, x_max, y_min = master_metadata["extent"] # Set the target values. target_bounds = (x_min, y_min, x_max, y_max) x_res = master_metadata["pixel_width"] y_res = master_metadata["pixel_height"] x_pixels = master_metadata["width"] y_pixels = master_metadata["height"] target_size = (x_res, y_res) target_in_pixels = False # We allow overwrite of parameters specifically set. # Handle projection if projection is not None: target_projection = parse_projection(projection) # If no projection is specified, other from master or parameters. The most common one is chosen. elif target_projection is None: # Sort and count the projections projection_counter: dict = {} for proj in used_projections: if proj in projection_counter: projection_counter[proj] += 1 else: projection_counter[proj] = 1 # Choose most common projection most_common_projection = sorted(projection_counter, key=projection_counter.get, reverse=True) target_projection = parse_projection(most_common_projection[0]) if target_size is not None: # If a raster is input, use it's pixel size as target values. if isinstance(target_size, (gdal.Dataset, str)): if isinstance(target_size, str) and not is_raster(target_size): raise ValueError( f"Unable to parse the raster used for target_size: {target_size}" ) # Reprojection is necessary to ensure the correct pixel_size reprojected_target_size = internal_reproject_raster( target_size, target_projection) target_size_raster = raster_to_metadata(reprojected_target_size) # Set the target values. x_res = target_size_raster["width"] y_res = target_size_raster["height"] else: # If a list, tuple, int or float is passed. Turn them into target values. x_res, y_res, x_pixels, y_pixels = raster_size_from_list( target_size, target_in_pixels) # If nothing has been specified, we will infer the pixel_size based on the median of all input rasters. elif x_res is None and y_res is None and x_pixels is None and y_pixels is None: # Ready numpy arrays for insertion x_res_arr = np.empty(len(raster_list), dtype="float32") y_res_arr = np.empty(len(raster_list), dtype="float32") for index, raster in enumerate(raster_list): # It is necessary to reproject each raster, as pixel height and width might be different after projection. reprojected = internal_reproject_raster(raster, target_projection) target_size_raster = raster_to_metadata(reprojected) # Add the pixel sizes to the numpy arrays x_res_arr[index] = target_size_raster["pixel_width"] y_res_arr[index] = target_size_raster["pixel_height"] # Keep track of the reprojected arrays so we only reproject rasters once. reprojected_rasters.append(reprojected) # Use the median values of pixel sizes as target values. x_res = np.median(x_res_arr) y_res = np.median(y_res_arr) if target_bounds is None: # If a bounding box is supplied, simply use that one. It must be in the target projection. if isinstance(bounding_box, (list, tuple)): if len(bounding_box) != 4: raise ValueError( "bounding_box as a list/tuple must have 4 values.") target_bounds = bounding_box # If the bounding box is a raster. Take the extent and reproject it to the target projection. elif is_raster(bounding_box): reprojected_bbox_raster = raster_to_metadata( internal_reproject_raster(bounding_box, target_projection)) x_min, y_max, x_max, y_min = reprojected_bbox_raster["extent"] # add to target values. target_bounds = (x_min, y_min, x_max, y_max) # If the bounding box is a raster. Take the extent and reproject it to the target projection. elif is_vector(bounding_box): reprojected_bbox_vector = internal_vector_to_metadata( internal_reproject_vector(bounding_box, target_projection)) x_min, y_max, x_max, y_min = reprojected_bbox_vector["extent"] # add to target values. target_bounds = (x_min, y_min, x_max, y_max) # If the bounding box is a string, we either take the union or the intersection of all the # bounding boxes of the input rasters. elif isinstance(bounding_box, str): if bounding_box == "intersection" or bounding_box == "union": extents = [] # If the rasters have not been reprojected, reproject them now. if len(reprojected_rasters) != len(raster_list): reprojected_rasters = [] for raster in raster_list: raster_metadata = raster_to_metadata(raster) if raster_metadata["projection_osr"].IsSame( target_projection): reprojected_rasters.append(raster) else: reprojected = internal_reproject_raster( raster, target_projection) reprojected_rasters.append(reprojected) # Add the extents of the reprojected rasters to the extents list. for reprojected_raster in reprojected_rasters: reprojected_raster_metadata = dict( raster_to_metadata(reprojected_raster)) extents.append(reprojected_raster_metadata["extent"]) # Placeholder values x_min, y_max, x_max, y_min = extents[0] # Loop the extents. Narrowing if intersection, expanding if union. for index, extent in enumerate(extents): if index == 0: continue if bounding_box == "intersection": if extent[0] > x_min: x_min = extent[0] if extent[1] < y_max: y_max = extent[1] if extent[2] < x_max: x_max = extent[2] if extent[3] > y_min: y_min = extent[3] elif bounding_box == "union": if extent[0] < x_min: x_min = extent[0] if extent[1] > y_max: y_max = extent[1] if extent[2] > x_max: x_max = extent[2] if extent[3] < y_min: y_min = extent[3] # Add to target values. target_bounds = (x_min, y_min, x_max, y_max) else: raise ValueError( f"Unable to parse or infer target_bounds: {target_bounds}") else: raise ValueError( f"Unable to parse or infer target_bounds: {target_bounds}") """ If the rasters have not been reprojected, we reproject them now. The reprojection is necessary as warp has to be a two step process in order to align the rasters properly. This might not be necessary in a future version of gdal. """ if len(reprojected_rasters) != len(raster_list): reprojected_rasters = [] for raster in raster_list: raster_metadata = raster_to_metadata(raster) # If the raster is already the correct projection, simply append the raster. if raster_metadata["projection_osr"].IsSame(target_projection): reprojected_rasters.append(raster) else: reprojected = internal_reproject_raster( raster, target_projection) reprojected_rasters.append(reprojected) # If any of the target values are still undefined. Throw an error! if target_projection is None or target_bounds is None: raise Exception( "Error while preparing the target projection or bounds.") if x_res is None and y_res is None and x_pixels is None and y_pixels is None: raise Exception("Error while preparing the target pixel size.") # This is the list of rasters to return. If output is not memory, it's a list of paths. return_list: List[str] = [] for index, raster in enumerate(reprojected_rasters): raster_metadata = raster_to_metadata(raster) out_name = path_list[index] out_format = path_to_driver_raster(out_name) if skip_existing and os.path.exists(out_name): return_list.append(out_name) continue # Handle nodata. out_src_nodata = None out_dst_nodata = None if src_nodata == "infer": out_src_nodata = raster_metadata["nodata_value"] if out_src_nodata is None: out_src_nodata = gdal_nodata_value_from_type( raster_metadata["datatype_gdal_raw"]) elif src_nodata == None: out_src_nodata = None elif not isinstance(src_nodata, str): out_src_nodata = src_nodata if dst_nodata == "infer": out_dst_nodata = out_src_nodata elif dst_nodata == False or dst_nodata == None: out_dst_nodata = None elif src_nodata == None: out_dst_nodata = None elif not isinstance(dst_nodata, str): out_dst_nodata = dst_nodata # Removes file if it exists and overwrite is True. remove_if_overwrite(out_name, overwrite) # Hand over to gdal.Warp to do the heavy lifting! warped = gdal.Warp( out_name, raster, xRes=x_res, yRes=y_res, width=x_pixels, height=y_pixels, dstSRS=target_projection, outputBounds=target_bounds, format=out_format, resampleAlg=translate_resample_method(resample_alg), creationOptions=default_options(creation_options), srcNodata=out_src_nodata, dstNodata=out_dst_nodata, targetAlignedPixels=False, cropToCutline=False, multithread=True, warpMemoryLimit=ram, ) if warped == None: raise Exception("Error while warping rasters.") return_list.append(out_name) if not rasters_are_aligned(return_list, same_extent=True): raise Exception("Error while aligning rasters. Output is not aligned") return return_list
def _clip_raster( raster: Union[str, gdal.Dataset], clip_geom: Union[str, ogr.DataSource, gdal.Dataset], out_path: Optional[str] = None, resample_alg: str = "nearest", crop_to_geom: bool = True, adjust_bbox: bool = True, all_touch: bool = True, overwrite: bool = True, creation_options: list = [], dst_nodata: Union[str, int, float] = "infer", layer_to_clip: int = 0, prefix: str = "", postfix: str = "_clipped", verbose: int = 1, uuid: bool = False, ram: int = 8000, ) -> str: """OBS: Internal. Single output. Clips a raster(s) using a vector geometry or the extents of a raster. """ type_check(raster, [str, gdal.Dataset], "raster") type_check(clip_geom, [str, ogr.DataSource, gdal.Dataset], "clip_geom") type_check(out_path, [str], "out_path", allow_none=True) type_check(resample_alg, [str], "resample_alg") type_check(crop_to_geom, [bool], "crop_to_geom") type_check(adjust_bbox, [bool], "adjust_bbox") type_check(all_touch, [bool], "all_touch") type_check(dst_nodata, [str, int, float], "dst_nodata") type_check(layer_to_clip, [int], "layer_to_clip") type_check(overwrite, [bool], "overwrite") type_check(creation_options, [list], "creation_options") type_check(prefix, [str], "prefix") type_check(postfix, [str], "postfix") type_check(verbose, [int], "verbose") type_check(uuid, [bool], "uuid") _, path_list = ready_io_raster(raster, out_path, overwrite=overwrite, prefix=prefix, postfix=postfix, uuid=uuid) if out_path is not None: if "vsimem" not in out_path: if not os.path.isdir(os.path.split(os.path.normpath(out_path))[0]): raise ValueError( f"out_path folder does not exists: {out_path}") # Input is a vector. if is_vector(clip_geom): clip_ds = open_vector(clip_geom) clip_metadata = internal_vector_to_metadata( clip_ds, process_layer=layer_to_clip) if clip_metadata["layer_count"] > 1: clip_ds = internal_vector_to_memory(clip_ds, layer_to_extract=layer_to_clip) if isinstance(clip_ds, ogr.DataSource): clip_ds = clip_ds.GetName() # Input is a raster (use extent) elif is_raster(clip_geom): clip_metadata = raster_to_metadata(clip_geom, create_geometry=True) clip_metadata["layer_count"] = 1 clip_ds = clip_metadata["extent_datasource"].GetName() else: if file_exists(clip_geom): raise ValueError(f"Unable to parse clip geometry: {clip_geom}") else: raise ValueError(f"Unable to locate clip geometry {clip_geom}") if layer_to_clip > (clip_metadata["layer_count"] - 1): raise ValueError("Requested an unable layer_to_clip.") if clip_ds is None: raise ValueError(f"Unable to parse input clip geom: {clip_geom}") clip_projection = clip_metadata["projection_osr"] clip_extent = clip_metadata["extent"] # options warp_options = [] if all_touch: warp_options.append("CUTLINE_ALL_TOUCHED=TRUE") else: warp_options.append("CUTLINE_ALL_TOUCHED=FALSE") origin_layer = open_raster(raster) raster_metadata = raster_to_metadata(raster) origin_projection = raster_metadata["projection_osr"] origin_extent = raster_metadata["extent"] # Check if projections match, otherwise reproject target geom. if not origin_projection.IsSame(clip_projection): clip_metadata["extent"] = reproject_extent( clip_metadata["extent"], clip_projection, origin_projection, ) # Fast check: Does the extent of the two inputs overlap? if not gdal_bbox_intersects(origin_extent, clip_extent): raise Exception("Geometries did not intersect.") output_bounds = raster_metadata["extent_gdal_warp"] if crop_to_geom: if adjust_bbox: output_bounds = align_bbox( raster_metadata["extent"], clip_metadata["extent"], raster_metadata["pixel_width"], raster_metadata["pixel_height"], warp_format=True, ) else: output_bounds = clip_metadata["extent_gdal_warp"] # formats out_name = path_list[0] out_format = path_to_driver_raster(out_name) out_creation_options = default_options(creation_options) # nodata src_nodata = raster_metadata["nodata_value"] out_nodata = None if src_nodata is not None: out_nodata = src_nodata else: if dst_nodata == "infer": out_nodata = gdal_nodata_value_from_type( raster_metadata["datatype_gdal_raw"]) elif dst_nodata is None: out_nodata = None elif isinstance(dst_nodata, (int, float)): out_nodata = dst_nodata else: raise ValueError(f"Unable to parse nodata_value: {dst_nodata}") # Removes file if it exists and overwrite is True. remove_if_overwrite(out_path, overwrite) if verbose == 0: gdal.PushErrorHandler("CPLQuietErrorHandler") clipped = gdal.Warp( out_name, origin_layer, format=out_format, resampleAlg=translate_resample_method(resample_alg), targetAlignedPixels=False, outputBounds=output_bounds, xRes=raster_metadata["pixel_width"], yRes=raster_metadata["pixel_height"], cutlineDSName=clip_ds, cropToCutline= False, # GDAL does this incorrectly when targetAlignedPixels is True. creationOptions=out_creation_options, warpMemoryLimit=ram, warpOptions=warp_options, srcNodata=raster_metadata["nodata_value"], dstNodata=out_nodata, multithread=True, ) if verbose == 0: gdal.PopErrorHandler() if clipped is None: raise Exception("Error while clipping raster.") return out_name
def extract_patches( raster_list, outdir, tile_size=32, zones=None, options=None, ): """ Generate patches for machine learning from rasters """ base_options = { "overlaps": True, "border_check": True, "merge_output": True, "force_align": True, "output_raster_labels": True, "label_geom": None, "label_res": 0.2, "label_mult": 100, "tolerance": 0.0, "fill_value": 0, "zone_layer_id": 0, "align_with_size": 20, "prefix": "", "postfix": "", } if options is None: options = base_options else: for key in options: if key not in base_options: raise ValueError(f"Invalid option: {key}") base_options[key] = options[key] options = base_options if zones is not None and not is_vector(zones): raise TypeError( "Clip geom is invalid. Did you input a valid geometry?") if not isinstance(raster_list, list): raster_list = [raster_list] for raster in raster_list: if not is_raster(raster): raise TypeError("raster_list is not a list of rasters.") if not os.path.isdir(outdir): raise ValueError( "Outdir does not exist. Please create before running the function." ) if not rasters_are_aligned(raster_list, same_extent=True): if options["force_align"]: print( "Rasters we not aligned. Realigning rasters due to force_align=True option." ) raster_list = align_rasters(raster_list) else: raise ValueError("Rasters in raster_list are not aligned.") offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]] raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True) pixel_size = min(raster_metadata["pixel_height"], raster_metadata["pixel_width"]) if zones is None: zones = raster_metadata["extent_datasource_path"] zones_meta = vector_to_metadata(zones) mem_driver = ogr.GetDriverByName("ESRI Shapefile") if zones_meta["layer_count"] == 0: raise ValueError("Vector contains no layers.") zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]] if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]: raise ValueError("clip geom is not Polygon or Multi Polygon.") zones_ogr = open_vector(zones) zones_layer = zones_ogr.GetLayer(options["zone_layer_id"]) feature_defn = zones_layer.GetLayerDefn() fids = vector_get_fids(zones_ogr, options["zone_layer_id"]) progress(0, len(fids) * len(raster_list), "processing fids") processed_fids = [] processed = 0 labels_processed = False for idx, raster in enumerate(raster_list): name = os.path.splitext(os.path.basename(raster))[0] list_extracted = [] list_masks = [] list_labels = [] for fid in fids: feature = zones_layer.GetFeature(fid) geom = feature.GetGeometryRef() fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp" fid_ds = mem_driver.CreateDataSource(fid_path) fid_ds_lyr = fid_ds.CreateLayer( "fid_layer", geom_type=ogr.wkbPolygon, srs=zones_layer_meta["projection_osr"], ) copied_feature = ogr.Feature(feature_defn) copied_feature.SetGeometry(geom) fid_ds_lyr.CreateFeature(copied_feature) fid_ds.FlushCache() fid_ds.SyncToDisk() valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif" rasterize_vector( fid_path, pixel_size, out_path=valid_path, extent=fid_path, ) valid_arr = raster_to_array(valid_path) if options["label_geom"] is not None and fid not in processed_fids: if not is_vector(options["label_geom"]): raise TypeError( "label geom is invalid. Did you input a valid geometry?" ) uuid = str(uuid4().int) label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp" label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif" label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif" intersect_vector(options["label_geom"], fid_ds, out_path=label_clip_path) try: rasterize_vector( label_clip_path, options["label_res"], out_path=label_ras_path, extent=valid_path, ) except Exception: array_to_raster( np.zeros(valid_arr.shape, dtype="float32"), valid_path, out_path=label_ras_path, ) resample_raster( label_ras_path, pixel_size, resample_alg="average", out_path=label_warp_path, ) labels_arr = (raster_to_array(label_warp_path) * options["label_mult"]).astype("float32") if options["output_raster_labels"]: array_to_raster( labels_arr, label_warp_path, out_path= f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif", ) raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif" try: clip_raster( raster, valid_path, raster_clip_path, all_touch=False, adjust_bbox=False, ) except Exception as e: print( f"Warning: {raster} did not intersect geom with fid: {fid}." ) print(e) if options["label_geom"] is not None: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) continue arr = raster_to_array(raster_clip_path) if arr.shape[:2] != valid_arr.shape[:2]: raise Exception( f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}" ) arr_offsets = get_overlaps(arr, offsets, tile_size, options["border_check"]) arr = np.concatenate(arr_offsets) valid_offsets = np.concatenate( get_overlaps(valid_arr, offsets, tile_size, options["border_check"])) valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) / (tile_size * tile_size))) <= options["tolerance"])[:, 0] arr = arr[valid_mask] valid_masked = valid_offsets[valid_mask] if options["label_geom"] is not None and not labels_processed: labels_masked = np.concatenate( get_overlaps(labels_arr, offsets, tile_size, options["border_check"]))[valid_mask] if options["merge_output"]: list_extracted.append(arr) list_masks.append(valid_masked) if options["label_geom"] is not None and not labels_processed: list_labels.append(labels_masked) else: np.save( f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy", arr.filled(options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy", valid_masked.filled(options["fill_value"]), ) if fid not in processed_fids: processed_fids.append(fid) processed += 1 progress(processed, len(fids) * len(raster_list), "processing fids") if not options["merge_output"]: gdal.Unlink(label_clip_path) gdal.Unlink(label_ras_path) gdal.Unlink(label_warp_path) gdal.Unlink(fid_path) gdal.Unlink(valid_path) if options["merge_output"]: np.save( f"{outdir}{options['prefix']}{name}{options['postfix']}.npy", np.ma.concatenate(list_extracted).filled( options["fill_value"]), ) np.save( f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy", np.ma.concatenate(list_masks).filled(options["fill_value"]), ) if options["label_geom"] is not None and not labels_processed: np.save( f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy", np.ma.concatenate(list_labels).filled( options["fill_value"]), ) labels_processed = True progress(1, 1, "processing fids") return 1
def download_s1_tile( scihub_username, scihub_password, onda_username, onda_password, destination, footprint, date=("20200601", "20210101"), orbitdirection="ASCENDING", # ASCENDING, DESCENDING min_overlap=0.50, producttype="GRD", sensoroperationalmode="IW", polarisationmode="VV VH", api_url="https://apihub.copernicus.eu/apihub/", ): api = SentinelAPI(scihub_username, scihub_password, api_url, timeout=60) if is_vector: geom = internal_vector_to_metadata(footprint, create_geometry=True) elif is_raster: geom = raster_to_metadata(footprint, create_geometry=True) products = api.query( geom["extent_wkt_latlng"], date=date, platformname="Sentinel-1", orbitdirection=orbitdirection, producttype=producttype, sensoroperationalmode=sensoroperationalmode, polarisationmode=polarisationmode, timeout=60, ) download_products = OrderedDict() download_ids = [] zero_contents = 0 geom_footprint = ogr.CreateGeometryFromWkt(geom["extent_wkt_latlng"]) for product in products: dic = products[product] img_footprint = ogr.CreateGeometryFromWkt(dic["footprint"]) img_area = img_footprint.GetArea() intersection = img_footprint.Intersection(geom_footprint) within = img_footprint.Intersection(intersection) within_area = within.GetArea() overlap_img = within_area / img_area overlap_geom = within_area / geom_footprint.GetArea() if max(overlap_img, overlap_geom) > min_overlap: download_products[product] = dic download_ids.append(product) if len(download_products) > 0: print(f"Downloading {len(download_products)} files.") downloaded = [] for img_id in download_ids: out_path = destination + download_products[img_id][ "filename"] + ".zip" if os.path.exists(out_path): print(f"Skipping {out_path}") continue download_url = f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/$value" try: content_size = get_content_size(download_url, out_path, auth=(onda_username, onda_password)) except Exception as e: print(f"Failed to get content size for {img_id}") print(e) continue if content_size == 0: zero_contents += 1 print( f"{img_id} requested from Archive but was not downloaded.") continue try: if content_size > 0: download( download_url, out_path, auth=(onda_username, onda_password), verbose=True, ) downloaded.append(img_id) except Exception as e: print(f"Error downloading {img_id}: {e}") return downloaded else: print("No images found") return []
def add_border_to_raster( input_raster, out_path=None, border_size=100, border_size_unit="px", border_value=0, overwrite: bool = True, creation_options: list = [], ): in_raster = open_raster(input_raster) metadata = raster_to_metadata(in_raster) # Parse the driver driver_name = "GTiff" if out_path is None else path_to_driver_raster( out_path) if driver_name is None: raise ValueError(f"Unable to parse filetype from path: {out_path}") driver = gdal.GetDriverByName(driver_name) if driver is None: raise ValueError( f"Error while creating driver from extension: {out_path}") output_name = None if out_path is None: output_name = f"/vsimem/raster_proximity_{uuid4().int}.tif" else: output_name = out_path in_arr = raster_to_array(in_raster) if border_size_unit == "px": border_size_y = border_size border_size_x = border_size new_shape = ( in_arr.shape[0] + (2 * border_size_y), in_arr.shape[1] + (2 * border_size_x), in_arr.shape[2], ) else: border_size_y = round(border_size / metadata["pixel_height"]) border_size_x = round(border_size / metadata["pixel_width"]) new_shape = ( in_arr.shape[0] + (2 * border_size_y), in_arr.shape[1] + (2 * border_size_x), in_arr.shape[2], ) new_arr = np.full(new_shape, border_value, dtype=in_arr.dtype) new_arr[border_size_y:-border_size_y, border_size_x:-border_size_x, :] = in_arr if isinstance(in_arr, np.ma.MaskedArray): mask = np.zeros(new_shape, dtype=bool) mask[border_size_y:-border_size_y, border_size_x:-border_size_x, :] = in_arr.mask new_arr = np.ma.array(new_arr, mask=mask) new_arr.fill_value = in_arr.fill_value remove_if_overwrite(out_path, overwrite) dest_raster = driver.Create( output_name, new_shape[1], new_shape[0], metadata["band_count"], numpy_to_gdal_datatype(in_arr.dtype), default_options(creation_options), ) og_transform = in_raster.GetGeoTransform() new_transform = [] for i in og_transform: new_transform.append(i) new_transform[0] -= border_size_x * og_transform[1] new_transform[3] -= border_size_y * og_transform[5] dest_raster.SetGeoTransform(new_transform) dest_raster.SetProjection(in_raster.GetProjectionRef()) for band_num in range(1, metadata["band_count"] + 1): dst_band = dest_raster.GetRasterBand(band_num) dst_band.WriteArray(new_arr[:, :, band_num - 1]) if metadata["has_nodata"]: dst_band.SetNoDataValue(metadata["nodata_value"]) return output_name
def raster_to_grid( raster: Union[str, gdal.Dataset], grid: Union[str, ogr.DataSource], out_dir: str, use_field: Optional[str] = None, generate_vrt: bool = True, overwrite: bool = True, process_layer: int = 0, creation_options: list = [], verbose: int = 1, ) -> Union[List[str], Tuple[Optional[List[str]], Optional[str]]]: """Clips a raster to a grid. Generate .vrt. Returns: The filepath for the newly created raster. """ type_check(raster, [str, gdal.Dataset], "raster") type_check(grid, [str, ogr.DataSource], "grid") type_check(out_dir, [str], "out_dir") type_check(overwrite, [bool], "overwrite") type_check(process_layer, [int], "process_layer") type_check(creation_options, [list], "creation_options") type_check(verbose, [int], "verbose") use_grid = open_vector(grid) grid_metadata = internal_vector_to_metadata(use_grid) raster_metadata = raster_to_metadata(raster, create_geometry=True) # Reproject raster if necessary. if not raster_metadata["projection_osr"].IsSame(grid_metadata["projection_osr"]): use_grid = reproject_vector(grid, raster_metadata["projection_osr"]) grid_metadata = internal_vector_to_metadata(use_grid) if not isinstance(grid_metadata, dict): raise Exception("Error while parsing metadata.") # Only use the polygons in the grid that intersect the extent of the raster. use_grid = intersect_vector(use_grid, raster_metadata["extent_datasource"]) ref = open_raster(raster) use_grid = open_vector(use_grid) layer = use_grid.GetLayer(process_layer) feature_count = layer.GetFeatureCount() raster_extent = raster_metadata["extent_ogr"] filetype = path_to_ext(raster) name = raster_metadata["name"] geom_type = grid_metadata["layers"][process_layer]["geom_type_ogr"] if use_field is not None: if use_field not in grid_metadata["layers"][process_layer]["field_names"]: names = grid_metadata["layers"][process_layer]["field_names"] raise ValueError( f"Requested field not found. Fields available are: {names}" ) generated = [] # For the sake of good reporting - lets first establish how many features intersect # the raster. if verbose: print("Finding intersections.") intersections = 0 for _ in range(feature_count): feature = layer.GetNextFeature() geom = feature.GetGeometryRef() if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()): continue intersections += 1 layer.ResetReading() if verbose: print(f"Found {intersections} intersections.") if intersections == 0: print("Warning: Found 0 intersections. Returning empty list.") return ([], None) # TODO: Replace this in gdal. 3.1 driver = ogr.GetDriverByName("Esri Shapefile") clipped = 0 for _ in range(feature_count): feature = layer.GetNextFeature() geom = feature.GetGeometryRef() if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()): continue if verbose == 1: progress(clipped, intersections - 1, "clip_grid") fid = feature.GetFID() test_ds_path = f"/vsimem/grid_{uuid4().int}.shp" test_ds = driver.CreateDataSource(test_ds_path) test_ds_lyr = test_ds.CreateLayer( "mem_layer_grid", geom_type=geom_type, srs=raster_metadata["projection_osr"], ) test_ds_lyr.CreateFeature(feature.Clone()) test_ds.FlushCache() out_name = None if use_field is not None: out_name = f"{out_dir}{feature.GetField(use_field)}{filetype}" else: out_name = f"{out_dir}{name}_{fid}{filetype}" clip_raster( ref, test_ds_path, out_path=out_name, adjust_bbox=True, crop_to_geom=True, all_touch=False, postfix="", prefix="", creation_options=default_options(creation_options), verbose=0, ) generated.append(out_name) clipped += 1 if generate_vrt: vrt_name = f"{out_dir}{name}.vrt" stack_rasters_vrt(generated, vrt_name, seperate=False) return (generated, vrt_name) return generated