def match_projections(
rasters,
master,
out_dir,
overwrite=False,
dst_nodata="infer",
copy_if_already_correct=True,
):
target_projection = parse_projection(master)
created = []
for raster in rasters:
metadata = raster_to_metadata(raster)
outname = out_dir + metadata["name"] + ".tif"
created.append(outname)
if os.path.exists(outname):
if not overwrite:
continue
internal_reproject_raster(
raster,
target_projection,
outname,
copy_if_already_correct=copy_if_already_correct,
dst_nodata=dst_nodata,
)
return created
def raster_get_nodata_value(
raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str],
) -> Union[List[Optional[Number]], Optional[Number]]:
"""Get the nodata value of a raster or a from a list of rasters.
Args:
raster (path | raster | list): The raster(s) to retrieve nodata values from.
Returns:
Returns the nodata value from a raster or a list of rasters
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
rasters = get_raster_path(raster, return_list=True)
nodata_values = []
for internal_raster in rasters:
if not is_raster(internal_raster):
raise ValueError(f"Input raster is invalid: {internal_raster}")
raster_metadata = raster_to_metadata(internal_raster)
if not isinstance(raster_metadata, dict):
raise Exception("Metadata is in the wrong format.")
raster_nodata = raster_metadata["nodata_value"]
nodata_values.append(raster_nodata)
if isinstance(raster, list):
return nodata_values
else:
return nodata_values[0]
def raster_has_nodata_value(
raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str],
) -> Union[bool, List[bool]]:
"""Check if a raster or a list of rasters contain nodata values
Args:
raster (path | raster | list): The raster(s) to check for nodata values.
Returns:
True if input raster has nodata values. If a list is the input, the output
is a list of booleans indicating if the input raster has nodata values.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
nodata_values = []
rasters = get_raster_path(raster, return_list=True)
for internal_raster in rasters:
if not is_raster(internal_raster):
raise ValueError(f"Input raster is invalid: {internal_raster}")
raster_metadata = raster_to_metadata(internal_raster)
if not isinstance(raster_metadata, dict):
raise Exception("Metadata is in the wrong format.")
raster_nodata = raster_metadata["nodata_value"]
if raster_nodata is not None:
nodata_values.append(True)
else:
nodata_values.append(False)
if isinstance(raster, list):
return nodata_values
else:
return nodata_values[0]
def raster_mask_values(
raster: Union[gdal.Dataset, str, list],
values_to_mask: list,
out_path: Union[list, str, None] = None,
include_original_nodata: bool = True,
dst_nodata: Union[float, int, str, list, None] = "infer",
in_place: bool = False,
overwrite: bool = True,
opened: bool = False,
prefix: str = "",
postfix: str = "_nodata_masked",
creation_options: list = [],
) -> Union[list, gdal.Dataset, str]:
"""Mask a raster with a list of values.
Args:
raster (path | raster | list): The raster(s) to retrieve nodata values from.
values_to_mask (list): The list of values to mask in the raster(s)
**kwargs:
include_original_nodata: (bool): If True, the nodata value of the raster(s) will be
included in the values to mask.
dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata
value is set based on the input datatype. A list of nodata values can be based matching
the amount of input rasters. If multiple nodata values should be set, use raster_mask_values.
out_path (path | list | None): The destination of the changed rasters. If out_paths
are specified, in_place is automatically set to False. The path can be a folder.
in_place (bool): Should the rasters be changed in_place or copied?
prefix (str): Prefix to add the the output if a folder is specified in out_path.
postfix (str): Postfix to add the the output if a folder is specified in out_path.
Returns:
Returns the rasters with nodata removed. If in_place is True a reference to the
changed orignal is returned, otherwise a copied memory raster or the path to the
generated raster is outputted.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(values_to_mask, [list], "values_to_mask")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(include_original_nodata, [bool], "include_original_nodata")
type_check(dst_nodata, [float, int, str, list],
"dst_nodata",
allow_none=True)
type_check(in_place, [bool], "in_place")
type_check(overwrite, [bool], "overwrite")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(opened, [bool], "opened")
type_check(creation_options, [list], "creation_options")
rasters_metadata = []
internal_in_place = in_place if out_path is None else False
internal_dst_nodata = None
for value in values_to_mask:
if not isinstance(value, (int, float)):
raise ValueError("Values in values_to_mask must be ints or floats")
if isinstance(dst_nodata, str) and dst_nodata != "infer":
raise ValueError(f"Invalid dst_nodata value. {dst_nodata}")
if isinstance(dst_nodata, list):
if not isinstance(raster, list) or len(dst_nodata) != len(raster):
raise ValueError(
"If dst_nodata is a list, raster must also be a list of equal length."
)
for value in dst_nodata:
if isinstance(value, (float, int, str, None)):
raise ValueError("Invalid type in dst_nodata list.")
if isinstance(value, str) and value != "infer":
raise ValueError(
"If dst_nodata is a string it must be 'infer'")
raster_list, out_names = ready_io_raster(raster, out_path, overwrite,
prefix, postfix)
output_rasters = []
for index, internal_raster in enumerate(raster_list):
raster_metadata = None
if len(rasters_metadata) == 0:
raster_metadata = raster_to_metadata(internal_raster)
rasters_metadata.append(raster_metadata)
else:
raster_metadata = rasters_metadata[index]
if dst_nodata == "infer":
internal_dst_nodata = gdal_nodata_value_from_type(
raster_metadata["dtype_gdal_raw"])
elif isinstance(dst_nodata, list):
internal_dst_nodata = dst_nodata[index]
else:
internal_dst_nodata = dst_nodata
mask_values = list(values_to_mask)
if include_original_nodata:
if raster_metadata["nodata_value"] is not None:
mask_values.append(raster_metadata["nodata_value"])
arr = raster_to_array(internal_raster, filled=True)
mask = None
for index, mask_value in enumerate(mask_values):
if index == 0:
mask = arr == mask_value
else:
mask = mask | arr == mask_value
arr = np.ma.masked_array(arr,
mask=mask,
fill_value=internal_dst_nodata)
if internal_in_place:
for band in range(raster_metadata["bands"]):
raster_band = internal_raster.GetRasterBand(band + 1)
raster_band.WriteArray(arr[:, :, band])
raster_band = None
else:
out_name = out_names[index]
remove_if_overwrite(out_name, overwrite)
output_rasters.append(
array_to_raster(arr, internal_raster, out_path=out_name))
if isinstance(raster, list):
return output_rasters
return output_rasters[0]
def raster_set_nodata(
raster: Union[List[Union[gdal.Dataset, str]], gdal.Dataset, str],
dst_nodata: Union[float, int, str, list, None],
out_path: Union[list, str, None] = None,
overwrite: bool = True,
in_place: bool = False,
prefix: str = "",
postfix: str = "_nodata_set",
opened: bool = False,
creation_options: list = [],
) -> Union[list, gdal.Dataset, str]:
"""Sets all the nodata from a raster or a list of rasters.
Args:
raster (path | raster | list): The raster(s) to retrieve nodata values from.
dst_nodata (float, int, str, None): The target nodata value. If 'infer' the nodata
value is set based on the input datatype. A list of nodata values can be based matching
the amount of input rasters. If multiple nodata values should be set, use raster_mask_values.
**kwargs:
out_path (path | list | None): The destination of the changed rasters. If out_paths
are specified, in_place is automatically set to False. The path can be a folder.
in_place (bool): Should the rasters be changed in_place or copied?
prefix (str): Prefix to add the the output if a folder is specified in out_path.
postfix (str): Postfix to add the the output if a folder is specified in out_path.
Returns:
Returns the rasters with nodata set. If in_place is True a reference to the
changed orignal is returned, otherwise a copied memory raster or the path to the
generated raster is outputted.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(dst_nodata, [float, int, str, list],
"dst_nodata",
allow_none=True)
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(overwrite, [bool], "overwrite")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(opened, [bool], "opened")
type_check(creation_options, [list], "creation_options")
rasters, out_names = ready_io_raster(raster, out_path, overwrite, prefix,
postfix)
rasters_metadata: List[Metadata_raster] = []
internal_dst_nodata = None
if isinstance(dst_nodata, str) and dst_nodata != "infer":
raise ValueError(f"Invalid dst_nodata value. {dst_nodata}")
if isinstance(dst_nodata, list):
if not isinstance(raster, list) or len(dst_nodata) != len(raster):
raise ValueError(
"If dst_nodata is a list, raster must also be a list of equal length."
)
for value in dst_nodata:
if isinstance(value, (float, int, str, None)):
raise ValueError("Invalid type in dst_nodata list.")
if isinstance(value, str) and value != "infer":
raise ValueError(
"If dst_nodata is a string it must be 'infer'")
output_rasters = []
for index, internal_raster in enumerate(rasters):
raster_metadata = None
if len(rasters_metadata) == 0:
raster_metadata = raster_to_metadata(internal_raster)
if not isinstance(raster_metadata, dict):
raise Exception("Metadata is in the wrong format.")
rasters_metadata.append(raster_metadata)
else:
raster_metadata = rasters_metadata[index]
if dst_nodata == "infer":
internal_dst_nodata = gdal_nodata_value_from_type(
raster_metadata["dtype_gdal_raw"])
elif isinstance(dst_nodata, list):
internal_dst_nodata = dst_nodata[index]
else:
internal_dst_nodata = dst_nodata
if in_place:
for band in range(raster_metadata["bands"]):
raster_band = internal_raster.GetRasterBand(band + 1)
raster_band.SetNodataValue(internal_dst_nodata)
raster_band = None
else:
if out_path is None:
raster_mem = raster_to_memory(internal_raster)
raster_mem_ref = raster_to_reference(raster_mem)
else:
remove_if_overwrite(out_names[index], overwrite)
raster_mem = raster_to_disk(internal_raster, out_names[index])
raster_mem_ref = raster_to_reference(raster_mem)
for band in range(raster_metadata["bands"]):
raster_band = raster_mem_ref.GetRasterBand(band + 1)
raster_band.SetNodataValue(internal_dst_nodata)
if isinstance(raster, list):
return output_rasters
return output_rasters[0]
def internal_reproject_raster(
raster: Union[str, gdal.Dataset],
projection: Union[int, str, gdal.Dataset, ogr.DataSource,
osr.SpatialReference],
out_path: Optional[str] = None,
resample_alg: str = "nearest",
copy_if_already_correct: bool = True,
overwrite: bool = True,
creation_options: list = [],
dst_nodata: Union[str, int, float] = "infer",
prefix: str = "",
postfix: str = "_reprojected",
) -> str:
"""OBS: Internal. Single output.
Reproject a raster(s) to a target coordinate reference system.
"""
type_check(raster, [str, gdal.Dataset], "raster")
type_check(
projection,
[int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference],
"projection",
)
type_check(out_path, [str], "out_path", allow_none=True)
type_check(resample_alg, [str], "resample_alg")
type_check(copy_if_already_correct, [bool], "copy_if_already_correct")
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
type_check(dst_nodata, [str, int, float], "dst_nodata")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
raster_list, path_list = ready_io_raster(raster, out_path, overwrite,
prefix, postfix)
out_name = path_list[0]
ref = open_raster(raster_list[0])
metadata = raster_to_metadata(ref)
out_creation_options = default_options(creation_options)
out_format = path_to_driver_raster(out_name)
original_projection = parse_projection(ref)
target_projection = parse_projection(projection)
if not isinstance(original_projection, osr.SpatialReference):
raise Exception("Error while parsing input projection.")
if not isinstance(target_projection, osr.SpatialReference):
raise Exception("Error while parsing target projection.")
if original_projection.IsSame(target_projection):
if not copy_if_already_correct:
return get_raster_path(ref)
src_nodata = metadata["nodata_value"]
out_nodata = None
if src_nodata is not None:
out_nodata = src_nodata
else:
if dst_nodata == "infer":
out_nodata = gdal_nodata_value_from_type(
metadata["datatype_gdal_raw"])
elif isinstance(dst_nodata, str):
raise TypeError(f"dst_nodata is in a wrong format: {dst_nodata}")
else:
out_nodata = dst_nodata
remove_if_overwrite(out_path, overwrite)
reprojected = gdal.Warp(
out_name,
ref,
format=out_format,
srcSRS=original_projection,
dstSRS=target_projection,
resampleAlg=translate_resample_method(resample_alg),
creationOptions=out_creation_options,
srcNodata=metadata["nodata_value"],
dstNodata=out_nodata,
multithread=True,
)
if reprojected is None:
raise Exception(f"Error while reprojecting raster: {raster}")
return out_name
def _warp_raster(
raster: Union[str, gdal.Dataset],
out_path: Optional[str] = None,
projection: Optional[Union[int, str, gdal.Dataset, ogr.DataSource,
osr.SpatialReference]] = None,
clip_geom: Optional[Union[str, ogr.DataSource]] = None,
target_size: Optional[Union[Tuple[Number], Number]] = None,
target_in_pixels: bool = False,
resample_alg: str = "nearest",
crop_to_geom: bool = True,
all_touch: bool = True,
adjust_bbox: bool = True,
overwrite: bool = True,
creation_options: Union[list, None] = None,
src_nodata: Union[str, int, float] = "infer",
dst_nodata: Union[str, int, float] = "infer",
layer_to_clip: int = 0,
prefix: str = "",
postfix: str = "_resampled",
) -> str:
"""WARNING: INTERNAL. DO NOT USE."""
raster_list, path_list = ready_io_raster(raster, out_path, overwrite,
prefix, postfix)
origin = open_raster(raster_list[0])
out_name = path_list[0]
raster_metadata = raster_to_metadata(origin, create_geometry=True)
# options
warp_options = []
if all_touch:
warp_options.append("CUTLINE_ALL_TOUCHED=TRUE")
else:
warp_options.append("CUTLINE_ALL_TOUCHED=FALSE")
origin_projection: osr.SpatialReference = raster_metadata["projection_osr"]
origin_extent: ogr.Geometry = raster_metadata["extent_geom_latlng"]
target_projection = origin_projection
if projection is not None:
target_projection = parse_projection(projection)
if clip_geom is not None:
if is_raster(clip_geom):
opened_raster = open_raster(clip_geom)
clip_metadata_raster = raster_to_metadata(opened_raster,
create_geometry=True)
clip_ds = clip_metadata_raster["extent_datasource"]
clip_metadata = internal_vector_to_metadata(clip_ds,
create_geometry=True)
elif is_vector(clip_geom):
clip_ds = open_vector(clip_geom)
clip_metadata = internal_vector_to_metadata(clip_ds,
create_geometry=True)
else:
if file_exists(clip_geom):
raise ValueError(f"Unable to parse clip geometry: {clip_geom}")
else:
raise ValueError(f"Unable to find clip geometry {clip_geom}")
if layer_to_clip > (clip_metadata["layer_count"] - 1):
raise ValueError("Requested an unable layer_to_clip.")
clip_projection = clip_metadata["projection_osr"]
clip_extent = clip_metadata["extent_geom_latlng"]
# Fast check: Does the extent of the two inputs overlap?
if not origin_extent.Intersects(clip_extent):
raise Exception("Clipping geometry did not intersect raster.")
# Check if projections match, otherwise reproject target geom.
if not target_projection.IsSame(clip_projection):
clip_metadata["extent"] = reproject_extent(
clip_metadata["extent"],
clip_projection,
target_projection,
)
# The extent needs to be reprojected to the target.
# this ensures that adjust_bbox works.
x_min_og, y_max_og, x_max_og, y_min_og = reproject_extent(
raster_metadata["extent"],
origin_projection,
target_projection,
)
output_bounds = (x_min_og, y_min_og, x_max_og, y_max_og
) # gdal_warp format
if crop_to_geom:
if adjust_bbox:
output_bounds = align_bbox(
raster_metadata["extent"],
clip_metadata["extent"],
raster_metadata["pixel_width"],
raster_metadata["pixel_height"],
warp_format=True,
)
else:
x_min_og, y_max_og, x_max_og, y_min_og = clip_metadata[
"extent"]
output_bounds = (
x_min_og,
y_min_og,
x_max_og,
y_max_og,
) # gdal_warp format
if clip_metadata["layer_count"] > 1:
clip_ds = vector_to_memory(
clip_ds,
memory_path=f"clip_geom_{uuid4().int}.gpkg",
layer_to_extract=layer_to_clip,
)
elif not isinstance(clip_ds, str):
clip_ds = vector_to_memory(
clip_ds,
memory_path=f"clip_geom_{uuid4().int}.gpkg",
)
if clip_ds is None:
raise ValueError(f"Unable to parse input clip geom: {clip_geom}")
x_res, y_res, x_pixels, y_pixels = raster_size_from_list(
target_size, target_in_pixels)
out_format = path_to_driver_raster(out_name)
out_creation_options = default_options(creation_options)
# nodata
out_nodata = None
if src_nodata is not None:
out_nodata = raster_metadata["nodata_value"]
else:
if dst_nodata == "infer":
out_nodata = gdal_nodata_value_from_type(
raster_metadata["datatype_gdal_raw"])
else:
out_nodata = dst_nodata
# Removes file if it exists and overwrite is True.
remove_if_overwrite(out_path, overwrite)
warped = gdal.Warp(
out_name,
origin,
xRes=x_res,
yRes=y_res,
width=x_pixels,
height=y_pixels,
cutlineDSName=clip_ds,
outputBounds=output_bounds,
format=out_format,
srcSRS=origin_projection,
dstSRS=target_projection,
resampleAlg=translate_resample_method(resample_alg),
creationOptions=out_creation_options,
warpOptions=warp_options,
srcNodata=src_nodata,
dstNodata=out_nodata,
targetAlignedPixels=False,
cropToCutline=False,
multithread=True,
)
if warped is None:
raise Exception(f"Error while warping raster: {raster}")
return out_name
def backscatter_step1(
zip_file,
out_path,
gpt_path="~/snap/bin/gpt",
extent=None,
tmp_folder=None,
):
graph = "backscatter_step1.xml"
# Get absolute location of graph processing tool
gpt = find_gpt(gpt_path)
out_path_ext = out_path + ".dim"
if os.path.exists(out_path_ext):
print(f"{out_path_ext} already processed")
return out_path_ext
xmlfile = os.path.join(os.path.dirname(__file__), f"./graphs/{graph}")
snap_graph_step1 = open(xmlfile, "r")
snap_graph_step1_str = snap_graph_step1.read()
snap_graph_step1.close()
if extent is not None:
if is_vector(extent):
metadata = vector_to_metadata(extent, create_geometry=True)
elif is_raster(extent):
metadata = raster_to_metadata(extent, create_geometry=True)
elif isinstance(extent, str):
metadata = raster_to_metadata(extent, create_geometry=True)
else:
raise ValueError("Extent must be a vector, raster or a path to a raster.")
interest_area = metadata["extent_wkt_latlng"]
else:
interest_area = "POLYGON ((-180.0 -90.0, 180.0 -90.0, 180.0 90.0, -180.0 90.0, -180.0 -90.0))"
snap_graph_step1_str = snap_graph_step1_str.replace("${extent}", interest_area)
snap_graph_step1_str = snap_graph_step1_str.replace("${inputfile}", zip_file)
snap_graph_step1_str = snap_graph_step1_str.replace("${outputfile}", out_path)
xmlfile = tmp_folder + os.path.basename(out_path) + "_graph.xml"
f = open(xmlfile, "w")
f.write(snap_graph_step1_str)
f.close()
command = [
gpt,
os.path.abspath(xmlfile),
f"-q {cpu_count()}",
]
if platform == "linux" or platform == "linux2":
cmd = " ".join(command)
else:
cmd = f'cmd /c {" ".join(command)}'
os.system(cmd)
return out_path_ext
def download_s2_tile(
scihub_username,
scihub_password,
onda_username,
onda_password,
destination,
aoi_vector,
date_start="20200601",
date_end="20210101",
clouds=10,
producttype="S2MSI2A",
tile=None,
retry_count=10,
retry_wait_min=30,
retry_current=0,
retry_downloaded=[],
api_url="http://apihub.copernicus.eu/apihub",
):
print("Downloading Sentinel-2 tiles")
try:
api = SentinelAPI(scihub_username,
scihub_password,
api_url,
timeout=60)
except Exception as e:
print(e)
raise Exception("Error connecting to SciHub")
if is_vector(aoi_vector):
geom = internal_vector_to_metadata(aoi_vector, create_geometry=True)
elif is_raster(aoi_vector):
geom = raster_to_metadata(aoi_vector, create_geometry=True)
geom_extent = geom["extent_wkt_latlng"]
download_products = OrderedDict()
download_ids = []
date = (date_start, date_end)
if tile is not None and tile != "":
kw = {"raw": f"tileid:{tile} OR filename:*_T{tile}_*"}
try:
products = api.query(
date=date,
platformname="Sentinel-2",
cloudcoverpercentage=(0, clouds),
producttype="S2MSI2A",
timeout=60,
**kw,
)
except Exception as e:
print(e)
raise Exception("Error connecting to SciHub")
else:
try:
products = api.query(
geom_extent,
date=date,
platformname="Sentinel-2",
cloudcoverpercentage=(0, clouds),
producttype=producttype,
)
except Exception as e:
print(e)
raise Exception("Error connecting to SciHub")
for product in products:
dic = products[product]
product_tile = dic["title"].split("_")[-2][1:]
if (tile is not None and tile != "") and product_tile != tile:
continue
download_products[product] = dic
download_ids.append(product)
print(f"Downloading {len(download_products)} tiles")
downloaded = [] + retry_downloaded
for img_id in download_ids:
out_path = destination + download_products[img_id]["filename"] + ".zip"
if out_path in downloaded:
continue
# /footprint url for.
download_url = (
f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/$value"
)
try:
content_size = get_content_size(download_url,
auth=(onda_username,
onda_password))
except Exception as e:
print(f"Failed to get content size for {img_id}")
print(e)
continue
try:
if content_size > 0:
if os.path.isfile(
out_path) and content_size == os.path.getsize(
out_path):
downloaded.append(out_path)
print(f"Skipping {img_id}")
else:
print(f"Downloading: {img_id}")
download(
download_url,
out_path,
auth=HTTPBasicAuth(onda_username, onda_password),
verbose=False,
skip_if_exists=True,
)
downloaded.append(out_path)
else:
print("Requesting from archive. Not downloaded.")
order_url = f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/Ens.Order"
order_response = order(order_url,
auth=(onda_username, onda_password))
except Exception as e:
print(f"Error downloading {img_id}: {e}")
if len(downloaded) >= len(download_ids):
return downloaded
elif retry_current < retry_count:
print(
f"Retrying {retry_current}/{retry_count}. Sleeping for {retry_wait_min} minutes."
)
sleep(retry_wait_min * 60)
download_s2_tile(
scihub_username,
scihub_password,
onda_username,
onda_password,
destination,
aoi_vector,
date_start=date_start,
date_end=date_end,
clouds=clouds,
producttype=producttype,
tile=tile,
retry_count=retry_count,
retry_wait_min=retry_wait_min,
retry_current=retry_current + 1,
retry_downloaded=retry_downloaded + downloaded,
)
else:
return retry_downloaded + downloaded
def internal_resample_raster(
raster: Union[str, gdal.Dataset],
target_size: Union[tuple, int, float, str, gdal.Dataset],
target_in_pixels: bool = False,
out_path: Optional[str] = None,
resample_alg: str = "nearest",
overwrite: bool = True,
creation_options: list = [],
dtype=None,
dst_nodata: Union[str, int, float] = "infer",
prefix: str = "",
postfix: str = "_resampled",
add_uuid: bool = False,
) -> str:
"""OBS: Internal. Single output.
Reprojects a raster given a target projection. Beware if your input is in
latitude and longitude, you'll need to specify the target_size in degrees as well.
"""
type_check(raster, [str, gdal.Dataset], "raster")
type_check(target_size, [tuple, int, float, str, gdal.Dataset],
"target_size")
type_check(target_in_pixels, [bool], "target_in_pixels")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(resample_alg, [str], "resample_alg")
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
type_check(dst_nodata, [str, int, float], "dst_nodata")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
raster_list, path_list = ready_io_raster(
raster,
out_path,
overwrite=overwrite,
prefix=prefix,
postfix=postfix,
uuid=add_uuid,
)
ref = open_raster(raster_list[0])
metadata = raster_to_metadata(ref)
out_name = path_list[0]
x_res, y_res, x_pixels, y_pixels = raster_size_from_list(
target_size, target_in_pixels)
out_creation_options = default_options(creation_options)
out_format = path_to_driver_raster(out_name)
src_nodata = metadata["nodata_value"]
out_nodata = None
if src_nodata is not None:
out_nodata = src_nodata
else:
if dst_nodata == "infer":
out_nodata = gdal_nodata_value_from_type(
metadata["datatype_gdal_raw"])
elif isinstance(dst_nodata, str):
raise TypeError(f"dst_nodata is in a wrong format: {dst_nodata}")
else:
out_nodata = dst_nodata
remove_if_overwrite(out_path, overwrite)
resampled = gdal.Warp(
out_name,
ref,
width=x_pixels,
height=y_pixels,
xRes=x_res,
yRes=y_res,
format=out_format,
outputType=translate_datatypes(dtype),
resampleAlg=translate_resample_method(resample_alg),
creationOptions=out_creation_options,
srcNodata=metadata["nodata_value"],
dstNodata=out_nodata,
multithread=True,
)
if resampled is None:
raise Exception(f"Error while resampling raster: {out_name}")
return out_name
def shift_raster(
raster: Union[gdal.Dataset, str],
shift: Union[Number, Tuple[Number, Number], List[Number]],
out_path: Optional[str] = None,
overwrite: bool = True,
creation_options: list = [],
) -> Union[gdal.Dataset, str]:
"""Shifts a raster in a given direction.
Returns:
An in-memory raster. If an out_path is given the output is a string containing
the path to the newly created raster.
"""
type_check(raster, [list, str, gdal.Dataset], "raster")
type_check(shift, [tuple, list], "shift")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
ref = open_raster(raster)
metadata = raster_to_metadata(ref)
x_shift: float = 0.0
y_shift: float = 0.0
if isinstance(shift, tuple) or isinstance(shift, list):
if len(shift) == 1:
if is_number(shift[0]):
x_shift = float(shift[0])
y_shift = float(shift[0])
else:
raise ValueError(
"shift is not a number or a list/tuple of numbers.")
elif len(shift) == 2:
if is_number(shift[0]) and is_number(shift[1]):
x_shift = float(shift[0])
y_shift = float(shift[1])
else:
raise ValueError("shift is either empty or larger than 2.")
elif is_number(shift):
x_shift = float(shift)
y_shift = float(shift)
else:
raise ValueError("shift is invalid.")
out_name = None
out_format = None
out_creation_options = []
if out_path is None:
raster_name = metadata["basename"]
out_name = f"/vsimem/{raster_name}_{uuid4().int}_resampled.tif"
out_format = "GTiff"
else:
out_creation_options = default_options(creation_options)
out_name = out_path
out_format = path_to_driver_raster(out_path)
remove_if_overwrite(out_path, overwrite)
driver = gdal.GetDriverByName(out_format)
shifted = driver.Create(
out_name, # Location of the saved raster, ignored if driver is memory.
metadata["width"], # Dataframe width in pixels (e.g. 1920px).
metadata["height"], # Dataframe height in pixels (e.g. 1280px).
metadata["band_count"], # The number of bands required.
metadata["datatype_gdal_raw"], # Datatype of the destination.
out_creation_options,
)
new_transform = list(metadata["transform"])
new_transform[0] += x_shift
new_transform[3] += y_shift
shifted.SetGeoTransform(new_transform)
shifted.SetProjection(metadata["projection"])
src_nodata = metadata["nodata_value"]
for band in range(metadata["band_count"]):
origin_raster_band = ref.GetRasterBand(band + 1)
target_raster_band = shifted.GetRasterBand(band + 1)
target_raster_band.WriteArray(origin_raster_band.ReadAsArray())
target_raster_band.SetNoDataValue(src_nodata)
if out_path is not None:
shifted = None
return out_path
else:
return shifted
def align_rasters(
rasters: List[Union[str, gdal.Dataset]],
out_path: Optional[Union[List[str], str]] = None,
master: Optional[Union[gdal.Dataset, str]] = None,
postfix: str = "_aligned",
bounding_box: Union[str, gdal.Dataset, ogr.DataSource, list,
tuple] = "intersection",
resample_alg: str = "nearest",
target_size: Optional[Union[tuple, list, int, float, str,
gdal.Dataset]] = None,
target_in_pixels: bool = False,
projection: Optional[Union[int, str, gdal.Dataset, ogr.DataSource,
osr.SpatialReference]] = None,
overwrite: bool = True,
creation_options: list = [],
src_nodata: Optional[Union[str, int, float]] = "infer",
dst_nodata: Optional[Union[str, int, float]] = "infer",
prefix: str = "",
ram=8000,
skip_existing=False,
) -> List[str]:
type_check(rasters, [list], "rasters")
type_check(out_path, [list, str], "out_path", allow_none=True)
type_check(master, [list, str], "master", allow_none=True)
type_check(bounding_box, [str, gdal.Dataset, ogr.DataSource, list, tuple],
"bounding_box")
type_check(resample_alg, [str], "resample_alg")
type_check(
target_size,
[tuple, list, int, float, str, gdal.Dataset],
"target_size",
allow_none=True,
)
type_check(
target_in_pixels,
[int, str, gdal.Dataset, ogr.DataSource, osr.SpatialReference],
"target_in_pixels",
allow_none=True,
)
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
type_check(src_nodata, [str, int, float], "src_nodata", allow_none=True)
type_check(dst_nodata, [str, int, float], "dst_nodata", allow_none=True)
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
raster_list, path_list = ready_io_raster(
rasters,
out_path,
overwrite=overwrite,
prefix=prefix,
postfix=postfix,
uuid=False,
)
x_pixels = None
y_pixels = None
x_res = None
y_res = None
target_projection = None
target_bounds = None
reprojected_rasters: List[str] = []
# Read the metadata for each raster.
# Catalogue the used projections, to choose the most common one if necessary.
used_projections: List[dict] = []
metadata: List[str] = []
for raster in rasters:
meta = raster_to_metadata(raster)
metadata.append(meta)
used_projections.append(meta["projection"])
# If there is a master layer, copy information from that layer.
if master is not None:
master_metadata = raster_to_metadata(master)
target_projection = master_metadata["projection_osr"]
x_min, y_max, x_max, y_min = master_metadata["extent"]
# Set the target values.
target_bounds = (x_min, y_min, x_max, y_max)
x_res = master_metadata["pixel_width"]
y_res = master_metadata["pixel_height"]
x_pixels = master_metadata["width"]
y_pixels = master_metadata["height"]
target_size = (x_res, y_res)
target_in_pixels = False
# We allow overwrite of parameters specifically set.
# Handle projection
if projection is not None:
target_projection = parse_projection(projection)
# If no projection is specified, other from master or parameters. The most common one is chosen.
elif target_projection is None:
# Sort and count the projections
projection_counter: dict = {}
for proj in used_projections:
if proj in projection_counter:
projection_counter[proj] += 1
else:
projection_counter[proj] = 1
# Choose most common projection
most_common_projection = sorted(projection_counter,
key=projection_counter.get,
reverse=True)
target_projection = parse_projection(most_common_projection[0])
if target_size is not None:
# If a raster is input, use it's pixel size as target values.
if isinstance(target_size, (gdal.Dataset, str)):
if isinstance(target_size, str) and not is_raster(target_size):
raise ValueError(
f"Unable to parse the raster used for target_size: {target_size}"
)
# Reprojection is necessary to ensure the correct pixel_size
reprojected_target_size = internal_reproject_raster(
target_size, target_projection)
target_size_raster = raster_to_metadata(reprojected_target_size)
# Set the target values.
x_res = target_size_raster["width"]
y_res = target_size_raster["height"]
else:
# If a list, tuple, int or float is passed. Turn them into target values.
x_res, y_res, x_pixels, y_pixels = raster_size_from_list(
target_size, target_in_pixels)
# If nothing has been specified, we will infer the pixel_size based on the median of all input rasters.
elif x_res is None and y_res is None and x_pixels is None and y_pixels is None:
# Ready numpy arrays for insertion
x_res_arr = np.empty(len(raster_list), dtype="float32")
y_res_arr = np.empty(len(raster_list), dtype="float32")
for index, raster in enumerate(raster_list):
# It is necessary to reproject each raster, as pixel height and width might be different after projection.
reprojected = internal_reproject_raster(raster, target_projection)
target_size_raster = raster_to_metadata(reprojected)
# Add the pixel sizes to the numpy arrays
x_res_arr[index] = target_size_raster["pixel_width"]
y_res_arr[index] = target_size_raster["pixel_height"]
# Keep track of the reprojected arrays so we only reproject rasters once.
reprojected_rasters.append(reprojected)
# Use the median values of pixel sizes as target values.
x_res = np.median(x_res_arr)
y_res = np.median(y_res_arr)
if target_bounds is None:
# If a bounding box is supplied, simply use that one. It must be in the target projection.
if isinstance(bounding_box, (list, tuple)):
if len(bounding_box) != 4:
raise ValueError(
"bounding_box as a list/tuple must have 4 values.")
target_bounds = bounding_box
# If the bounding box is a raster. Take the extent and reproject it to the target projection.
elif is_raster(bounding_box):
reprojected_bbox_raster = raster_to_metadata(
internal_reproject_raster(bounding_box, target_projection))
x_min, y_max, x_max, y_min = reprojected_bbox_raster["extent"]
# add to target values.
target_bounds = (x_min, y_min, x_max, y_max)
# If the bounding box is a raster. Take the extent and reproject it to the target projection.
elif is_vector(bounding_box):
reprojected_bbox_vector = internal_vector_to_metadata(
internal_reproject_vector(bounding_box, target_projection))
x_min, y_max, x_max, y_min = reprojected_bbox_vector["extent"]
# add to target values.
target_bounds = (x_min, y_min, x_max, y_max)
# If the bounding box is a string, we either take the union or the intersection of all the
# bounding boxes of the input rasters.
elif isinstance(bounding_box, str):
if bounding_box == "intersection" or bounding_box == "union":
extents = []
# If the rasters have not been reprojected, reproject them now.
if len(reprojected_rasters) != len(raster_list):
reprojected_rasters = []
for raster in raster_list:
raster_metadata = raster_to_metadata(raster)
if raster_metadata["projection_osr"].IsSame(
target_projection):
reprojected_rasters.append(raster)
else:
reprojected = internal_reproject_raster(
raster, target_projection)
reprojected_rasters.append(reprojected)
# Add the extents of the reprojected rasters to the extents list.
for reprojected_raster in reprojected_rasters:
reprojected_raster_metadata = dict(
raster_to_metadata(reprojected_raster))
extents.append(reprojected_raster_metadata["extent"])
# Placeholder values
x_min, y_max, x_max, y_min = extents[0]
# Loop the extents. Narrowing if intersection, expanding if union.
for index, extent in enumerate(extents):
if index == 0:
continue
if bounding_box == "intersection":
if extent[0] > x_min:
x_min = extent[0]
if extent[1] < y_max:
y_max = extent[1]
if extent[2] < x_max:
x_max = extent[2]
if extent[3] > y_min:
y_min = extent[3]
elif bounding_box == "union":
if extent[0] < x_min:
x_min = extent[0]
if extent[1] > y_max:
y_max = extent[1]
if extent[2] > x_max:
x_max = extent[2]
if extent[3] < y_min:
y_min = extent[3]
# Add to target values.
target_bounds = (x_min, y_min, x_max, y_max)
else:
raise ValueError(
f"Unable to parse or infer target_bounds: {target_bounds}")
else:
raise ValueError(
f"Unable to parse or infer target_bounds: {target_bounds}")
"""
If the rasters have not been reprojected, we reproject them now.
The reprojection is necessary as warp has to be a two step process
in order to align the rasters properly. This might not be necessary
in a future version of gdal.
"""
if len(reprojected_rasters) != len(raster_list):
reprojected_rasters = []
for raster in raster_list:
raster_metadata = raster_to_metadata(raster)
# If the raster is already the correct projection, simply append the raster.
if raster_metadata["projection_osr"].IsSame(target_projection):
reprojected_rasters.append(raster)
else:
reprojected = internal_reproject_raster(
raster, target_projection)
reprojected_rasters.append(reprojected)
# If any of the target values are still undefined. Throw an error!
if target_projection is None or target_bounds is None:
raise Exception(
"Error while preparing the target projection or bounds.")
if x_res is None and y_res is None and x_pixels is None and y_pixels is None:
raise Exception("Error while preparing the target pixel size.")
# This is the list of rasters to return. If output is not memory, it's a list of paths.
return_list: List[str] = []
for index, raster in enumerate(reprojected_rasters):
raster_metadata = raster_to_metadata(raster)
out_name = path_list[index]
out_format = path_to_driver_raster(out_name)
if skip_existing and os.path.exists(out_name):
return_list.append(out_name)
continue
# Handle nodata.
out_src_nodata = None
out_dst_nodata = None
if src_nodata == "infer":
out_src_nodata = raster_metadata["nodata_value"]
if out_src_nodata is None:
out_src_nodata = gdal_nodata_value_from_type(
raster_metadata["datatype_gdal_raw"])
elif src_nodata == None:
out_src_nodata = None
elif not isinstance(src_nodata, str):
out_src_nodata = src_nodata
if dst_nodata == "infer":
out_dst_nodata = out_src_nodata
elif dst_nodata == False or dst_nodata == None:
out_dst_nodata = None
elif src_nodata == None:
out_dst_nodata = None
elif not isinstance(dst_nodata, str):
out_dst_nodata = dst_nodata
# Removes file if it exists and overwrite is True.
remove_if_overwrite(out_name, overwrite)
# Hand over to gdal.Warp to do the heavy lifting!
warped = gdal.Warp(
out_name,
raster,
xRes=x_res,
yRes=y_res,
width=x_pixels,
height=y_pixels,
dstSRS=target_projection,
outputBounds=target_bounds,
format=out_format,
resampleAlg=translate_resample_method(resample_alg),
creationOptions=default_options(creation_options),
srcNodata=out_src_nodata,
dstNodata=out_dst_nodata,
targetAlignedPixels=False,
cropToCutline=False,
multithread=True,
warpMemoryLimit=ram,
)
if warped == None:
raise Exception("Error while warping rasters.")
return_list.append(out_name)
if not rasters_are_aligned(return_list, same_extent=True):
raise Exception("Error while aligning rasters. Output is not aligned")
return return_list
def _clip_raster(
raster: Union[str, gdal.Dataset],
clip_geom: Union[str, ogr.DataSource, gdal.Dataset],
out_path: Optional[str] = None,
resample_alg: str = "nearest",
crop_to_geom: bool = True,
adjust_bbox: bool = True,
all_touch: bool = True,
overwrite: bool = True,
creation_options: list = [],
dst_nodata: Union[str, int, float] = "infer",
layer_to_clip: int = 0,
prefix: str = "",
postfix: str = "_clipped",
verbose: int = 1,
uuid: bool = False,
ram: int = 8000,
) -> str:
"""OBS: Internal. Single output.
Clips a raster(s) using a vector geometry or the extents of
a raster.
"""
type_check(raster, [str, gdal.Dataset], "raster")
type_check(clip_geom, [str, ogr.DataSource, gdal.Dataset], "clip_geom")
type_check(out_path, [str], "out_path", allow_none=True)
type_check(resample_alg, [str], "resample_alg")
type_check(crop_to_geom, [bool], "crop_to_geom")
type_check(adjust_bbox, [bool], "adjust_bbox")
type_check(all_touch, [bool], "all_touch")
type_check(dst_nodata, [str, int, float], "dst_nodata")
type_check(layer_to_clip, [int], "layer_to_clip")
type_check(overwrite, [bool], "overwrite")
type_check(creation_options, [list], "creation_options")
type_check(prefix, [str], "prefix")
type_check(postfix, [str], "postfix")
type_check(verbose, [int], "verbose")
type_check(uuid, [bool], "uuid")
_, path_list = ready_io_raster(raster,
out_path,
overwrite=overwrite,
prefix=prefix,
postfix=postfix,
uuid=uuid)
if out_path is not None:
if "vsimem" not in out_path:
if not os.path.isdir(os.path.split(os.path.normpath(out_path))[0]):
raise ValueError(
f"out_path folder does not exists: {out_path}")
# Input is a vector.
if is_vector(clip_geom):
clip_ds = open_vector(clip_geom)
clip_metadata = internal_vector_to_metadata(
clip_ds, process_layer=layer_to_clip)
if clip_metadata["layer_count"] > 1:
clip_ds = internal_vector_to_memory(clip_ds,
layer_to_extract=layer_to_clip)
if isinstance(clip_ds, ogr.DataSource):
clip_ds = clip_ds.GetName()
# Input is a raster (use extent)
elif is_raster(clip_geom):
clip_metadata = raster_to_metadata(clip_geom, create_geometry=True)
clip_metadata["layer_count"] = 1
clip_ds = clip_metadata["extent_datasource"].GetName()
else:
if file_exists(clip_geom):
raise ValueError(f"Unable to parse clip geometry: {clip_geom}")
else:
raise ValueError(f"Unable to locate clip geometry {clip_geom}")
if layer_to_clip > (clip_metadata["layer_count"] - 1):
raise ValueError("Requested an unable layer_to_clip.")
if clip_ds is None:
raise ValueError(f"Unable to parse input clip geom: {clip_geom}")
clip_projection = clip_metadata["projection_osr"]
clip_extent = clip_metadata["extent"]
# options
warp_options = []
if all_touch:
warp_options.append("CUTLINE_ALL_TOUCHED=TRUE")
else:
warp_options.append("CUTLINE_ALL_TOUCHED=FALSE")
origin_layer = open_raster(raster)
raster_metadata = raster_to_metadata(raster)
origin_projection = raster_metadata["projection_osr"]
origin_extent = raster_metadata["extent"]
# Check if projections match, otherwise reproject target geom.
if not origin_projection.IsSame(clip_projection):
clip_metadata["extent"] = reproject_extent(
clip_metadata["extent"],
clip_projection,
origin_projection,
)
# Fast check: Does the extent of the two inputs overlap?
if not gdal_bbox_intersects(origin_extent, clip_extent):
raise Exception("Geometries did not intersect.")
output_bounds = raster_metadata["extent_gdal_warp"]
if crop_to_geom:
if adjust_bbox:
output_bounds = align_bbox(
raster_metadata["extent"],
clip_metadata["extent"],
raster_metadata["pixel_width"],
raster_metadata["pixel_height"],
warp_format=True,
)
else:
output_bounds = clip_metadata["extent_gdal_warp"]
# formats
out_name = path_list[0]
out_format = path_to_driver_raster(out_name)
out_creation_options = default_options(creation_options)
# nodata
src_nodata = raster_metadata["nodata_value"]
out_nodata = None
if src_nodata is not None:
out_nodata = src_nodata
else:
if dst_nodata == "infer":
out_nodata = gdal_nodata_value_from_type(
raster_metadata["datatype_gdal_raw"])
elif dst_nodata is None:
out_nodata = None
elif isinstance(dst_nodata, (int, float)):
out_nodata = dst_nodata
else:
raise ValueError(f"Unable to parse nodata_value: {dst_nodata}")
# Removes file if it exists and overwrite is True.
remove_if_overwrite(out_path, overwrite)
if verbose == 0:
gdal.PushErrorHandler("CPLQuietErrorHandler")
clipped = gdal.Warp(
out_name,
origin_layer,
format=out_format,
resampleAlg=translate_resample_method(resample_alg),
targetAlignedPixels=False,
outputBounds=output_bounds,
xRes=raster_metadata["pixel_width"],
yRes=raster_metadata["pixel_height"],
cutlineDSName=clip_ds,
cropToCutline=
False, # GDAL does this incorrectly when targetAlignedPixels is True.
creationOptions=out_creation_options,
warpMemoryLimit=ram,
warpOptions=warp_options,
srcNodata=raster_metadata["nodata_value"],
dstNodata=out_nodata,
multithread=True,
)
if verbose == 0:
gdal.PopErrorHandler()
if clipped is None:
raise Exception("Error while clipping raster.")
return out_name
def extract_patches(
raster_list,
outdir,
tile_size=32,
zones=None,
options=None,
):
"""
Generate patches for machine learning from rasters
"""
base_options = {
"overlaps": True,
"border_check": True,
"merge_output": True,
"force_align": True,
"output_raster_labels": True,
"label_geom": None,
"label_res": 0.2,
"label_mult": 100,
"tolerance": 0.0,
"fill_value": 0,
"zone_layer_id": 0,
"align_with_size": 20,
"prefix": "",
"postfix": "",
}
if options is None:
options = base_options
else:
for key in options:
if key not in base_options:
raise ValueError(f"Invalid option: {key}")
base_options[key] = options[key]
options = base_options
if zones is not None and not is_vector(zones):
raise TypeError(
"Clip geom is invalid. Did you input a valid geometry?")
if not isinstance(raster_list, list):
raster_list = [raster_list]
for raster in raster_list:
if not is_raster(raster):
raise TypeError("raster_list is not a list of rasters.")
if not os.path.isdir(outdir):
raise ValueError(
"Outdir does not exist. Please create before running the function."
)
if not rasters_are_aligned(raster_list, same_extent=True):
if options["force_align"]:
print(
"Rasters we not aligned. Realigning rasters due to force_align=True option."
)
raster_list = align_rasters(raster_list)
else:
raise ValueError("Rasters in raster_list are not aligned.")
offsets = get_offsets(tile_size) if options["overlaps"] else [[0, 0]]
raster_metadata = raster_to_metadata(raster_list[0], create_geometry=True)
pixel_size = min(raster_metadata["pixel_height"],
raster_metadata["pixel_width"])
if zones is None:
zones = raster_metadata["extent_datasource_path"]
zones_meta = vector_to_metadata(zones)
mem_driver = ogr.GetDriverByName("ESRI Shapefile")
if zones_meta["layer_count"] == 0:
raise ValueError("Vector contains no layers.")
zones_layer_meta = zones_meta["layers"][options["zone_layer_id"]]
if zones_layer_meta["geom_type"] not in ["Multi Polygon", "Polygon"]:
raise ValueError("clip geom is not Polygon or Multi Polygon.")
zones_ogr = open_vector(zones)
zones_layer = zones_ogr.GetLayer(options["zone_layer_id"])
feature_defn = zones_layer.GetLayerDefn()
fids = vector_get_fids(zones_ogr, options["zone_layer_id"])
progress(0, len(fids) * len(raster_list), "processing fids")
processed_fids = []
processed = 0
labels_processed = False
for idx, raster in enumerate(raster_list):
name = os.path.splitext(os.path.basename(raster))[0]
list_extracted = []
list_masks = []
list_labels = []
for fid in fids:
feature = zones_layer.GetFeature(fid)
geom = feature.GetGeometryRef()
fid_path = f"/vsimem/fid_mem_{uuid4().int}_{str(fid)}.shp"
fid_ds = mem_driver.CreateDataSource(fid_path)
fid_ds_lyr = fid_ds.CreateLayer(
"fid_layer",
geom_type=ogr.wkbPolygon,
srs=zones_layer_meta["projection_osr"],
)
copied_feature = ogr.Feature(feature_defn)
copied_feature.SetGeometry(geom)
fid_ds_lyr.CreateFeature(copied_feature)
fid_ds.FlushCache()
fid_ds.SyncToDisk()
valid_path = f"/vsimem/{options['prefix']}validmask_{str(fid)}{options['postfix']}.tif"
rasterize_vector(
fid_path,
pixel_size,
out_path=valid_path,
extent=fid_path,
)
valid_arr = raster_to_array(valid_path)
if options["label_geom"] is not None and fid not in processed_fids:
if not is_vector(options["label_geom"]):
raise TypeError(
"label geom is invalid. Did you input a valid geometry?"
)
uuid = str(uuid4().int)
label_clip_path = f"/vsimem/fid_{uuid}_{str(fid)}_clipped.shp"
label_ras_path = f"/vsimem/fid_{uuid}_{str(fid)}_rasterized.tif"
label_warp_path = f"/vsimem/fid_{uuid}_{str(fid)}_resampled.tif"
intersect_vector(options["label_geom"],
fid_ds,
out_path=label_clip_path)
try:
rasterize_vector(
label_clip_path,
options["label_res"],
out_path=label_ras_path,
extent=valid_path,
)
except Exception:
array_to_raster(
np.zeros(valid_arr.shape, dtype="float32"),
valid_path,
out_path=label_ras_path,
)
resample_raster(
label_ras_path,
pixel_size,
resample_alg="average",
out_path=label_warp_path,
)
labels_arr = (raster_to_array(label_warp_path) *
options["label_mult"]).astype("float32")
if options["output_raster_labels"]:
array_to_raster(
labels_arr,
label_warp_path,
out_path=
f"{outdir}{options['prefix']}label_{str(fid)}{options['postfix']}.tif",
)
raster_clip_path = f"/vsimem/raster_{uuid}_{str(idx)}_clipped.tif"
try:
clip_raster(
raster,
valid_path,
raster_clip_path,
all_touch=False,
adjust_bbox=False,
)
except Exception as e:
print(
f"Warning: {raster} did not intersect geom with fid: {fid}."
)
print(e)
if options["label_geom"] is not None:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
continue
arr = raster_to_array(raster_clip_path)
if arr.shape[:2] != valid_arr.shape[:2]:
raise Exception(
f"Error while matching array shapes. Raster: {arr.shape}, Valid: {valid_arr.shape}"
)
arr_offsets = get_overlaps(arr, offsets, tile_size,
options["border_check"])
arr = np.concatenate(arr_offsets)
valid_offsets = np.concatenate(
get_overlaps(valid_arr, offsets, tile_size,
options["border_check"]))
valid_mask = ((1 - (valid_offsets.sum(axis=(1, 2)) /
(tile_size * tile_size))) <=
options["tolerance"])[:, 0]
arr = arr[valid_mask]
valid_masked = valid_offsets[valid_mask]
if options["label_geom"] is not None and not labels_processed:
labels_masked = np.concatenate(
get_overlaps(labels_arr, offsets, tile_size,
options["border_check"]))[valid_mask]
if options["merge_output"]:
list_extracted.append(arr)
list_masks.append(valid_masked)
if options["label_geom"] is not None and not labels_processed:
list_labels.append(labels_masked)
else:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_{name}{options['postfix']}.npy",
arr.filled(options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}{str(fid)}_mask_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}{str(fid)}_label_{name}{options['postfix']}.npy",
valid_masked.filled(options["fill_value"]),
)
if fid not in processed_fids:
processed_fids.append(fid)
processed += 1
progress(processed,
len(fids) * len(raster_list), "processing fids")
if not options["merge_output"]:
gdal.Unlink(label_clip_path)
gdal.Unlink(label_ras_path)
gdal.Unlink(label_warp_path)
gdal.Unlink(fid_path)
gdal.Unlink(valid_path)
if options["merge_output"]:
np.save(
f"{outdir}{options['prefix']}{name}{options['postfix']}.npy",
np.ma.concatenate(list_extracted).filled(
options["fill_value"]),
)
np.save(
f"{outdir}{options['prefix']}mask_{name}{options['postfix']}.npy",
np.ma.concatenate(list_masks).filled(options["fill_value"]),
)
if options["label_geom"] is not None and not labels_processed:
np.save(
f"{outdir}{options['prefix']}label_{name}{options['postfix']}.npy",
np.ma.concatenate(list_labels).filled(
options["fill_value"]),
)
labels_processed = True
progress(1, 1, "processing fids")
return 1
def download_s1_tile(
scihub_username,
scihub_password,
onda_username,
onda_password,
destination,
footprint,
date=("20200601", "20210101"),
orbitdirection="ASCENDING", # ASCENDING, DESCENDING
min_overlap=0.50,
producttype="GRD",
sensoroperationalmode="IW",
polarisationmode="VV VH",
api_url="https://apihub.copernicus.eu/apihub/",
):
api = SentinelAPI(scihub_username, scihub_password, api_url, timeout=60)
if is_vector:
geom = internal_vector_to_metadata(footprint, create_geometry=True)
elif is_raster:
geom = raster_to_metadata(footprint, create_geometry=True)
products = api.query(
geom["extent_wkt_latlng"],
date=date,
platformname="Sentinel-1",
orbitdirection=orbitdirection,
producttype=producttype,
sensoroperationalmode=sensoroperationalmode,
polarisationmode=polarisationmode,
timeout=60,
)
download_products = OrderedDict()
download_ids = []
zero_contents = 0
geom_footprint = ogr.CreateGeometryFromWkt(geom["extent_wkt_latlng"])
for product in products:
dic = products[product]
img_footprint = ogr.CreateGeometryFromWkt(dic["footprint"])
img_area = img_footprint.GetArea()
intersection = img_footprint.Intersection(geom_footprint)
within = img_footprint.Intersection(intersection)
within_area = within.GetArea()
overlap_img = within_area / img_area
overlap_geom = within_area / geom_footprint.GetArea()
if max(overlap_img, overlap_geom) > min_overlap:
download_products[product] = dic
download_ids.append(product)
if len(download_products) > 0:
print(f"Downloading {len(download_products)} files.")
downloaded = []
for img_id in download_ids:
out_path = destination + download_products[img_id][
"filename"] + ".zip"
if os.path.exists(out_path):
print(f"Skipping {out_path}")
continue
download_url = f"https://catalogue.onda-dias.eu/dias-catalogue/Products({img_id})/$value"
try:
content_size = get_content_size(download_url,
out_path,
auth=(onda_username,
onda_password))
except Exception as e:
print(f"Failed to get content size for {img_id}")
print(e)
continue
if content_size == 0:
zero_contents += 1
print(
f"{img_id} requested from Archive but was not downloaded.")
continue
try:
if content_size > 0:
download(
download_url,
out_path,
auth=(onda_username, onda_password),
verbose=True,
)
downloaded.append(img_id)
except Exception as e:
print(f"Error downloading {img_id}: {e}")
return downloaded
else:
print("No images found")
return []
def add_border_to_raster(
input_raster,
out_path=None,
border_size=100,
border_size_unit="px",
border_value=0,
overwrite: bool = True,
creation_options: list = [],
):
in_raster = open_raster(input_raster)
metadata = raster_to_metadata(in_raster)
# Parse the driver
driver_name = "GTiff" if out_path is None else path_to_driver_raster(
out_path)
if driver_name is None:
raise ValueError(f"Unable to parse filetype from path: {out_path}")
driver = gdal.GetDriverByName(driver_name)
if driver is None:
raise ValueError(
f"Error while creating driver from extension: {out_path}")
output_name = None
if out_path is None:
output_name = f"/vsimem/raster_proximity_{uuid4().int}.tif"
else:
output_name = out_path
in_arr = raster_to_array(in_raster)
if border_size_unit == "px":
border_size_y = border_size
border_size_x = border_size
new_shape = (
in_arr.shape[0] + (2 * border_size_y),
in_arr.shape[1] + (2 * border_size_x),
in_arr.shape[2],
)
else:
border_size_y = round(border_size / metadata["pixel_height"])
border_size_x = round(border_size / metadata["pixel_width"])
new_shape = (
in_arr.shape[0] + (2 * border_size_y),
in_arr.shape[1] + (2 * border_size_x),
in_arr.shape[2],
)
new_arr = np.full(new_shape, border_value, dtype=in_arr.dtype)
new_arr[border_size_y:-border_size_y,
border_size_x:-border_size_x, :] = in_arr
if isinstance(in_arr, np.ma.MaskedArray):
mask = np.zeros(new_shape, dtype=bool)
mask[border_size_y:-border_size_y,
border_size_x:-border_size_x, :] = in_arr.mask
new_arr = np.ma.array(new_arr, mask=mask)
new_arr.fill_value = in_arr.fill_value
remove_if_overwrite(out_path, overwrite)
dest_raster = driver.Create(
output_name,
new_shape[1],
new_shape[0],
metadata["band_count"],
numpy_to_gdal_datatype(in_arr.dtype),
default_options(creation_options),
)
og_transform = in_raster.GetGeoTransform()
new_transform = []
for i in og_transform:
new_transform.append(i)
new_transform[0] -= border_size_x * og_transform[1]
new_transform[3] -= border_size_y * og_transform[5]
dest_raster.SetGeoTransform(new_transform)
dest_raster.SetProjection(in_raster.GetProjectionRef())
for band_num in range(1, metadata["band_count"] + 1):
dst_band = dest_raster.GetRasterBand(band_num)
dst_band.WriteArray(new_arr[:, :, band_num - 1])
if metadata["has_nodata"]:
dst_band.SetNoDataValue(metadata["nodata_value"])
return output_name
def raster_to_grid(
raster: Union[str, gdal.Dataset],
grid: Union[str, ogr.DataSource],
out_dir: str,
use_field: Optional[str] = None,
generate_vrt: bool = True,
overwrite: bool = True,
process_layer: int = 0,
creation_options: list = [],
verbose: int = 1,
) -> Union[List[str], Tuple[Optional[List[str]], Optional[str]]]:
"""Clips a raster to a grid. Generate .vrt.
Returns:
The filepath for the newly created raster.
"""
type_check(raster, [str, gdal.Dataset], "raster")
type_check(grid, [str, ogr.DataSource], "grid")
type_check(out_dir, [str], "out_dir")
type_check(overwrite, [bool], "overwrite")
type_check(process_layer, [int], "process_layer")
type_check(creation_options, [list], "creation_options")
type_check(verbose, [int], "verbose")
use_grid = open_vector(grid)
grid_metadata = internal_vector_to_metadata(use_grid)
raster_metadata = raster_to_metadata(raster, create_geometry=True)
# Reproject raster if necessary.
if not raster_metadata["projection_osr"].IsSame(grid_metadata["projection_osr"]):
use_grid = reproject_vector(grid, raster_metadata["projection_osr"])
grid_metadata = internal_vector_to_metadata(use_grid)
if not isinstance(grid_metadata, dict):
raise Exception("Error while parsing metadata.")
# Only use the polygons in the grid that intersect the extent of the raster.
use_grid = intersect_vector(use_grid, raster_metadata["extent_datasource"])
ref = open_raster(raster)
use_grid = open_vector(use_grid)
layer = use_grid.GetLayer(process_layer)
feature_count = layer.GetFeatureCount()
raster_extent = raster_metadata["extent_ogr"]
filetype = path_to_ext(raster)
name = raster_metadata["name"]
geom_type = grid_metadata["layers"][process_layer]["geom_type_ogr"]
if use_field is not None:
if use_field not in grid_metadata["layers"][process_layer]["field_names"]:
names = grid_metadata["layers"][process_layer]["field_names"]
raise ValueError(
f"Requested field not found. Fields available are: {names}"
)
generated = []
# For the sake of good reporting - lets first establish how many features intersect
# the raster.
if verbose:
print("Finding intersections.")
intersections = 0
for _ in range(feature_count):
feature = layer.GetNextFeature()
geom = feature.GetGeometryRef()
if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()):
continue
intersections += 1
layer.ResetReading()
if verbose:
print(f"Found {intersections} intersections.")
if intersections == 0:
print("Warning: Found 0 intersections. Returning empty list.")
return ([], None)
# TODO: Replace this in gdal. 3.1
driver = ogr.GetDriverByName("Esri Shapefile")
clipped = 0
for _ in range(feature_count):
feature = layer.GetNextFeature()
geom = feature.GetGeometryRef()
if not ogr_bbox_intersects(raster_extent, geom.GetEnvelope()):
continue
if verbose == 1:
progress(clipped, intersections - 1, "clip_grid")
fid = feature.GetFID()
test_ds_path = f"/vsimem/grid_{uuid4().int}.shp"
test_ds = driver.CreateDataSource(test_ds_path)
test_ds_lyr = test_ds.CreateLayer(
"mem_layer_grid",
geom_type=geom_type,
srs=raster_metadata["projection_osr"],
)
test_ds_lyr.CreateFeature(feature.Clone())
test_ds.FlushCache()
out_name = None
if use_field is not None:
out_name = f"{out_dir}{feature.GetField(use_field)}{filetype}"
else:
out_name = f"{out_dir}{name}_{fid}{filetype}"
clip_raster(
ref,
test_ds_path,
out_path=out_name,
adjust_bbox=True,
crop_to_geom=True,
all_touch=False,
postfix="",
prefix="",
creation_options=default_options(creation_options),
verbose=0,
)
generated.append(out_name)
clipped += 1
if generate_vrt:
vrt_name = f"{out_dir}{name}.vrt"
stack_rasters_vrt(generated, vrt_name, seperate=False)
return (generated, vrt_name)
return generated
