def test_bytes_rio():
# Test RGB
output_profile = dict(
driver="GTiff",
dtype=img_rgb_true.dtype,
count=img_rgb_true.shape[0],
height=img_rgb_true.shape[1],
width=img_rgb_true.shape[2],
)
with rasterio.MemoryFile() as memfile:
with memfile.open(**output_profile) as dst:
dst.write(img_rgb_true)
bytes_buffer = memfile.read()
pesto_bytes_rgb = PestoImage.from_bytes(bytes_buffer).to_array()
# Test RGBN
output_profile = dict(
driver="GTiff",
dtype=img_rgbn_true.dtype,
count=img_rgbn_true.shape[0],
height=img_rgbn_true.shape[1],
width=img_rgbn_true.shape[2],
)
with rasterio.MemoryFile() as memfile:
with memfile.open(**output_profile) as dst:
dst.write(img_rgbn_true)
bytes_buffer = memfile.read()
pesto_bytes_rgbn = PestoImage.from_bytes(bytes_buffer).to_array()
assert np.all(img_rgb_true == pesto_bytes_rgb)
assert np.all(img_rgbn_true == pesto_bytes_rgbn)
def to_dataset(lyr: str) -> xr.DataArray:
with rio.MemoryFile() as memfile:
memfile.write(r_dict[lyr])
with memfile.open() as src:
geom = [_geometry.intersection(box(*src.bounds))]
if geom[0].is_empty:
msk, transform, _ = rio_mask.raster_geometry_mask(
src, [_geometry], invert=True)
else:
msk, transform, _ = rio_mask.raster_geometry_mask(
src, geom, invert=True)
meta = src.meta
meta.update({
"driver": "GTiff",
"height": msk.shape[0],
"width": msk.shape[1],
"transform": transform,
"nodata": nodata,
})
with rio.vrt.WarpedVRT(src, **meta) as vrt:
ds = xr.open_rasterio(vrt)
try:
ds = ds.squeeze("band", drop=True)
except ValueError:
pass
coords = {
ds_dims[0]: ds.coords[ds_dims[0]],
ds_dims[1]: ds.coords[ds_dims[1]]
}
msk_da = xr.DataArray(msk, coords, dims=ds_dims)
ds = ds.where(msk_da, drop=True)
ds.attrs["crs"] = r_crs.to_string()
ds.name = var_name[lyr]
return ds
def calc_region(region, Crd_reg, res_desired, GeoRef):
"""
This function reads the region geometry, and returns a masking raster equal to 1 for pixels within and 0 outside of
the region.
:param region: Region geometry
:type region: Geopandas series
:param Crd_reg: Coordinates of the region
:type Crd_reg: list
:param res_desired: Desired high resolution of the output raster
:type res_desired: list
:param GeoRef: Georeference dictionary containing *RasterOrigin*, *RasterOrigin_alt*, *pixelWidth*, and *pixelHeight*.
:type GeoRef: dict
:return A_region: Masking raster of the region.
:rtype: numpy array
"""
latlim = Crd_reg[2] - Crd_reg[0]
lonlim = Crd_reg[3] - Crd_reg[1]
M = int(math.fabs(latlim) / res_desired[0])
N = int(math.fabs(lonlim) / res_desired[1])
A_region = np.ones((M, N))
origin = [Crd_reg[3], Crd_reg[2]]
if region['geometry'].geom_type == "MultiPolygon":
features = [feature for feature in region['geometry']]
else:
features = [region['geometry']]
west = origin[0]
south = origin[1]
profile = {
"driver":
"GTiff",
"height":
M,
"width":
N,
"count":
1,
"dtype":
rasterio.float64,
"crs":
"EPSG:4326",
"transform":
rasterio.transform.from_origin(west, south, GeoRef["pixelWidth"],
GeoRef["pixelHeight"]),
}
with rasterio.MemoryFile() as memfile:
with memfile.open(**profile) as f:
f.write(A_region, 1)
out_image, out_transform = rasterio.mask.mask(f,
features,
crop=False,
nodata=0,
all_touched=False,
filled=True)
A_region = out_image[0]
return A_region
def _compress_image(im: np.ndarray,
driver='PNG',
**opts) -> bytes:
import rasterio
import warnings
if im.dtype != np.uint8:
raise ValueError("Only support uint8 images on input")
if im.ndim == 3:
h, w, nc = im.shape
bands = np.transpose(im, axes=(2, 0, 1))
elif im.ndim == 2:
(h, w), nc = im.shape, 1
bands = im.reshape(nc, h, w)
else:
raise ValueError('Expect 2 or 3 dimensional array got: {}'.format(im.ndim))
rio_opts = dict(width=w,
height=h,
count=nc,
driver=driver,
dtype='uint8',
**opts)
with warnings.catch_warnings():
warnings.simplefilter('ignore', rasterio.errors.NotGeoreferencedWarning)
with rasterio.MemoryFile() as mem:
with mem.open(**rio_opts) as dst:
dst.write(bands)
return mem.read()
def test_plot_grid_data():
x0 = (0, 0)
n = 32
dx = 1.0 / n
transform = rasterio.transform.from_origin(west=0.0,
north=1.0,
xsize=dx,
ysize=dx)
# Interpolate a scalar field
array = np.array([[dx * (i + j) for j in range(n + 1)]
for i in range(n + 1)])
missing = -9999.0
array[0, 0] = missing
array = np.flipud(array)
memfile = rasterio.MemoryFile(ext='.tif')
opts = {
'driver': 'GTiff',
'count': 1,
'width': n,
'height': n,
'transform': transform,
'nodata': -9999
}
with memfile.open(**opts) as dataset:
dataset.write(array, indexes=1)
dataset = memfile.open()
levels = np.linspace(-0.5, 0.5, 5)
contours = icepack.plot.contourf(dataset, levels=levels)
assert contours is not None
colorbar = plt.colorbar(contours)
assert colorbar is not None
def _create_id_grid(
xmin: float,
ymin: float,
xmax: float,
ymax: float,
resolution: float,
crs: str = "epsg:4326",
) -> rasterio.io.DatasetWriter:
"""
Creates an in-memory raster with a grid where each pixel has a unique
ID. The unique IDs start at 0 in the upper left corner and increments
from left to right and top to bottom. The max value for unique ID
will be (height * width) - 1.
Parameters
----------
xmin : float
Upper-left corner x coordinate.
ymin : float
Lower-right corner y coordinate.
xmax : float
Lower-left corner x coordinate.
ymax : float
Upper-left corner y coordinate.
resolution : float
Pixel resolution.
crs : str
Coordinate Reference System. Must be in the form epsg:code.
Returns
-------
DatasetWriter
In-memory raster with unique IDs.
Notes
-----
Coordinates and resolution should match with the reference system
passed in crs.
"""
height = np.ceil((ymax - ymin) / resolution).astype(int)
width = np.ceil((xmax - xmin) / resolution).astype(int)
transform = rasterio.transform.from_origin(xmin, ymax, resolution,
resolution)
arr = np.arange(height * width, dtype=np.uint32).reshape(height, width)
memfile = rasterio.MemoryFile()
grid = memfile.open(
driver="MEM",
height=height,
width=width,
count=1,
crs=crs,
transform=transform,
dtype=rasterio.uint32,
)
grid.write(arr, 1)
return grid
def _as_rasterio_dataset(self):
# create in-memory rasterio dataset
prof = self._default_rasterio_profile()
memfile = rasterio.MemoryFile()
with memfile.open(**prof) as ds:
ds.write(self._data(), 1)
ds = memfile.open()
return ds
def _ssebop(url: str) -> List[np.ndarray]: # type: ignore
r = session.get(url)
z = zipfile.ZipFile(io.BytesIO(r.content))
with rio.MemoryFile() as memfile:
memfile.write(z.read(z.filelist[0].filename))
with memfile.open() as src:
return list(src.sample(co_list))
def test_notgeoref_warning():
with rasterio.MemoryFile() as mem:
with mem.open(driver='GTiff',
width=10,
height=10,
dtype='uint8',
count=1) as src:
pass
with pytest.warns(NotGeoreferencedWarning):
with mem.open() as dst:
pass
def _ssebop(urls):
dt, url = urls
r = session.get(url)
z = zipfile.ZipFile(io.BytesIO(r.content))
with rio.MemoryFile() as memfile:
memfile.write(z.read(z.filelist[0].filename))
with memfile.open() as src:
return {
"dt": dt,
"eta": [e[0] for e in src.sample([(lon, lat)])][0],
}
def _sample_tiff(
content: bytes,
coords: Union[List[Tuple[float, float]], Iterator[Tuple[float, float]]],
crs: str,
resampling: rio_warp.Resampling,
) -> np.ndarray:
"""Sample a tiff response for a list of coordinates.
Parameters
----------
content : bytes
coords : list of tuples
A list containing x- and y-coordinates of a mesh, [(x, y), ...].
crs : str
The spatial reference system of the input grid, defaults to epsg:4326.
resolution : float
resampling : rasterio.warp.Resampling
The reasmpling method to use if the input crs is not in the supported
3DEP's CRS list which are epsg:4326 and epsg:3857. It defaults to bilinear.
The available methods can be found `here <https://rasterio.readthedocs.io/en/latest/api/rasterio.enums.html#rasterio.enums.Resampling>`__
Returns
-------
numpy.ndarray
An array of elevations where its index matches the input coords list
"""
with rio.MemoryFile() as memfile:
memfile.write(content)
with memfile.open() as src:
transform, width, height = rio_warp.calculate_default_transform(
src.crs, crs, src.width, src.height, *src.bounds)
kwargs = src.meta.copy()
kwargs.update({
"crs": crs,
"transform": transform,
"width": width,
"height": height
})
with rio.vrt.WarpedVRT(src, **kwargs) as vrt:
if crs != src.crs:
for i in range(1, src.count + 1):
rio_warp.reproject(
source=rio.band(src, i),
destination=rio.band(vrt, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
crs=crs,
resampling=resampling,
)
return np.array([e.item() for e in vrt.sample(coords)])
def create_image(template, data, transform=None):
log.debug('Creating new image')
transform = transform or template.transform
profile = template.profile.copy()
profile.update({DRIVER: GTIFF,
COUNT: data.shape[0],
HEIGHT: data.shape[1],
WIDTH: data.shape[2],
TRANSFORM: transform,
PHOTOMETRIC: RGB})
new_image = rio.MemoryFile().open(**profile)
new_image.write(data)
return new_image
def convex_hull_exact(src):
kwargs = dict(bidx=1, band=False, as_mask=True, geographic=True)
data = src.read()
if np.any(np.isnan(data)) and src.nodata is not None:
# hack: replace NaNs with nodata to make sure they are excluded
with rasterio.MemoryFile() as memfile, memfile.open(**src.profile) as tmpsrc:
data[np.isnan(data)] = src.nodata
tmpsrc.write(data)
dataset_shape = list(rasterio.features.dataset_features(tmpsrc, **kwargs))
else:
dataset_shape = list(rasterio.features.dataset_features(src, **kwargs))
convex_hull = MultiPolygon([shape(s['geometry']) for s in dataset_shape]).convex_hull
return convex_hull
def _recompress_tar_member(
readable_member: ReadableMember,
out_tar: tarfile.TarFile,
compress_args: Dict,
verify: PackageChecksum,
tmpdir: Path,
):
member, open_member = readable_member
new_member = copy.copy(member)
# Copy with a minimum 664 permission, which is used by USGS tars.
# (some of our repacked datasets have only user read permission.)
new_member.mode = new_member.mode | 0o664
# If it's a tif, check whether it's compressed.
if member.name.lower().endswith(".tif"):
with open_member() as input_fp, rasterio.open(input_fp) as ds:
if not ds.profile.get("compress"):
# No compression: let's compress it
with rasterio.MemoryFile(filename=member.name) as memory_file:
try:
_recompress_image(ds, memory_file, **compress_args)
except Exception as e:
raise RecompressFailure(
f"Error during {member.name}") from e
new_member.size = memory_file.getbuffer().nbytes
out_tar.addfile(new_member, memory_file)
# Image has been written. Seek to beginning to take a checksum.
memory_file.seek(0)
verify.add(memory_file, tmpdir / new_member.name)
return
else:
# It's already compressed, we'll fall through and copy it verbatim.
pass
if member.size == 0:
# Typically a directory entry.
out_tar.addfile(new_member)
return
# Copy unchanged into target (typically text/metadata files).
with open_member() as member:
file_contents = member.read()
out_tar.addfile(new_member, io.BytesIO(file_contents))
verify.add(io.BytesIO(file_contents), tmpdir / new_member.name)
del file_contents
def to_src(arr, metadata):
if metadata['driver'] != 'GTiff':
metadata['driver'] = 'GTiff'
with ExitStack() as stack:
memfile = stack.enter_context(rasterio.MemoryFile())
with memfile.open(**metadata) as data: # Open as DatasetWriter
if arr.ndim == 2:
data.write_band(1, arr)
logger.debug(f"Saved band and metadata as DataSetWriter")
else:
data.write(arr.astype(metadata['dtype']))
logger.debug(f"Saved array and metadata as DataSetWriter")
del arr
with memfile.open() as data: # Reopen as DatasetReader
yield data
def get_elevation_bybbox(bbox, coords):
"""Get elevation from DEM data for a list of coordinates.
The elevations are extracted from SRTM1 (30-m resolution) data.
This function is intended for getting elevations for a gridded dataset.
Parameters
----------
bbox : list
Bounding box with coordinates in [west, south, east, north] format.
coords : list of tuples
A list of coordinates in (lon, lat) format to extract the elevation.
Returns
-------
array_like
A numpy array of elevations in meters
"""
import rasterio
west, south, east, north = bbox
url = "https://portal.opentopography.org/otr/getdem"
payload = dict(
demtype="SRTMGL1",
west=round(west, 6),
south=round(south, 6),
east=round(east, 6),
north=round(north, 6),
outputFormat="GTiff",
)
session = retry_requests()
try:
r = session.get(url, params=payload)
except HTTPError or ConnectionError or Timeout or RequestException:
raise
with rasterio.MemoryFile() as memfile:
memfile.write(r.content)
with memfile.open() as src:
elevations = np.array([e[0] for e in src.sample(coords)],
dtype=np.float32)
return elevations
def to_bytes(self) -> bytes:
driver = ImageDriver.TIFF if self.bands() > 3 else ImageDriver.PNG
output_profile = dict(
driver=driver.driver,
width=self.width(),
height=self.height(),
count=self.bands(),
dtype=self.array.dtype,
)
with rasterio.MemoryFile() as memfile:
with memfile.open(**output_profile) as dst:
dst.write(self.array)
buffer = memfile.read()
return buffer
def force_same_scaling(images, choose=first):
footprints = {image: shape_to_polygon(next(shapes(image.dataset_mask(), transform=image.transform)))
for image in images}
reference, rest = split(images, choose)
scaled = [reference]
while rest:
candidate = most_intersecting(footprints, scaled, rest)
rest.remove(candidate)
scale = measure_scaling(footprints, scaled, candidate)
log.info(f'Scaling {candidate} by {scale}')
profile = candidate.profile
profile.update({'photometric': 'RGB'})
copy = rio.MemoryFile().open(**profile)
types = {i: dtype for i, dtype in zip(candidate.indexes, candidate.dtypes)}
for i in range(1, 4):
copy.write((candidate.read(i) * scale).astype(types[i]), i)
scaled.append(copy)
return scaled
def reproject_to_memory(image, crs):
# https://rasterio.readthedocs.io/en/stable/topics/reproject.html
transform, width, height = calculate_default_transform(image.crs, crs, image.width, image.height, *image.bounds)
profile = image.profile.copy()
profile.update({CRS: crs,
TRANSFORM: transform,
WIDTH: width,
HEIGHT: height})
transformed = rio.MemoryFile().open(**profile)
for i in range(1, image.count + 1):
reproject(source=rio.band(image, i),
destination=rio.band(transformed, i),
src_transform=image.transform,
src_crs=image.crs,
dst_transform=transform,
dst_crs=crs,
resampling=Resampling.nearest)
return transformed
def test_no_notgeoref_warning(transform, gcps, rpcs):
with rasterio.MemoryFile() as mem:
with mem.open(driver='GTiff',
width=10,
height=10,
dtype='uint8',
count=1,
transform=transform) as src:
if gcps:
src.gcps = (gcps, rasterio.crs.CRS.from_epsg(4326))
if rpcs:
src.rpcs = rpcs
with pytest.warns(None) as record:
with mem.open() as dst:
pass
assert len(record) == 0
def test_init(self):
"""
Test that all possible inputs work properly in Raster class init
"""
# first, filename
r = gr.Raster(datasets.get_path("landsat_B4"))
assert isinstance(r, gr.Raster)
# second, passing a Raster itself (points back to Raster passed)
r2 = gr.Raster(r)
assert isinstance(r2, gr.Raster)
# third, rio.Dataset
ds = rio.open(datasets.get_path("landsat_B4"))
r3 = gr.Raster(ds)
assert isinstance(r3, gr.Raster)
assert r3.filename is not None
# finally, as memoryfile
memfile = rio.MemoryFile(open(datasets.get_path("landsat_B4"), 'rb'))
r4 = gr.Raster(memfile)
assert isinstance(r4, gr.Raster)
assert np.logical_and.reduce((np.array_equal(r.data,
r2.data,
equal_nan=True),
np.array_equal(r2.data,
r3.data,
equal_nan=True),
np.array_equal(r3.data,
r4.data,
equal_nan=True)))
assert np.logical_and.reduce((np.all(r.data.mask == r2.data.mask),
np.all(r2.data.mask == r3.data.mask),
np.all(r3.data.mask == r4.data.mask)))
# the data will not be copied, immutable objects will
r.data[0, 0, 0] += 5
assert r2.data[0, 0, 0] == r.data[0, 0, 0]
r.nbands = 2
assert r.nbands != r2.nbands
def make_rio_dataset(array, missing=-9999.0):
ny = array.shape[0] - 1
nx = array.shape[1] - 1
transform = rasterio.transform.from_origin(
west=0.0, north=1.0, xsize=1 / nx, ysize=1 / ny
)
memfile = rasterio.MemoryFile(ext=".tif")
opts = {
"driver": "GTiff",
"count": 1,
"width": nx + 1,
"height": ny + 1,
"dtype": array.dtype,
"transform": transform,
"nodata": missing,
}
with memfile.open(**opts) as dataset:
dataset.write(array, indexes=1)
return memfile.open()
def make_rio_dataset(array, missing=-9999.):
ny = array.shape[0] - 1
nx = array.shape[1] - 1
transform = rasterio.transform.from_origin(west=0.0,
north=1.0,
xsize=1 / nx,
ysize=1 / ny)
memfile = rasterio.MemoryFile(ext='.tif')
opts = {
'driver': 'GTiff',
'count': 1,
'width': nx + 1,
'height': ny + 1,
'dtype': array.dtype,
'transform': transform,
'nodata': missing
}
with memfile.open(**opts) as dataset:
dataset.write(array, indexes=1)
return memfile.open()
def reproject(content):
with rio.MemoryFile() as memfile:
memfile.write(content)
with memfile.open() as src:
transform, width, height = rio_warp.calculate_default_transform(
src.crs, crs, src.width, src.height, *src.bounds
)
kwargs = src.meta.copy()
kwargs.update(
{"crs": crs, "transform": transform, "width": width, "height": height}
)
with rio.vrt.WarpedVRT(src, **kwargs) as vrt:
if crs != src.crs:
for i in range(1, src.count + 1):
rio_warp.reproject(
source=rio.band(src, i),
destination=rio.band(vrt, i),
src_transform=src.transform,
src_crs=src.crs,
dst_transform=transform,
crs=crs,
resampling=resampling,
)
da = xr.open_rasterio(vrt)
try:
da = da.squeeze("band", drop=True)
except ValueError:
pass
da.name = "elevation"
da.attrs["transform"] = transform
da.attrs["res"] = (transform[0], transform[4])
da.attrs["bounds"] = tuple(vrt.bounds)
da.attrs["nodatavals"] = vrt.nodatavals
da.attrs["crs"] = vrt.crs.to_string()
return da
def gridme(csvfile, outfile, chunksize, decimal_comma=False, **kwargs):
"""Grid point cloud to UTM grid"""
from pointtogrid import peskycsv
import rasterio
from rio_cogeo import cogeo
# define csv parsing settings
csvkw = dict(chunksize=chunksize)
if decimal_comma:
csvkw.update(decimal=',')
data, profile = peskycsv.flow(path=csvfile,
show_pbar=True,
csvkw=csvkw,
**kwargs)
profile = dict(profile, **COG_PROFILE)
with rasterio.MemoryFile() as memfile:
with memfile.open(**profile) as mem:
mem.write(data, indexes=1)
cogeo.cog_translate(mem, outfile, profile)
from citycatio import Model
import pandas as pd
import unittest
import rasterio as rio
import numpy as np
from rasterio.transform import Affine
import geopandas as gpd
from shapely.geometry import Polygon
dem_file = rio.MemoryFile()
x_min, y_max = 100, 500
res = 5
height, width = 100, 200
x_max = x_min + width * res
y_min = y_max - height * res
array = np.round(np.random.random((height, width)), 3)
transform = Affine.translation(x_min, y_max) * Affine.scale(res, -res)
with rio.open(
dem_file,
'w',
driver='GTiff',
height=height,
width=width,
count=1,
dtype=array.dtype,
transform=transform,
nodata=-9999
) as dst:
dst.write(array, 1)
def write_cog(fname,
pix,
overwrite=False,
blocksize=None,
overview_resampling=None,
overview_levels=None,
**extra_rio_opts):
""" Write xarray.Array to GeoTiff file.
"""
from pathlib import Path
import rasterio
from rasterio.shutil import copy as rio_copy
if blocksize is None:
blocksize = 512
if overview_levels is None:
overview_levels = [2**i for i in range(1, 6)]
if overview_resampling is None:
overview_resampling = 'nearest'
nodata = pix.attrs.get('nodata', None)
resampling = rasterio.enums.Resampling[overview_resampling]
if pix.ndim == 2:
h, w = pix.shape
nbands = 1
band = 1
elif pix.ndim == 3:
nbands, h, w = pix.shape
band = tuple(i for i in range(1, nbands+1))
else:
raise ValueError('Need 2d or 3d ndarray on input')
if not isinstance(fname, Path):
fname = Path(fname)
if fname.exists():
if overwrite:
fname.unlink()
else:
raise IOError("File exists")
gbox = pix.geobox
if gbox is None:
raise ValueError("Not geo-registered: check crs attribute")
assert gbox.shape == (h, w)
A = gbox.transform
crs = str(gbox.crs)
rio_opts = dict(width=w,
height=h,
count=nbands,
dtype=pix.dtype.name,
crs=crs,
transform=A,
tiled=True,
blockxsize=min(blocksize, w),
blockysize=min(blocksize, h),
zlevel=9,
predictor=3 if pix.dtype.kind == 'f' else 2,
compress='DEFLATE')
if nodata is not None:
rio_opts.update(nodata=nodata)
rio_opts.update(extra_rio_opts)
# copy re-compresses anyway so skip compression for temp image
tmp_opts = rio_opts.copy()
tmp_opts.pop('compress')
tmp_opts.pop('predictor')
tmp_opts.pop('zlevel')
with rasterio.Env(GDAL_TIFF_OVR_BLOCKSIZE=blocksize):
with rasterio.MemoryFile() as mem:
with mem.open(driver='GTiff', **tmp_opts) as tmp:
tmp.write(pix.values, band)
tmp.build_overviews(overview_levels, resampling)
rio_copy(tmp,
fname,
driver='GTiff',
copy_src_overviews=True,
**rio_opts)
def test_interpolating_to_mesh():
# Make the mesh the square `[1/4, 3/4] x [1/4, 3/4]`
nx, ny = 32, 32
mesh = firedrake.UnitSquareMesh(nx, ny)
x, y = firedrake.SpatialCoordinate(mesh)
Vc = mesh.coordinates.function_space()
f = firedrake.interpolate(
firedrake.as_vector((x / 2 + 1 / 4, y / 2 + 1 / 4)), Vc)
mesh.coordinates.assign(f)
# Set up the geometry of the gridded data set
x0 = (0, 0)
n = 32
dx = 1.0 / n
transform = rasterio.transform.from_origin(west=0.0,
north=1.0,
xsize=dx,
ysize=dx)
# Interpolate a scalar field
array = np.array([[dx * (i + j) for j in range(n + 1)]
for i in range(n + 1)])
missing = -9999.0
array[0, 0] = missing
array = np.flipud(array)
memfile = rasterio.MemoryFile(ext='.tif')
opts = {
'driver': 'GTiff',
'count': 1,
'width': n,
'height': n,
'transform': transform,
'nodata': -9999
}
with memfile.open(**opts) as dataset:
dataset.write(array, indexes=1)
dataset = memfile.open()
Q = firedrake.FunctionSpace(mesh, family='CG', degree=1)
p = firedrake.interpolate(x + y, Q)
q = icepack.interpolate(dataset, Q)
assert firedrake.norm(p - q) / firedrake.norm(p) < 1 / n
# Interpolate a vector field
array_vx = np.copy(array)
array_vy = np.array([[dx * (j - i) for j in range(n + 1)]
for i in range(n + 1)])
array_vy[-1, -1] = -9999.0
array_vy = np.flipud(array_vy)
memfile_vx, memfile_vy = rasterio.MemoryFile(), rasterio.MemoryFile()
with memfile_vx.open(**opts) as vx, memfile_vy.open(**opts) as vy:
vx.write(array_vx, indexes=1)
vy.write(array_vy, indexes=1)
vx, vy = memfile_vx.open(), memfile_vy.open()
V = firedrake.VectorFunctionSpace(mesh, family='CG', degree=1)
u = firedrake.interpolate(firedrake.as_vector((x + y, x - y)), V)
v = icepack.interpolate((vx, vy), V)
assert firedrake.norm(u - v) / firedrake.norm(u) < 1 / n
def ssebopeta_bygeom(geometry,
start=None,
end=None,
years=None,
resolution=None):
"""Gridded data from the SSEBop database.
Note
----
Since there's still no web service available for subsetting, the data first
needs to be downloads for the requested period then the data is masked by the
region interest locally. Therefore, it's not as fast as other functions and
the bottleneck could be download speed.
Parameters
----------
geometry : Geometry
The geometry for downloading clipping the data. For a box geometry,
the order should be as follows:
geom = box(minx, miny, maxx, maxy)
start : string or datetime
Starting date
end : string or datetime
Ending date
years : list
List of years
resolution : float
The desired output resolution for the output in decimal degree,
defaults to no resampling. The resampling is done using bilinear method
Returns
-------
xarray.DataArray
The actual ET for the requested region.
"""
from shapely.geometry import Polygon
import socket
from unittest.mock import patch
import zipfile
import io
if not isinstance(geometry, Polygon):
raise TypeError("Geometry should be of type Shapely Polygon.")
if years is None and start is not None and end is not None:
if pd.to_datetime(start) < pd.to_datetime("2000-01-01"):
raise ValueError("SSEBop database ranges from 2000 till 2018.")
elif years is not None and start is None and end is None:
years = years if isinstance(years, list) else [years]
dates = [pd.date_range(f"{year}0101", f"{year}1231") for year in years]
for d in dates:
if d[0] < pd.to_datetime("2000-01-01"):
raise ValueError("SSEBop database ranges from 2000 till 2018.")
else:
raise ValueError(
"Either years or start and end arguments should be provided.")
base_url = "https://edcintl.cr.usgs.gov/downloads/sciweb1/shared//uswem/web/conus/eta/modis_eta/daily/downloads"
f_list = [(d,
f"{base_url}/det{d.strftime('%Y%j')}.modisSSEBopETactual.zip")
for d in pd.date_range(start, end)]
print(
f"[CNT: ({geometry.centroid.x:.2f}, {geometry.centroid.y:.2f})] ".
ljust(MARGINE) + f"Downloading the data from SSEBop",
end=" >>> ",
)
orig_getaddrinfo = socket.getaddrinfo
session = utils.retry_requests()
def getaddrinfoIPv4(host, port, family=0, type=0, proto=0, flags=0):
return orig_getaddrinfo(
host=host,
port=port,
family=socket.AF_INET,
type=type,
proto=proto,
flags=flags,
)
# disable IPv6 to speedup the download
with patch("socket.getaddrinfo", side_effect=getaddrinfoIPv4):
# find the mask using the first dataset
dt, url = f_list[0]
try:
r = session.get(url)
except HTTPError or ConnectionError or Timeout or RequestException:
raise
z = zipfile.ZipFile(io.BytesIO(r.content))
with rasterio.MemoryFile() as memfile:
memfile.write(z.read(z.filelist[0].filename))
with memfile.open() as src:
ras_msk, _ = rasterio.mask.mask(src, [geometry])
nodata = src.nodata
with xr.open_rasterio(src) as ds:
ds.data = ras_msk
msk = ds < nodata if nodata > 0.0 else ds > nodata
ds = ds.where(msk, drop=True)
ds = ds.expand_dims(dict(time=[dt]))
ds = ds.squeeze("band", drop=True)
ds.name = "eta"
data = ds * 1e-3
# apply the mask to the rest of the data and merge
for dt, url in f_list[1:]:
try:
r = session.get(url)
except HTTPError or ConnectionError or Timeout or RequestException:
raise
z = zipfile.ZipFile(io.BytesIO(r.content))
with rasterio.MemoryFile() as memfile:
memfile.write(z.read(z.filelist[0].filename))
with memfile.open() as src:
with xr.open_rasterio(src) as ds:
ds = ds.where(msk, drop=True)
ds = ds.expand_dims(dict(time=[dt]))
ds = ds.squeeze("band", drop=True)
ds.name = "eta"
data = xr.merge([data, ds * 1e-3])
data["eta"].attrs["units"] = "mm/day"
if resolution is not None:
fac = resolution * 3600.0 / 30.0 # from degree to 1 km
new_x = np.linspace(data.x[0], data.x[-1], int(data.dims["x"] / fac))
new_y = np.linspace(data.y[0], data.y[-1], int(data.dims["y"] / fac))
data = data.interp(x=new_x, y=new_y, method="linear")
print("finished.")
return data
def dem_bygeom(geometry, demtype="SRTMGL1", resolution=None, output=None):
"""Get DEM data from `OpenTopography <https://opentopography.org/>`_ service.
Parameters
----------
geometry : Geometry
A shapely Polygon.
demtype : string, optional
The type of DEM to be downloaded, default to SRTMGL1 for 30 m resolution.
Available options are 'SRTMGL3' for SRTM GL3 (3 arc-sec or ~90m) and 'SRTMGL1' for
SRTM GL1 (1 arc-sec or ~30m).
resolution : float, optional
The desired output resolution for the output in decimal degree,
defaults to no resampling. The resampling is done using cubic convolution method
output : string or Path, optional
The path to save the data as raster, defaults to None.
Returns
-------
xarray.DataArray
DEM in meters.
"""
import rasterio
import rasterio.mask
from shapely.geometry import Polygon
from rasterio.enums import Resampling
import os
if not isinstance(geometry, Polygon):
raise TypeError("Geometry should be of type Shapely Polygon.")
west, south, east, north = geometry.bounds
url = "https://portal.opentopography.org/otr/getdem"
payload = dict(
demtype=demtype,
west=round(west, 6),
south=round(south, 6),
east=round(east, 6),
north=round(north, 6),
outputFormat="GTiff",
)
print(
f"[CNT: ({geometry.centroid.x:.2f}, {geometry.centroid.y:.2f})] ".
ljust(MARGINE) + f"Downloading DEM data from OpenTopography",
end=" >>> ",
)
session = utils.retry_requests()
try:
r = session.get(url, params=payload)
except ConnectionError or Timeout or RequestException:
raise
with rasterio.MemoryFile() as memfile:
memfile.write(r.content)
with memfile.open() as src:
if resolution is not None:
fac = resolution * 3600.0 # degree to arc-sec since res is 1 arc-sec
data = src.read(
out_shape=(src.count, int(src.width / fac),
int(src.height / fac)),
resampling=Resampling.bilinear,
)
transform = src.transform * src.transform.scale(
(src.width / data.shape[1]), (src.height / data.shape[2]))
meta = src.meta
meta.update({
"driver": "GTiff",
"width": data.shape[1],
"height": data.shape[2],
"transform": transform,
})
with rasterio.open("/tmp/resampled.tif", "w", **meta) as tmp:
tmp.write(data)
with rasterio.open("/tmp/resampled.tif", "r") as tmp:
ras_msk, transform = rasterio.mask.mask(tmp, [geometry])
nodata = tmp.nodata
dest = "/tmp/resampled.tif"
meta = tmp.meta
meta.update({
"driver": "GTiff",
"width": ras_msk.shape[1],
"height": ras_msk.shape[2],
"transform": transform,
})
else:
ras_msk, transform = rasterio.mask.mask(src, [geometry])
nodata = src.nodata
dest = src
meta = src.meta
meta.update({
"driver": "GTiff",
"width": ras_msk.shape[1],
"height": ras_msk.shape[2],
"transform": transform,
})
if output is not None:
data_dir = Path(output).parent
if not data_dir.is_dir():
try:
os.makedirs(data_dir)
except OSError:
print(
f"[CNT: ({geometry.centroid.x:.2f}, {geometry.centroid.y:.2f})] "
.ljust(MARGINE) +
f"Input directory cannot be created: {data_dir}")
with rasterio.open(output, "w", **meta) as f:
f.write(ras_msk)
with xr.open_rasterio(dest) as ds:
ds.data = ras_msk
msk = ds < nodata if nodata > 0.0 else ds > nodata
ds = ds.where(msk, drop=True)
ds = ds.squeeze("band", drop=True)
ds.name = "elevation"
ds.attrs["units"] = "meters"
print("finished.")
return ds