def _recompress_image(
input_image: rasterio.DatasetReader,
output_fp: rasterio.MemoryFile,
zlevel=9,
block_size=(512, 512),
):
"""
Read an image from given file pointer, and write as a compressed GeoTIFF.
"""
# noinspection PyUnusedLocal
block_size_y, block_size_x = block_size
if len(input_image.indexes) != 1:
raise ValueError(
f"Expecting one-band-per-tif input (USGS packages). "
f"Input has multiple layers {repr(input_image.indexes)}")
array: numpy.ndarray = input_image.read(1)
profile = input_image.profile
profile.update(
driver="GTiff",
predictor=_PREDICTOR_TABLE[array.dtype.name],
compress="deflate",
zlevel=zlevel,
blockxsize=block_size_x,
blockysize=block_size_y,
tiled=True,
)
with output_fp.open(**profile) as output_dataset:
output_dataset.write(array, 1)
# Copy gdal metadata
output_dataset.update_tags(**input_image.tags())
output_dataset.update_tags(1, **input_image.tags(1))
def _write_array(self, dataset: rio.DatasetReader, window: Box,
arr: np.ndarray) -> None:
"""Write array out to a rasterio dataset. Array must be of shape
(C, H, W).
"""
window = window.rasterio_format()
if len(arr.shape) == 2:
dataset.write_band(1, arr, window=window)
else:
for i, band in enumerate(arr, start=1):
dataset.write_band(i, band, window=window)
def get_landsat8_ndvi(
tile_reader: rasterio.DatasetReader,
window: Union[rasterio.windows.Window,
Tuple] = None) -> np.ndarray:
"""Computes the NDVI (Normalized Difference Vegetation Index) over a tile or a section
on the tile specified by the window, for Landsat 8 tiles.
NDVI values are between -1.0 and 1.0 (hence "normalized"), mostly representing greenness, where any
negative values are mainly generated from clouds, water, and snow, and values
near zero are mainly generated from rock and bare soil. Very low values (0.1 and below)
of NDVI correspond to barren areas of rock, sand, or snow. Moderate values (0.2 to 0.3)
represent shrub and grassland, while high values (0.6 to 0.8) indicate temperate
and tropical rainforests. Source: https://desktop.arcgis.com/
For surface reflectance products, there are some out of range values in the red and near infrared bands
for areas around water/clouds. We should get rid of these before NDVI calculation,
otherwise the NDVI will also be out of range.
Source: https://www.researchgate.net/post/Why_Landsat_8_NDVI_Values_are_out_of_Range_Not_in_between-1_to_1
If we decide not to get rid of invalid values in the red and NIR bands and see some out-of-range NDVI values,
we can check to see if these out-of-range NDVI values are a small minority in all pixels in the dataset.
Args:
tile_reader: a rasterio.io.DatasetReader object returned by rasterio.open()
window: a tuple of four (col_off x, row_off y, width delta_x, height delta_y)
to specify the section of the tile to compute NDVI over, or a rasterio Window object
Returns:
2D numpy array of dtype float32 of the NDVI values at each pixel
Pixel value is set to 0 if the sum of the red and NIR value there is 0 (empty).
"""
if window and isinstance(window, Tuple):
window = rasterio.windows.Window(window[0], window[1], window[2],
window[3])
band_red = tile_reader.read(4,
window=window,
boundless=True,
fill_value=0).squeeze()
band_nir = tile_reader.read(5,
window=window,
boundless=True,
fill_value=0).squeeze()
sum_red_nir = band_nir + band_red
# sum of the NIR and red bands being zero is most likely because this section is empty
# this workaround means that the final NDVI at such pixels are 0.
sum_red_nir[sum_red_nir == 0.0] = 1
ndvi = (band_nir - band_red) / sum_red_nir
return ndvi
def write_measurement_rio(
self,
name: str,
ds: DatasetReader,
overviews=images.DEFAULT_OVERVIEWS,
overview_resampling=Resampling.average,
expand_valid_data=True,
file_id=None,
):
"""
Write a measurement by reading it an open rasterio dataset
:param ds: An open rasterio dataset
See :func:`write_measurement` for other parameters.
"""
if len(ds.indexes) != 1:
raise NotImplementedError(
f"TODO: Multi-band images not currently implemented (have {len(ds.indexes)})"
)
self._write_measurement(
name,
ds.read(1),
images.GridSpec.from_rio(ds),
self.names.measurement_file_path(
self._work_path, name, "tif", file_id=file_id
),
expand_valid_data=expand_valid_data,
nodata=ds.nodata,
overview_resampling=overview_resampling,
overviews=overviews,
)
def get_window_and_affine(geom: BasePolygon,
raster_src: DatasetReader) -> Tuple[Window, Affine]:
"""
Get a rasterio window block from the bounding box of a vector feature and
calculates the affine transformation needed to map the coordinates of the
geometry onto a resulting array defined by the shape of the window.
Args:
geom (Shapely geometry): A geometry in the spatial reference system
of the raster to be read.
raster_src (rasterio file-like object): A rasterio raster source which
will have the window operation performed and contains the base
affine transformation.
Returns:
A pair of tuples which define a rectangular range that can be provided
to rasterio for a windowed read
See: https://mapbox.github.io/rasterio/windowed-rw.html#windowrw
An Affine object used to transform geometry coordinates to cell values
"""
# Create a window range from the bounds
window: Window = raster_src.window(*geom.bounds).round_lengths(
pixel_precision=5).round_offsets(pixel_precision=5)
# Create a transform relative to this window
affine = rasterio.windows.transform(window, raster_src.transform)
return window, affine
def _crop_img_to_shp(img: rasterio.DatasetReader, shape: shapefile.Shape,
out_path: _OutPath) -> bool:
# Get the bbox to crop to
shp_bbox = _BBox(*[round(v) for v in shape.bbox])
img_bbox = _BBox(*list(img.bounds))
bbox = shp_bbox.intersect(img_bbox)
if not bbox.is_valid:
return False
# Crop the image
window = bbox.to_window(img)
data = img.read(window=window)
# Write to the output directory
out_path = out_path.crop_path(shp_bbox.left, shp_bbox.bottom)
x_res, y_res = img.res
transform = Affine.translation(bbox.left, bbox.top) * Affine.scale(
x_res, -y_res)
profile = img.profile
profile.update(transform=transform,
height=window.height,
width=window.width)
with rasterio.open(out_path, 'w', **profile) as writer:
writer.write(data)
# Fixes band 4 being labelled as alpha channel
writer.colorinterp = img.colorinterp
print(f'Created: {out_path}')
return True
def read_raster_from_polygon(
src: rio.DatasetReader,
poly: Union[Polygon, MultiPolygon]) -> np.ma.MaskedArray:
"""Read valid pixel values from all locations inside a polygon
Uses the polygon as a mask in addition to the existing raster mask
Args:
src: an open rasterio dataset to read from
poly: a shapely Polygon or MultiPolygon
Returns:
masked array of shape (nbands, nrows, ncols)
"""
# get the read parameters
window = rio.windows.from_bounds(*poly.bounds, src.transform)
transform = rio.windows.transform(window, src.transform)
# get the data
data = src.read(window=window, masked=True, boundless=True)
bands, rows, cols = data.shape
poly_mask = geometry_mask([poly],
transform=transform,
out_shape=(rows, cols))
# update the mask
data[:, poly_mask] = np.ma.masked
return data
def show_patch(
tile_reader: rasterio.DatasetReader,
bands: Union[Iterable, int],
window: Union[rasterio.windows.Window, Tuple] = None,
band_min: Numeric = 0,
band_max: Numeric = 7000,
gamma: Numeric = 1.0,
size: Tuple[Numeric, Numeric] = (256, 256),
return_array: bool = False) -> Union[np.ndarray, Image.Image]:
"""Show a patch of imagery.
Args:
tile_reader: a rasterio.io.DatasetReader object returned by rasterio.open()
bands: list or tuple of ints, or a single int, indicating which band(s) to read. See notes
below regarding the order of band numbers to pass in
window: a tuple of four (col_off x, row_off y, width delta_x, height delta_y)
to specify the section of the tile to return, or a rasterio Window object
band_min: minimum value the pixels are clipped tp
band_max: maximum value the pixels are clipped to
gamma: the gamma value to use in gamma correction. Output is darker for gamma > 1, lighter if gamma < 1.
This is not the same as GEE visParams' gamma!
size: Used when this function is called to produce a PIL Image, i.e. only when
`return_array` is False.
None if do not resize, otherwise a (w, h) tuple in pixel unit.
Default is (256, 256). (500, 500) looks better in notebooks
return_array: True will cause this function to return a numpy array, with dtype the same as
the original data;
False (default) to get a PIL Image object (values scaled to be uint8 values)
Returns:
a PIL Image object, resized to `size`, or the (not resized, original data type) numpy array
if `return_array` is true. The dims start with height and width, optionally
with the channel dim at the end if greater than 1.
rasterio read() does not pad with 0. The array returned may be smaller than the window specified
Notes:
- PIL renders the bands in RGB order; keep that in mind when passing in `bands` as a list or tuple
so that the bands are mapped to Red, Green and Blue in the desired order.
- Band index starts with 1
"""
if isinstance(bands, int):
bands = [
bands
] # otherwise rasterio read will return a 2D array instead of 3D
if window and isinstance(window, Tuple):
window = rasterio.windows.Window(window[0], window[1], window[2],
window[3])
# read as (bands, rows, columns) or (c, h, w)
bands = tile_reader.read(bands,
window=window,
boundless=True,
fill_value=0)
bands = ImageryVisualizer.norm_band(bands,
band_min=band_min,
band_max=band_max,
gamma=gamma)
# need to rearrange to (h, w, channel/bands)
bands = np.transpose(bands, axes=[1, 2, 0])
bands = bands.squeeze(
) # PIL accepts (h, w, 3) or (h, w), not (h, w, 1)
if return_array:
return bands
# skimage.img_as_ubyte: negative input values will be clipped. Positive values are scaled between 0 and 255
# fine to use here as we already got rid of negative values by normalizing above
bands = img_as_ubyte(bands)
im = Image.fromarray(bands)
if size:
im = im.resize(size)
return im
def __make_rastertiles_Z__(src_dataset: rio.DatasetReader, world_size: float,
tile_size: int, zoom: int) -> list():
# get bands
src_bands = src_dataset.read()
# structure for store tiles
tiles = []
# get bounds
src_bbox = src_dataset.bounds
src_bbox = [src_bbox.left, src_bbox.top, src_bbox.right, src_bbox.bottom]
# get pixel size
pixel_size = __pixel_size__(world_size, tile_size, zoom)
# get all quadrant
quadrants = __make_quadrants__(src_bbox, zoom, world_size, 1)
for xmin, ymin, xmax, ymax in quadrants:
# get bbox of quadrant
Xmin, Ymin, Xmax, Ymax = list(
__tile_world_bbox__(xmin, ymin, zoom, world_size, tile_size))
# get pixel size
pixel_size = __pixel_size__(world_size, tile_size, zoom)
# make dst shape (3, tsize, tsize), 3 is fix because it's an image RGB
dst_shape = (3, tile_size, tile_size)
# make transform with orig (Xmin, Ymin) and scale (psize, -psize)
dst_transform = A.translation(Xmin, Ymin) * A.scale(
pixel_size, -pixel_size)
dtype = src_dataset.dtypes[0]
if dtype == rio.uint8:
datatype = 1
elif dtype == rio.uint16:
datatype = 2
elif dtype == rio.int16:
datatype = 3
elif dtype == rio.uint32:
datatype = 4
elif dtype == rio.int32:
datatype = 5
elif dtype == rio.float32:
datatype = 6
elif dtype == rio.float64:
datatype = 7
else:
assert False
# init dst bands
dst_bands = np.zeros(dst_shape, dtype=dtype)
count = dst_bands.shape[0]
nodata = 0 if src_dataset.nodata is None else src_dataset.nodata
# make reprojection for each bands
for i in range(count):
try:
reproject(source=src_bands[i],
destination=dst_bands[i],
src_transform=src_dataset.transform,
src_crs=src_dataset.crs,
src_nodata=nodata,
dst_transform=dst_transform,
dst_crs=src_dataset.crs)
except IndexError:
continue
gdal_bands = [{
'data': dst_bands[x],
'nodata_value': nodata
} for x in range(count)]
gdal_raster = GDALRaster({
'srid': WEB_MERCATOR_SRID,
'width': tile_size,
'height': tile_size,
'datatype': datatype,
'nr_of_bands': count,
'origin': [Xmin, Ymin],
'scale': [pixel_size, -pixel_size],
'bands': gdal_bands
})
tiles.append((zoom, xmin, ymin, gdal_raster))
del src_bands
# return structure
return tiles
def __make_imagetiles_Z__(src_dataset: rio.DatasetReader, world_size: float,
tile_size: int, zoom: int) -> list():
# structure for store tiles
tiles = []
# get bounding box
src_bbox = src_dataset.bounds
src_bbox = [src_bbox.left, src_bbox.top, src_bbox.right, src_bbox.bottom]
# get pixel size
pixel_size = __pixel_size__(world_size, tile_size, zoom)
# get all quadrant
quadrants = __make_quadrants__(src_bbox, zoom, world_size, 1)
for xmin, ymin, xmax, ymax in quadrants:
# get bbox of quadrant
Xmin, Ymin, Xmax, Ymax = list(
__tile_world_bbox__(xmin, ymin, zoom, world_size, tile_size))
# get pixel size
pixel_size = __pixel_size__(world_size, tile_size, zoom)
# make dst shape (3, tsize, tsize), 3 is fix because it's an image RGB
dst_shape = (3, tile_size, tile_size)
# make transform with orig (Xmin, Ymin) and scale (psize, -psize)
dst_transform = A.translation(Xmin, Ymin) * A.scale(
pixel_size, -pixel_size)
# init dst bands
dst_bands = np.zeros(dst_shape, dtype=np.uint8)
# make reprojection for each bands
for i in range(3):
reproject(source=src_dataset.read(i + 1),
destination=dst_bands[i],
src_transform=src_dataset.transform,
src_crs=src_dataset.crs,
dst_transform=dst_transform,
dst_crs=src_dataset.crs)
# switch channel fst to channel last
dst_bands = np.rollaxis(dst_bands, 0, 3)
# make alpha band for no data
dst_sum = np.sum(dst_bands, axis=2)
alpha = np.zeros((tile_size, tile_size, 3))
alpha[dst_sum > 0] = np.array([255, 255, 255])
# convert alpha as pilimage
pil_alpha = Image.fromarray(alpha.astype(dtype=np.uint8)).convert('L')
# convert dst_bands as pilimage & put alpha
pil_tile = Image.fromarray(dst_bands)
pil_tile.putalpha(pil_alpha)
# write in a buffer as bytes
buffer = BytesIO()
pil_tile.save(fp=buffer, format="PNG")
# push all in ret structure
tiles.append((zoom, xmin, ymin, buffer))
# return structure
return tiles
def _read_chip(self, ds: rio.DatasetReader, kwargs) -> np.ndarray:
"""Reads from the given dataset with the parameters in kwargs. Created
mostly to utilize retry functionality"""
return ds.read(**kwargs)
def _tile_segments(self, dataset_directory: str,
ds: rasterio.DatasetReader,
bbox_filtering_function) -> List[List[Dict[str, any]]]:
"""
:param dataset_directory:
:param ds:
:return: annotations: List[List[Dict]]
"""
# Use rasterio to compute geometric transformation parameters
ulx, xres, _, uly, _, yres = ds.get_transform()
# Size of Ortho in geometric scale
img_h, img_w = ds.height, ds.width
orthox = xres * img_w
orthoy = yres * img_h
# Locate ShapeFile in datasets/dataset/Segments/ directory
shapefile_name = glob(os.path.join(dataset_directory, 'Segments',
'*'))[0][:-4]
print(f'Importing Shapefile {shapefile_name}')
shpf = shapefile.Reader(shapefile_name)
shape_recs = shpf.shapeRecords()
bbox = shpf.bbox
minx = bbox[0]
maxy = bbox[3]
# Offset of Shapefile from top left corner of Ortho [0, width/height] in geometric scale
offsetx = minx - ulx
offsety = maxy - uly
# Scale is ratio of image pixel width/height to geometric width/height in raster
scalex = img_w / orthox
scaley = img_h / orthoy
rescale_x = lambda x: (x - minx + offsetx) * scalex
rescale_y = lambda y: (y - maxy + offsety) * scaley
# Compute the number of image tiles and initialize annotation
x_tiles = ceil(img_w / self.dx)
num_tiles = x_tiles * ceil(img_h / self.dy)
annotations = [[] for _ in range(num_tiles)]
bad_segments = 0
print(f'Tiling Shapes for {os.path.basename(dataset_directory)}')
for shape_rec in tqdm(shape_recs):
shp = shape_rec.shape
if shp.shapeType == 5: # 5 - polygon
class_name = shape_rec.record.segClass
if class_name[-1] == '2':
class_name = class_name[:-1]
# Update class dict
if class_name not in self.classes:
self.classes[class_name] = len(self.classes)
# transform polygon and bounding into image coordinate system
rescaled_poly = [[rescale_x(x), rescale_y(y)]
for x, y in fix_polygon_tail(shp.points)]
if len(rescaled_poly) < 3:
bad_segments += 1
continue
shape_bbox = [
rescale_x(shp.bbox[0]),
rescale_y(shp.bbox[3]),
rescale_x(shp.bbox[2]),
rescale_y(shp.bbox[1])
]
if not bbox_filtering_function(shape_bbox):
bad_segments += 1
continue
x_min, y_min, x_max, y_max = shape_bbox
x_pos, y_pos = (x_min // self.dx), (y_min // self.dy)
# Compute tiles that shape belongs to
for y_shift in (0, self.dy):
for x_shift in (0, self.dx):
x, y = self.dx * x_pos - x_shift, self.dy * y_pos - y_shift
x_c = x + self.tile_width if x + self.tile_width < img_w else img_w
y_c = y + self.tile_height if y + self.tile_height < img_h else img_h
if box_in_box(shape_bbox, [x, y, x_c, y_c]):
x_pos_c, y_pos_c = x_pos - if_non_zero(
x_shift), y_pos - if_non_zero(y_shift)
annotations[int(
x_tiles * y_pos_c + x_pos_c)].append(
create_annotation(
poly=rescaled_poly,
bbox=shape_bbox,
rescale_corner=(x, y),
is_crowd=0,
category_id=self.classes[class_name]))
print(f'{bad_segments} of {len(shape_recs)} segments filtered')
return annotations
def img_coords(self, img: rasterio.DatasetReader) -> Dict[str, int]:
(top, left), (bottom, right) = img.index(self.left,
self.top), img.index(
self.right, self.bottom)
return {'left': left, 'bottom': bottom, 'right': right, 'top': top}