def _crop_img_to_shp(img: rasterio.DatasetReader, shape: shapefile.Shape,
out_path: _OutPath) -> bool:
# Get the bbox to crop to
shp_bbox = _BBox(*[round(v) for v in shape.bbox])
img_bbox = _BBox(*list(img.bounds))
bbox = shp_bbox.intersect(img_bbox)
if not bbox.is_valid:
return False
# Crop the image
window = bbox.to_window(img)
data = img.read(window=window)
# Write to the output directory
out_path = out_path.crop_path(shp_bbox.left, shp_bbox.bottom)
x_res, y_res = img.res
transform = Affine.translation(bbox.left, bbox.top) * Affine.scale(
x_res, -y_res)
profile = img.profile
profile.update(transform=transform,
height=window.height,
width=window.width)
with rasterio.open(out_path, 'w', **profile) as writer:
writer.write(data)
# Fixes band 4 being labelled as alpha channel
writer.colorinterp = img.colorinterp
print(f'Created: {out_path}')
return True
def test_complex_nodata(tmpdir):
"""A complex dataset can be created with a real nodata value"""
import numpy as np
import rasterio
from rasterio.transform import Affine
x = np.linspace(-4.0, 4.0, 240)
y = np.linspace(-3.0, 3.0, 180)
X, Y = np.meshgrid(x, y)
Z1 = np.ones_like(X) + 1j
res = (x[-1] - x[0]) / 240.0
transform1 = Affine.translation(x[0] - res / 2,
y[-1] - res / 2) * Affine.scale(res, -res)
tempfile = str(tmpdir.join("test.tif"))
with rasterio.open(tempfile,
'w',
driver='GTiff',
height=Z1.shape[0],
width=Z1.shape[1],
nodata=0,
count=1,
dtype=Z1.dtype,
crs='+proj=latlong',
transform=transform1) as dst:
dst.write(Z1, 1)
with rasterio.open(tempfile) as dst:
assert dst.nodata == 0
def __call__(self):
with rasterio.open(input) as src:
if bands:
if sampling == 1:
img = src.read_band(bidx)
transform = src.transform
# Decimate the band.
else:
img = numpy.zeros(
(src.height // sampling, src.width // sampling),
dtype=src.dtypes[src.indexes.index(bidx)])
img = src.read_band(bidx, img)
transform = src.affine * Affine.scale(float(sampling))
else:
if sampling == 1:
img = src.read_mask()
transform = src.transform
# Decimate the mask.
else:
img = numpy.zeros(
(src.height // sampling, src.width // sampling),
dtype=numpy.uint8)
img = src.read_mask(img)
transform = src.affine * Affine.scale(float(sampling))
bounds = src.bounds
xs = [bounds[0], bounds[2]]
ys = [bounds[1], bounds[3]]
if projected == 'geographic':
xs, ys = rasterio.warp.transform(src.crs,
{'init': 'epsg:4326'}, xs,
ys)
if precision >= 0:
xs = [round(v, precision) for v in xs]
ys = [round(v, precision) for v in ys]
self._xs = xs
self._ys = ys
kwargs = {'transform': transform}
if not bands and not with_nodata:
kwargs['mask'] = (img == 255)
for g, i in rasterio.features.shapes(img, **kwargs):
if projected == 'geographic':
g = rasterio.warp.transform_geom(
src.crs,
'EPSG:4326',
g,
antimeridian_cutting=True,
precision=precision)
xs, ys = zip(*coords(g))
yield {
'type': 'Feature',
'id': str(i),
'properties': {
'val': i
},
'bbox': [min(xs), min(ys),
max(xs), max(ys)],
'geometry': g
}
def tiffs(tmpdir):
data = np.ones((1, 1, 1), 'uint8')
kwargs = {
'count': '1',
'driver': 'GTiff',
'dtype': 'uint8',
'height': 1,
'width': 1
}
kwargs['transform'] = Affine(1, 0, 1, 0, -1, 1)
with rasterio.open(str(tmpdir.join('a-sw.tif')), 'w', **kwargs) as r:
r.write(data * 40)
kwargs['transform'] = Affine(1, 0, 2, 0, -1, 2)
with rasterio.open(str(tmpdir.join('b-ct.tif')), 'w', **kwargs) as r:
r.write(data * 60)
kwargs['transform'] = Affine(2, 0, 3, 0, -2, 4)
with rasterio.open(str(tmpdir.join('c-ne.tif')), 'w', **kwargs) as r:
r.write(data * 90)
kwargs['transform'] = Affine(2, 0, 2, 0, -2, 4)
with rasterio.open(str(tmpdir.join('d-ne.tif')), 'w', **kwargs) as r:
r.write(data * 120)
return tmpdir
def selectInterestArea(imgName, x, y):
out_folder = os.path.join(workingDir, 'Split', imgName[:-4])
with rio.open(os.path.join(imageDir, imgName)) as inds:
# tile_width, tile_height = int(inds.width/16),int(inds.height/16)
tile_width, tile_height = 1220, 1220
inv_transform = Affine.scale(
1 / inds.transform.a, 1 / inds.transform.e) * Affine.translation(
-inds.transform.xoff, -inds.transform.yoff)
meta = inds.meta.copy()
rasterio.warp.transform(crs0, inds.crs, [y], [x])
a = rasterio.warp.transform(crs0, inds.crs, [y], [x])
xPixel, yPixel = pixel_location = inv_transform * (a[0][0], a[1][0])
print(pixel_location)
# print(count_sliding_window(inds.read(1)))
for window, transform in get_tiles(inds, tile_width, tile_height):
print(window)
meta['transform'] = transform
meta['width'], meta['height'] = window.width, window.height
outpath = os.path.join(
out_folder,
output_filename.format(int(window.col_off),
int(window.row_off)))
if ((xPixel > window.col_off) &
(xPixel < window.col_off + window.width) &
(yPixel > window.row_off) &
(yPixel < window.row_off + window.height)):
with rio.open(outpath, 'w', **meta) as outds:
outds.write(inds.read(window=window))
def basic_flattening(target_folder, raster, res, origin, size, tin = False):
"""Reads some pre-determined vector files, tiles them using
Lisa's code and "burns" them into the output raster. The flat
elevation of the polygons is estimated by Laplace-interpolating
at the locations of the polygon vertices. The underlying TIN
is constructed from the centre points of the raster pixels.
Rasterisation takes place via rasterio's interface.
"""
import startin
from rasterio.features import rasterize
from rasterio.transform import Affine
transform = (Affine.translation(origin[0], origin[1])
* Affine.scale(size, size))
x0, x1 = origin[0] + size / 2, origin[0] + ((res[0] - 0.5) * size)
y0, y1 = origin[1] + size / 2, origin[1] + ((res[1] - 0.5) * size)
poly_fpaths = [
'rest_bodies/bbg_rest_of_the_water.shp',
'sea_bodies/bbg_sea_and_big_bodies.shp',
# You can add more resources here.
]
wfs_urls = [
#('http://3dbag.bk.tudelft.nl/data/wfs', 'BAG3D:pand3d'),
# You can add more resources here.
]
in_vecs = []
for fpath in poly_fpaths:
vec = vector_prepare([[x0, x1], [y0, y1]], target_folder + fpath)
if len(vec) != 0: in_vecs.append(vec)
for wfs in wfs_urls:
vec = wfs_prepare([[x0, x1], [y0, y1]], wfs[0], wfs[1])
if len(vec) != 0: in_vecs.append(vec)
if len(in_vecs) == 0: return
if tin is False:
xs, ys = np.linspace(x0, x1, res[0]), np.linspace(y0, y1, res[1])
xg, yg = np.meshgrid(xs, ys); xg = xg.flatten(); yg = yg.flatten()
cs = np.vstack((xg, yg, raster.flatten())).transpose()
data = cs[cs[:,2] != -9999]
tin = startin.DT(); tin.insert(data)
elevations = []
for polys in in_vecs:
for poly, i in zip(polys, range(len(polys))):
els = []
for vx in poly.exterior.coords:
try: els += [tin.interpolate_laplace(vx[0], vx[1])]
except: pass
for interior in poly.interiors:
for vx in interior.coords:
try: els += [tin.interpolate_laplace(vx[0], vx[1])]
except: pass
elevations.append(np.median(els))
shapes = []
for polys in in_vecs:
shapes += [(p, v) for p, v in zip(polys, elevations)]
raspolys = rasterize(shapes, raster.shape, -9999, transform = transform)
for yi in range(res[1]):
for xi in range(res[0]):
if raspolys[yi, xi] != -9999: raster[yi, xi] = raspolys[yi, xi]
return tin
def latlon2transform(lat, lon, cell_center=True):
lat = np.asarray(lat)
lon = np.asarray(lon)
resX = (lon[-1] - lon[0]) / (len(lon) - 1)
resY = (lat[-1] - lat[0]) / (len(lat) - 1)
trans = Affine.translation(lon[0] - resX / 2. * cell_center,
lat[0] - resY / 2. * cell_center)
scale = Affine.scale(lon[1] - lon[0], lat[1] - lat[0])
return trans * scale
def __call__(self):
with rasterio.open(input) as src:
img = None
nodata_mask = None
if bands:
if sampling == 1:
img = src.read(bidx, masked=False)
transform = src.transform
# Decimate the band.
else:
img = numpy.zeros(
(src.height//sampling, src.width//sampling),
dtype=src.dtypes[src.indexes.index(bidx)])
img = src.read(bidx, img, masked=False)
transform = src.affine * Affine.scale(float(sampling))
if not bands or not with_nodata:
if sampling == 1:
nodata_mask = src.read_masks(bidx)
transform = src.transform
# Decimate the mask.
else:
nodata_mask = numpy.zeros(
(src.height//sampling, src.width//sampling),
dtype=numpy.uint8)
nodata_mask = src.read_masks(bidx, nodata_mask)
transform = src.affine * Affine.scale(float(sampling))
bounds = src.bounds
xs = [bounds[0], bounds[2]]
ys = [bounds[1], bounds[3]]
if projection == 'geographic':
xs, ys = rasterio.warp.transform(
src.crs, {'init': 'epsg:4326'}, xs, ys)
if precision >= 0:
xs = [round(v, precision) for v in xs]
ys = [round(v, precision) for v in ys]
self._xs = xs
self._ys = ys
kwargs = {'transform': transform}
# Default is to exclude nodata features.
if nodata_mask is not None:
kwargs['mask'] = (nodata_mask==255)
if img is None:
img = nodata_mask
for g, i in rasterio.features.shapes(img, **kwargs):
if projection == 'geographic':
g = rasterio.warp.transform_geom(
src.crs, 'EPSG:4326', g,
antimeridian_cutting=True, precision=precision)
xs, ys = zip(*coords(g))
yield {
'type': 'Feature',
'id': str(i),
'properties': {'val': i},
'bbox': [min(xs), min(ys), max(xs), max(ys)],
'geometry': g }
def transform_lat_lon(lat, lon):
"""
Transforms latitude and longitude arrays.
:param lat: numpy array latitude coordinates.
:param lon: numpy array longitude coordinates.
:return:
"""
lat = np.asarray(lat)
lon = np.asarray(lon)
trans = Affine.translation(lon[0], lat[0])
scale = Affine.scale(lon[1] - lon[0], lat[1] - lat[0])
return trans * scale
def pad(array, transform, pad_width, mode=None, **kwargs):
"""pad array and adjust affine transform matrix.
Parameters
----------
array: ndarray
Numpy ndarray, for best results a 2D array
transform: Affine transform
transform object mapping pixel space to coordinates
pad_width: int
number of pixels to pad array on all four
mode: str or function
define the method for determining padded values
Returns
-------
(array, transform): tuple
Tuple of new array and affine transform
Notes
-----
See numpy docs for details on mode and other kwargs:
http://docs.scipy.org/doc/numpy-1.10.0/reference/generated/numpy.pad.html
"""
import numpy as np
transform = guard_transform(transform)
padded_array = np.pad(array, pad_width, mode, **kwargs)
padded_trans = list(transform)
padded_trans[2] -= pad_width * padded_trans[0]
padded_trans[5] -= pad_width * padded_trans[4]
return padded_array, Affine(*padded_trans[:6])
def getTransform(xvar=None, x=None, y=None, lcheck=True):
''' generate an affine transformation from xarray coordinate axes '''
from rasterio.transform import Affine # to generate Affine transform
if isinstance(xvar,(xr.DataArray,xr.Dataset,nc.Dataset)):
x,y = getGeoCoords(xvar, lraise=True)
elif xvar is None and isinstance(x,(xr.DataArray,nc.Variable)) and isinstance(y,(xr.DataArray,nc.Variable)):
pass # x and y axes are supplied directly
elif xvar:
raise TypeError('Can only infer GeoTransform from xarray Dataset or DataArray or netCDF4 Dataset\n - not from {}.'.format(xvar))
# check X-axis
if isinstance(x,xr.DataArray): x = x.data
elif isinstance(x,nc.Variable): x = x[:]
if not isinstance(x,np.ndarray):
raise TypeError(x)
diff_x = np.diff(x); dx = diff_x.min()
if lcheck and not np.isclose(dx, diff_x.max(), rtol=1.e-2):
raise ValueError("X-axis is not regular: {} - {}".format(dx, diff_x.max()))
# check Y-axis
if isinstance(y,xr.DataArray): y = y.data
elif isinstance(y,nc.Variable): y = y[:]
if not isinstance(y,np.ndarray):
raise TypeError(y)
diff_y = np.diff(y); dy = diff_y.min()
if lcheck and not np.isclose(dy, diff_y.max(), rtol=1.e-2):
raise ValueError("Y-axis is not regular. {} - {}".format(dy, diff_y.max()))
# generate transform
return Affine.from_gdal(x[0]-dx/2.,dx,0.,y[0]-dy/2.,0.,dy), (len(x),len(y))
def __init__(self, pixel_size, raster_data=None, **kwargs):
logger.info("Preparing a base dataframe...")
x_min, y_min, x_max, y_max = -180, -90, 180, 90 # global
self.pixel_size = pixel_size
self.x_res = int((x_max - x_min) / self.pixel_size)
self.y_res = int((y_max - y_min) / self.pixel_size)
self.base_layer = RasterEnvironmentalLayer()
self.base_layer.raster_affine = Affine(pixel_size, 0.0, x_min, 0.0,
-pixel_size, y_max)
logger.info("Base layer: Computing world coordinates...")
if raster_data is not None:
all_coordinates = self.base_layer.pixel_to_world_coordinates(
raster_data=raster_data)
else:
all_coordinates = self.base_layer.pixel_to_world_coordinates(
raster_data=np.ones((self.y_res, self.x_res), dtype=np.uint8))
# self.base_dataframe = pd.DataFrame([all_coordinates[0], all_coordinates[1]]).T
# self.base_dataframe.columns = ['decimallatitude', 'decimallongitude']
self.base_dataframe = pd.DataFrame(
columns=['decimallatitude', 'decimallongitude'])
self.base_dataframe['decimallatitude'] = np.array(all_coordinates[0])
self.base_dataframe['decimallongitude'] = np.array(all_coordinates[1])
self.base_dataframe.set_index(['decimallatitude', 'decimallongitude'],
inplace=True,
drop=True)
del all_coordinates
gc.collect()
def getTransform(xvar=None, x=None, y=None, lcheck=True):
''' generate an affine transformation from xarray coordinate axes '''
from rasterio.transform import Affine # to generate Affine transform
if isinstance(xvar,(xr.DataArray,xr.Dataset)):
x,y = getGeoCoords(xvar, lraise=True)
elif xvar:
raise TypeError('Can only infer GeoTransform from xarray Dataset or DataArray - not from {}.'.format(xvar))
# check X-axis
if isinstance(x,xr.DataArray): x = x.data
if not isinstance(x,np.ndarray):
raise TypeError(x)
diff_x = np.diff(x); dx = diff_x.min()
if lcheck and not np.isclose(dx, diff_x.max(), rtol=1.e-2):
raise ValueError("X-axis is not regular: {} - {}".format(dx, diff_x.max()))
# check Y-axis
if isinstance(y,xr.DataArray): y = y.data
if not isinstance(y,np.ndarray):
raise TypeError(y)
diff_y = np.diff(y); dy = diff_y.min()
if lcheck and not np.isclose(dy, diff_y.max(), rtol=1.e-2):
raise ValueError("Y-axis is not regular. {} - {}".format(dy, diff_y.max()))
# generate transform
return Affine.from_gdal(x[0]-dx/2.,dx,0.,y[0]-dy/2.,0.,dy), (len(x),len(y))
def rasterizeShapes(pshapes, geodict, all_touched=True):
"""
Rasterizing a shape
Args:
pshapes: Sequence of orthographically projected shapes.
geodict: Mapio geodictionary.
all_touched: Turn pixel "on" if shape touches pixel, otherwise turn it
on if the center of the pixel is contained within the shape. Note
that the footprint of the shape is inflated and the amount of
inflation depends on the pixel size if all_touched=True.
Returns:
Rasterio grid.
"""
outshape = (geodict.ny, geodict.nx)
txmin = geodict.xmin - geodict.dx / 2.0
tymax = geodict.ymax + geodict.dy / 2.0
transform = Affine.from_gdal(txmin, geodict.dx, 0.0, tymax, 0.0,
-geodict.dy)
imgs = rasterio.features.rasterize(pshapes,
out_shape=outshape,
fill=0.0,
transform=transform,
all_touched=all_touched,
default_value=1.0)
return imgs
def preprocess_bands(self, band_stack, subset_box=None):
bands_rescaled = band_stack[0:4].copy()
bands_rescaled[np.isnan(bands_rescaled)] = 0
bands_rescaled = rescale_s2(bands_rescaled)
bands_rescaled[bands_rescaled == 0] = np.nan
band_stack = None
self.lat, self.lon = lat_from_meta(self.meta), lon_from_meta(self.meta)
if subset_box is not None:
ymin, ymax, xmin, xmax = subset_box["ymin"], subset_box[
"ymax"], subset_box["xmin"], subset_box["xmax"]
bands_rescaled = bands_rescaled[:, ymin:ymax, xmin:xmax]
self.lat, self.lon = self.lat[ymin:ymax], self.lon[xmin:xmax]
self.meta["height"], self.meta["width"] = bands_rescaled.shape[
1], bands_rescaled.shape[2]
t = list(self.meta["transform"])
t[2], t[5] = self.lon[0], self.lat[0]
self.meta["transform"] = Affine(t[0], t[1], t[2], t[3], t[4], t[5])
bbox_epsg4326 = list(np.flip(metadata_to_bbox_epsg4326(self.meta)))
osm_mask = self._get_osm_mask(
bbox_epsg4326, "EPSG:" + str(self.meta["crs"].to_epsg()),
bands_rescaled[0], {
"lat": self.lat,
"lon": self.lon
}, dirs["osm"])
if np.count_nonzero(osm_mask) == 0:
raise ValueError("No OSM roads of requested road types in aoi")
bands_rescaled *= osm_mask
bands_rescaled[bands_rescaled == 0] = np.nan
osm_mask = None
self._build_variables(bands_rescaled)
def download_tiles(self, scene, grid):
bbox, shape = grid
x, y = map(lambda coor: int(coor / self.resolution), shape)
if scene.geometry.intersects(bbox):
array = self.request(scene, bbox, (x, y))
diff = bbox.difference(scene.geometry)
if diff.area != 0:
x0, _, _, ye = bbox.bounds
transform = Affine(a=self.resolution,
b=0,
c=x0,
d=0,
e=-self.resolution,
f=ye)
mask = rasterize([(diff, True)],
out_shape=(y, x),
transform=transform,
fill=False,
all_touched=True)
array = numpy.where(mask, self.nodata, array)
else:
array = None
if array is not None:
return array
else:
return self.nodata_tile((x, y))
def write_cog(layout, location, data, filename_template):
"""
Write a GPS tile out as a Cloud-Optimized GeoTIFF
Uses GDAL to write a COG out to disk, utilizing the given layout and location
to generate the GeoTIFF header.
Args:
layout (``gps.LayoutDefinition``): The layout for the source layer
location (``gps.SpatialKey`` or ``gps.SpaceTimeKey``): The cell identifier
in the layer corresponding to the image data
data (``gps.Tile``): The image data
filename_template (str): A pattern giving the destination for the target
image. Contains two '{}' tokens which will be ``.format``ed with the
column and row of the ``location``, in that order. May be an S3 uri or
local path.
"""
bands, w, h = data.cells.shape
nodata = data.no_data_value
dtype = data.cells.dtype
cw, ch = cell_size(layout)
ex = extent_for_cell(layout, location)
overview_level = int(log(w) / log(2) - 8)
with rstr.io.MemoryFile() as memfile:
with memfile.open(driver='GTiff',
count=bands,
width=w,
height=h,
transform=Affine(cw, 0.0, ex.xmin, 0.0, -ch,
ex.ymax),
crs=rstr.crs.CRS.from_epsg(4326),
nodata=nodata,
dtype=dtype,
compress='lzw',
tiled=True) as mem:
windows = list(mem.block_windows(1))
for _, w in windows:
segment = data.cells[:, w.row_off:(w.row_off + w.height),
w.col_off:(w.col_off + w.width)]
mem.write(segment, window=w)
mask_value = np.all(segment != nodata, axis=0).astype(
np.uint8) * 255
mem.write_mask(mask_value, window=w)
overviews = [2**j for j in range(1, overview_level + 1)]
mem.build_overviews(overviews, rwarp.Resampling.nearest)
mem.update_tags(ns='rio_oveview',
resampling=rwarp.Resampling.nearest.value)
uri = urlparse.urlparse(filename_template)
if uri.scheme == 's3':
boto3.resource('s3').Bucket(uri.netloc).upload_fileobj(
memfile,
uri.path.format(location.col, location.row)[1:])
else:
rshutil.copy(mem,
filename_template.format(location.col,
location.row),
copy_src_overviews=True)
def get_web_optimized_params(
src_dst,
tilesize=256,
latitude_adjustment: bool = True,
warp_resampling: str = "nearest",
grid_crs=CRS.from_epsg(3857),
) -> Dict:
"""Return VRT parameters for a WebOptimized COG."""
bounds = list(
transform_bounds(src_dst.crs,
CRS.from_epsg(4326),
*src_dst.bounds,
densify_pts=21))
center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2]
lat = 0 if latitude_adjustment else center[1]
_, max_zoom = get_zooms(src_dst, lat=lat, tilesize=tilesize)
extrema = tile_extrema(bounds, max_zoom)
left, _, _, top = mercantile.xy_bounds(extrema["x"]["min"],
extrema["y"]["min"], max_zoom)
vrt_res = _meters_per_pixel(max_zoom, 0, tilesize=tilesize)
vrt_transform = Affine(vrt_res, 0, left, 0, -vrt_res, top)
vrt_width = (extrema["x"]["max"] - extrema["x"]["min"]) * tilesize
vrt_height = (extrema["y"]["max"] - extrema["y"]["min"]) * tilesize
return dict(
crs=grid_crs,
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
resampling=ResamplingEnums[warp_resampling],
)
def predictFromTreeDEM(fp, dtree, outfn):
hh_msk, hv_msk = readBands(fp)
#print('HH read msk contains nan: '+str(np.isnan(hh_msk).any()))
#print('HV read msk contains nan: '+str(np.isnan(hv_msk).any()))
hh_msk_flat = hh_msk.flatten()
hv_msk_flat = hv_msk.flatten()
hhhvDEM_stacked = np.stack((hh_msk_flat, hv_msk_flat, dem_flat), axis=1)
pred = dtree.predict(hhhvDEM_stacked)
pred_reshaped = pred.reshape(-1, 11908)
meta_outFile = {
'driver':
'GTiff',
'dtype':
'float32',
'nodata':
-9999.0,
'width':
11908,
'height':
12600,
'count':
1,
'crs':
CRS.from_epsg(3413),
'transform':
Affine(10.0, 0.0, -384702.1263054441, 0.0, -10.0, -2122443.806211936)
}
with rio.open(outfn, 'w', **meta_outFile) as outTif:
outTif.write(pred_reshaped, indexes=1)
def geopandas_to_raster(gpd_frame, raster_file, gpd_column, geo_grid: GeoGrid,
datatype, all_touched):
""" Write geopandas data frame to raster
:param gpd_frame:
:param raster_file:
:param gpd_column:
:param geo_grid: GeoGrid instance
:param datatype: data type ('uint8', 'float32', etc.)
:param all_touched:
:return:
"""
# _ = _create_raster_file(raster_file, geo_grid, geo_crs, 0)
# Open temp raster file with rasterio and copy metadata
# raster_fcn = rasterio.open(raster_temp_file.name)
# meta = raster_fcn.meta.copy()
# meta.update(compress='lzw')
with rasterio.open(
raster_file,
'w',
driver="GTiff",
width=geo_grid.num_x,
height=geo_grid.num_y,
count=1,
dtype=datatype,
crs=gpd_frame.crs,
transform=Affine.from_gdal(*geo_grid.geo_transform)) as out:
# this is where we create a generator of geom, value pairs to use in rasterizing
shapes = (
(geom, value)
for geom, value in zip(gpd_frame.geometry, gpd_frame[gpd_column]))
burned = features.rasterize(
shapes=shapes,
fill=0,
out_shape=(geo_grid.num_y, geo_grid.num_x),
dtype=datatype,
transform=Affine.from_gdal(*geo_grid.geo_transform),
all_touched=all_touched)
out.write_band(1, burned)
return 0
def test_tiled_dataset_blocksize_guard(tmp_path):
"""Tiled datasets with dimensions less than blocksize are not permitted"""
pytest.importorskip("pathlib")
tmp_file = tmp_path / "test.tif"
with pytest.raises(ValueError):
rasterio.open(
tmp_file, "w", driver="GTiff", count=1, height=13, width=13, dtype="uint8", crs="epsg:3857",
transform=Affine.identity(), tiled=True, blockxsize=256, blockysize=256)
def rasterizeShapes(pshapes,geodict):
outshape = (geodict.ny,geodict.nx)
txmin = geodict.xmin - geodict.dx/2.0
tymax = geodict.ymax + geodict.dy/2.0
transform = Affine.from_gdal(txmin,geodict.dx,0.0,tymax,0.0,-geodict.dy)
img = rasterio.features.rasterize(pshapes,out_shape=outshape,fill=0.0,transform=transform,
all_touched=True,default_value=1.0)
return img
def get_params():
return {
'rain': {
'height': 129,
'width': 135,
'affine': Affine(0.25, 0, 66.375, 0, -0.25, 38.625)
},
'tmin': {
'height': 31,
'width': 31,
'affine': Affine(1, 0, 67, 0, -1, 38)
},
'tmax': {
'height': 31,
'width': 31,
'affine': Affine(1, 0, 67, 0, -1, 38)
}
}
def save_geotiff(img, coords, filename):
height, width, channels = img.shape
xres = (coords[1][0] - coords[0][0]) / width
yres = (coords[0][1] - coords[1][1]) / height
transform = Affine.translation(coords[0][0] - xres / 2, coords[0][1] +
yres / 2) * Affine.scale(xres, -yres)
profile = {
'driver': 'GTiff',
'width': width,
'height': height,
'count': channels,
'crs': '+proj=latlong',
'transform': transform,
'dtype': img.dtype,
'compress': 'JPEG'
}
with rasterio.open(filename, 'w', **profile) as f:
f.write(img.transpose(2, 0, 1))
def read_buffered_window(uri, metadata, px_buffer, cell, resampling):
"""
Read a region from a source image, reprojecting in the process.
Given the metadata for a layer (including a layout), a cell id, and a buffer
size, read a window from the supplied image while reprojecting the source image
to the CRS specified by the metadata.
In concept, we specify a grid in the target CRS using the cell and resolution in
the metadata, padding with px_buffer additional cells on all sides of the cell.
For each cell center, we read from the source image after back-projecting the
target location to the source image's coordinate system and applying the chosen
resampling kernel.
Args:
uri (str): The source image URI; any scheme interpretable by GDAL is allowed
metadata (``gps.Metadata``): The source layer metadata
px_buffer (int): The number of pixels worth of padding that should be added
to the extent to be read and added to the tile.
cell (``gps.SpatialKey`` or ``gps.SpaceTimeKey``): The key of the target
region
resampling (``rasterio.warp.Resampling``): The resampling method to be used
while reading from the source image
Returns:
``gps.Tile``
"""
if ("GDAL_DATA" not in os.environ) and '_GDAL_DATA' in vars():
os.environ["GDAL_DATA"] = _GDAL_DATA
layout = metadata.layout_definition
dest_bounds = buffered_cell_extent(layout, px_buffer, cell)
res = cell_size(metadata.layout_definition)
dest_transform = Affine(res[0], 0, dest_bounds.xmin, 0, -res[1],
dest_bounds.ymax)
dest_width = max(int(ceil((dest_bounds.xmax - dest_bounds.xmin) / res[0])),
1)
dest_height = max(
int(ceil((dest_bounds.ymax - dest_bounds.ymin) / res[1])), 1)
dest_transform, dest_width, dest_height = rwarp.aligned_target(
dest_transform, dest_width, dest_height, res)
with rstr.open(uri) as src:
tile = np.zeros((src.count, layout.tileLayout.tileCols + 2 * px_buffer,
layout.tileLayout.tileRows + 2 * px_buffer))
rwarp.reproject(source=rstr.band(src, list(range(1, src.count + 1))),
destination=tile,
src_transform=src.transform,
src_crs=src.crs,
src_nodata=src.nodata,
dst_transform=dest_transform,
dst_crs=CRS.from_epsg(4326),
dst_nodata=src.nodata,
resampling=resampling,
num_threads=1)
return gps.Tile.from_numpy_array(tile)
def get_web_optimized_params(
src_dst,
zoom_level_strategy: str = "auto",
zoom_level: Optional[int] = None,
aligned_levels: Optional[int] = None,
tms: morecantile.TileMatrixSet = morecantile.tms.get("WebMercatorQuad"),
) -> Dict:
"""Return VRT parameters for a WebOptimized COG."""
if src_dst.crs != tms.rasterio_crs:
with WarpedVRT(src_dst, crs=tms.rasterio_crs) as vrt:
bounds = vrt.bounds
aff = list(vrt.transform)
else:
bounds = src_dst.bounds
aff = list(src_dst.transform)
resolution = max(abs(aff[0]), abs(aff[4]))
if zoom_level is None:
# find max zoom (closest to the raster resolution)
max_zoom = tms.zoom_for_res(
resolution,
max_z=30,
zoom_level_strategy=zoom_level_strategy,
)
else:
max_zoom = zoom_level
# defined the zoom level we want to align the raster
aligned_levels = aligned_levels or 0
base_zoom = max_zoom - aligned_levels
# find new raster bounds (bounds of UL tile / LR tile)
ul_tile = tms._tile(bounds[0], bounds[3], base_zoom)
w, _, _, n = tms.xy_bounds(ul_tile)
# The output resolution should match the TMS resolution at MaxZoom
vrt_res = tms._resolution(tms.matrix(max_zoom))
# Output transform is built from the origin (UL tile) and output resolution
vrt_transform = Affine(vrt_res, 0, w, 0, -vrt_res, n)
lr_tile = tms._tile(bounds[2], bounds[1], base_zoom)
e, _, _, s = tms.xy_bounds(
morecantile.Tile(lr_tile.x + 1, lr_tile.y + 1, lr_tile.z)
)
vrt_width = max(1, round((e - w) / vrt_transform.a))
vrt_height = max(1, round((s - n) / vrt_transform.e))
return dict(
crs=tms.rasterio_crs,
transform=vrt_transform,
width=vrt_width,
height=vrt_height,
)
def write_geotiff(raster, origin, size, fpath):
"""Writes data in an n by n numpy array to disk as a
GeoTIFF raster. The header is based on the raster array
and a manual definition of the coordinate system and an
affine transform.
"""
transform = (Affine.translation(origin[0], origin[1]) *
Affine.scale(size, size))
with rasterio.Env():
with rasterio.open(fpath,
'w',
driver='GTiff',
height=raster.shape[0],
width=raster.shape[1],
count=1,
dtype=rasterio.float32,
crs='EPSG:28992',
transform=transform) as out_file:
out_file.write(raster.astype(rasterio.float32), 1)
def temp_to_tif(input, output):
filepath = '/home/sg/Projects/FIrehub-model/Bartsotas/example.tif'
with rasterio.open(filepath) as src:
print(src.crs)
metadata = src.profile
metadata.update(transform=Affine(0.02, 0.0, 19.2, 0.0, 0.02, 34))
with rasterio.open(
'/home/sg/Projects/FIrehub-model/Bartsotas/dataset_07072019/' +
output + '.tif', 'w', **metadata) as dst:
dst.write(input.astype(rasterio.float64), 1)
def main(args):
data = np.load(args.in_file).astype(args.type)
if len(data.shape) > 3 or len(data.shape) < 2:
raise UnsupportedNumberDimsError(
f"Number of dims must be 2 or 3. Got {len(data.shape)}."
)
elif len(data.shape) == 3:
nbands = data.shape[0]
else:
nbands = 1
# Add extra axis for iteration purposes
data = np.expand_dims(data, axis=0)
trans = REGION_TO_TRANS[args.region]
if args.mask is not None:
mask = trans(np.load(args.mask).astype(bool))
data[..., mask] = args.missing_value
crs = eg.GRID_NAME_TO_V1_PROJ[eg.ML]
x, y = [
trans(xi)
for xi in eg.v1_lonlat_to_meters(*eg.v1_get_full_grid_lonlat(eg.ML))
]
x = x[0]
y = y[:, 0]
xres = (x[-1] - x[0]) / len(x)
yres = (y[-1] - y[0]) / len(y)
t = Affine.translation(x[0], y[0]) * Affine.scale(xres, yres)
ds = rio.open(
args.out_file,
"w",
driver="GTiff",
height=data.shape[1],
width=data.shape[2],
count=nbands,
dtype=args.type,
crs=crs.srs,
transform=t,
compress="lzw",
nodata=args.missing_value,
)
for i, band in enumerate(data):
ds.write(band, i + 1)
ds.close()
def exec(self) -> dict:
data_array = self.ndarray[self.data_attr].values
x_min = min(self.ndarray.coords["X"])
x_max = max(self.ndarray.coords["X"])
y_min = min(self.ndarray.coords["Y"])
y_max = max(self.ndarray.coords["Y"])
x_res = (x_max - x_min) / len(self.ndarray.coords["X"])
y_res = (y_max - y_min) / len(self.ndarray.coords["Y"])
transform = Affine.translation(
x_min - x_res / 2, y_min - y_res / 2) * Affine.scale(x_res, y_res)
if self.is_multiple_files:
os.makedirs(self.output_file, exist_ok=True)
for i, time in enumerate(self.ndarray.coords["time"].values):
with rasterio.open(
str(self.output_file /
f"{time.replace(':', '-')}.tif"),
'w',
driver='GTiff',
height=data_array.shape[2],
width=data_array.shape[1],
count=1,
dtype=data_array.dtype,
crs='+init=epsg:4326',
transform=transform,
) as dst:
array = data_array[i].transpose()
dst.write(np.expand_dims(array, 0))
else:
with rasterio.open(
str(self.output_file),
'w',
driver='GTiff',
height=data_array.shape[2],
width=data_array.shape[1],
count=12,
dtype=data_array.dtype,
crs='+init=epsg:4326',
transform=transform,
) as dst:
dst.write(data_array.swapaxes(1, 2))
return {"result": True}
def test_instantiate_scene(self):
self.assertEqual(self.l7.mtl['L1_METADATA_FILE']['PRODUCT_METADATA']['FILE_NAME_BAND_1'],
'LE07_L1TP_039028_20100702_20160915_01_T1_B1.TIF')
self.assertEqual(self.l7.utm_zone, 12)
self.assertEqual(self.l7.ex_atm_irrad, (1970.0, 1842.0, 1547.0, 1044.0,
255.700, np.nan, 82.06, 1369.00))
self.assertEqual(self.l7.rasterio_geometry['height'], 727)
self.assertEqual(self.l7.rasterio_geometry['driver'], 'GTiff')
self.assertEqual(self.l7.rasterio_geometry['dtype'], 'uint8')
self.assertEqual(self.l7.rasterio_geometry['transform'], Affine(30.0, 0.0, 367035.0,
0.0, -30.0, 5082585.0))
def test_instantiate_scene(self):
l8 = Landsat8(self.dirname_cloud)
self.assertEqual(l8.mtl['L1_METADATA_FILE']['PRODUCT_METADATA']['FILE_NAME_BAND_1'],
'LC80400282014193LGN00_B1.TIF')
self.assertEqual(l8.utm_zone, 12)
self.assertEqual(l8.rasterio_geometry['height'], 727)
self.assertEqual(l8.rasterio_geometry['driver'], 'GTiff')
self.assertEqual(l8.rasterio_geometry['dtype'], 'uint16')
self.assertEqual(l8.rasterio_geometry['transform'], Affine(30.0, 0.0, 367035.0,
0.0, -30.0, 5082585.0))
def pad(array, transform, pad_width, mode=None, **kwargs):
"""Returns a padded array and shifted affine transform matrix.
Array is padded using `numpy.pad()`."""
import numpy
transform = guard_transform(transform)
padded_array = numpy.pad(array, pad_width, mode, **kwargs)
padded_trans = list(transform)
padded_trans[2] -= pad_width*padded_trans[0]
padded_trans[5] -= pad_width*padded_trans[4]
return padded_array, Affine(*padded_trans[:6])
def write_geotiff(raster, origin, size, fpath):
"""Writes the interpolated TIN-linear and Laplace rasters
to disk using the GeoTIFF format. The header is based on
the raster array and a manual definition of the coordinate
system and an identity affine transform.
"""
import rasterio
from rasterio.transform import Affine
transform = (Affine.translation(origin[0], origin[1])
* Affine.scale(size, size))
with rasterio.Env():
with rasterio.open(fpath, 'w', driver = 'GTiff',
height = raster.shape[0],
width = raster.shape[1],
count = 1,
dtype = rasterio.float32,
crs='EPSG:28992',
transform = transform
) as out_file:
out_file.write(raster.astype(rasterio.float32), 1)
def test_untiled_dataset_blocksize(tmp_path):
"""Blocksize is not relevant to untiled datasets (see #1689)"""
pytest.importorskip("pathlib")
tmp_file = tmp_path / "test.tif"
with rasterio.open(
tmp_file, "w", driver="GTiff", count=1, height=13, width=13, dtype="uint8", crs="epsg:3857",
transform=Affine.identity(), blockxsize=256, blockysize=256) as dataset:
pass
with rasterio.open(tmp_file) as dataset:
assert not dataset.profile["tiled"]
assert dataset.shape == (13, 13)
def __call__(self):
with rasterio.open(input) as src:
img = None
msk = None
if band:
if sampling == 1:
img = src.read(bidx, masked=False)
transform = src.affine
# Decimate the band.
else:
img = numpy.zeros(
(src.height//sampling, src.width//sampling),
dtype=src.dtypes[src.indexes.index(bidx)])
img = src.read(bidx, img, masked=False)
transform = src.affine * Affine.scale(float(sampling))
if as_mask:
tmp = numpy.ones_like(img, 'uint8') * 255
tmp[img == 0] = 0
img = tmp
msk = tmp
if not band or not with_nodata:
if sampling == 1:
msk = src.read_masks(bidx)
if bidx is None:
msk = numpy.logical_or.reduce(msk).astype('uint8')
transform = src.affine
# Decimate the mask.
else:
msk_shape = src.height//sampling, src.width//sampling
if bidx is None:
msk = numpy.zeros(
(src.count,) + msk_shape, 'uint8')
else:
msk = numpy.zeros(msk_shape, 'uint8')
msk = src.read_masks(bidx, msk)
if bidx is None:
msk = numpy.logical_or.reduce(msk).astype('uint8')
transform = src.affine * Affine.scale(float(sampling))
bounds = src.bounds
xs = [bounds[0], bounds[2]]
ys = [bounds[1], bounds[3]]
if projection == 'geographic':
xs, ys = rasterio.warp.transform(
src.crs, {'init': 'epsg:4326'}, xs, ys)
if precision >= 0:
xs = [round(v, precision) for v in xs]
ys = [round(v, precision) for v in ys]
self._xs = xs
self._ys = ys
kwargs = {'transform': transform}
# Default is to exclude nodata features.
if msk is not None:
kwargs['mask'] = msk #(msk > 0)
if img is None:
img = msk
for g, i in rasterio.features.shapes(img, **kwargs):
if projection == 'geographic':
g = rasterio.warp.transform_geom(
src.crs, 'EPSG:4326', g,
antimeridian_cutting=True, precision=precision)
xs, ys = zip(*coords(g))
yield {
'type': 'Feature',
'id': str(i),
'properties': {
'val': i, 'filename': os.path.basename(src.name)
},
'bbox': [min(xs), min(ys), max(xs), max(ys)],
'geometry': g
}
def __call__(self):
with rasterio.open(input) as src:
if bidx is not None and bidx > src.count:
raise ValueError('bidx is out of range for raster')
img = None
msk = None
# Adjust transforms.
transform = src.affine
if sampling > 1:
# Decimation of the raster produces a georeferencing
# shift that we correct with a translation.
transform *= Affine.translation(
src.width%sampling, src.height%sampling)
# And follow by scaling.
transform *= Affine.scale(float(sampling))
# Most of the time, we'll use the valid data mask.
# We skip reading it if we're extracting every possible
# feature (even invalid data features) from a band.
if not band or (band and not as_mask and not with_nodata):
if sampling == 1:
msk = src.read_masks(bidx)
else:
msk_shape = (
src.height//sampling, src.width//sampling)
if bidx is None:
msk = numpy.zeros(
(src.count,) + msk_shape, 'uint8')
else:
msk = numpy.zeros(msk_shape, 'uint8')
msk = src.read_masks(bidx, msk)
if bidx is None:
msk = numpy.logical_or.reduce(msk).astype('uint8')
# Possibly overidden below.
img = msk
# Read the band data unless the --mask option is given.
if band:
if sampling == 1:
img = src.read(bidx, masked=False)
else:
img = numpy.zeros(
(src.height//sampling, src.width//sampling),
dtype=src.dtypes[src.indexes.index(bidx)])
img = src.read(bidx, img, masked=False)
# If --as-mask option was given, convert the image
# to a binary image. This reduces the number of shape
# categories to 2 and likely reduces the number of
# shapes.
if as_mask:
tmp = numpy.ones_like(img, 'uint8') * 255
tmp[img == 0] = 0
img = tmp
if not with_nodata:
msk = tmp
# Transform the raster bounds.
bounds = src.bounds
xs = [bounds[0], bounds[2]]
ys = [bounds[1], bounds[3]]
if projection == 'geographic':
xs, ys = rasterio.warp.transform(
src.crs, {'init': 'epsg:4326'}, xs, ys)
if precision >= 0:
xs = [round(v, precision) for v in xs]
ys = [round(v, precision) for v in ys]
self._xs = xs
self._ys = ys
# Prepare keyword arguments for shapes().
kwargs = {'transform': transform}
if not with_nodata:
kwargs['mask'] = msk
src_basename = os.path.basename(src.name)
# Yield GeoJSON features.
for i, (g, val) in enumerate(
rasterio.features.shapes(img, **kwargs)):
if projection == 'geographic':
g = rasterio.warp.transform_geom(
src.crs, 'EPSG:4326', g,
antimeridian_cutting=True, precision=precision)
xs, ys = zip(*coords(g))
yield {
'type': 'Feature',
'id': "{0}:{1}".format(src_basename, i),
'properties': {
'val': val, 'filename': src_basename
},
'bbox': [min(xs), min(ys), max(xs), max(ys)],
'geometry': g
}
def merge_rgba_tool(sources, outtif, bounds=None, res=None, precision=7,
creation_options={}):
"""A windowed, top-down approach to merging.
For each block window, it loops through the sources,
reads the corresponding source window until the block
is filled with data or we run out of sources.
Uses more disk IO but is faster* and
consumes significantly less memory
* The read efficiencies comes from using
RGBA tifs where we can assume band 4 is the sole
determinant of nodata. This avoids the use of
expensive masked reads but, of course, limits
what data can used. Hence merge_rgba.
"""
first = sources[0]
first_res = first.res
dtype = first.dtypes[0]
profile = first.profile
# Extent from option or extent of all inputs.
if bounds:
dst_w, dst_s, dst_e, dst_n = bounds
else:
# scan input files.
# while we're at it, validate assumptions about inputs
xs = []
ys = []
for src in sources:
left, bottom, right, top = src.bounds
xs.extend([left, right])
ys.extend([bottom, top])
if src.profile['count'] != 4: # TODO, how to test for alpha?
raise ValueError("Inputs must be 4-band RGBA rasters")
dst_w, dst_s, dst_e, dst_n = min(xs), min(ys), max(xs), max(ys)
logger.debug("Output bounds: %r", (dst_w, dst_s, dst_e, dst_n))
output_transform = Affine.translation(dst_w, dst_n)
logger.debug("Output transform, before scaling: %r", output_transform)
# Resolution/pixel size.
if not res:
res = first_res
elif not np.iterable(res):
res = (res, res)
elif len(res) == 1:
res = (res[0], res[0])
output_transform *= Affine.scale(res[0], -res[1])
logger.debug("Output transform, after scaling: %r", output_transform)
# Compute output array shape. We guarantee it will cover the output
# bounds completely.
output_width = int(math.ceil((dst_e - dst_w) / res[0]))
output_height = int(math.ceil((dst_n - dst_s) / res[1]))
# Adjust bounds to fit.
dst_e, dst_s = output_transform * (output_width, output_height)
logger.debug("Output width: %d, height: %d", output_width, output_height)
logger.debug("Adjusted bounds: %r", (dst_w, dst_s, dst_e, dst_n))
profile['transform'] = output_transform
profile['height'] = output_height
profile['width'] = output_width
profile['nodata'] = None # rely on alpha mask
# Creation opts
profile.update(creation_options)
# create destination file
with rasterio.open(outtif, 'w', **profile) as dstrast:
for idx, dst_window in dstrast.block_windows():
left, bottom, right, top = dstrast.window_bounds(dst_window)
blocksize = ((dst_window[0][1] - dst_window[0][0]) *
(dst_window[1][1] - dst_window[1][0]))
# initialize array destined for the block
dst_count = first.count
dst_rows, dst_cols = tuple(b - a for a, b in dst_window)
dst_shape = (dst_count, dst_rows, dst_cols)
logger.debug("Temp shape: %r", dst_shape)
dstarr = np.zeros(dst_shape, dtype=dtype)
# Read up srcs until
# a. everything is data; i.e. no nodata
# b. no sources left
for src in sources:
# The full_cover behavior is problematic here as it includes
# extra pixels along the bottom right when the sources are
# slightly misaligned
#
# src_window = get_window(left, bottom, right, top,
# src.transform, precision=precision)
#
# With rio merge this just adds an extra row, but when the
# imprecision occurs at each block, you get artifacts
# Alternative, custom get_window using rounding
window_start = rowcol(
src.transform, left, top, op=round, precision=precision)
window_stop = rowcol(
src.transform, right, bottom, op=round, precision=precision)
src_window = tuple(zip(window_start, window_stop))
temp = np.zeros(dst_shape, dtype=dtype)
temp = src.read(out=temp, window=src_window,
boundless=True, masked=False)
# pixels without data yet are available to write
write_region = np.logical_and(
(dstarr[3] == 0), # 0 is nodata
(temp[3] != 0))
np.copyto(dstarr, temp, where=write_region)
# check if dest has any nodata pixels available
if np.count_nonzero(dstarr[3]) == blocksize:
break
dstrast.write(dstarr, window=dst_window)
return output_transform
def test_open_specific_driver_with_options():
"""Open a GeoTIFF with the GTiff driver and GEOREF_SOURCES option"""
with rasterio.open(
'tests/data/RGB.byte.tif', driver='GTiff', GEOREF_SOURCES='NONE') as src:
assert src.transform == Affine.identity()
def pixel_to_world_coordinates(self,
raster_data=None,
filter_no_data_value=True,
no_data_value=0,
band_number=1):
"""
Map the pixel coordinates to world coordinates. The affine transformation matrix is used for this purpose.
The convention is to reference the pixel corner. To reference the pixel center instead, we translate each pixel by 50%.
The "no value" pixels (cells) can be filtered out.
A dataset's pixel coordinate system has its origin at the "upper left" (imagine it displayed on your screen).
Column index increases to the right, and row index increases downward. The mapping of these coordinates to
"world" coordinates in the dataset's reference system is done with an affine transformation matrix.
param string raster_data: the raster data (2-dimensional array) to translate to world coordinates. If not provided, \
it tries to load existing rasterized data about the RasterEnvironmentalLayer.
:param int no_data_value: The pixel values depicting non-burned cells. Default is 0.
:param bool filter_no_data_value: Whether to filter-out the no-data pixel values. Default is true. If set to \
false, all pixels in a 2-dimensional array will be converted to world coordinates. Typically this option is used \
to get a "base" map of the coordinates of all pixels in an image (map).
:returns: A tuple of numpy ndarrays. The first array contains the latitude values for each \
non-zero cell, the second array contains the longitude values for each non-zero cell.
TODO: document raster-affine
:rtype: tuple(np.ndarray, np.ndarray)
"""
if raster_data is None:
logger.info("No raster data provided, attempting to load default...")
try:
raster_data = self.load_data(self.file_path).read(band_number) # we work on one layer, the first
logger.info("Succesfully loaded existing raster data from %s." % self.file_path)
except AttributeError as e:
logger.error("Could not open raster file. %s " % str(e))
raise AttributeError(e)
logger.info("Transforming to world coordinates...")
# first get the original Affine transformation matrix
if hasattr(self, "raster_affine"):
T0 = self.raster_affine
else: # default from arguments
# try to deduce it
logger.info("No Affine translation defined for this layer, trying to deduce it.")
x_min, y_min, x_max, y_max = -180, -90, 180, 90
pixel_size = (x_max - x_min) / raster_data.shape[1]
if pixel_size != (y_max - y_min) / raster_data.shape[0]:
logger.error("Could not deduce Affine transformation...possibly the pixel is not a square.")
return
T0 = Affine(pixel_size, 0.0, x_min, 0.0, -pixel_size, y_max)
# convert it to gdal format (it is otherwise flipped)
T0 = Affine(*reversed(T0.to_gdal()))
logger.debug("Affine transformation T0:\n %s " % (T0,))
# shift by 50% to get the pixel center
T1 = T0 * Affine.translation(0.5, 0.5)
# apply the shift, filtering out no_data_value values
logger.debug("Raster data shape: %s " % (raster_data.shape,))
logger.debug("Affine transformation T1:\n %s " % (T1,))
if filter_no_data_value:
logger.info("Filtering out no_data pixels.")
raster_data = np.where(raster_data != no_data_value, raster_data, np.nan)
coordinates = (T1 * np.where(~np.isnan(raster_data)))
else:
coordinates = (T1 * np.where(raster_data))
logger.info("Transformation to world coordinates completed.")
return coordinates
def rasterize(self, raster_file=None,
pixel_size=None,
all_touched=False,
no_data_value=0,
default_value=1,
crs=None,
cropped=False,
classifier_column=None,
*args, **kwargs):
"""
Rasterize (burn) the environment rangemaps (geometrical shapes) into pixels (cells), i.e., a 2-dimensional image array
of type numpy ndarray. Uses the `Rasterio <https://mapbox.github.io/rasterio/_modules/rasterio/features.html>`_ library
for this purpose. All the shapes from the ``VectorEnvironmentalLayer`` object data are burned in a single *band* of the image.
Rasterio datasets can generally have one or more bands, or layers. Following the GDAL convention, these are indexed starting with 1.
:param string raster_file: The full path to the targed GeoTIFF raster file (including the directory and filename in one string).
:param int pixel_size: The size of the pixel in degrees, i.e., the resolution to use for rasterizing.
:param bool all_touched: If true, all pixels touched by geometries, will be burned in. If false, only pixels \
whose center is within the polygon or that are selected by Bresenham's line algorithm, will be burned in.
:param int no_data_value: Used as value of the pixels which are not burned in. Default is 0.
:param int default_value: Used as value of the pixels which are burned in. Default is 1.
:param crs: The Coordinate Reference System to use. Default is "ESPG:4326"
:param bool cropped: If true, the resulting pixel array (image) is cropped to the region borders, which contain \
the burned pixels (i.e., an envelope within the range). Otherwise, a "global world map" is used, i.e., the boundaries \
are set to (-180, -90, 180, 90) for the resulting array.
:returns: Rasterio RasterReader file object which can be used to read individual bands from the raster file.
:rtype: rasterio._io.RasterReader
"""
if not (pixel_size or raster_file):
raise AttributeError("Please provide pixel_size and a target raster_file.")
if not hasattr(self, 'data_full'):
raise AttributeError("You have not loaded the data.")
if crs is None:
crs = {'init': "EPSG:4326"}
# crop to the boundaries of the shape?
if cropped:
# cascaded_union_geometry = shapely.ops.cascaded_union(self.data_full.geometry)
x_min, y_min, x_max, y_max = self.data_full.geometry.total_bounds
# else global map
else:
x_min, y_min, x_max, y_max = -180, -90, 180, 90
x_res = int((x_max - x_min) / pixel_size)
y_res = int((y_max - y_min) / pixel_size)
logger.info("Will rasterize using pixel_size=%s, all_touched=%s, no_data_value=%s, fill_value=%s "
% (pixel_size, all_touched, no_data_value, default_value))
transform = Affine.translation(x_min, y_max) * Affine.scale(pixel_size, -pixel_size)
if classifier_column:
logger.info("Will rasterize using classifier: %s." % classifier_column)
classifier_categories = self.data_full[classifier_column].unique()
stacked_layers = []
for category_name in classifier_categories:
if category_name:
logger.info("Rasterizing category %s " % category_name)
result = features.rasterize(self.data_full.geometry[self.data_full[classifier_column] == category_name],
transform=transform,
out_shape=(y_res, x_res),
all_touched=all_touched,
fill=no_data_value,
default_value=default_value
)
stacked_layers.append(result)
stacked_layers = np.stack(stacked_layers)
for i, band in enumerate(stacked_layers, 1):
with rasterio.open(raster_file, 'w', driver='GTiff', width=x_res, height=y_res,
count=stacked_layers.shape[0],
dtype=np.uint8,
nodata=no_data_value,
transform=transform,
crs=crs) as out:
out.write(band.astype(np.uint8), indexes=i)
result_final = stacked_layers
else:
logger.info("Will rasterize everything on a single band.")
result_final = features.rasterize(self.data_full.geometry,
transform=transform,
out_shape=(y_res, x_res),
all_touched=all_touched,
fill=no_data_value,
default_value=default_value
)
with rasterio.open(raster_file, 'w', driver='GTiff', width=x_res, height=y_res,
count=1,
dtype=np.uint8,
nodata=no_data_value,
transform=transform,
crs=crs) as out:
out.write(result_final.astype(np.uint8), indexes=1)
out.close()
logger.info("RASTERIO: Data rasterized into file %s " % raster_file)
logger.info("RASTERIO: Resolution: x_res={0} y_res={1}".format(x_res, y_res))
self.raster_file = raster_file
self.raster_affine = transform
self.stacked_layers = stacked_layers
return result_final
def merge(sources, bounds=None, res=None, nodata=None, precision=7):
"""Copy valid pixels from input files to an output file.
All files must have the same number of bands, data type, and
coordinate reference system.
Input files are merged in their listed order using the reverse
painter's algorithm. If the output file exists, its values will be
overwritten by input values.
Geospatial bounds and resolution of a new output file in the
units of the input file coordinate reference system may be provided
and are otherwise taken from the first input file.
Parameters
----------
sources: list of source datasets
Open rasterio RasterReader objects to be merged.
bounds: tuple, optional
Bounds of the output image (left, bottom, right, top).
If not set, bounds are determined from bounds of input rasters.
res: tuple, optional
Output resolution in units of coordinate reference system. If not set,
the resolution of the first raster is used. If a single value is passed,
output pixels will be square.
nodata: float, optional
nodata value to use in output file. If not set, uses the nodata value
in the first input raster.
Returns
-------
dest: numpy ndarray
Contents of all input rasters in single array.
out_transform: affine object
Information for mapping pixel coordinates in `dest` to another
coordinate system
"""
first = sources[0]
first_res = first.res
nodataval = first.nodatavals[0]
dtype = first.dtypes[0]
# Extent from option or extent of all inputs.
if bounds:
dst_w, dst_s, dst_e, dst_n = bounds
else:
# scan input files.
xs = []
ys = []
for src in sources:
left, bottom, right, top = src.bounds
xs.extend([left, right])
ys.extend([bottom, top])
dst_w, dst_s, dst_e, dst_n = min(xs), min(ys), max(xs), max(ys)
logger.debug("Output bounds: %r", (dst_w, dst_s, dst_e, dst_n))
output_transform = Affine.translation(dst_w, dst_n)
logger.debug("Output transform, before scaling: %r", output_transform)
# Resolution/pixel size.
if not res:
res = first_res
elif not np.iterable(res):
res = (res, res)
elif len(res) == 1:
res = (res[0], res[0])
output_transform *= Affine.scale(res[0], -res[1])
logger.debug("Output transform, after scaling: %r", output_transform)
# Compute output array shape. We guarantee it will cover the output
# bounds completely.
output_width = int(math.ceil((dst_e - dst_w) / res[0]))
output_height = int(math.ceil((dst_n - dst_s) / res[1]))
# Adjust bounds to fit.
dst_e, dst_s = output_transform * (output_width, output_height)
logger.debug("Output width: %d, height: %d", output_width, output_height)
logger.debug("Adjusted bounds: %r", (dst_w, dst_s, dst_e, dst_n))
# create destination array
dest = np.zeros((first.count, output_height, output_width), dtype=dtype)
if nodata is not None:
nodataval = nodata
logger.debug("Set nodataval: %r", nodataval)
if nodataval is not None:
# Only fill if the nodataval is within dtype's range.
inrange = False
if np.dtype(dtype).kind in ('i', 'u'):
info = np.iinfo(dtype)
inrange = (info.min <= nodataval <= info.max)
elif np.dtype(dtype).kind == 'f':
info = np.finfo(dtype)
inrange = (info.min <= nodataval <= info.max)
if inrange:
dest.fill(nodataval)
else:
warnings.warn(
"Input file's nodata value, %s, is beyond the valid "
"range of its data type, %s. Consider overriding it "
"using the --nodata option for better results." % (
nodataval, dtype))
else:
nodataval = 0
for src in sources:
# Real World (tm) use of boundless reads.
# This approach uses the maximum amount of memory to solve the problem.
# Making it more efficient is a TODO.
# 1. Compute spatial intersection of destination and source.
src_w, src_s, src_e, src_n = src.bounds
int_w = src_w if src_w > dst_w else dst_w
int_s = src_s if src_s > dst_s else dst_s
int_e = src_e if src_e < dst_e else dst_e
int_n = src_n if src_n < dst_n else dst_n
# 2. Compute the source window.
src_window = get_window(
int_w, int_s, int_e, int_n, src.affine, precision=precision)
logger.debug("Src %s window: %r", src.name, src_window)
# 3. Compute the destination window.
dst_window = get_window(
int_w, int_s, int_e, int_n, output_transform, precision=precision)
logger.debug("Dst window: %r", dst_window)
# 4. Initialize temp array.
tcount = first.count
trows, tcols = tuple(b - a for a, b in dst_window)
temp_shape = (tcount, trows, tcols)
logger.debug("Temp shape: %r", temp_shape)
temp = np.zeros(temp_shape, dtype=dtype)
temp = src.read(out=temp, window=src_window, boundless=False,
masked=True)
# 5. Copy elements of temp into dest.
roff, coff = dst_window[0][0], dst_window[1][0]
region = dest[:, roff:roff + trows, coff:coff + tcols]
np.copyto(
region, temp,
where=np.logical_and(region == nodataval, temp.mask == False))
return dest, output_transform
def dataset_features(
src,
bidx=None,
sampling=1,
band=True,
as_mask=False,
with_nodata=False,
geographic=True,
precision=-1):
"""Yield GeoJSON features for the dataset
The geometries are polygons bounding contiguous regions of the same raster value.
Parameters
----------
src: Rasterio Dataset
bidx: int
band index
sampling: int (DEFAULT: 1)
Inverse of the sampling fraction; a value of 10 decimates
band: boolean (DEFAULT: True)
extract features from a band (True) or a mask (False)
as_mask: boolean (DEFAULT: False)
Interpret band as a mask and output only one class of valid data shapes?
with_nodata: boolean (DEFAULT: False)
Include nodata regions?
geographic: str (DEFAULT: True)
Output shapes in EPSG:4326? Otherwise use the native CRS.
precision: int (DEFAULT: -1)
Decimal precision of coordinates. -1 for full float precision output
Yields
------
GeoJSON-like Feature dictionaries for shapes found in the given band
"""
if bidx is not None and bidx > src.count:
raise ValueError('bidx is out of range for raster')
img = None
msk = None
# Adjust transforms.
transform = src.transform
if sampling > 1:
# Determine the target shape (to decimate)
shape = (int(math.ceil(src.height / sampling)),
int(math.ceil(src.width / sampling)))
# Calculate independent sampling factors
x_sampling = src.width / shape[1]
y_sampling = src.height / shape[0]
# Decimation of the raster produces a georeferencing
# shift that we correct with a translation.
transform *= Affine.translation(
src.width % x_sampling, src.height % y_sampling)
# And follow by scaling.
transform *= Affine.scale(x_sampling, y_sampling)
# Most of the time, we'll use the valid data mask.
# We skip reading it if we're extracting every possible
# feature (even invalid data features) from a band.
if not band or (band and not as_mask and not with_nodata):
if sampling == 1:
msk = src.read_masks(bidx)
else:
msk_shape = shape
if bidx is None:
msk = np.zeros(
(src.count,) + msk_shape, 'uint8')
else:
msk = np.zeros(msk_shape, 'uint8')
msk = src.read_masks(bidx, msk)
if bidx is None:
msk = np.logical_or.reduce(msk).astype('uint8')
# Possibly overridden below.
img = msk
# Read the band data unless the --mask option is given.
if band:
if sampling == 1:
img = src.read(bidx, masked=False)
else:
img = np.zeros(
shape,
dtype=src.dtypes[src.indexes.index(bidx)])
img = src.read(bidx, img, masked=False)
# If as_mask option was given, convert the image
# to a binary image. This reduces the number of shape
# categories to 2 and likely reduces the number of
# shapes.
if as_mask:
tmp = np.ones_like(img, 'uint8') * 255
tmp[img == 0] = 0
img = tmp
if not with_nodata:
msk = tmp
# Prepare keyword arguments for shapes().
kwargs = {'transform': transform}
if not with_nodata:
kwargs['mask'] = msk
src_basename = os.path.basename(src.name)
# Yield GeoJSON features.
for i, (g, val) in enumerate(
rasterio.features.shapes(img, **kwargs)):
if geographic:
g = warp.transform_geom(
src.crs, 'EPSG:4326', g,
antimeridian_cutting=True, precision=precision)
xs, ys = zip(*coords(g))
yield {
'type': 'Feature',
'id': "{0}:{1}".format(src_basename, i),
'properties': {
'val': val,
'filename': src_basename
},
'bbox': [min(xs), min(ys), max(xs), max(ys)],
'geometry': g
}
def affine_from_corner(ulx, uly, dx, dy):
return Affine.translation(ulx, uly)*Affine.scale(dx, -dy)
def merge(ctx, files, driver, bounds, res, nodata):
"""Copy valid pixels from input files to an output file.
All files must have the same number of bands, data type, and
coordinate reference system.
Input files are merged in their listed order using the reverse
painter's algorithm. If the output file exists, its values will be
overwritten by input values.
Geospatial bounds and resolution of a new output file in the
units of the input file coordinate reference system may be provided
and are otherwise taken from the first input file.
"""
import numpy as np
verbosity = (ctx.obj and ctx.obj.get('verbosity')) or 1
logger = logging.getLogger('rio')
try:
with rasterio.drivers(CPL_DEBUG=verbosity>2):
output = files[-1]
files = files[:-1]
with rasterio.open(files[0]) as first:
first_res = first.res
kwargs = first.meta
kwargs.pop('affine')
nodataval = first.nodatavals[0]
dtype = first.dtypes[0]
if os.path.exists(output):
# TODO: prompt user to update existing file (-i option) like:
# overwrite b.tif? (y/n [n]) n
# not overwritten
dst = rasterio.open(output, 'r+')
nodataval = dst.nodatavals[0]
dtype = dst.dtypes[0]
dest = np.zeros((dst.count,) + dst.shape, dtype=dtype)
else:
# Create new output file.
# Extent from option or extent of all inputs.
if not bounds:
# scan input files.
xs = []
ys = []
for f in files:
with rasterio.open(f) as src:
left, bottom, right, top = src.bounds
xs.extend([left, right])
ys.extend([bottom, top])
bounds = min(xs), min(ys), max(xs), max(ys)
output_transform = Affine.translation(bounds[0], bounds[3])
# Resolution/pixel size.
if not res:
res = first_res
output_transform *= Affine.scale(res[0], -res[1])
# Dataset shape.
output_width = int(math.ceil((bounds[2]-bounds[0])/res[0]))
output_height = int(math.ceil((bounds[3]-bounds[1])/res[1]))
kwargs['driver'] == driver
kwargs['transform'] = output_transform
kwargs['width'] = output_width
kwargs['height'] = output_height
logger.debug("Kwargs: %r", kwargs)
logger.debug("bounds: %r", bounds)
logger.debug("Res: %r", res)
dst = rasterio.open(output, 'w', **kwargs)
dest = np.zeros((first.count, output_height, output_width),
dtype=dtype)
logger.debug("In merge, dest shape: %r", dest.shape)
if nodata is not None:
nodataval = nodata
if nodataval is not None:
# Only fill if the nodataval is within dtype's range.
inrange = False
if np.dtype(dtype).kind in ('i', 'u'):
info = np.iinfo(dtype)
inrange = (info.min <= nodataval <= info.max)
elif np.dtype(dtype).kind == 'f':
info = np.finfo(dtype)
inrange = (info.min <= nodataval <= info.max)
if inrange:
dest.fill(nodataval)
else:
warnings.warn(
"Input file's nodata value, %s, is beyond the valid "
"range of its data type, %s. Consider overriding it "
"using the --nodata option for better results." % (
nodataval, dtype))
else:
nodataval = 0
dst_w, dst_s, dst_e, dst_n = dst.bounds
for fname in reversed(files):
with rasterio.open(fname) as src:
# Real World (tm) use of boundless reads.
# This approach uses the maximum amount of memory to solve
# the problem. Making it more efficient is a TODO.
# 1. Compute spatial intersection of destination
# and source.
src_w, src_s, src_e, src_n = src.bounds
int_w = src_w if src_w > dst_w else dst_w
int_s = src_s if src_s > dst_s else dst_s
int_e = src_e if src_e < dst_e else dst_e
int_n = src_n if src_n < dst_n else dst_n
# 2. Compute the source window.
src_window = src.window(int_w, int_s, int_e, int_n)
# 3. Compute the destination window.
dst_window = dst.window(int_w, int_s, int_e, int_n)
# 4. Initialize temp array.
temp = np.zeros(
(first.count,) + tuple(b - a for a, b in dst_window),
dtype=dtype)
temp = src.read(
out=temp,
window=src_window,
boundless=False,
masked=True)
# 5. Copy elements of temp into dest.
roff, coff = dst.index(int_w, int_n)
h, w = temp.shape[-2:]
region = dest[:,roff:roff+h,coff:coff+w]
np.copyto(region, temp,
where=np.logical_and(
region==nodataval, temp.mask==False))
if dst.mode == 'r+':
temp = dst.read(masked=True)
np.copyto(dest, temp,
where=np.logical_and(
dest==nodataval, temp.mask==False))
dst.write(dest)
dst.close()
sys.exit(0)
except Exception:
logger.exception("Failed. Exception caught")
sys.exit(1)